suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-13 11:55:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

moving leading content processing logic into yaml decoder

Browse Source
This commit is contained in:
Mike Farah 2022-10-25 15:35:49 +11:00
parent 4138c3b005
commit 2e588a11a0
36 changed files with 230 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/constant.go
View File

@ -1,6 +1,5 @@
package cmd
var leadingContentPreProcessing = true
var unwrapScalar = true
var writeInplace = false

4
cmd/evaluate_all_command.go
View File

@ -109,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
switch len(args) {
case 0:
if nullInput {
err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "")
err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)
} else {
cmd.Println(cmd.UsageString())
return nil
}
default:
err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
}
completedSuccessfully = err == nil

4
cmd/evalute_sequence_command.go
View File

@ -123,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
switch len(args) {
case 0:
if nullInput {
err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "")
err = streamEvaluator.EvaluateNew(processExpression(expression), printer)
} else {
cmd.Println(cmd.UsageString())
return nil
}
default:
err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
}
completedSuccessfully = err == nil

7
cmd/root.go
View File

@ -59,6 +59,11 @@ yq -P sample.json
"naming conflicts with the default content name, directive name and proc inst prefix. If you need to keep " +
"`+` please set that value explicityly with --xml-attribute-prefix.")
}
//copy preference form global setting
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
},
}
@ -97,7 +102,7 @@ yq -P sample.json
rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")

4
cmd/utils.go
View File

@ -74,7 +74,7 @@ func configureDecoder() (yqlib.Decoder, error) {
return yqlib.NewCSVObjectDecoder('\t'), nil
}
return yqlib.NewYamlDecoder(), nil
return yqlib.NewYamlDecoder(yqlib.ConfiguredYamlPreferences), nil
}
func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@ -105,7 +105,7 @@ func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
case yqlib.TSVOutputFormat:
return yqlib.NewCsvEncoder('\t')
case yqlib.YamlOutputFormat:
return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
case yqlib.XMLOutputFormat:
return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
}

1
examples/small.properties
View File

@ -1 +1,2 @@
# great huh
this.is = a properties file

15
pkg/yqlib/all_at_once_evaluator.go
View File

@ -8,7 +8,7 @@ import (
// A yaml expression evaluator that runs the expression once against all files/nodes in memory.
type Evaluator interface {
EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error
EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@ -46,21 +46,16 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
return context.MatchingNodes, nil
}
func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
fileIndex := 0
firstFileLeadingContent := ""
var allDocuments = list.New()
for _, filename := range filenames {
reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
reader, err := readStream(filename)
if err != nil {
return err
}
if fileIndex == 0 {
firstFileLeadingContent = leadingContent
}
fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
if err != nil {
return err
@ -75,11 +70,9 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
Filename: "",
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
LeadingContent: firstFileLeadingContent,
LeadingContent: "",
}
allDocuments.PushBack(candidateNode)
} else {
allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
}
matches, err := e.EvaluateCandidateNodes(expression, allDocuments)

12
pkg/yqlib/csv_test.go
View File

@ -136,13 +136,13 @@ var csvScenarios = []formatScenario{
func testCSVScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "encode-csv":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
case "encode-tsv":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
case "decode-csv-object":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "decode-tsv-object":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-csv":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
default:
@ -171,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
}
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
)
}
@ -203,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
}
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
)
}

22
pkg/yqlib/decoder_base64.go
View File

@ -19,21 +19,22 @@ func NewBase64Decoder() Decoder {
return &base64Decoder{finished: false, encoding: *base64.StdEncoding}
}
func (dec *base64Decoder) Init(reader io.Reader) {
func (dec *base64Decoder) Init(reader io.Reader) error {
dec.reader = reader
dec.readAnything = false
dec.finished = false
return nil
}
func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
func (dec *base64Decoder) Decode() (*CandidateNode, error) {
if dec.finished {
return io.EOF
return nil, io.EOF
}
base64Reader := base64.NewDecoder(&dec.encoding, dec.reader)
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(base64Reader); err != nil {
return err
return nil, err
}
if buf.Len() == 0 {
dec.finished = true
@ -42,12 +43,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
// otherwise if we've already read some bytes, and now we get
// an empty string, then we are done.
if dec.readAnything {
return io.EOF
return nil, io.EOF
}
}
dec.readAnything = true
rootYamlNode.Kind = yaml.ScalarNode
rootYamlNode.Tag = "!!str"
rootYamlNode.Value = buf.String()
return nil
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: buf.String(),
},
}, nil
}

23
pkg/yqlib/decoder_csv_object.go
View File

@ -19,12 +19,13 @@ func NewCSVObjectDecoder(separator rune) Decoder {
return &csvObjectDecoder{separator: separator}
}
func (dec *csvObjectDecoder) Init(reader io.Reader) {
func (dec *csvObjectDecoder) Init(reader io.Reader) error {
cleanReader, enc := utfbom.Skip(reader)
log.Debugf("Detected encoding: %s\n", enc)
dec.reader = *csv.NewReader(cleanReader)
dec.reader.Comma = dec.separator
dec.finished = false
return nil
}
func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
@ -47,14 +48,14 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
return objectNode
}
func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
if dec.finished {
return io.EOF
return nil, io.EOF
}
headerRow, err := dec.reader.Read()
log.Debugf(": headerRow%v", headerRow)
if err != nil {
return err
return nil, err
}
rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
@ -68,13 +69,13 @@ func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
log.Debugf("Read next contentRow: %v, %v", contentRow, err)
}
if !errors.Is(err, io.EOF) {
return err
return nil, err
}
log.Debugf("finished, contentRow%v", contentRow)
log.Debugf("err: %v", err)
rootYamlNode.Kind = yaml.DocumentNode
rootYamlNode.Content = []*yaml.Node{rootArray}
return nil
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{rootArray},
},
}, nil
}

19
pkg/yqlib/decoder_json.go
View File

@ -16,26 +16,31 @@ func NewJSONDecoder() Decoder {
return &jsonDecoder{}
}
func (dec *jsonDecoder) Init(reader io.Reader) {
func (dec *jsonDecoder) Init(reader io.Reader) error {
dec.decoder = *json.NewDecoder(reader)
return nil
}
func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
var dataBucket orderedMap
log.Debug("going to decode")
err := dec.decoder.Decode(&dataBucket)
if err != nil {
return err
return nil, err
}
node, err := dec.convertToYamlNode(&dataBucket)
if err != nil {
return err
return nil, err
}
rootYamlNode.Kind = yaml.DocumentNode
rootYamlNode.Content = []*yaml.Node{node}
return nil
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{node},
},
}, nil
}
func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {

26
pkg/yqlib/decoder_properties.go
View File

@ -20,9 +20,10 @@ func NewPropertiesDecoder() Decoder {
return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
}
func (dec *propertiesDecoder) Init(reader io.Reader) {
func (dec *propertiesDecoder) Init(reader io.Reader) error {
dec.reader = reader
dec.finished = false
return nil
}
func parsePropKey(key string) []interface{} {
@ -78,22 +79,22 @@ func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, c
return err
}
func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
if dec.finished {
return io.EOF
return nil, io.EOF
}
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(dec.reader); err != nil {
return err
return nil, err
}
if buf.Len() == 0 {
dec.finished = true
return io.EOF
return nil, io.EOF
}
properties, err := properties.LoadString(buf.String())
if err != nil {
return err
return nil, err
}
properties.DisableExpansion = true
@ -109,14 +110,17 @@ func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
for _, key := range properties.Keys() {
if err := dec.applyProperty(properties, context, key); err != nil {
return err
return nil, err
}
}
rootYamlNode.Kind = yaml.DocumentNode
rootYamlNode.Content = []*yaml.Node{rootMap.Node}
dec.finished = true
return nil
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{rootMap.Node},
},
}, nil
}

2
pkg/yqlib/decoder_test.go
View File

@ -23,7 +23,7 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) s
writer := bufio.NewWriter(&output)
if decoder == nil {
decoder = NewYamlDecoder()
decoder = NewYamlDecoder(ConfiguredYamlPreferences)
}
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)

23
pkg/yqlib/decoder_xml.go
View File

@ -25,10 +25,11 @@ func NewXMLDecoder(prefs XmlPreferences) Decoder {
}
}
func (dec *xmlDecoder) Init(reader io.Reader) {
func (dec *xmlDecoder) Init(reader io.Reader) error {
dec.reader = reader
dec.readAnything = false
dec.finished = false
return nil
}
func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
@ -118,32 +119,36 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
return scalar, nil
}
func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
if dec.finished {
return io.EOF
return nil, io.EOF
}
root := &xmlNode{}
// cant use xj - it doesn't keep map order.
err := dec.decodeXML(root)
if err != nil {
return err
return nil, err
}
firstNode, err := dec.convertToYamlNode(root)
if err != nil {
return err
return nil, err
} else if firstNode.Tag == "!!null" {
dec.finished = true
if dec.readAnything {
return io.EOF
return nil, io.EOF
}
}
dec.readAnything = true
rootYamlNode.Kind = yaml.DocumentNode
rootYamlNode.Content = []*yaml.Node{firstNode}
dec.finished = true
return nil
return &CandidateNode{
Node: &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{firstNode},
},
}, nil
}
type xmlNode struct {

6
pkg/yqlib/decoder_yaml.go
View File

@ -13,11 +13,11 @@ import (
type yamlDecoder struct {
decoder yaml.Decoder
// work around of various parsing issues by yaml.v3 with document headers
prefs yamlPreferences
prefs YamlPreferences
leadingContent string
}
func NewYamlDecoder(prefs yamlPreferences) Decoder {
func NewYamlDecoder(prefs YamlPreferences) Decoder {
return &yamlDecoder{prefs: prefs}
}
@ -57,7 +57,7 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
readerToUse := reader
leadingContent := ""
var err error
if dec.leadingContentPreProcessing {
if dec.prefs.LeadingContentPreProcessing {
readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))
if err != nil {
return err

2
pkg/yqlib/doc/operators/comment-operators.md
View File

@ -246,6 +246,7 @@ Note the use of `...` to ensure key nodes are included.
Given a sample.yml file of:
```yaml
# hi
a: cat # comment
# great
b: # key comment
@ -263,6 +264,7 @@ b:
## Get line comment
Given a sample.yml file of:
```yaml
# welcome!
a: cat # meow
# have a great day
```

13
pkg/yqlib/doc/usage/properties.md
View File

@ -9,7 +9,7 @@ Note that empty arrays and maps are not encoded by default.
Given a sample.yml file of:
```yaml
# block comments don't come through
# block comments come through
person: # neither do comments on maps
name: Mike Wazowski # comments on values appear
pets:
@ -25,6 +25,7 @@ yq -o=props sample.yml
```
will output
```properties
# block comments come through
# comments on values appear
person.name = Mike Wazowski
@ -38,7 +39,7 @@ Note that string values with blank characters in them are encapsulated with doub
Given a sample.yml file of:
```yaml
# block comments don't come through
# block comments come through
person: # neither do comments on maps
name: Mike Wazowski # comments on values appear
pets:
@ -54,6 +55,7 @@ yq -o=props --unwrapScalar=false sample.yml
```
will output
```properties
# block comments come through
# comments on values appear
person.name = "Mike Wazowski"
@ -65,7 +67,7 @@ person.food.0 = pizza
## Encode properties: no comments
Given a sample.yml file of:
```yaml
# block comments don't come through
# block comments come through
person: # neither do comments on maps
name: Mike Wazowski # comments on values appear
pets:
@ -91,7 +93,7 @@ Use a yq expression to set the empty maps and sequences to your desired value.
Given a sample.yml file of:
```yaml
# block comments don't come through
# block comments come through
person: # neither do comments on maps
name: Mike Wazowski # comments on values appear
pets:
@ -107,6 +109,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) =
```
will output
```properties
# block comments come through
# comments on values appear
person.name = Mike Wazowski
@ -120,6 +123,7 @@ emptyMap =
## Decode properties
Given a sample.properties file of:
```properties
# block comments come through
# comments on values appear
person.name = Mike Wazowski
@ -145,6 +149,7 @@ person:
## Roundtrip
Given a sample.properties file of:
```properties
# block comments come through
# comments on values appear
person.name = Mike Wazowski

2
pkg/yqlib/doc/usage/xml.md
View File

@ -402,7 +402,7 @@ yq -o=xml '.' sample.yml
```
will output
```xml
<!-- above_cat inline_cat --><cat><!-- above_array inline_array -->
<!-- inline_cat --><cat><!-- above_array inline_array -->
<array>val1<!-- inline_val1 --></array>
<array><!-- above_val2 -->val2<!-- inline_val2 --></array>
</cat><!-- below_cat -->

2
pkg/yqlib/encoder_properties_test.go
View File

@ -14,7 +14,7 @@ func yamlToProps(sampleYaml string, unwrapScalar bool) string {
writer := bufio.NewWriter(&output)
var propsEncoder = NewPropertiesEncoder(unwrapScalar)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}

2
pkg/yqlib/encoder_test.go
View File

@ -14,7 +14,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
writer := bufio.NewWriter(&output)
var jsonEncoder = NewJSONEncoder(indent, false)
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}

10
pkg/yqlib/encoder_yaml.go
View File

@ -13,14 +13,14 @@ import (
type yamlEncoder struct {
indent int
colorise bool
prefs yamlPreferences
prefs YamlPreferences
}
func NewYamlEncoder(indent int, colorise bool, printDocSeparators bool, unwrapScalar bool) Encoder {
func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder {
if indent < 0 {
indent = 0
}
return &yamlEncoder{indent, colorise, printDocSeparators, unwrapScalar}
return &yamlEncoder{indent, colorise, prefs}
}
func (ye *yamlEncoder) CanHandleAliases() bool {
@ -28,7 +28,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool {
}
func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
if ye.printDocSeparators {
if ye.prefs.PrintDocSeparators {
log.Debug("-- writing doc sep")
if err := writeString(writer, "---\n"); err != nil {
return err
@ -75,7 +75,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
if node.Kind == yaml.ScalarNode && ye.unwrapScalar {
if node.Kind == yaml.ScalarNode && ye.prefs.UnwrapScalar {
return writeString(writer, node.Value+"\n")
}

10
pkg/yqlib/json_test.go
View File

@ -262,11 +262,11 @@ func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, true, true))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
}
func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
docs, err := readDocumentWithLeadingContent(jsonString, "sample.json", 0)
docs, err := readDocument(jsonString, "sample.json", 0)
if err != nil {
t.Error(err)
@ -293,12 +293,12 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
func testJSONScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "encode", "decode":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false)), s.description)
case "":
var actual = resultToString(t, decodeJSON(t, s.input))
test.AssertResultWithContext(t, s.expected, actual, s.description)
case "decode-ndjson":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-ndjson":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false)), s.description)
case "roundtrip-multi":
@ -385,7 +385,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJSONEncoder(s.indent, false))))
writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false))))
}
func TestJSONScenarios(t *testing.T) {

2
pkg/yqlib/lexer_participle.go
View File

@ -84,7 +84,7 @@ var participleYqRules = []*participleYqRule{
{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
{"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0},
{"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0},
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},

22
pkg/yqlib/lib.go
View File

@ -21,14 +21,6 @@ func InitExpressionParser() {
}
}
type yamlPreferences struct {
LeadingContentPreProcessing bool
printDocSeparators bool
unwrapScalar bool
}
var YamlPreferences = NewDefaultYamlPreferences()
var log = logging.MustGetLogger("yq-lib")
var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`
@ -256,14 +248,16 @@ func guessTagFromCustomType(node *yaml.Node) string {
}
func parseSnippet(value string) (*yaml.Node, error) {
decoder := NewYamlDecoder()
decoder.Init(strings.NewReader(value))
var dataBucket yaml.Node
err := decoder.Decode(&dataBucket)
if len(dataBucket.Content) == 0 {
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
err := decoder.Init(strings.NewReader(value))
if err != nil {
return nil, err
}
parsedNode, err := decoder.Decode()
if len(parsedNode.Node.Content) == 0 {
return nil, fmt.Errorf("bad data")
}
return dataBucket.Content[0], err
return unwrapDoc(parsedNode.Node), err
}
func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {

6
pkg/yqlib/operator_comments.go
View File

@ -83,6 +83,10 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
log.Debugf("GetComments operator!")
var results = list.New()
yamlPrefs := NewDefaultYamlPreferences()
yamlPrefs.PrintDocSeparators = false
yamlPrefs.UnwrapScalar = false
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
comment := ""
@ -92,7 +96,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
var chompRegexp = regexp.MustCompile(`\n$`)
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
var encoder = NewYamlEncoder(2, false, false, false)
var encoder = NewYamlEncoder(2, false, yamlPrefs)
if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {
return Context{}, err
}

12
pkg/yqlib/operator_datetime.go
View File

@ -4,7 +4,6 @@ import (
"container/list"
"errors"
"fmt"
"strings"
"time"
"gopkg.in/yaml.v3"
@ -54,7 +53,6 @@ func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode
func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
format, err := getStringParamter("format", d, context, expressionNode.RHS)
layout := context.GetDateTimeLayout()
decoder := NewYamlDecoder()
if err != nil {
return Context{}, err
@ -69,19 +67,15 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err)
}
formattedTimeStr := parsedTime.Format(format)
decoder.Init(strings.NewReader(formattedTimeStr))
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
var node *yaml.Node
node, errorReading := parseSnippet(formattedTimeStr)
if errorReading != nil {
log.Debugf("could not parse %v - lets just leave it as a string", formattedTimeStr)
log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading)
node = &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: formattedTimeStr,
}
} else {
node = unwrapDoc(&dataBucket)
}
results.PushBack(candidate.CreateReplacement(node))

14
pkg/yqlib/operator_encoder_decoder.go
View File

@ -21,7 +21,7 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
case TSVOutputFormat:
return NewCsvEncoder('\t')
case YamlOutputFormat:
return NewYamlEncoder(indent, false, true, true)
return NewYamlEncoder(indent, false, ConfiguredYamlPreferences)
case XMLOutputFormat:
return NewXMLEncoder(indent, ConfiguredXMLPreferences)
case Base64OutputFormat:
@ -102,7 +102,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
var decoder Decoder
switch preferences.format {
case YamlInputFormat:
decoder = NewYamlDecoder()
decoder = NewYamlDecoder(ConfiguredYamlPreferences)
case XMLInputFormat:
decoder = NewXMLDecoder(ConfiguredXMLPreferences)
case Base64InputFormat:
@ -121,17 +121,19 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList())
var dataBucket yaml.Node
log.Debugf("got: [%v]", candidate.Node.Value)
decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value))
err := decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value))
if err != nil {
return Context{}, err
}
errorReading := decoder.Decode(&dataBucket)
decodedNode, errorReading := decoder.Decode()
if errorReading != nil {
return Context{}, errorReading
}
//first node is a doc
node := unwrapDoc(&dataBucket)
node := unwrapDoc(decodedNode.Node)
results.PushBack(candidate.CreateReplacement(node))
}

29
pkg/yqlib/operators_test.go
View File

@ -40,21 +40,16 @@ func TestMain(m *testing.M) {
}
func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
return NewPrinter(NewYamlEncoder(indent, colorsEnabled, printDocSeparators, unwrapScalar), NewSinglePrinterWriter(writer))
prefs := NewDefaultYamlPreferences()
prefs.PrintDocSeparators = printDocSeparators
prefs.UnwrapScalar = unwrapScalar
return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer))
}
func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
if err != nil {
return nil, err
}
func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
reader := bufio.NewReader(strings.NewReader(content))
inputs, err := readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder())
if err != nil {
return nil, err
}
inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
return inputs, nil
return readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder(ConfiguredYamlPreferences))
}
func testScenario(t *testing.T, s *expressionScenario) {
@ -67,7 +62,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
inputs := list.New()
if s.document != "" {
inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0)
inputs, err = readDocument(s.document, "sample.yml", 0)
if err != nil {
t.Error(err, s.document, s.expression)
@ -75,7 +70,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
}
if s.document2 != "" {
moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1)
moreInputs, err := readDocument(s.document2, "another.yml", 1)
if err != nil {
t.Error(err, s.document2, s.expression)
return
@ -176,7 +171,7 @@ func formatYaml(yaml string, filename string) string {
panic(err)
}
streamEvaluator := NewStreamEvaluator()
_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "", NewYamlDecoder())
_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -322,13 +317,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
if s.document != "" {
inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0)
inputs, err = readDocument(formattedDoc, "sample.yml", 0)
if err != nil {
t.Error(err, s.document, s.expression)
return
}
if s.document2 != "" {
moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1)
moreInputs, err := readDocument(formattedDoc2, "another.yml", 1)
if err != nil {
t.Error(err, s.document, s.expression)
return

18
pkg/yqlib/printer_test.go
View File

@ -38,7 +38,7 @@ func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -76,7 +76,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -118,7 +118,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -165,7 +165,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -215,7 +215,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -234,7 +234,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -263,7 +263,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -294,7 +294,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) {
panic(err)
}
streamEvaluator := NewStreamEvaluator()
_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n", NewYamlDecoder())
_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}
@ -316,7 +316,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
// when outputing JSON.
printer := NewPrinter(NewJSONEncoder(0, false), NewSinglePrinterWriter(writer))
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder())
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
if err != nil {
panic(err)
}

23
pkg/yqlib/properties_test.go
View File

@ -8,7 +8,7 @@ import (
"github.com/mikefarah/yq/v4/test"
)
const samplePropertiesYaml = `# block comments don't come through
const samplePropertiesYaml = `# block comments come through
person: # neither do comments on maps
name: Mike Wazowski # comments on values appear
pets:
@ -18,7 +18,8 @@ emptyArray: []
emptyMap: []
`
const expectedPropertiesUnwrapped = `# comments on values appear
const expectedPropertiesUnwrapped = `# block comments come through
# comments on values appear
person.name = Mike Wazowski
# comments on array values appear
@ -26,7 +27,8 @@ person.pets.0 = cat
person.food.0 = pizza
`
const expectedPropertiesWrapped = `# comments on values appear
const expectedPropertiesWrapped = `# block comments come through
# comments on values appear
person.name = "Mike Wazowski"
# comments on array values appear
@ -55,7 +57,8 @@ person.pets.0 = cat
person.food.0 = pizza
`
const expectedPropertiesWithEmptyMapsAndArrays = `# comments on values appear
const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through
# comments on values appear
person.name = Mike Wazowski
# comments on array values appear
@ -143,7 +146,7 @@ func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario)
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true))))
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true))))
}
func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
@ -168,7 +171,7 @@ func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false))))
writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false))))
}
func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
@ -193,7 +196,7 @@ func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, true, true))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))))
}
func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {
@ -245,11 +248,11 @@ func TestPropertyScenarios(t *testing.T) {
for _, s := range propertyScenarios {
switch s.scenarioType {
case "":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "encode-wrapped":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewPropertiesEncoder(false)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description)

20
pkg/yqlib/stream_evaluator.go
View File

@ -14,9 +14,9 @@ import (
// Uses less memory than loading all documents and running the expression once, but this cannot process
// cross document expressions.
type StreamEvaluator interface {
Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string, decoder Decoder) (uint, error)
EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error
EvaluateNew(expression string, printer Printer, leadingContent string) error
Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error)
EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
EvaluateNew(expression string, printer Printer) error
}
type streamEvaluator struct {
@ -28,7 +28,7 @@ func NewStreamEvaluator() StreamEvaluator {
return &streamEvaluator{treeNavigator: NewDataTreeNavigator()}
}
func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
node, err := ExpressionParser.ParseExpression(expression)
if err != nil {
return err
@ -38,7 +38,6 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
Filename: "",
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
LeadingContent: leadingContent,
}
inputList := list.New()
inputList.PushBack(candidateNode)
@ -50,15 +49,13 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
return printer.PrintResults(result.MatchingNodes)
}
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
var totalProcessDocs uint
node, err := ExpressionParser.ParseExpression(expression)
if err != nil {
return err
}
var firstFileLeadingContent string
for _, filename := range filenames {
reader, err := readStream(filename)
@ -78,7 +75,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
}
if totalProcessDocs == 0 {
return s.EvaluateNew(expression, printer, firstFileLeadingContent)
return s.EvaluateNew(expression, printer)
}
return nil
@ -87,7 +84,10 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) {
var currentIndex uint
decoder.Init(reader)
err := decoder.Init(reader)
if err != nil {
return 0, err
}
for {
candidateNode, errorReading := decoder.Decode()

31
pkg/yqlib/string_evaluator.go
View File

@ -6,12 +6,10 @@ import (
"errors"
"fmt"
"io"
yaml "gopkg.in/yaml.v3"
)
type StringEvaluator interface {
Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error)
Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error)
}
type stringEvaluator struct {
@ -25,7 +23,7 @@ func NewStringEvaluator() StringEvaluator {
}
}
func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, leadingContentPreProcessing bool, decoder Decoder) (string, error) {
func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {
// Use bytes.Buffer for output of string
out := new(bytes.Buffer)
@ -37,16 +35,18 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
return "", err
}
reader, leadingContent, err := readString(input, leadingContentPreProcessing)
reader, err := readString(input)
if err != nil {
return "", err
}
var currentIndex uint
decoder.Init(reader)
err = decoder.Init(reader)
if err != nil {
return "", err
}
for {
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
candidateNode, errorReading := decoder.Decode()
if errors.Is(errorReading, io.EOF) {
s.fileIndex = s.fileIndex + 1
@ -54,20 +54,9 @@ func (s *stringEvaluator) Evaluate(expression string, input string, encoder Enco
} else if errorReading != nil {
return "", fmt.Errorf("bad input '%v': %w", input, errorReading)
}
candidateNode.Document = currentIndex
candidateNode.FileIndex = s.fileIndex
candidateNode := &CandidateNode{
Document: currentIndex,
Node: &dataBucket,
FileIndex: s.fileIndex,
}
// move document comments into candidate node
// otherwise unwrap drops them.
candidateNode.TrailingContent = dataBucket.FootComment
dataBucket.FootComment = ""
if currentIndex == 0 {
candidateNode.LeadingContent = leadingContent
}
inputList := list.New()
inputList.PushBack(candidateNode)

6
pkg/yqlib/string_evaluator_test.go
View File

@ -18,10 +18,10 @@ func TestStringEvaluator_Evaluate_Nominal(t *testing.T) {
`---` + "\n" +
` - name: jq` + "\n" +
` description: Command-line JSON processor` + "\n"
encoder := NewYamlEncoder(2, true, true, true)
decoder := NewYamlDecoder()
encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences)
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
result, err := NewStringEvaluator().Evaluate(expression, input, encoder, true, decoder)
result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder)
if err != nil {
t.Error(err)
}

5
pkg/yqlib/utils.go
View File

@ -37,7 +37,10 @@ func writeString(writer io.Writer, txt string) error {
}
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
decoder.Init(reader)
err := decoder.Init(reader)
if err != nil {
return nil, err
}
inputList := list.New()
var currentIndex uint

12
pkg/yqlib/xml_test.go
View File

@ -414,19 +414,19 @@ var xmlScenarios = []formatScenario{
func testXMLScenario(t *testing.T, s formatScenario) {
switch s.scenarioType {
case "", "decode":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description)
case "decode-keep-ns":
prefs := NewDefaultXmlPreferences()
prefs.KeepNamespace = true
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "decode-raw-token":
prefs := NewDefaultXmlPreferences()
prefs.UseRawToken = true
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
case "roundtrip-skip-directives":
prefs := NewDefaultXmlPreferences()
prefs.SkipDirectives = true
@ -480,7 +480,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, true, true))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences))))
}
func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {
@ -549,7 +549,7 @@ func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n")
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, ConfiguredXMLPreferences))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences))))
}
func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {

17
pkg/yqlib/yaml.go Normal file
View File

@ -0,0 +1,17 @@
package yqlib
type YamlPreferences struct {
LeadingContentPreProcessing bool
PrintDocSeparators bool
UnwrapScalar bool
}
func NewDefaultYamlPreferences() YamlPreferences {
return YamlPreferences{
LeadingContentPreProcessing: true,
PrintDocSeparators: true,
UnwrapScalar: true,
}
}
var ConfiguredYamlPreferences = NewDefaultYamlPreferences()