suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Disable strict XML parsing by default #1155

This commit is contained in:
Mike Farah 2022-03-28 14:05:10 +11:00
parent 3a1e2c7518
commit bbeae229ca
11 changed files with 86 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
acceptance_tests/inputs-format.sh
View File

@ -58,6 +58,27 @@ EOM
assertEquals "$expected" "$X"
}
testInputXmlStrict() {
cat >test.yml <<EOL
<?xml version="1.0"?>
<!DOCTYPE root [
<!ENTITY writer "Catherine.">
<!ENTITY copyright "(r) Great">
]>
<root>
<item>&writer;&copyright;</item>
</root>
EOL
X=$(./yq -p=xml --xml-strict-mode test.yml 2>&1)
assertEquals 1 $?
assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
X=$(./yq ea -p=xml --xml-strict-mode test.yml 2>&1)
assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
}
testInputXmlGithubAction() {
cat >test.yml <<EOL
<cat legs="4">BiBi</cat>

1
cmd/constant.go
View File

@ -10,6 +10,7 @@ var inputFormat = "yaml"
var xmlAttributePrefix = "+"
var xmlContentName = "+content"
var xmlStrictMode = false
var exitStatus = false
var forceColor = false

2
cmd/root.go
View File

@ -54,6 +54,7 @@ yq -P sample.json
yqlib.InitExpressionParser()
yqlib.XMLPreferences.AttributePrefix = xmlAttributePrefix
yqlib.XMLPreferences.ContentName = xmlContentName
yqlib.XMLPreferences.StrictMode = xmlStrictMode
},
}
@ -70,6 +71,7 @@ yq -P sample.json
rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")

2
cmd/utils.go
View File

@ -53,7 +53,7 @@ func configureDecoder() (yqlib.Decoder, error) {
}
switch yqlibInputFormat {
case yqlib.XMLInputFormat:
return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName), nil
return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode), nil
case yqlib.PropertiesInputFormat:
return yqlib.NewPropertiesDecoder(), nil
}

10
pkg/yqlib/decoder_xml.go
View File

@ -2,6 +2,7 @@ package yqlib
import (
"encoding/xml"
"errors"
"io"
"strings"
"unicode"
@ -14,14 +15,15 @@ type xmlDecoder struct {
reader io.Reader
attributePrefix string
contentName string
strictMode bool
finished bool
}
func NewXMLDecoder(attributePrefix string, contentName string) Decoder {
func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool) Decoder {
if contentName == "" {
contentName = "content"
}
return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false, strictMode: strictMode}
}
func (dec *xmlDecoder) Init(reader io.Reader) {
@ -189,7 +191,7 @@ type element struct {
// of the map keys.
func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
xmlDec := xml.NewDecoder(dec.reader)
xmlDec.Strict = dec.strictMode
// That will convert the charset if the provided XML is non-UTF-8
xmlDec.CharsetReader = charset.NewReaderLabel
@ -201,7 +203,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
for {
t, e := xmlDec.Token()
if e != nil {
if e != nil && !errors.Is(e, io.EOF) {
return e
}
if t == nil {

25
pkg/yqlib/doc/usage/xml.md
View File

@ -120,6 +120,31 @@ cat:
+legs: "4"
```
## Parse xml: custom dtd
DTD entities are ignored.
Given a sample.xml file of:
```xml
<?xml version="1.0"?>
<!DOCTYPE root [
<!ENTITY writer "Blah.">
<!ENTITY copyright "Blah">
]>
<root>
<item>&writer;&copyright;</item>
</root>
```
then
```bash
yq -p=xml '.' sample.xml
```
will output
```yaml
root:
item: '&writer;&copyright;'
```
## Parse xml: with comments
A best attempt is made to preserve comments.

2
pkg/yqlib/encoder_xml.go
View File

@ -9,7 +9,7 @@ import (
yaml "gopkg.in/yaml.v3"
)
var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content", StrictMode: false}
type xmlEncoder struct {
attributePrefix string

6
pkg/yqlib/expression_tokeniser.go
View File

@ -413,9 +413,9 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()}))
lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)}))
lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)}))
lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)}))
lexer.Add([]byte(`load_base64`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewBase64Decoder()}))

1
pkg/yqlib/lib.go
View File

@ -24,6 +24,7 @@ func InitExpressionParser() {
type xmlPreferences struct {
AttributePrefix string
ContentName string
StrictMode bool
}
var log = logging.MustGetLogger("yq-lib")

2
pkg/yqlib/operator_encoder_decoder.go
View File

@ -104,7 +104,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
case YamlInputFormat:
decoder = NewYamlDecoder()
case XMLInputFormat:
decoder = NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)
decoder = NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode)
case Base64InputFormat:
decoder = NewBase64Decoder()
}

28
pkg/yqlib/xml_test.go
View File

@ -153,6 +153,20 @@ var expectedXMLWithComments = `<!-- above_cat inline_cat --><cat><!-- above_arra
</cat><!-- below_cat -->
`
var xmlWithCustomDtd = `
<?xml version="1.0"?>
<!DOCTYPE root [
<!ENTITY writer "Blah.">
<!ENTITY copyright "Blah">
]>
<root>
<item>&writer;&copyright;</item>
</root>`
var expectedDtd = `root:
item: '&writer;&copyright;'
`
var xmlScenarios = []formatScenario{
{
description: "Parse xml: simple",
@ -185,6 +199,12 @@ var xmlScenarios = []formatScenario{
input: "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
expected: "cat:\n +content: meow\n +legs: \"4\"\n",
},
{
description: "Parse xml: custom dtd",
subdescription: "DTD entities are ignored.",
input: xmlWithCustomDtd,
expected: expectedDtd,
},
{
description: "Parse xml: with comments",
subdescription: "A best attempt is made to preserve comments.",
@ -286,9 +306,9 @@ func testXMLScenario(t *testing.T, s formatScenario) {
if s.scenarioType == "encode" {
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewXMLEncoder(2, "+", "+content")), s.description)
} else if s.scenarioType == "roundtrip" {
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content"), NewXMLEncoder(2, "+", "+content")), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewXMLEncoder(2, "+", "+content")), s.description)
} else {
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content"), NewYamlEncoder(4, false, true, true)), s.description)
test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewYamlEncoder(4, false, true, true)), s.description)
}
}
@ -327,7 +347,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content"), NewYamlEncoder(2, false, true, true))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewYamlEncoder(2, false, true, true))))
}
func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {
@ -363,7 +383,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n")
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content"), NewXMLEncoder(2, "+", "+content"))))
writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", processFormatScenario(s, NewXMLDecoder("+", "+content", false), NewXMLEncoder(2, "+", "+content"))))
}
func TestXMLScenarios(t *testing.T) {