mirror of
https://github.com/mikefarah/yq.git
synced 2026-08-24 16:39:58 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Bhargav RavuriandGitHub
|
32af402508 | ||
|
Mike Farah
|
285b257e0e |
@@ -38,11 +38,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -67,4 +67,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@v2
|
||||
|
||||
@@ -14,16 +14,16 @@ jobs:
|
||||
IMAGE_NAME: mikefarah/yq
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v2
|
||||
with:
|
||||
platforms: all
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v2
|
||||
with:
|
||||
version: latest
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
docker buildx build \
|
||||
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
|
||||
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
|
||||
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
|
||||
--label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
|
||||
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
|
||||
--label "org.opencontainers.image.licenses=MIT" \
|
||||
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
docker buildx build \
|
||||
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
|
||||
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
|
||||
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
|
||||
--label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
|
||||
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
|
||||
--label "org.opencontainers.image.licenses=MIT" \
|
||||
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
|
||||
|
||||
@@ -10,14 +10,14 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
- name: Set up Go 1.19
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '^1.20'
|
||||
go-version: 1.19
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
@@ -2,17 +2,17 @@ name: Release YQ
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v4.*'
|
||||
- 'v*'
|
||||
- 'draft-*'
|
||||
|
||||
jobs:
|
||||
publishGitRelease:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '^1.20'
|
||||
go-version: '^1.19'
|
||||
check-latest: true
|
||||
- name: Compile man page markup
|
||||
id: gen-man-page-md
|
||||
|
||||
@@ -12,7 +12,7 @@ jobs:
|
||||
environment: snap
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v3
|
||||
- uses: snapcore/action-build@v1
|
||||
id: build
|
||||
- uses: snapcore/action-publish@v1
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v3
|
||||
- name: Get test
|
||||
id: get_value
|
||||
uses: mikefarah/yq@master
|
||||
|
||||
@@ -41,8 +41,6 @@ yq*.snap
|
||||
|
||||
test.yml
|
||||
test*.yml
|
||||
test*.xml
|
||||
test*.toml
|
||||
test*.yaml
|
||||
0.yml
|
||||
1.yml
|
||||
|
||||
+6
-9
@@ -20,15 +20,12 @@ linters:
|
||||
- unparam
|
||||
linters-settings:
|
||||
depguard:
|
||||
rules:
|
||||
prevent_unmaintained_packages:
|
||||
list-mode: lax
|
||||
files:
|
||||
- $all
|
||||
- "!$test"
|
||||
deny:
|
||||
- pkg: io/ioutil
|
||||
desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
|
||||
list-type: blacklist
|
||||
include-go-root: true
|
||||
packages:
|
||||
- io/ioutil
|
||||
packages-with-error-message:
|
||||
- io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead."
|
||||
issues:
|
||||
exclude-rules:
|
||||
- linters:
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Development
|
||||
|
||||
1. Install (Golang)[https://golang.org/]
|
||||
1. Install (golang)[https://golang.org/]
|
||||
1. Run `scripts/devtools.sh` to install the required devtools
|
||||
2. Run `make [local] vendor` to install the vendor dependencies
|
||||
2. Run `make [local] test` to ensure you can run the existing tests
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
# Documentation
|
||||
|
||||
The documentation is a bit of a mixed bag (sorry in advance, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
|
||||
The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
|
||||
|
||||
Documentation is written in markdown, and is published in the 'gitbook' branch.
|
||||
|
||||
|
||||
+2
-2
@@ -1,10 +1,10 @@
|
||||
FROM golang:1.22.1 as builder
|
||||
FROM golang:1.19.4 as builder
|
||||
|
||||
WORKDIR /go/src/mikefarah/yq
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN CGO_ENABLED=0 go build -ldflags "-s -w" .
|
||||
RUN CGO_ENABLED=0 go build .
|
||||
# RUN ./scripts/test.sh -- this too often times out in the github pipeline.
|
||||
RUN ./scripts/acceptance.sh
|
||||
|
||||
|
||||
+1
-3
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22.1
|
||||
FROM golang:1.19.4
|
||||
|
||||
COPY scripts/devtools.sh /opt/devtools.sh
|
||||
|
||||
@@ -6,7 +6,5 @@ RUN set -e -x \
|
||||
&& /opt/devtools.sh
|
||||
ENV PATH=/go/bin:$PATH
|
||||
|
||||
RUN apt-get update && apt-get install -y npm && npm install -g npx cspell@latest
|
||||
|
||||
ENV CGO_ENABLED 0
|
||||
ENV GOPATH /go:/yq
|
||||
|
||||
@@ -84,12 +84,8 @@ format: vendor
|
||||
${ENGINERUN} bash ./scripts/format.sh
|
||||
|
||||
|
||||
.PHONY: spelling
|
||||
spelling: format
|
||||
${ENGINERUN} bash ./scripts/spelling.sh
|
||||
|
||||
.PHONY: secure
|
||||
secure: spelling
|
||||
secure: format
|
||||
${ENGINERUN} bash ./scripts/secure.sh
|
||||
|
||||
.PHONY: check
|
||||
|
||||
@@ -7,7 +7,6 @@ export GIT_DESCRIBE = $(shell git describe --tags --always)
|
||||
LDFLAGS :=
|
||||
LDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}
|
||||
LDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE}
|
||||
LDFLAGS += -w -s
|
||||
|
||||
GITHUB_TOKEN ?=
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ Pipe from STDIN:
|
||||
yq '.a.b[0].c' < file.yaml
|
||||
```
|
||||
|
||||
Update a yaml file, in place
|
||||
Update a yaml file, inplace
|
||||
```bash
|
||||
yq -i '.a.b[0].c = "cool"' file.yaml
|
||||
```
|
||||
@@ -31,10 +31,6 @@ NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml
|
||||
|
||||
Merge multiple files
|
||||
```bash
|
||||
# merge two files
|
||||
yq -n 'load("file1.yaml") * load("file2.yaml")'
|
||||
|
||||
# merge using globs:
|
||||
# note the use of `ea` to evaluate all the files at once
|
||||
# instead of in sequence
|
||||
yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
|
||||
@@ -49,17 +45,12 @@ yq -i '
|
||||
' file.yaml
|
||||
```
|
||||
|
||||
Find and update an item in an array:
|
||||
```bash
|
||||
yq '(.[] | select(.name == "foo") | .address) = "12 cat st"'
|
||||
```
|
||||
|
||||
Convert JSON to YAML
|
||||
```bash
|
||||
yq -Poy sample.json
|
||||
yq -P sample.json
|
||||
```
|
||||
|
||||
See [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information.
|
||||
See the [documentation](https://mikefarah.gitbook.io/yq/) for more examples.
|
||||
|
||||
Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell)
|
||||
|
||||
@@ -238,8 +229,6 @@ go install github.com/mikefarah/yq/v4@latest
|
||||
## Community Supported Installation methods
|
||||
As these are supported by the community :heart: - however, they may be out of date with the officially supported releases.
|
||||
|
||||
_Please note that the Debian package (previously supported by @rmescandon) is no longer maintained. Please use an alternative installation method._
|
||||
|
||||
|
||||
### Nix
|
||||
|
||||
@@ -266,9 +255,6 @@ pacman -S go-yq
|
||||
```
|
||||
|
||||
### Windows:
|
||||
|
||||
Using [Chocolatey](https://chocolatey.org)
|
||||
|
||||
[](https://chocolatey.org/packages/yq)
|
||||
[](https://chocolatey.org/packages/yq)
|
||||
```
|
||||
@@ -276,15 +262,12 @@ choco install yq
|
||||
```
|
||||
Supported by @chillum (https://chocolatey.org/packages/yq)
|
||||
|
||||
Using [scoop](https://scoop.sh/)
|
||||
```
|
||||
scoop install main/yq
|
||||
```
|
||||
and
|
||||
|
||||
Using [winget](https://learn.microsoft.com/en-us/windows/package-manager/)
|
||||
```
|
||||
winget install --id MikeFarah.yq
|
||||
```
|
||||
### Winget
|
||||
winget install yq
|
||||
|
||||
https://winget.run/pkg/MikeFarah/yq
|
||||
|
||||
### Mac:
|
||||
Using [MacPorts](https://www.macports.org/)
|
||||
@@ -303,6 +286,15 @@ Supported by Tuan Hoang
|
||||
https://pkgs.alpinelinux.org/package/edge/community/x86/yq
|
||||
|
||||
|
||||
### On Ubuntu 16.04 or higher from Debian package:
|
||||
```sh
|
||||
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
|
||||
sudo add-apt-repository ppa:rmescandon/yq
|
||||
sudo apt update
|
||||
sudo apt install yq -y
|
||||
```
|
||||
Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
|
||||
|
||||
## Features
|
||||
- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
|
||||
- Written in portable go, so you can download a lovely dependency free binary
|
||||
@@ -314,7 +306,7 @@ https://pkgs.alpinelinux.org/package/edge/community/x86/yq
|
||||
- [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read)
|
||||
- [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
|
||||
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
|
||||
- [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
|
||||
- [Update inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
|
||||
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
|
||||
- Keeps yaml formatting and comments when updating (though there are issues with whitespace)
|
||||
- [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)
|
||||
@@ -341,7 +333,7 @@ Examples:
|
||||
# yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
|
||||
yq '.stuff' < myfile.yml # outputs the data at the "stuff" node from "myfile.yml"
|
||||
|
||||
yq -i '.stuff = "foo"' myfile.yml # update myfile.yml in place
|
||||
yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
|
||||
|
||||
|
||||
Available Commands:
|
||||
@@ -358,7 +350,7 @@ Flags:
|
||||
--header-preprocess Slurp any header comments and separators before processing expression. (default true)
|
||||
-h, --help help for yq
|
||||
-I, --indent int sets indent level for output (default 2)
|
||||
-i, --inplace update the file in place of first file given.
|
||||
-i, --inplace update the file inplace of first file given.
|
||||
-p, --input-format string [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml")
|
||||
-M, --no-colors force print with no colors
|
||||
-N, --no-doc Don't print document separators (---)
|
||||
@@ -377,6 +369,5 @@ Use "yq [command] --help" for more information about a command.
|
||||
## Known Issues / Missing Features
|
||||
- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
|
||||
- Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell)
|
||||
- "yes", "no" were dropped as boolean values in the yaml 1.2 standard - which is the standard yq assumes.
|
||||
|
||||
See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.
|
||||
|
||||
@@ -3,25 +3,25 @@
|
||||
testWriteInPlacePipeIn() {
|
||||
result=$(./yq e -i -n '.a' 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result"
|
||||
assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result"
|
||||
}
|
||||
|
||||
testWriteInPlacePipeInEvalall() {
|
||||
result=$(./yq ea -i -n '.a' 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result"
|
||||
assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result"
|
||||
}
|
||||
|
||||
testWriteInPlaceWithSplit() {
|
||||
result=$(./yq e -s "cat" -i '.a = "thing"' test.yml 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: write in place cannot be used with split file" "$result"
|
||||
assertEquals "Error: write inplace cannot be used with split file" "$result"
|
||||
}
|
||||
|
||||
testWriteInPlaceWithSplitEvalAll() {
|
||||
result=$(./yq ea -s "cat" -i '.a = "thing"' test.yml 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: write in place cannot be used with split file" "$result"
|
||||
assertEquals "Error: write inplace cannot be used with split file" "$result"
|
||||
}
|
||||
|
||||
testNullWithFiles() {
|
||||
|
||||
@@ -96,17 +96,6 @@ testBasicExpressionFromFile() {
|
||||
assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
|
||||
}
|
||||
|
||||
testBasicExpressionFromFileDos() {
|
||||
./yq -n ".xyz = 123" > test.yml
|
||||
echo '.xyz = "meow" | .cool = "frog"' | sed 's/$'"/`echo \\\r`/" > instructions.txt
|
||||
|
||||
X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)
|
||||
assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
|
||||
|
||||
X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)
|
||||
assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
|
||||
}
|
||||
|
||||
testBasicGitHubAction() {
|
||||
./yq -n ".a = 123" > test.yml
|
||||
X=$(cat /dev/null | ./yq test.yml)
|
||||
@@ -154,7 +143,7 @@ testBasicCatWithFilesNoDash() {
|
||||
}
|
||||
|
||||
# when the nullinput flag is used
|
||||
# don't automatically read STDIN (this breaks github actions)
|
||||
# dont automatically read STDIN (this breaks github actions)
|
||||
testBasicCreateFileGithubAction() {
|
||||
cat /dev/null | ./yq -n ".a = 123" > test.yml
|
||||
}
|
||||
@@ -313,7 +302,7 @@ testBasicExitStatusNoEval() {
|
||||
assertEquals 1 "$?"
|
||||
}
|
||||
|
||||
testBasicExtractFieldWithSeparator() {
|
||||
testBasicExtractFieldWithSeperator() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
name: chart-name
|
||||
@@ -323,7 +312,7 @@ EOL
|
||||
assertEquals "chart-name" "$X"
|
||||
}
|
||||
|
||||
testBasicExtractMultipleFieldWithSeparator() {
|
||||
testBasicExtractMultipleFieldWithSeperator() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
name: chart-name
|
||||
@@ -362,12 +351,6 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testBasicClosedStdIn() {
|
||||
cat >test.yml <<EOL
|
||||
a: 1
|
||||
EOL
|
||||
X=$(./yq e '.a' test.yml <&-)
|
||||
assertEquals "1" "$X"
|
||||
}
|
||||
|
||||
|
||||
source ./scripts/shunit2
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
setUp() {
|
||||
rm test*.yml || true
|
||||
cat >test.yml <<EOL
|
||||
# comment
|
||||
EOL
|
||||
}
|
||||
|
||||
testStringInterpolation() {
|
||||
X=$(./yq -n '"Mike \(3 + 4)"')
|
||||
assertEquals "Mike 7" "$X"
|
||||
}
|
||||
|
||||
testNoStringInterpolation() {
|
||||
X=$(./yq --string-interpolation=f -n '"Mike \(3 + 4)"')
|
||||
assertEquals "Mike \(3 + 4)" "$X"
|
||||
}
|
||||
|
||||
|
||||
source ./scripts/shunit2
|
||||
@@ -1,249 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
setUp() {
|
||||
rm test*.yml 2>/dev/null || true
|
||||
rm test*.toml 2>/dev/null || true
|
||||
rm test*.tfstate 2>/dev/null || true
|
||||
rm test*.json 2>/dev/null || true
|
||||
rm test*.properties 2>/dev/null || true
|
||||
rm test*.csv 2>/dev/null || true
|
||||
rm test*.tsv 2>/dev/null || true
|
||||
rm test*.xml 2>/dev/null || true
|
||||
}
|
||||
|
||||
testInputJson() {
|
||||
cat >test.json <<EOL
|
||||
{ "mike" : { "things": "cool" } }
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
{
|
||||
"mike": {
|
||||
"things": "cool"
|
||||
}
|
||||
}
|
||||
EOM
|
||||
|
||||
X=$(./yq test.json)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.json)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputToml() {
|
||||
cat >test.toml <<EOL
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
owner:
|
||||
name: Tom Preston-Werner
|
||||
dob: 1979-05-27T07:32:00-08:00
|
||||
EOM
|
||||
|
||||
X=$(./yq -oy test.toml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea -oy test.toml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputTfstate() {
|
||||
cat >test.tfstate <<EOL
|
||||
{ "mike" : { "things": "cool" } }
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
{"mike": {"things": "cool"}}
|
||||
EOM
|
||||
|
||||
X=$(./yq test.tfstate)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.tfstate)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputJsonOutputYaml() {
|
||||
cat >test.json <<EOL
|
||||
{ "mike" : { "things": "cool" } }
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
mike:
|
||||
things: cool
|
||||
EOM
|
||||
|
||||
X=$(./yq test.json -oy)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.json -oy)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputProperties() {
|
||||
cat >test.properties <<EOL
|
||||
mike.things = hello
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
mike.things = hello
|
||||
EOM
|
||||
|
||||
X=$(./yq e test.properties)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq test.properties)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.properties)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputPropertiesGitHubAction() {
|
||||
cat >test.properties <<EOL
|
||||
mike.things = hello
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
mike.things = hello
|
||||
EOM
|
||||
|
||||
X=$(cat /dev/null | ./yq e test.properties)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(cat /dev/null | ./yq ea test.properties)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputCSV() {
|
||||
cat >test.csv <<EOL
|
||||
fruit,yumLevel
|
||||
apple,5
|
||||
banana,4
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
fruit,yumLevel
|
||||
apple,5
|
||||
banana,4
|
||||
EOM
|
||||
|
||||
X=$(./yq e test.csv)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.csv)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputCSVUTF8() {
|
||||
read -r -d '' expected << EOM
|
||||
id,first,last
|
||||
1,john,smith
|
||||
1,jane,smith
|
||||
EOM
|
||||
|
||||
X=$(./yq utf8.csv)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputTSV() {
|
||||
cat >test.tsv <<EOL
|
||||
fruit yumLevel
|
||||
apple 5
|
||||
banana 4
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
fruit yumLevel
|
||||
apple 5
|
||||
banana 4
|
||||
EOM
|
||||
|
||||
X=$(./yq e test.tsv)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.tsv)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
testInputXml() {
|
||||
cat >test.xml <<EOL
|
||||
<cat legs="4">BiBi</cat>
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
<cat legs="4">BiBi</cat>
|
||||
EOM
|
||||
|
||||
X=$(./yq e test.xml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.xml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputXmlNamespaces() {
|
||||
cat >test.xml <<EOL
|
||||
<?xml version="1.0"?>
|
||||
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
|
||||
</map>
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
<?xml version="1.0"?>
|
||||
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
|
||||
EOM
|
||||
|
||||
X=$(./yq e test.xml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea test.xml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
|
||||
|
||||
testInputXmlStrict() {
|
||||
cat >test.xml <<EOL
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE root [
|
||||
<!ENTITY writer "Catherine.">
|
||||
<!ENTITY copyright "(r) Great">
|
||||
]>
|
||||
<root>
|
||||
<item>&writer;©right;</item>
|
||||
</root>
|
||||
EOL
|
||||
|
||||
X=$(./yq --xml-strict-mode test.xml 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
|
||||
|
||||
X=$(./yq ea --xml-strict-mode test.xml 2>&1)
|
||||
assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
|
||||
}
|
||||
|
||||
testInputXmlGithubAction() {
|
||||
cat >test.xml <<EOL
|
||||
<cat legs="4">BiBi</cat>
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
<cat legs="4">BiBi</cat>
|
||||
EOM
|
||||
|
||||
X=$(cat /dev/null | ./yq e test.xml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(cat /dev/null | ./yq ea test.xml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
source ./scripts/shunit2
|
||||
@@ -63,61 +63,6 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputCSVCustomSeparator() {
|
||||
cat >test.csv <<EOL
|
||||
fruit;yumLevel
|
||||
apple;5
|
||||
banana;4
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
- fruit: apple
|
||||
yumLevel: 5
|
||||
- fruit: banana
|
||||
yumLevel: 4
|
||||
EOM
|
||||
|
||||
X=$(./yq -p=csv --csv-separator ";" test.csv)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea -p=csv --csv-separator ";" test.csv)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputCSVNoAuto() {
|
||||
cat >test.csv <<EOL
|
||||
thing1
|
||||
name: cat
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
- thing1: 'name: cat'
|
||||
EOM
|
||||
|
||||
X=$(./yq --csv-auto-parse=f test.csv -oy)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea --csv-auto-parse=f test.csv -oy)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputTSVNoAuto() {
|
||||
cat >test.tsv <<EOL
|
||||
thing1
|
||||
name: cat
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
- thing1: 'name: cat'
|
||||
EOM
|
||||
|
||||
X=$(./yq --tsv-auto-parse=f test.tsv -oy)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea --tsv-auto-parse=f test.tsv -oy)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testInputCSVUTF8() {
|
||||
read -r -d '' expected << EOM
|
||||
- id: 1
|
||||
@@ -175,7 +120,7 @@ EOM
|
||||
}
|
||||
|
||||
testInputXmlNamespaces() {
|
||||
cat >test.xml <<EOL
|
||||
cat >test.yml <<EOL
|
||||
<?xml version="1.0"?>
|
||||
<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
|
||||
</map>
|
||||
@@ -189,10 +134,10 @@ map:
|
||||
+@xsi:schemaLocation: some-url
|
||||
EOM
|
||||
|
||||
X=$(./yq e -p=xml test.xml)
|
||||
X=$(./yq e -p=xml test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea -p=xml test.xml)
|
||||
X=$(./yq ea -p=xml test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ a: test
|
||||
EOL
|
||||
}
|
||||
|
||||
testLeadingSeparatorWithDoc() {
|
||||
testLeadingSeperatorWithDoc() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -35,7 +35,7 @@ EOM
|
||||
}
|
||||
|
||||
|
||||
testLeadingSeparatorWithNewlinesNewDoc() {
|
||||
testLeadingSeperatorWithNewlinesNewDoc() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -62,7 +62,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorWithNewlinesMoreComments() {
|
||||
testLeadingSeperatorWithNewlinesMoreComments() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -92,7 +92,7 @@ EOM
|
||||
}
|
||||
|
||||
|
||||
testLeadingSeparatorWithDirective() {
|
||||
testLeadingSeperatorWithDirective() {
|
||||
cat >test.yml <<EOL
|
||||
%YAML 1.1
|
||||
---
|
||||
@@ -110,18 +110,18 @@ EOM
|
||||
}
|
||||
|
||||
|
||||
testLeadingSeparatorPipeIntoEvalSeq() {
|
||||
testLeadingSeperatorPipeIntoEvalSeq() {
|
||||
X=$(./yq e - < test.yml)
|
||||
expected=$(cat test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorExtractField() {
|
||||
testLeadingSeperatorExtractField() {
|
||||
X=$(./yq e '.a' - < test.yml)
|
||||
assertEquals "test" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorExtractFieldWithCommentsAfterSep() {
|
||||
testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -132,7 +132,7 @@ EOL
|
||||
assertEquals "test" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorExtractFieldWithCommentsBeforeSep() {
|
||||
testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -144,7 +144,7 @@ EOL
|
||||
}
|
||||
|
||||
|
||||
testLeadingSeparatorExtractFieldMultiDoc() {
|
||||
testLeadingSeperatorExtractFieldMultiDoc() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
a: test
|
||||
@@ -161,13 +161,13 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorExtractFieldMultiDocWithComments() {
|
||||
testLeadingSeperatorExtractFieldMultiDocWithComments() {
|
||||
cat >test.yml <<EOL
|
||||
# here
|
||||
---
|
||||
# there
|
||||
a: test
|
||||
# wherever
|
||||
# whereever
|
||||
---
|
||||
# you are
|
||||
a: test2
|
||||
@@ -184,26 +184,26 @@ EOM
|
||||
}
|
||||
|
||||
|
||||
testLeadingSeparatorEvalSeq() {
|
||||
testLeadingSeperatorEvalSeq() {
|
||||
X=$(./yq e test.yml)
|
||||
expected=$(cat test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorPipeIntoEvalAll() {
|
||||
testLeadingSeperatorPipeIntoEvalAll() {
|
||||
X=$(./yq ea - < test.yml)
|
||||
expected=$(cat test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
|
||||
testLeadingSeparatorEvalAll() {
|
||||
testLeadingSeperatorEvalAll() {
|
||||
X=$(./yq ea test.yml)
|
||||
expected=$(cat test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalSimple() {
|
||||
testLeadingSeperatorMultiDocEvalSimple() {
|
||||
read -r -d '' expected << EOM
|
||||
---
|
||||
a: test
|
||||
@@ -217,7 +217,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocInOneFile() {
|
||||
testLeadingSeperatorMultiDocInOneFile() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -231,7 +231,7 @@ EOL
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocInOneFileEvalAll() {
|
||||
testLeadingSeperatorMultiDocInOneFileEvalAll() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -245,7 +245,7 @@ EOL
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalComments() {
|
||||
testLeadingSeperatorMultiDocEvalComments() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -273,7 +273,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalCommentsTrailingSep() {
|
||||
testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -305,7 +305,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiMultiDocEvalCommentsTrailingSep() {
|
||||
testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
|
||||
cat >test.yml <<EOL
|
||||
# hi peeps
|
||||
# cool
|
||||
@@ -345,7 +345,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalCommentsLeadingSep() {
|
||||
testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -405,7 +405,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalCommentsStripComments() {
|
||||
testLeadingSeperatorMultiDocEvalCommentsStripComments() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -428,7 +428,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
|
||||
testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -454,7 +454,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalJsonFlag() {
|
||||
testLeadingSeperatorMultiDocEvalJsonFlag() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -483,7 +483,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalAllJsonFlag() {
|
||||
testLeadingSeperatorMultiDocEvalAllJsonFlag() {
|
||||
cat >test.yml <<EOL
|
||||
---
|
||||
# hi peeps
|
||||
@@ -512,7 +512,7 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLeadingSeparatorMultiDocEvalAll() {
|
||||
testLeadingSeperatorMultiDocEvalAll() {
|
||||
read -r -d '' expected << EOM
|
||||
---
|
||||
a: test
|
||||
@@ -9,7 +9,7 @@ testLoadFileNotExist() {
|
||||
testLoadFileExpNotExist() {
|
||||
result=$(./yq e -n 'load(.a)' 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: filename expression returned nil" "$result"
|
||||
assertEquals "Error: Filename expression returned nil" "$result"
|
||||
}
|
||||
|
||||
testStrLoadFileNotExist() {
|
||||
@@ -21,7 +21,7 @@ testStrLoadFileNotExist() {
|
||||
testStrLoadFileExpNotExist() {
|
||||
result=$(./yq e -n 'strload(.a)' 2>&1)
|
||||
assertEquals 1 $?
|
||||
assertEquals "Error: filename expression returned nil" "$result"
|
||||
assertEquals "Error: Filename expression returned nil" "$result"
|
||||
}
|
||||
|
||||
source ./scripts/shunit2
|
||||
@@ -1,286 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
setUp() {
|
||||
rm test*.yml || true
|
||||
}
|
||||
|
||||
## Convenient bash shortcut to read records of NUL separated values
|
||||
## from stdin the safe way. See example usage in the next tests.
|
||||
read-0() {
|
||||
local eof="" IFS=''
|
||||
while [ "$1" ]; do
|
||||
## - The `-r` avoids bad surprise with '\n' and other interpreted
|
||||
## sequences that can be read.
|
||||
## - The `-d ''` is the (strange?) way to refer to NUL delimiter.
|
||||
## - The `--` is how to avoid unpleasant surprises if your
|
||||
## "$1" starts with "-" (minus) sign. This protection also
|
||||
## will produce a readable error if you want to try to start
|
||||
## your variable names with a "-".
|
||||
read -r -d '' -- "$1" || eof=1
|
||||
shift
|
||||
done
|
||||
[ -z "$eof" ] ## fail on EOF
|
||||
}
|
||||
|
||||
## Convenient bash shortcut to be used with the next function `p-err`
|
||||
## to read NUL separated values the safe way AND catch any errors from
|
||||
## the process creating the stream of NUL separated data. See example
|
||||
## usage in the tests.
|
||||
read-0-err() {
|
||||
local ret="$1" eof="" idx=0 last=
|
||||
read -r -- "${ret?}" <<<"0"
|
||||
shift
|
||||
while [ "$1" ]; do
|
||||
last=$idx
|
||||
read -r -d '' -- "$1" || {
|
||||
## Put this last value in ${!ret}
|
||||
eof="$1"
|
||||
read -r -- "$ret" <<<"${!eof}"
|
||||
break
|
||||
}
|
||||
((idx++))
|
||||
shift
|
||||
done
|
||||
[ -z "$eof" ] || {
|
||||
if [ "$last" != 0 ]; then
|
||||
## Uhoh, we have no idea if the errorlevel of the internal
|
||||
## command was properly delimited with a NUL char, and
|
||||
## anyway something went really wrong at least about the
|
||||
## number of fields separated by NUL char and the one
|
||||
## expected.
|
||||
echo "Error: read-0-err couldn't fill all value $ret = '${!ret}', '$eof', '${!eof}'" >&2
|
||||
read -r -- "$ret" <<<"not-enough-values"
|
||||
else
|
||||
if ! [[ "${!ret}" =~ ^[0-9]+$ && "${!ret}" -ge 0 && "${!ret}" -le 127 ]]; then
|
||||
## This could happen if you don't use `p-err` wrapper,
|
||||
## or used stdout in unexpected ways in your inner
|
||||
## command.
|
||||
echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2
|
||||
read -r -- "$ret" <<<"last-value-not-a-number"
|
||||
fi
|
||||
fi
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
## Simply runs command given as argument and adds errorlevel in the
|
||||
## standard output. Is expected to be used in tandem with
|
||||
## `read-0-err`.
|
||||
p-err() {
|
||||
local exp="$1"
|
||||
"$@"
|
||||
printf "%s" "$?"
|
||||
}
|
||||
|
||||
wyq-r() {
|
||||
local exp="$1"
|
||||
./yq e -0 -r=false "$1"
|
||||
printf "%s" "$?"
|
||||
}
|
||||
|
||||
testBasicUsageRaw() {
|
||||
cat >test.yml <<EOL
|
||||
a: foo
|
||||
b: bar
|
||||
EOL
|
||||
|
||||
printf "foo\0bar\0" > expected.out
|
||||
|
||||
## We need to compare binary content here. We have to filter the compared
|
||||
## content through a representation that gets rid of NUL chars but accurately
|
||||
## transcribe the content.
|
||||
## Also as it would be nice to have a pretty output in case the test fails,
|
||||
## we use here 'hd': a widely available shortcut to 'hexdump' that will
|
||||
## pretty-print any binary to it's hexadecimal representation.
|
||||
##
|
||||
## Note that the standard `assertEquals` compare its arguments
|
||||
## value, but they can't hold NUL characters (this comes from the
|
||||
## limitation of the C API of `exec*(..)` functions that requires
|
||||
## `const char *arv[]`). And these are NUL terminated strings. As a
|
||||
## consequence, the NUL characters gets removed in bash arguments.
|
||||
assertEquals "$(hd expected.out)" \
|
||||
"$(./yq e -0 '.a, .b' test.yml | hd)"
|
||||
|
||||
rm expected.out
|
||||
}
|
||||
|
||||
testBasicUsage() {
|
||||
local a b
|
||||
cat >test.yml <<EOL
|
||||
a: foo
|
||||
b: bar
|
||||
EOL
|
||||
|
||||
## We provide 2 values, and ask to fill 2 variables.
|
||||
read-0 a b < <(./yq e -0 '.a, .b' test.yml)
|
||||
assertEquals "$?" "0" ## Everything is fine
|
||||
assertEquals "foo" "$a" ## Values are correctly parsed
|
||||
assertEquals "bar" "$b"
|
||||
|
||||
a=YYY ; b=XXX
|
||||
## Not enough values provided to fill `a` and `b`.
|
||||
read-0 a b < <(./yq e -0 '.a' test.yml)
|
||||
assertEquals "$?" "1" ## An error was emitted
|
||||
assertEquals "foo" "$a" ## First value was correctly parsed
|
||||
assertEquals "" "$b" ## Second was still reset
|
||||
|
||||
## Error from inner command are not catchable !. Use
|
||||
## `read-0-err`/`p-err` for that.
|
||||
read-0 a < <(printf "\0"; ./yq e -0 'xxx' test.yml; )
|
||||
assertEquals "$?" "0"
|
||||
|
||||
}
|
||||
|
||||
testBasicUsageJson() {
|
||||
cat >test.yml <<EOL
|
||||
a:
|
||||
x: foo
|
||||
b: bar
|
||||
EOL
|
||||
|
||||
read-0 a b < <(./yq e -0 -o=json '.a, .b' test.yml)
|
||||
|
||||
assertEquals '{
|
||||
"x": "foo"
|
||||
}' "$a"
|
||||
assertEquals '"bar"' "$b"
|
||||
|
||||
}
|
||||
|
||||
testFailWithValueContainingNUL() {
|
||||
local a b c
|
||||
## Note that value of field 'a' actually contains a NUL char !
|
||||
cat >test.yml <<EOL
|
||||
a: "foo\u0000bar"
|
||||
b: 1
|
||||
c: |
|
||||
wiz
|
||||
boom
|
||||
EOL
|
||||
|
||||
## We are looking for trouble with asking to separated fields with NUL
|
||||
## char and requested value `.a` actually contains itself a NUL char !
|
||||
read-0 a b c < <(./yq e -0 '.a, .b, .c' test.yml)
|
||||
assertNotEquals "0" "$?" ## read-0 failed to fill all values
|
||||
|
||||
## But here, we can request for one value, even if `./yq` fails
|
||||
read-0 b < <(./yq e -0 '.b, .a' test.yml)
|
||||
assertEquals "0" "$?" ## read-0 succeeds at feeding the first value
|
||||
## Note: to catch the failure of `yq`, see in the next tests the usage
|
||||
## of `read-0-err`.
|
||||
|
||||
## using -r=false solves any NUL containing value issues, but keeps
|
||||
## all in YAML representation:
|
||||
read-0 a b c < <(./yq e -0 -r=false '.a, .b, .c' test.yml)
|
||||
assertEquals "0" "$?" ## All goes well despite asking for `a` value
|
||||
|
||||
assertEquals '"foo\0bar"' "$a" ## This is a YAML string representation
|
||||
assertEquals '1' "$b"
|
||||
assertEquals '|
|
||||
wiz
|
||||
boom' "$c"
|
||||
}
|
||||
|
||||
testStandardLoop() {
|
||||
local E a b res
|
||||
|
||||
## Here everything is normal: 4 values, that will be paired
|
||||
## in key/values.
|
||||
cat >test.yml <<EOL
|
||||
- yay
|
||||
- wiz
|
||||
- hop
|
||||
- pow
|
||||
EOL
|
||||
|
||||
res=""
|
||||
while read-0-err E a b; do
|
||||
res+="$a: $b;"
|
||||
done < <(p-err ./yq -0 '.[]' test.yml)
|
||||
|
||||
assertEquals "0" "$E" ## errorlevel of internal command
|
||||
assertEquals "yay: wiz;hop: pow;" "$res" ## expected result
|
||||
}
|
||||
|
||||
testStandardLoopWithoutEnoughValues() {
|
||||
local E a b res
|
||||
|
||||
## Here 5 values, there will be a missing value when reading
|
||||
## pairs of value.
|
||||
cat >test.yml <<EOL
|
||||
- yay
|
||||
- wiz
|
||||
- hop
|
||||
- pow
|
||||
- kwak
|
||||
EOL
|
||||
|
||||
res=""
|
||||
## The loop will succeed 2 times then fail
|
||||
while read-0-err E a b; do
|
||||
res+="$a: $b;"
|
||||
done < <(p-err ./yq -0 '.[]' test.yml)
|
||||
|
||||
assertEquals "not-enough-values" "$E" ## Not enough value error
|
||||
assertEquals "yay: wiz;hop: pow;" "$res" ## the 2 full key/value pairs
|
||||
|
||||
}
|
||||
|
||||
testStandardLoopWithInternalCmdError() {
|
||||
local E a b res
|
||||
|
||||
## Note the third value contains a NUL char !
|
||||
cat >test.yml <<EOL
|
||||
- yay
|
||||
- wiz
|
||||
- "foo\0bar"
|
||||
- hop
|
||||
- pow
|
||||
EOL
|
||||
|
||||
res=""
|
||||
## It should be only upon the second pass in the loop that
|
||||
## read-0-err will catch the fact that there is an error !
|
||||
while read-0-err E a b; do
|
||||
res+="$a: $b;"
|
||||
done < <(p-err ./yq -0 '.[]' test.yml)
|
||||
assertEquals "1" "$E" ## Internal command errorlevel (from `./yq`)
|
||||
assertEquals "yay: wiz;" "$res" ## first 2 values were ok at least
|
||||
|
||||
}
|
||||
|
||||
testStandardLoopNotEnoughErrorEatsCmdError() {
|
||||
local E a b res
|
||||
|
||||
## Because of possible edge cases where the internal errorlevel
|
||||
## reported by `p-err` in the standard output might be mangled
|
||||
## with the unfinished record, `read-0-err E ...` will NOT report
|
||||
## the internal command error in the variable E and instead will
|
||||
## store the value 'not-enough-values'. In real world, anyway, you
|
||||
## will want to react the same if the internal command failed
|
||||
## and/or you didn't get as much values as expected while
|
||||
## reading. Keep in mind also that standard error is not
|
||||
## swallowed, so you can read reports from the inner command AND
|
||||
## from `read-0-err`.
|
||||
|
||||
## Here, note that the fourth value contains a NUL char !
|
||||
cat >test.yml <<EOL
|
||||
- yay
|
||||
- wiz
|
||||
- hop
|
||||
- "foo\0bar"
|
||||
- pow
|
||||
EOL
|
||||
|
||||
res=""
|
||||
## It should be only upon the second loop that read-0-err will catch
|
||||
## the fact that there are not enough data to fill the requested variables
|
||||
while read-0-err E a b; do
|
||||
res+="$a: $b;"
|
||||
done < <(p-err ./yq -0 '.[]' test.yml)
|
||||
assertEquals "not-enough-values" "$E" ## Not enough values error eats internal error !
|
||||
assertEquals "yay: wiz;" "$res" ## first 2 values were ok at least
|
||||
}
|
||||
|
||||
|
||||
source ./scripts/shunit2
|
||||
@@ -72,15 +72,6 @@ EOL
|
||||
assertEquals "\"cat\"" "$X"
|
||||
}
|
||||
|
||||
testOutputYamlRawOnRoot() {
|
||||
cat >test.yml <<EOL
|
||||
'a'
|
||||
EOL
|
||||
|
||||
X=$(./yq e -r '.' test.yml)
|
||||
assertEquals "a" "$X"
|
||||
}
|
||||
|
||||
testOutputJsonRaw() {
|
||||
cat >test.yml <<EOL
|
||||
a: cat
|
||||
@@ -198,27 +189,6 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testOutputCSVCustomSeparator() {
|
||||
cat >test.yml <<EOL
|
||||
- fruit: apple
|
||||
yumLevel: 5
|
||||
- fruit: banana
|
||||
yumLevel: 4
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
fruit;yumLevel
|
||||
apple;5
|
||||
banana;4
|
||||
EOM
|
||||
|
||||
X=$(./yq -oc --csv-separator ";" test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq ea -o=csv --csv-separator ";" test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testOutputTSV() {
|
||||
cat >test.yml <<EOL
|
||||
- fruit: apple
|
||||
@@ -301,43 +271,4 @@ EOM
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
testLuaOutputPretty() {
|
||||
cat >test.yml <<EOL
|
||||
animals:
|
||||
cat: meow
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
return {
|
||||
["animals"] = {
|
||||
["cat"] = "meow";
|
||||
};
|
||||
};
|
||||
EOM
|
||||
|
||||
X=$(./yq e --output-format=lua test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
X=$(./yq e --output-format=lua --prettyPrint test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
}
|
||||
|
||||
testLuaOutputSubset() {
|
||||
cat >test.yml <<EOL
|
||||
animals:
|
||||
cat: meow
|
||||
EOL
|
||||
|
||||
read -r -d '' expected << EOM
|
||||
return {
|
||||
["cat"] = "meow";
|
||||
};
|
||||
EOM
|
||||
|
||||
X=$(./yq e --output-format=lua '.animals' test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
|
||||
}
|
||||
|
||||
source ./scripts/shunit2
|
||||
source ./scripts/shunit2
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
setUp() {
|
||||
rm test*.yq || true
|
||||
cat >test.yq <<EOL
|
||||
#!./yq
|
||||
.a.b
|
||||
EOL
|
||||
chmod +x test.yq
|
||||
|
||||
rm test*.yml || true
|
||||
cat >test.yml <<EOL
|
||||
a: {b: apple}
|
||||
EOL
|
||||
}
|
||||
|
||||
testCanExecYqFile() {
|
||||
read -r -d '' expected << EOM
|
||||
apple
|
||||
EOM
|
||||
X=$(./test.yq test.yml)
|
||||
assertEquals "$expected" "$X"
|
||||
}
|
||||
|
||||
source ./scripts/shunit2
|
||||
|
||||
+2
-5
@@ -6,10 +6,8 @@ var unwrapScalar = false
|
||||
|
||||
var writeInplace = false
|
||||
var outputToJSON = false
|
||||
|
||||
var outputFormat = ""
|
||||
|
||||
var inputFormat = ""
|
||||
var outputFormat = "yaml"
|
||||
var inputFormat = "yaml"
|
||||
|
||||
var exitStatus = false
|
||||
var forceColor = false
|
||||
@@ -18,7 +16,6 @@ var colorsEnabled = false
|
||||
var indent = 2
|
||||
var noDocSeparators = false
|
||||
var nullInput = false
|
||||
var nulSepOutput = false
|
||||
var verbose = false
|
||||
var version = false
|
||||
var prettyPrint = false
|
||||
|
||||
@@ -12,14 +12,8 @@ func createEvaluateAllCommand() *cobra.Command {
|
||||
Use: "eval-all [expression] [yaml_file1]...",
|
||||
Aliases: []string{"ea"},
|
||||
Short: "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
|
||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||
if len(args) == 0 {
|
||||
return nil, cobra.ShellCompDirectiveNoFileComp
|
||||
}
|
||||
return nil, cobra.ShellCompDirectiveDefault
|
||||
},
|
||||
Example: `
|
||||
# Merge f2.yml into f1.yml (in place)
|
||||
# Merge f2.yml into f1.yml (inplace)
|
||||
yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
|
||||
## the same command and expression using shortened names:
|
||||
yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
|
||||
@@ -32,7 +26,7 @@ yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
|
||||
## use '-' as a filename to pipe from STDIN
|
||||
cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
|
||||
`,
|
||||
Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/)
|
||||
Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/)
|
||||
See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
|
||||
|
||||
## Evaluate All ##
|
||||
@@ -76,7 +70,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
}()
|
||||
}
|
||||
|
||||
format, err := yqlib.FormatFromString(outputFormat)
|
||||
format, err := yqlib.OutputFormatFromString(outputFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -90,15 +84,9 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoder, err := configureEncoder()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoder := configureEncoder(format)
|
||||
|
||||
printer := yqlib.NewPrinter(encoder, printerWriter)
|
||||
if nulSepOutput {
|
||||
printer.SetNulSepOutput(true)
|
||||
}
|
||||
|
||||
if frontMatter != "" {
|
||||
frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
|
||||
|
||||
@@ -13,12 +13,6 @@ func createEvaluateSequenceCommand() *cobra.Command {
|
||||
Use: "eval [expression] [yaml_file1]...",
|
||||
Aliases: []string{"e"},
|
||||
Short: "(default) Apply the expression to each document in each yaml file in sequence",
|
||||
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||
if len(args) == 0 {
|
||||
return nil, cobra.ShellCompDirectiveNoFileComp
|
||||
}
|
||||
return nil, cobra.ShellCompDirectiveDefault
|
||||
},
|
||||
Example: `
|
||||
# Reads field under the given path for each file
|
||||
yq e '.a.b' f1.yml f2.yml
|
||||
@@ -34,10 +28,10 @@ cat file2.yml | yq e '.a.b' file1.yml - file3.yml
|
||||
## Note that editing an empty file does not work.
|
||||
yq e -n '.a.b.c = "cat"'
|
||||
|
||||
# Update a file in place
|
||||
# Update a file inplace
|
||||
yq e '.a.b = "cool"' -i file.yaml
|
||||
`,
|
||||
Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/)
|
||||
Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/)
|
||||
See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
|
||||
|
||||
## Evaluate Sequence ##
|
||||
@@ -90,7 +84,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
}()
|
||||
}
|
||||
|
||||
format, err := yqlib.FormatFromString(outputFormat)
|
||||
format, err := yqlib.OutputFormatFromString(outputFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -99,15 +93,9 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoder, err := configureEncoder()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoder := configureEncoder(format)
|
||||
|
||||
printer := yqlib.NewPrinter(encoder, printerWriter)
|
||||
if nulSepOutput {
|
||||
printer.SetNulSepOutput(true)
|
||||
}
|
||||
|
||||
decoder, err := configureDecoder(false)
|
||||
if err != nil {
|
||||
+43
-127
@@ -1,54 +1,21 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/properties"
|
||||
"github.com/mikefarah/yq/v4/pkg/xml"
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"github.com/spf13/cobra"
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
type runeValue rune
|
||||
|
||||
func newRuneVar(p *rune) *runeValue {
|
||||
return (*runeValue)(p)
|
||||
}
|
||||
|
||||
func (r *runeValue) String() string {
|
||||
return string(*r)
|
||||
}
|
||||
|
||||
func (r *runeValue) Set(rawVal string) error {
|
||||
val := strings.ReplaceAll(rawVal, "\\n", "\n")
|
||||
val = strings.ReplaceAll(val, "\\t", "\t")
|
||||
val = strings.ReplaceAll(val, "\\r", "\r")
|
||||
val = strings.ReplaceAll(val, "\\f", "\f")
|
||||
val = strings.ReplaceAll(val, "\\v", "\v")
|
||||
if len(val) != 1 {
|
||||
return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
|
||||
}
|
||||
*r = runeValue(rune(val[0]))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runeValue) Type() string {
|
||||
return "char"
|
||||
}
|
||||
|
||||
func New() *cobra.Command {
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "yq",
|
||||
Short: "yq is a lightweight and portable command-line data file processor.",
|
||||
Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/)
|
||||
Short: "yq is a lightweight and portable command-line YAML processor.",
|
||||
Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/)
|
||||
See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
|
||||
Example: `
|
||||
# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)
|
||||
# Use the '-p/--input-format' flag to specify a format type.
|
||||
cat file.xml | yq -p xml
|
||||
# yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
|
||||
|
||||
# read the "stuff" node from "myfile.yml"
|
||||
yq '.stuff' < myfile.yml
|
||||
@@ -57,7 +24,7 @@ yq '.stuff' < myfile.yml
|
||||
yq -i '.stuff = "foo"' myfile.yml
|
||||
|
||||
# print contents of sample.json as idiomatic YAML
|
||||
yq -P -oy sample.json
|
||||
yq -P sample.json
|
||||
`,
|
||||
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
@@ -70,23 +37,41 @@ yq -P -oy sample.json
|
||||
},
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
cmd.SetOut(cmd.OutOrStdout())
|
||||
level := logging.WARNING
|
||||
stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
|
||||
|
||||
if verbose {
|
||||
level = logging.DEBUG
|
||||
stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
|
||||
}
|
||||
|
||||
var format = logging.MustStringFormatter(stringFormat)
|
||||
var format = logging.MustStringFormatter(
|
||||
`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
|
||||
)
|
||||
var backend = logging.AddModuleLevel(
|
||||
logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
|
||||
|
||||
backend.SetLevel(level, "")
|
||||
if verbose {
|
||||
backend.SetLevel(logging.DEBUG, "")
|
||||
} else {
|
||||
backend.SetLevel(logging.WARNING, "")
|
||||
}
|
||||
|
||||
logging.SetBackend(backend)
|
||||
yqlib.InitExpressionParser()
|
||||
|
||||
outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if outputFormatType == yqlib.YamlOutputFormat ||
|
||||
outputFormatType == yqlib.PropsOutputFormat {
|
||||
unwrapScalar = true
|
||||
}
|
||||
if unwrapScalarFlag.IsExplicitySet() {
|
||||
unwrapScalar = unwrapScalarFlag.IsSet()
|
||||
}
|
||||
|
||||
//copy preference form global setting
|
||||
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
|
||||
|
||||
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -99,81 +84,27 @@ yq -P -oy sample.json
|
||||
panic(err)
|
||||
}
|
||||
|
||||
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString()))
|
||||
var outputCompletions = []string{"auto"}
|
||||
for _, formats := range yqlib.GetAvailableOutputFormats() {
|
||||
outputCompletions = append(outputCompletions, formats.FormalName)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
|
||||
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
|
||||
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString()))
|
||||
|
||||
var inputCompletions = []string{"auto"}
|
||||
for _, formats := range yqlib.GetAvailableInputFormats() {
|
||||
inputCompletions = append(inputCompletions, formats.FormalName)
|
||||
}
|
||||
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
|
||||
rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
|
||||
rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
|
||||
rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
|
||||
rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
|
||||
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
|
||||
rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
|
||||
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
|
||||
rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
|
||||
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
|
||||
|
||||
rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
|
||||
rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
|
||||
|
||||
rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
|
||||
rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.")
|
||||
rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
|
||||
rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
|
||||
rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true"
|
||||
rootCmd.PersistentFlags().BoolVarP(&nulSepOutput, "nul-output", "0", false, "Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.")
|
||||
|
||||
rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
|
||||
rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
|
||||
@@ -181,28 +112,13 @@ yq -P -oy sample.json
|
||||
rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
|
||||
rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
|
||||
rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
|
||||
|
||||
rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
|
||||
if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
|
||||
if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
|
||||
if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
rootCmd.AddCommand(
|
||||
createEvaluateSequenceCommand(),
|
||||
|
||||
@@ -47,7 +47,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
|
||||
var err error = nil
|
||||
switch args[0] {
|
||||
case "bash":
|
||||
err = cmd.Root().GenBashCompletionV2(os.Stdout, true)
|
||||
err = cmd.Root().GenBashCompletion(os.Stdout)
|
||||
case "zsh":
|
||||
err = cmd.Root().GenZshCompletion(os.Stdout)
|
||||
case "fish":
|
||||
|
||||
+6
-6
@@ -8,21 +8,21 @@ import (
|
||||
|
||||
type boolFlag interface {
|
||||
pflag.Value
|
||||
IsExplicitlySet() bool
|
||||
IsExplicitySet() bool
|
||||
IsSet() bool
|
||||
}
|
||||
|
||||
type unwrapScalarFlagStrc struct {
|
||||
explicitlySet bool
|
||||
value bool
|
||||
explicitySet bool
|
||||
value bool
|
||||
}
|
||||
|
||||
func newUnwrapFlag() boolFlag {
|
||||
return &unwrapScalarFlagStrc{value: true}
|
||||
}
|
||||
|
||||
func (f *unwrapScalarFlagStrc) IsExplicitlySet() bool {
|
||||
return f.explicitlySet
|
||||
func (f *unwrapScalarFlagStrc) IsExplicitySet() bool {
|
||||
return f.explicitySet
|
||||
}
|
||||
|
||||
func (f *unwrapScalarFlagStrc) IsSet() bool {
|
||||
@@ -37,7 +37,7 @@ func (f *unwrapScalarFlagStrc) Set(value string) error {
|
||||
|
||||
v, err := strconv.ParseBool(value)
|
||||
f.value = v
|
||||
f.explicitlySet = true
|
||||
f.explicitySet = true
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+37
-108
@@ -4,19 +4,12 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/properties"
|
||||
"github.com/mikefarah/yq/v4/pkg/xml"
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
func isAutomaticOutputFormat() bool {
|
||||
return outputFormat == "" || outputFormat == "auto" || outputFormat == "a"
|
||||
}
|
||||
|
||||
func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
|
||||
cmd.SilenceUsage = true
|
||||
|
||||
@@ -45,7 +38,7 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
|
||||
}
|
||||
|
||||
if writeInplace && (len(args) == 0 || args[0] == "-") {
|
||||
return "", nil, fmt.Errorf("write in place flag only applicable when giving an expression and at least one file")
|
||||
return "", nil, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file")
|
||||
}
|
||||
|
||||
if frontMatter != "" && len(args) == 0 {
|
||||
@@ -53,82 +46,39 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
|
||||
}
|
||||
|
||||
if writeInplace && splitFileExp != "" {
|
||||
return "", nil, fmt.Errorf("write in place cannot be used with split file")
|
||||
return "", nil, fmt.Errorf("write inplace cannot be used with split file")
|
||||
}
|
||||
|
||||
if nullInput && len(args) > 0 {
|
||||
return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
|
||||
}
|
||||
|
||||
inputFilename := ""
|
||||
if len(args) > 0 {
|
||||
inputFilename = args[0]
|
||||
}
|
||||
if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
|
||||
|
||||
inputFormat = yqlib.FormatStringFromFilename(inputFilename)
|
||||
|
||||
_, err := yqlib.FormatFromString(inputFormat)
|
||||
if err != nil {
|
||||
// unknown file type, default to yaml
|
||||
yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
|
||||
inputFormat = "yaml"
|
||||
if isAutomaticOutputFormat() {
|
||||
outputFormat = "yaml"
|
||||
}
|
||||
} else if isAutomaticOutputFormat() {
|
||||
// automatic input worked, we can do it for output too unless specified
|
||||
if inputFormat == "json" {
|
||||
yqlib.GetLogger().Warning("JSON file output is now JSON by default (instead of yaml). Use '-oy' or '--output-format=yaml' for yaml output")
|
||||
}
|
||||
outputFormat = inputFormat
|
||||
}
|
||||
} else if isAutomaticOutputFormat() {
|
||||
// backwards compatibility -
|
||||
// before this was introduced, `yq -pcsv things.csv`
|
||||
// would produce *yaml* output.
|
||||
//
|
||||
outputFormat = yqlib.FormatStringFromFilename(inputFilename)
|
||||
if inputFilename != "-" {
|
||||
yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
|
||||
}
|
||||
outputFormat = "yaml"
|
||||
}
|
||||
|
||||
outputFormatType, err := yqlib.FormatFromString(outputFormat)
|
||||
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
yqlib.GetLogger().Debug("Using input format %v", inputFormat)
|
||||
yqlib.GetLogger().Debug("Using output format %v", outputFormat)
|
||||
|
||||
if outputFormatType == yqlib.YamlFormat ||
|
||||
outputFormatType == properties.PropertiesFormat {
|
||||
unwrapScalar = true
|
||||
}
|
||||
if unwrapScalarFlag.IsExplicitlySet() {
|
||||
unwrapScalar = unwrapScalarFlag.IsSet()
|
||||
}
|
||||
|
||||
return expression, args, nil
|
||||
}
|
||||
|
||||
func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
|
||||
format, err := yqlib.FormatFromString(inputFormat)
|
||||
yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether
|
||||
|
||||
yqlibDecoder := format.DecoderFactory()
|
||||
if yqlibDecoder == nil {
|
||||
return nil, fmt.Errorf("no support for %s input format", inputFormat)
|
||||
switch yqlibInputFormat {
|
||||
case yqlib.XMLInputFormat:
|
||||
return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
|
||||
case yqlib.PropertiesInputFormat:
|
||||
return yqlib.NewPropertiesDecoder(), nil
|
||||
case yqlib.JsonInputFormat:
|
||||
return yqlib.NewJSONDecoder(), nil
|
||||
case yqlib.CSVObjectInputFormat:
|
||||
return yqlib.NewCSVObjectDecoder(','), nil
|
||||
case yqlib.TSVObjectInputFormat:
|
||||
return yqlib.NewCSVObjectDecoder('\t'), nil
|
||||
}
|
||||
return yqlibDecoder, nil
|
||||
prefs := yqlib.ConfiguredYamlPreferences
|
||||
prefs.EvaluateTogether = evaluateTogether
|
||||
return yqlib.NewYamlDecoder(prefs), nil
|
||||
}
|
||||
|
||||
func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {
|
||||
func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
|
||||
|
||||
var printerWriter yqlib.PrinterWriter
|
||||
|
||||
@@ -145,30 +95,22 @@ func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterW
|
||||
return printerWriter, nil
|
||||
}
|
||||
|
||||
func configureEncoder() (yqlib.Encoder, error) {
|
||||
yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
|
||||
switch format {
|
||||
case yqlib.JSONOutputFormat:
|
||||
return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
|
||||
case yqlib.PropsOutputFormat:
|
||||
return yqlib.NewPropertiesEncoder(unwrapScalar)
|
||||
case yqlib.CSVOutputFormat:
|
||||
return yqlib.NewCsvEncoder(',')
|
||||
case yqlib.TSVOutputFormat:
|
||||
return yqlib.NewCsvEncoder('\t')
|
||||
case yqlib.YamlOutputFormat:
|
||||
return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
|
||||
case yqlib.XMLOutputFormat:
|
||||
return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
|
||||
}
|
||||
xml.ConfiguredXMLPreferences.Indent = indent
|
||||
yqlib.ConfiguredYamlPreferences.Indent = indent
|
||||
yqlib.ConfiguredJSONPreferences.Indent = indent
|
||||
|
||||
yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
|
||||
properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
|
||||
yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
|
||||
|
||||
yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
|
||||
yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
|
||||
|
||||
yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
|
||||
|
||||
encoder := yqlibOutputFormat.EncoderFactory()
|
||||
|
||||
if encoder == nil {
|
||||
return nil, fmt.Errorf("no support for %s output format", outputFormat)
|
||||
}
|
||||
return encoder, err
|
||||
panic("invalid encoder")
|
||||
}
|
||||
|
||||
// this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
|
||||
@@ -192,11 +134,8 @@ func maybeFile(str string) bool {
|
||||
}
|
||||
|
||||
func processStdInArgs(args []string) []string {
|
||||
stat, err := os.Stdin.Stat()
|
||||
if err != nil {
|
||||
yqlib.GetLogger().Debugf("error getting stdin: %v", err)
|
||||
}
|
||||
pipingStdin := stat != nil && (stat.Mode()&os.ModeCharDevice) == 0
|
||||
stat, _ := os.Stdin.Stat()
|
||||
pipingStdin := (stat.Mode() & os.ModeCharDevice) == 0
|
||||
|
||||
// if we've been given a file, don't automatically
|
||||
// read from stdin.
|
||||
@@ -220,25 +159,15 @@ func processStdInArgs(args []string) []string {
|
||||
|
||||
func processArgs(originalArgs []string) (string, []string, error) {
|
||||
expression := forceExpression
|
||||
args := processStdInArgs(originalArgs)
|
||||
maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])
|
||||
|
||||
if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
|
||||
// lets check if an expression file was given
|
||||
yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
|
||||
expressionFile = args[0]
|
||||
args = args[1:]
|
||||
}
|
||||
|
||||
if expressionFile != "" {
|
||||
expressionBytes, err := os.ReadFile(expressionFile)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
//replace \r\n (windows) with good ol' unix file endings.
|
||||
expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n")
|
||||
expression = string(expressionBytes)
|
||||
}
|
||||
|
||||
args := processStdInArgs(originalArgs)
|
||||
yqlib.GetLogger().Debugf("processed args: %v", args)
|
||||
if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
|
||||
yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ var (
|
||||
GitDescribe string
|
||||
|
||||
// Version is main version number that is being run at the moment.
|
||||
Version = "v4.42.1"
|
||||
Version = "v4.30.5"
|
||||
|
||||
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
|
||||
// then it means that it is a final release. Otherwise, this is a pre-release
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
---
|
||||
$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
|
||||
version: '0.2'
|
||||
language: en-GB
|
||||
dictionaryDefinitions:
|
||||
- name: project-words
|
||||
path: './project-words.txt'
|
||||
addWords: true
|
||||
dictionaries:
|
||||
- project-words
|
||||
ignorePaths:
|
||||
- 'vendor'
|
||||
- 'bin'
|
||||
- '/project-words.txt'
|
||||
Vendored
+3
-3
@@ -103,7 +103,7 @@ yq (4.16.2) focal; urgency=medium
|
||||
* Improved extract-checksum.sh
|
||||
* Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039)
|
||||
* enable more linters (#1043)
|
||||
* Bump Golang compiler #1037
|
||||
* Bump golang compiler #1037
|
||||
|
||||
-- Roberto Mier Escandon <rmescandon@gmail.com> Tue, 21 Dec 2021 09:41:44 +0000
|
||||
|
||||
@@ -127,7 +127,7 @@ yq (4.9.6) focal; urgency=medium
|
||||
|
||||
* Added darwin/arm64 build, thanks @alecthomas
|
||||
* Incremented docker alpine base version, thanks @da6d6i7-bronga
|
||||
* Bug fix: multiline expression
|
||||
* Bug fix: multine expression
|
||||
* Bug fix: special character
|
||||
|
||||
-- Roberto Mier Escandon <rmescandon@gmail.com> Tue, 29 Jun 2021 21:32:14 +0200
|
||||
@@ -135,7 +135,7 @@ yq (4.9.6) focal; urgency=medium
|
||||
yq (3.3.2) focal; urgency=medium
|
||||
|
||||
* Bug fix: existStatus bug (#459)
|
||||
* Automatically makes an os temp directory if it does not exist (#461)
|
||||
* Automatically makes a os temp directory if it does not exist (#461)
|
||||
|
||||
-- Roberto Mier Escandon <rmescandon@gmail.com> Fri, 07 Aug 2020 18:53:01 +0200
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -16,7 +16,7 @@ XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
|
||||
Package: yq
|
||||
Architecture: any
|
||||
Depends: ${shlibs:Depends}, ${misc:Depends}
|
||||
Description: lightweight and portable command-line data file processor
|
||||
Description: lightweight and portable command-line YAML processor
|
||||
.
|
||||
The aim of the project is to be the
|
||||
[jq](https://github.com/stedolan/jq) or sed of yaml files.
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
return {
|
||||
["country"] = "Australia"; -- this place
|
||||
["cities"] = {
|
||||
"Sydney",
|
||||
"Melbourne",
|
||||
"Brisbane",
|
||||
"Perth",
|
||||
};
|
||||
};
|
||||
+1
-11
@@ -1,11 +1 @@
|
||||
# block comments come through
|
||||
person: # neither do comments on maps
|
||||
name: Mike Wazowski # comments on values appear
|
||||
pets:
|
||||
- cat # comments on array values appear
|
||||
- dog # comments on array values appear
|
||||
- things:
|
||||
- frog
|
||||
food: [pizza] # comments on arrays do not
|
||||
emptyArray: []
|
||||
emptyMap: []
|
||||
["foobar", "foobaz", "blarp"]
|
||||
@@ -1,5 +0,0 @@
|
||||
#! yq
|
||||
.[] |(
|
||||
( select(kind == "scalar") | key + "='" + . + "'"),
|
||||
( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")")
|
||||
)
|
||||
+5
-1
@@ -1 +1,5 @@
|
||||
<zoo><thing><frog>boing</frog></thing></zoo>
|
||||
date: "2022-11-25"
|
||||
raw:
|
||||
id: 1
|
||||
XML: text_outside_1<Tag1 />text_outside_2<Tag2Kling Klong</Tag2>text_outside_3
|
||||
time: 09:19:00
|
||||
@@ -1,26 +0,0 @@
|
||||
|
||||
|
||||
# This is a TOML document
|
||||
|
||||
title = "TOML Example"
|
||||
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00
|
||||
|
||||
[database]
|
||||
enabled = true
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
data = [ ["delta", "phi"], [3.14] ]
|
||||
temp_targets = { cpu = 79.5, case = 72.0 }
|
||||
|
||||
[servers]
|
||||
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
role = "frontend"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
role = "backend"
|
||||
|
||||
+11
-2
@@ -1,2 +1,11 @@
|
||||
# things
|
||||
a: apple
|
||||
# Some doc
|
||||
|
||||
a: true
|
||||
b:
|
||||
c: 2
|
||||
d: [3, 4, 5]
|
||||
e:
|
||||
- name: fred
|
||||
value: 3
|
||||
- name: sam
|
||||
value: 4
|
||||
@@ -1,33 +1,31 @@
|
||||
module github.com/mikefarah/yq/v4
|
||||
|
||||
require (
|
||||
github.com/a8m/envsubst v1.4.2
|
||||
github.com/alecthomas/participle/v2 v2.1.1
|
||||
github.com/alecthomas/repr v0.4.0
|
||||
github.com/a8m/envsubst v1.3.0
|
||||
github.com/alecthomas/participle/v2 v2.0.0-beta.5
|
||||
github.com/alecthomas/repr v0.1.1
|
||||
github.com/dimchansky/utfbom v1.1.1
|
||||
github.com/elliotchance/orderedmap v1.5.1
|
||||
github.com/fatih/color v1.16.0
|
||||
github.com/goccy/go-json v0.10.2
|
||||
github.com/goccy/go-yaml v1.11.3
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/elliotchance/orderedmap v1.5.0
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/goccy/go-json v0.10.0
|
||||
github.com/goccy/go-yaml v1.9.7
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/magiconair/properties v1.8.7
|
||||
github.com/pelletier/go-toml/v2 v2.1.1
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/spf13/cobra v1.6.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/yuin/gopher-lua v1.1.1
|
||||
golang.org/x/net v0.22.0
|
||||
golang.org/x/text v0.14.0
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
|
||||
golang.org/x/text v0.3.8 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
)
|
||||
|
||||
go 1.21
|
||||
go 1.19
|
||||
|
||||
@@ -1,85 +1,83 @@
|
||||
github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
|
||||
github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
|
||||
github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
|
||||
github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
|
||||
github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=
|
||||
github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
|
||||
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
|
||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
|
||||
github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
|
||||
github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
|
||||
github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
|
||||
github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
|
||||
github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs=
|
||||
github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/elliotchance/orderedmap v1.5.1 h1:G1X4PYlljzimbdQ3RXmtIZiQ9d6aRQ3sH1nzjq5mECE=
|
||||
github.com/elliotchance/orderedmap v1.5.1/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
|
||||
github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
|
||||
github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
|
||||
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
|
||||
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
|
||||
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
|
||||
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
|
||||
github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
|
||||
github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-yaml v1.9.7 h1:D/Vx+JITklB1ugSkncB4BNR67M3X6AKs9+rqVeo3ddw=
|
||||
github.com/goccy/go-yaml v1.9.7/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
|
||||
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
|
||||
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
|
||||
github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
|
||||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
|
||||
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
||||
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
|
||||
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
|
||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
||||
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
|
||||
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo=
|
||||
golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ It pipes the current, lets call it 'root' context through the `lhs` expression o
|
||||
cat
|
||||
```
|
||||
|
||||
Side note: this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
|
||||
Sidenote: this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
|
||||
|
||||
The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node
|
||||
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package properties
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/magiconair/properties"
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
)
|
||||
|
||||
type propertiesDecoder struct {
|
||||
reader io.Reader
|
||||
finished bool
|
||||
d yqlib.DataTreeNavigator
|
||||
}
|
||||
|
||||
func NewPropertiesDecoder() yqlib.Decoder {
|
||||
return &propertiesDecoder{d: yqlib.NewDataTreeNavigator(), finished: false}
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) Init(reader io.Reader) error {
|
||||
dec.reader = reader
|
||||
dec.finished = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func parsePropKey(key string) []interface{} {
|
||||
pathStrArray := strings.Split(key, ".")
|
||||
path := make([]interface{}, len(pathStrArray))
|
||||
for i, pathStr := range pathStrArray {
|
||||
num, err := strconv.ParseInt(pathStr, 10, 32)
|
||||
if err == nil {
|
||||
path[i] = num
|
||||
} else {
|
||||
path[i] = pathStr
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) processComment(c string) string {
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
return "# " + c
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error {
|
||||
rhsCandidateNode := &yqlib.CandidateNode{
|
||||
Tag: "!!str",
|
||||
Value: fmt.Sprintf("%v", path[len(path)-1]),
|
||||
HeadComment: dec.processComment(strings.Join(comments, "\n")),
|
||||
Kind: yqlib.ScalarNode,
|
||||
}
|
||||
rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType()
|
||||
return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode)
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error {
|
||||
value, _ := properties.Get(key)
|
||||
path := parsePropKey(key)
|
||||
|
||||
propertyComments := properties.GetComments(key)
|
||||
if len(propertyComments) > 0 {
|
||||
err := dec.applyPropertyComments(context, path, propertyComments)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
rhsNode := yqlib.CreateStringScalarNode(value)
|
||||
rhsNode.Tag = rhsNode.GuessTagFromCustomType()
|
||||
|
||||
return dec.d.DeeplyAssign(context, path, rhsNode)
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
if _, err := buf.ReadFrom(dec.reader); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if buf.Len() == 0 {
|
||||
dec.finished = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
properties, err := properties.LoadString(buf.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
properties.DisableExpansion = true
|
||||
|
||||
rootMap := &yqlib.CandidateNode{
|
||||
Kind: yqlib.MappingNode,
|
||||
Tag: "!!map",
|
||||
}
|
||||
|
||||
context := yqlib.Context{}
|
||||
context = context.SingleChildContext(rootMap)
|
||||
|
||||
for _, key := range properties.Keys() {
|
||||
if err := dec.applyProperty(context, properties, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
}
|
||||
dec.finished = true
|
||||
|
||||
return rootMap, nil
|
||||
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
package properties
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
type keyValuePair struct {
|
||||
key string
|
||||
value string
|
||||
comment string
|
||||
}
|
||||
|
||||
func (kv *keyValuePair) String(unwrap bool, sep string) string {
|
||||
builder := strings.Builder{}
|
||||
|
||||
if kv.comment != "" {
|
||||
builder.WriteString(kv.comment)
|
||||
builder.WriteString("\n")
|
||||
}
|
||||
|
||||
builder.WriteString(kv.key)
|
||||
builder.WriteString(sep)
|
||||
|
||||
if unwrap {
|
||||
builder.WriteString(kv.value)
|
||||
} else {
|
||||
builder.WriteString("\"")
|
||||
builder.WriteString(kv.value)
|
||||
builder.WriteString("\"")
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
type testProperties struct {
|
||||
pairs []keyValuePair
|
||||
}
|
||||
|
||||
func (tp *testProperties) String(unwrap bool, sep string) string {
|
||||
kvs := []string{}
|
||||
|
||||
for _, kv := range tp.pairs {
|
||||
kvs = append(kvs, kv.String(unwrap, sep))
|
||||
}
|
||||
|
||||
return strings.Join(kvs, "\n")
|
||||
}
|
||||
|
||||
func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
var propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar})
|
||||
inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
node := inputs.Front().Value.(*CandidateNode)
|
||||
err = propsEncoder.Encode(writer, node)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
writer.Flush()
|
||||
|
||||
return strings.TrimSuffix(output.String(), "\n")
|
||||
}
|
||||
|
||||
func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) {
|
||||
wraps := []bool{}
|
||||
if testUnwrapped {
|
||||
wraps = append(wraps, true)
|
||||
}
|
||||
if testWrapped {
|
||||
wraps = append(wraps, false)
|
||||
}
|
||||
|
||||
for _, unwrap := range wraps {
|
||||
for _, sep := range []string{" = ", ";", "=", " "} {
|
||||
var actualProps = yamlToProps(sampleYaml, unwrap, sep)
|
||||
test.AssertResult(t, props.String(unwrap, sep), actualProps)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPropertiesEncoderSimple(t *testing.T) {
|
||||
var sampleYaml = `a: 'bob cool'`
|
||||
|
||||
doTest(
|
||||
t, sampleYaml,
|
||||
testProperties{
|
||||
pairs: []keyValuePair{
|
||||
{
|
||||
key: "a",
|
||||
value: "bob cool",
|
||||
},
|
||||
},
|
||||
},
|
||||
true, true,
|
||||
)
|
||||
}
|
||||
|
||||
func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
|
||||
var sampleYaml = `a: 'bob cool' # line`
|
||||
|
||||
doTest(
|
||||
t, sampleYaml,
|
||||
testProperties{
|
||||
pairs: []keyValuePair{
|
||||
{
|
||||
key: "a",
|
||||
value: "bob cool",
|
||||
comment: "# line",
|
||||
},
|
||||
},
|
||||
},
|
||||
true, true,
|
||||
)
|
||||
}
|
||||
|
||||
func TestPropertiesEncoderDeep(t *testing.T) {
|
||||
var sampleYaml = `a:
|
||||
b: "bob cool"
|
||||
`
|
||||
|
||||
doTest(
|
||||
t, sampleYaml,
|
||||
testProperties{
|
||||
pairs: []keyValuePair{
|
||||
{
|
||||
key: "a.b",
|
||||
value: "bob cool",
|
||||
},
|
||||
},
|
||||
},
|
||||
true, true,
|
||||
)
|
||||
}
|
||||
|
||||
func TestPropertiesEncoderDeepWithComments(t *testing.T) {
|
||||
var sampleYaml = `a: # a thing
|
||||
b: "bob cool" # b thing
|
||||
`
|
||||
|
||||
doTest(
|
||||
t, sampleYaml,
|
||||
testProperties{
|
||||
pairs: []keyValuePair{
|
||||
{
|
||||
key: "a.b",
|
||||
value: "bob cool",
|
||||
comment: "# b thing",
|
||||
},
|
||||
},
|
||||
},
|
||||
true, true,
|
||||
)
|
||||
}
|
||||
|
||||
func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
|
||||
var sampleYaml = `a:
|
||||
b: [{c: dog}, {c: cat}]
|
||||
`
|
||||
|
||||
doTest(
|
||||
t, sampleYaml,
|
||||
testProperties{
|
||||
pairs: []keyValuePair{
|
||||
{
|
||||
key: "a.b.0.c",
|
||||
value: "dog",
|
||||
},
|
||||
{
|
||||
key: "a.b.1.c",
|
||||
value: "cat",
|
||||
},
|
||||
},
|
||||
},
|
||||
true, false,
|
||||
)
|
||||
}
|
||||
|
||||
func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
|
||||
var sampleYaml = `a:
|
||||
b: [{c: dog named jim}, {c: cat named jim}]
|
||||
`
|
||||
|
||||
doTest(
|
||||
t, sampleYaml,
|
||||
testProperties{
|
||||
pairs: []keyValuePair{
|
||||
{
|
||||
key: "a.b.0.c",
|
||||
value: "dog named jim",
|
||||
},
|
||||
{
|
||||
key: "a.b.1.c",
|
||||
value: "cat named jim",
|
||||
},
|
||||
},
|
||||
},
|
||||
false, true,
|
||||
)
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package properties
|
||||
|
||||
import "github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
|
||||
type PropertiesPreferences struct {
|
||||
UnwrapScalar bool
|
||||
KeyValueSeparator string
|
||||
UseArrayBrackets bool
|
||||
}
|
||||
|
||||
func NewDefaultPropertiesPreferences() PropertiesPreferences {
|
||||
return PropertiesPreferences{
|
||||
UnwrapScalar: true,
|
||||
KeyValueSeparator: " = ",
|
||||
UseArrayBrackets: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PropertiesPreferences) Copy() PropertiesPreferences {
|
||||
return PropertiesPreferences{
|
||||
UnwrapScalar: p.UnwrapScalar,
|
||||
KeyValueSeparator: p.KeyValueSeparator,
|
||||
UseArrayBrackets: p.UseArrayBrackets,
|
||||
}
|
||||
}
|
||||
|
||||
var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties",
|
||||
func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
|
||||
func() yqlib.Decoder { return NewPropertiesDecoder() },
|
||||
nil,
|
||||
}
|
||||
|
||||
var propertyYqRules = []*yqlib.ParticipleYqRule{
|
||||
{"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0},
|
||||
{"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0},
|
||||
{"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0},
|
||||
}
|
||||
|
||||
func RegisterPropertiesFormat() {
|
||||
yqlib.RegisterFormat(PropertiesFormat)
|
||||
yqlib.RegisterRules(propertyYqRules)
|
||||
|
||||
}
|
||||
|
||||
var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()
|
||||
@@ -1,72 +0,0 @@
|
||||
package xml
|
||||
|
||||
import "github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
|
||||
type XmlPreferences struct {
|
||||
Indent int
|
||||
AttributePrefix string
|
||||
ContentName string
|
||||
StrictMode bool
|
||||
KeepNamespace bool
|
||||
UseRawToken bool
|
||||
ProcInstPrefix string
|
||||
DirectiveName string
|
||||
SkipProcInst bool
|
||||
SkipDirectives bool
|
||||
}
|
||||
|
||||
func NewDefaultXmlPreferences() XmlPreferences {
|
||||
return XmlPreferences{
|
||||
Indent: 2,
|
||||
AttributePrefix: "+@",
|
||||
ContentName: "+content",
|
||||
StrictMode: false,
|
||||
KeepNamespace: true,
|
||||
UseRawToken: true,
|
||||
ProcInstPrefix: "+p_",
|
||||
DirectiveName: "+directive",
|
||||
SkipProcInst: false,
|
||||
SkipDirectives: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *XmlPreferences) Copy() XmlPreferences {
|
||||
return XmlPreferences{
|
||||
Indent: p.Indent,
|
||||
AttributePrefix: p.AttributePrefix,
|
||||
ContentName: p.ContentName,
|
||||
StrictMode: p.StrictMode,
|
||||
KeepNamespace: p.KeepNamespace,
|
||||
UseRawToken: p.UseRawToken,
|
||||
ProcInstPrefix: p.ProcInstPrefix,
|
||||
DirectiveName: p.DirectiveName,
|
||||
SkipProcInst: p.SkipProcInst,
|
||||
SkipDirectives: p.SkipDirectives,
|
||||
}
|
||||
}
|
||||
|
||||
var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
|
||||
|
||||
var XMLFormat = &yqlib.Format{"xml", []string{"x"}, "xml",
|
||||
func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
|
||||
func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
|
||||
func(indent int) yqlib.Encoder {
|
||||
prefs := ConfiguredXMLPreferences.Copy()
|
||||
prefs.Indent = indent
|
||||
return NewXMLEncoder(prefs)
|
||||
},
|
||||
}
|
||||
|
||||
var xmlYqRules = []*yqlib.ParticipleYqRule{
|
||||
{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, yqlib.CreateEncodeYqActionParsingIndent(XMLFormat), 0},
|
||||
{"XmlDecode", `from_?xml|@xmld`, yqlib.CreateDecodeOpYqAction(XMLFormat), 0},
|
||||
{"XMLEncode", `to_?xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 2), 0},
|
||||
{"XMLEncodeNoIndent", `@xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 0), 0},
|
||||
{"LoadXML", `load_?xml|xml_?load`, yqlib.CreateLoadOpYqAction(NewXMLDecoder(ConfiguredXMLPreferences)), 0},
|
||||
}
|
||||
|
||||
func RegisterXmlFormat() {
|
||||
yqlib.RegisterFormat(XMLFormat)
|
||||
yqlib.RegisterRules(xmlYqRules)
|
||||
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package yqlib
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// A yaml expression evaluator that runs the expression once against all files/nodes in memory.
|
||||
@@ -9,7 +11,7 @@ type Evaluator interface {
|
||||
EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
|
||||
|
||||
// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
|
||||
EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error)
|
||||
EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
|
||||
|
||||
// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes
|
||||
EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)
|
||||
@@ -24,10 +26,10 @@ func NewAllAtOnceEvaluator() Evaluator {
|
||||
return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
|
||||
}
|
||||
|
||||
func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) {
|
||||
func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
|
||||
inputCandidates := list.New()
|
||||
for _, node := range nodes {
|
||||
inputCandidates.PushBack(node)
|
||||
inputCandidates.PushBack(&CandidateNode{Node: node})
|
||||
}
|
||||
return e.EvaluateCandidateNodes(expression, inputCandidates)
|
||||
}
|
||||
@@ -63,7 +65,13 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
|
||||
}
|
||||
|
||||
if allDocuments.Len() == 0 {
|
||||
candidateNode := CreateScalarNode(nil, "")
|
||||
candidateNode := &CandidateNode{
|
||||
Document: 0,
|
||||
Filename: "",
|
||||
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
|
||||
FileIndex: 0,
|
||||
LeadingContent: "",
|
||||
}
|
||||
allDocuments.PushBack(candidateNode)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
@@ -20,7 +18,7 @@ var evaluateNodesScenario = []expressionScenario{
|
||||
document: `a: hello`,
|
||||
expression: `.`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: hello\n",
|
||||
"D0, P[], (doc)::a: hello\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -34,23 +32,9 @@ var evaluateNodesScenario = []expressionScenario{
|
||||
|
||||
func TestAllAtOnceEvaluateNodes(t *testing.T) {
|
||||
var evaluator = NewAllAtOnceEvaluator()
|
||||
// logging.SetLevel(logging.DEBUG, "")
|
||||
for _, tt := range evaluateNodesScenario {
|
||||
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
|
||||
reader := bufio.NewReader(strings.NewReader(tt.document))
|
||||
err := decoder.Init(reader)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
candidateNode, errorReading := decoder.Decode()
|
||||
|
||||
if errorReading != nil {
|
||||
t.Error(errorReading)
|
||||
return
|
||||
}
|
||||
|
||||
list, _ := evaluator.EvaluateNodes(tt.expression, candidateNode)
|
||||
node := test.ParseData(tt.document)
|
||||
list, _ := evaluator.EvaluateNodes(tt.expression, &node)
|
||||
test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
|
||||
}
|
||||
}
|
||||
|
||||
+100
-372
@@ -3,189 +3,51 @@ package yqlib
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jinzhu/copier"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Kind uint32
|
||||
|
||||
const (
|
||||
SequenceNode Kind = 1 << iota
|
||||
MappingNode
|
||||
ScalarNode
|
||||
AliasNode
|
||||
)
|
||||
|
||||
type Style uint32
|
||||
|
||||
const (
|
||||
TaggedStyle Style = 1 << iota
|
||||
DoubleQuotedStyle
|
||||
SingleQuotedStyle
|
||||
LiteralStyle
|
||||
FoldedStyle
|
||||
FlowStyle
|
||||
)
|
||||
|
||||
func CreateStringScalarNode(stringValue string) *CandidateNode {
|
||||
var node = &CandidateNode{Kind: ScalarNode}
|
||||
node.Value = stringValue
|
||||
node.Tag = "!!str"
|
||||
return node
|
||||
}
|
||||
|
||||
func CreateScalarNode(value interface{}, stringValue string) *CandidateNode {
|
||||
var node = &CandidateNode{Kind: ScalarNode}
|
||||
node.Value = stringValue
|
||||
|
||||
switch value.(type) {
|
||||
case float32, float64:
|
||||
node.Tag = "!!float"
|
||||
case int, int64, int32:
|
||||
node.Tag = "!!int"
|
||||
case bool:
|
||||
node.Tag = "!!bool"
|
||||
case string:
|
||||
node.Tag = "!!str"
|
||||
case nil:
|
||||
node.Tag = "!!null"
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
type CandidateNode struct {
|
||||
Kind Kind
|
||||
Style Style
|
||||
|
||||
Tag string
|
||||
Value string
|
||||
Anchor string
|
||||
Alias *CandidateNode
|
||||
Content []*CandidateNode
|
||||
|
||||
HeadComment string
|
||||
LineComment string
|
||||
FootComment string
|
||||
|
||||
Node *yaml.Node // the actual node
|
||||
Parent *CandidateNode // parent node
|
||||
Key *CandidateNode // node key, if this is a value from a map (or index in an array)
|
||||
Key *yaml.Node // node key, if this is a value from a map (or index in an array)
|
||||
|
||||
LeadingContent string
|
||||
LeadingContent string
|
||||
TrailingContent string
|
||||
|
||||
document uint // the document index of this node
|
||||
filename string
|
||||
|
||||
Line int
|
||||
Column int
|
||||
|
||||
fileIndex int
|
||||
Path []interface{} /// the path we took to get to this node
|
||||
Document uint // the document index of this node
|
||||
Filename string
|
||||
FileIndex int
|
||||
// when performing op against all nodes given, this will treat all the nodes as one
|
||||
// (e.g. top level cross document merge). This property does not propagate to child nodes.
|
||||
EvaluateTogether bool
|
||||
IsMapKey bool
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CreateChild() *CandidateNode {
|
||||
return &CandidateNode{
|
||||
Parent: n,
|
||||
}
|
||||
}
|
||||
|
||||
func (n *CandidateNode) SetDocument(idx uint) {
|
||||
n.document = idx
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetDocument() uint {
|
||||
// defer to parent
|
||||
if n.Parent != nil {
|
||||
return n.Parent.GetDocument()
|
||||
}
|
||||
return n.document
|
||||
}
|
||||
|
||||
func (n *CandidateNode) SetFilename(name string) {
|
||||
n.filename = name
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetFilename() string {
|
||||
if n.Parent != nil {
|
||||
return n.Parent.GetFilename()
|
||||
}
|
||||
return n.filename
|
||||
}
|
||||
|
||||
func (n *CandidateNode) SetFileIndex(idx int) {
|
||||
n.fileIndex = idx
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetFileIndex() int {
|
||||
if n.Parent != nil {
|
||||
return n.Parent.GetFileIndex()
|
||||
}
|
||||
return n.fileIndex
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetKey() string {
|
||||
keyPrefix := ""
|
||||
if n.IsMapKey {
|
||||
keyPrefix = fmt.Sprintf("key-%v-", n.Value)
|
||||
keyPrefix = "key-"
|
||||
}
|
||||
key := ""
|
||||
if n.Key != nil {
|
||||
key = n.Key.Value
|
||||
}
|
||||
return fmt.Sprintf("%v%v - %v", keyPrefix, n.GetDocument(), key)
|
||||
return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) getParsedKey() interface{} {
|
||||
if n.IsMapKey {
|
||||
return n.Value
|
||||
}
|
||||
if n.Key == nil {
|
||||
return nil
|
||||
}
|
||||
if n.Key.Tag == "!!str" {
|
||||
return n.Key.Value
|
||||
}
|
||||
index, err := parseInt(n.Key.Value)
|
||||
if err != nil {
|
||||
return n.Key.Value
|
||||
}
|
||||
return index
|
||||
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetPath() []interface{} {
|
||||
key := n.getParsedKey()
|
||||
if n.Parent != nil && key != nil {
|
||||
return append(n.Parent.GetPath(), key)
|
||||
}
|
||||
|
||||
if key != nil {
|
||||
return []interface{}{key}
|
||||
}
|
||||
return make([]interface{}, 0)
|
||||
func (n *CandidateNode) GetNiceTag() string {
|
||||
return unwrapDoc(n.Node).Tag
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetNicePath() string {
|
||||
var sb strings.Builder
|
||||
path := n.GetPath()
|
||||
for i, element := range path {
|
||||
elementStr := fmt.Sprintf("%v", element)
|
||||
switch element.(type) {
|
||||
case int:
|
||||
sb.WriteString("[" + elementStr + "]")
|
||||
default:
|
||||
if i == 0 {
|
||||
sb.WriteString(elementStr)
|
||||
} else if strings.ContainsRune(elementStr, '.') {
|
||||
sb.WriteString("[" + elementStr + "]")
|
||||
} else {
|
||||
sb.WriteString("." + elementStr)
|
||||
}
|
||||
if n.Path != nil && len(n.Path) >= 0 {
|
||||
pathStr := make([]string, len(n.Path))
|
||||
for i, v := range n.Path {
|
||||
pathStr[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
return strings.Join(pathStr, ".")
|
||||
}
|
||||
return sb.String()
|
||||
return ""
|
||||
}
|
||||
|
||||
func (n *CandidateNode) AsList() *list.List {
|
||||
@@ -194,268 +56,134 @@ func (n *CandidateNode) AsList() *list.List {
|
||||
return elMap
|
||||
}
|
||||
|
||||
func (n *CandidateNode) SetParent(parent *CandidateNode) {
|
||||
n.Parent = parent
|
||||
}
|
||||
|
||||
func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) {
|
||||
key := rawKey.Copy()
|
||||
key.SetParent(n)
|
||||
key.IsMapKey = true
|
||||
|
||||
value := rawValue.Copy()
|
||||
value.SetParent(n)
|
||||
value.IsMapKey = false // force this, incase we are creating a value from a key
|
||||
value.Key = key
|
||||
|
||||
n.Content = append(n.Content, key, value)
|
||||
return key, value
|
||||
}
|
||||
|
||||
func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
|
||||
value := rawChild.Copy()
|
||||
value.SetParent(n)
|
||||
if value.Key != nil {
|
||||
value.Key.SetParent(n)
|
||||
} else {
|
||||
index := len(n.Content)
|
||||
keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index))
|
||||
keyNode.SetParent(n)
|
||||
value.Key = keyNode
|
||||
func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode {
|
||||
var value interface{}
|
||||
if key != nil {
|
||||
value = key.Value
|
||||
}
|
||||
n.Content = append(n.Content, value)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) AddChildren(children []*CandidateNode) {
|
||||
if n.Kind == MappingNode {
|
||||
for i := 0; i < len(children); i += 2 {
|
||||
key := children[i]
|
||||
value := children[i+1]
|
||||
n.AddKeyValueChild(key, value)
|
||||
}
|
||||
|
||||
} else {
|
||||
for _, rawChild := range children {
|
||||
n.AddChild(rawChild)
|
||||
}
|
||||
return &CandidateNode{
|
||||
Node: node,
|
||||
Path: n.createChildPath(value),
|
||||
Parent: n,
|
||||
Key: key,
|
||||
Document: n.Document,
|
||||
Filename: n.Filename,
|
||||
FileIndex: n.FileIndex,
|
||||
}
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GetValueRep() (interface{}, error) {
|
||||
log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
|
||||
realTag := n.GuessTagFromCustomType()
|
||||
|
||||
switch realTag {
|
||||
case "!!int":
|
||||
_, val, err := parseInt64(n.Value)
|
||||
return val, err
|
||||
case "!!float":
|
||||
// need to test this
|
||||
return strconv.ParseFloat(n.Value, 64)
|
||||
case "!!bool":
|
||||
return isTruthyNode(n), nil
|
||||
case "!!null":
|
||||
return nil, nil
|
||||
func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode {
|
||||
return &CandidateNode{
|
||||
Node: node,
|
||||
Path: n.createChildPath(index),
|
||||
Parent: n,
|
||||
Key: &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"},
|
||||
Document: n.Document,
|
||||
Filename: n.Filename,
|
||||
FileIndex: n.FileIndex,
|
||||
}
|
||||
|
||||
return n.Value, nil
|
||||
}
|
||||
|
||||
func (n *CandidateNode) GuessTagFromCustomType() string {
|
||||
if strings.HasPrefix(n.Tag, "!!") {
|
||||
return n.Tag
|
||||
} else if n.Value == "" {
|
||||
log.Debug("GuessTagFromCustomType: node has no value to guess the type with")
|
||||
return n.Tag
|
||||
func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
|
||||
return &CandidateNode{
|
||||
Node: node,
|
||||
Path: n.createChildPath(nil),
|
||||
Parent: n.Parent,
|
||||
Key: n.Key,
|
||||
IsMapKey: n.IsMapKey,
|
||||
Document: n.Document,
|
||||
Filename: n.Filename,
|
||||
FileIndex: n.FileIndex,
|
||||
}
|
||||
dataBucket, errorReading := parseSnippet(n.Value)
|
||||
|
||||
if errorReading != nil {
|
||||
log.Debug("GuessTagFromCustomType: could not guess underlying tag type %v", errorReading)
|
||||
return n.Tag
|
||||
}
|
||||
guessedTag := dataBucket.Tag
|
||||
log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
|
||||
return guessedTag
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) *CandidateNode {
|
||||
node := &CandidateNode{
|
||||
Kind: kind,
|
||||
Tag: tag,
|
||||
Value: value,
|
||||
}
|
||||
return n.CopyAsReplacement(node)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode {
|
||||
newCopy := replacement.Copy()
|
||||
newCopy.Parent = n.Parent
|
||||
|
||||
if n.IsMapKey {
|
||||
newCopy.Key = n
|
||||
} else {
|
||||
newCopy.Key = n.Key
|
||||
}
|
||||
|
||||
return newCopy
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode {
|
||||
replacement := n.CreateReplacement(kind, tag, "")
|
||||
func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
|
||||
replacement := n.CreateReplacement(node)
|
||||
replacement.LeadingContent = n.LeadingContent
|
||||
replacement.HeadComment = n.HeadComment
|
||||
replacement.LineComment = n.LineComment
|
||||
replacement.FootComment = n.FootComment
|
||||
replacement.Style = style
|
||||
replacement.TrailingContent = n.TrailingContent
|
||||
return replacement
|
||||
}
|
||||
|
||||
func (n *CandidateNode) Copy() *CandidateNode {
|
||||
return n.doCopy(true)
|
||||
func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
|
||||
if path == nil {
|
||||
newPath := make([]interface{}, len(n.Path))
|
||||
copy(newPath, n.Path)
|
||||
return newPath
|
||||
}
|
||||
|
||||
//don't use append as they may actually modify the path of the orignal node!
|
||||
newPath := make([]interface{}, len(n.Path)+1)
|
||||
copy(newPath, n.Path)
|
||||
newPath[len(n.Path)] = path
|
||||
return newPath
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CopyWithoutContent() *CandidateNode {
|
||||
return n.doCopy(false)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
|
||||
var content []*CandidateNode
|
||||
|
||||
var copyKey *CandidateNode
|
||||
if n.Key != nil {
|
||||
copyKey = n.Key.Copy()
|
||||
func (n *CandidateNode) Copy() (*CandidateNode, error) {
|
||||
clone := &CandidateNode{}
|
||||
err := copier.Copy(clone, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clone := &CandidateNode{
|
||||
Kind: n.Kind,
|
||||
Style: n.Style,
|
||||
|
||||
Tag: n.Tag,
|
||||
Value: n.Value,
|
||||
Anchor: n.Anchor,
|
||||
|
||||
// ok not to clone this,
|
||||
// as its a reference to somewhere else.
|
||||
Alias: n.Alias,
|
||||
Content: content,
|
||||
|
||||
HeadComment: n.HeadComment,
|
||||
LineComment: n.LineComment,
|
||||
FootComment: n.FootComment,
|
||||
|
||||
Parent: n.Parent,
|
||||
Key: copyKey,
|
||||
|
||||
LeadingContent: n.LeadingContent,
|
||||
|
||||
document: n.document,
|
||||
filename: n.filename,
|
||||
fileIndex: n.fileIndex,
|
||||
|
||||
Line: n.Line,
|
||||
Column: n.Column,
|
||||
|
||||
EvaluateTogether: n.EvaluateTogether,
|
||||
IsMapKey: n.IsMapKey,
|
||||
}
|
||||
|
||||
if cloneContent {
|
||||
clone.AddChildren(n.Content)
|
||||
}
|
||||
|
||||
return clone
|
||||
clone.Node = deepClone(n.Node)
|
||||
return clone, nil
|
||||
}
|
||||
|
||||
// updates this candidate from the given candidate node
|
||||
func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {
|
||||
if n == other {
|
||||
log.Debugf("UpdateFrom, no need to update from myself.")
|
||||
return
|
||||
}
|
||||
|
||||
// if this is an empty map or empty array, use the style of other node.
|
||||
if (n.Kind != ScalarNode && len(n.Content) == 0) ||
|
||||
if (n.Node.Kind != yaml.ScalarNode && len(n.Node.Content) == 0) ||
|
||||
// if the tag has changed (e.g. from str to bool)
|
||||
(n.GuessTagFromCustomType() != other.GuessTagFromCustomType()) {
|
||||
n.Style = other.Style
|
||||
(guessTagFromCustomType(n.Node) != guessTagFromCustomType(other.Node)) {
|
||||
n.Node.Style = other.Node.Style
|
||||
}
|
||||
|
||||
n.Content = make([]*CandidateNode, 0)
|
||||
n.Kind = other.Kind
|
||||
n.AddChildren(other.Content)
|
||||
|
||||
n.Value = other.Value
|
||||
n.Node.Content = deepCloneContent(other.Node.Content)
|
||||
n.Node.Kind = other.Node.Kind
|
||||
n.Node.Value = other.Node.Value
|
||||
|
||||
n.UpdateAttributesFrom(other, prefs)
|
||||
|
||||
}
|
||||
|
||||
func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
|
||||
log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
|
||||
if n.Kind != other.Kind {
|
||||
log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
|
||||
if n.Node.Kind != other.Node.Kind {
|
||||
// clear out the contents when switching to a different type
|
||||
// e.g. map to array
|
||||
n.Content = make([]*CandidateNode, 0)
|
||||
n.Value = ""
|
||||
n.Node.Content = make([]*yaml.Node, 0)
|
||||
n.Node.Value = ""
|
||||
}
|
||||
n.Kind = other.Kind
|
||||
n.Node.Kind = other.Node.Kind
|
||||
|
||||
// don't clobber custom tags...
|
||||
if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" {
|
||||
n.Tag = other.Tag
|
||||
if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
|
||||
n.Node.Tag = other.Node.Tag
|
||||
}
|
||||
|
||||
n.Alias = other.Alias
|
||||
n.Node.Alias = other.Node.Alias
|
||||
|
||||
if !prefs.DontOverWriteAnchor {
|
||||
n.Anchor = other.Anchor
|
||||
n.Node.Anchor = other.Node.Anchor
|
||||
}
|
||||
|
||||
// merge will pickup the style of the new thing
|
||||
// when autocreating nodes
|
||||
|
||||
if n.Style == 0 {
|
||||
n.Style = other.Style
|
||||
if n.Node.Style == 0 {
|
||||
n.Node.Style = other.Node.Style
|
||||
}
|
||||
|
||||
if other.FootComment != "" {
|
||||
n.FootComment = other.FootComment
|
||||
if other.Node.FootComment != "" {
|
||||
n.Node.FootComment = other.Node.FootComment
|
||||
}
|
||||
if other.HeadComment != "" {
|
||||
n.HeadComment = other.HeadComment
|
||||
if other.TrailingContent != "" {
|
||||
n.TrailingContent = other.TrailingContent
|
||||
}
|
||||
if other.LineComment != "" {
|
||||
n.LineComment = other.LineComment
|
||||
if other.Node.HeadComment != "" {
|
||||
n.Node.HeadComment = other.Node.HeadComment
|
||||
}
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CleanHeadAndLineComment() string {
|
||||
return n.CleanHeadComment() + n.CleanLineComment()
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CleanHeadComment() string {
|
||||
return strings.Replace(n.HeadComment, "#", "", 1)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CleanLineComment() string {
|
||||
return strings.Replace(n.LineComment, "#", "", 1)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CleanFootComment() string {
|
||||
return strings.Replace(n.FootComment, "#", "", 1)
|
||||
}
|
||||
|
||||
func (n *CandidateNode) ConvertKeysToStrings() {
|
||||
|
||||
if n.Kind == MappingNode {
|
||||
for index, child := range n.Content {
|
||||
if index%2 == 0 { // its a map key
|
||||
child.Tag = "!!str"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, child := range n.Content {
|
||||
child.ConvertKeysToStrings()
|
||||
if other.Node.LineComment != "" {
|
||||
n.Node.LineComment = other.Node.LineComment
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/goccy/go-yaml/ast"
|
||||
goccyToken "github.com/goccy/go-yaml/token"
|
||||
)
|
||||
|
||||
func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.CommentMap) (*CandidateNode, error) {
|
||||
newChild := o.CreateChild()
|
||||
|
||||
err := newChild.UnmarshalGoccyYAML(childNode, cm)
|
||||
return newChild, err
|
||||
}
|
||||
|
||||
func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap) error {
|
||||
log.Debugf("UnmarshalYAML %v", node)
|
||||
log.Debugf("UnmarshalYAML %v", node.Type().String())
|
||||
log.Debugf("UnmarshalYAML Node Value: %v", node.String())
|
||||
log.Debugf("UnmarshalYAML Node GetComment: %v", node.GetComment())
|
||||
|
||||
if node.GetComment() != nil {
|
||||
commentMapComments := cm[node.GetPath()]
|
||||
for _, comment := range node.GetComment().Comments {
|
||||
// need to use the comment map to find the position :/
|
||||
log.Debugf("%v has a comment of [%v]", node.GetPath(), comment.Token.Value)
|
||||
for _, commentMapComment := range commentMapComments {
|
||||
commentMapValue := strings.Join(commentMapComment.Texts, "\n")
|
||||
if commentMapValue == comment.Token.Value {
|
||||
log.Debug("found a matching entry in comment map")
|
||||
// we found the comment in the comment map,
|
||||
// now we can process the position
|
||||
switch commentMapComment.Position {
|
||||
case yaml.CommentHeadPosition:
|
||||
o.HeadComment = comment.String()
|
||||
log.Debug("its a head comment %v", comment.String())
|
||||
case yaml.CommentLinePosition:
|
||||
o.LineComment = comment.String()
|
||||
log.Debug("its a line comment %v", comment.String())
|
||||
case yaml.CommentFootPosition:
|
||||
o.FootComment = comment.String()
|
||||
log.Debug("its a foot comment %v", comment.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
o.Value = node.String()
|
||||
switch node.Type() {
|
||||
case ast.IntegerType:
|
||||
o.Kind = ScalarNode
|
||||
o.Tag = "!!int"
|
||||
case ast.FloatType:
|
||||
o.Kind = ScalarNode
|
||||
o.Tag = "!!float"
|
||||
case ast.BoolType:
|
||||
o.Kind = ScalarNode
|
||||
o.Tag = "!!bool"
|
||||
case ast.NullType:
|
||||
o.Kind = ScalarNode
|
||||
o.Tag = "!!null"
|
||||
o.Value = node.GetToken().Value
|
||||
case ast.StringType:
|
||||
o.Kind = ScalarNode
|
||||
o.Tag = "!!str"
|
||||
switch node.GetToken().Type {
|
||||
case goccyToken.SingleQuoteType:
|
||||
o.Style = SingleQuotedStyle
|
||||
case goccyToken.DoubleQuoteType:
|
||||
o.Style = DoubleQuotedStyle
|
||||
}
|
||||
o.Value = node.(*ast.StringNode).Value
|
||||
log.Debugf("string value %v", node.(*ast.StringNode).Value)
|
||||
case ast.LiteralType:
|
||||
o.Kind = ScalarNode
|
||||
o.Tag = "!!str"
|
||||
o.Style = LiteralStyle
|
||||
astLiteral := node.(*ast.LiteralNode)
|
||||
log.Debugf("astLiteral.Start.Type %v", astLiteral.Start.Type)
|
||||
if astLiteral.Start.Type == goccyToken.FoldedType {
|
||||
log.Debugf("folded Type %v", astLiteral.Start.Type)
|
||||
o.Style = FoldedStyle
|
||||
}
|
||||
log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value)
|
||||
log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type)
|
||||
// TODO: here I could put the original value with line breaks
|
||||
// to solve the multiline > problem
|
||||
o.Value = astLiteral.Value.Value
|
||||
case ast.TagType:
|
||||
if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm); err != nil {
|
||||
return err
|
||||
}
|
||||
o.Tag = node.(*ast.TagNode).Start.Value
|
||||
case ast.MappingType:
|
||||
log.Debugf("UnmarshalYAML - a mapping node")
|
||||
o.Kind = MappingNode
|
||||
o.Tag = "!!map"
|
||||
|
||||
mappingNode := node.(*ast.MappingNode)
|
||||
if mappingNode.IsFlowStyle {
|
||||
o.Style = FlowStyle
|
||||
}
|
||||
for _, mappingValueNode := range mappingNode.Values {
|
||||
err := o.goccyProcessMappingValueNode(mappingValueNode, cm)
|
||||
if err != nil {
|
||||
return ast.ErrInvalidAnchorName
|
||||
}
|
||||
}
|
||||
if mappingNode.FootComment != nil {
|
||||
log.Debugf("mapping node has a foot comment of: %v", mappingNode.FootComment)
|
||||
o.FootComment = mappingNode.FootComment.String()
|
||||
}
|
||||
case ast.MappingValueType:
|
||||
log.Debugf("UnmarshalYAML - a mapping node")
|
||||
o.Kind = MappingNode
|
||||
o.Tag = "!!map"
|
||||
mappingValueNode := node.(*ast.MappingValueNode)
|
||||
err := o.goccyProcessMappingValueNode(mappingValueNode, cm)
|
||||
if err != nil {
|
||||
return ast.ErrInvalidAnchorName
|
||||
}
|
||||
case ast.SequenceType:
|
||||
log.Debugf("UnmarshalYAML - a sequence node")
|
||||
o.Kind = SequenceNode
|
||||
o.Tag = "!!seq"
|
||||
sequenceNode := node.(*ast.SequenceNode)
|
||||
if sequenceNode.IsFlowStyle {
|
||||
o.Style = FlowStyle
|
||||
}
|
||||
astSeq := sequenceNode.Values
|
||||
o.Content = make([]*CandidateNode, len(astSeq))
|
||||
for i := 0; i < len(astSeq); i++ {
|
||||
keyNode := o.CreateChild()
|
||||
keyNode.IsMapKey = true
|
||||
keyNode.Tag = "!!int"
|
||||
keyNode.Kind = ScalarNode
|
||||
keyNode.Value = fmt.Sprintf("%v", i)
|
||||
|
||||
valueNode, err := o.goccyDecodeIntoChild(astSeq[i], cm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valueNode.Key = keyNode
|
||||
o.Content[i] = valueNode
|
||||
}
|
||||
|
||||
default:
|
||||
log.Debugf("UnmarshalYAML - node idea of the type!!")
|
||||
}
|
||||
log.Debugf("KIND: %v", o.Kind)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap) error {
|
||||
log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
|
||||
keyNode, err := o.goccyDecodeIntoChild(mappingEntry.Key, cm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keyNode.IsMapKey = true
|
||||
|
||||
log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
|
||||
valueNode, err := o.goccyDecodeIntoChild(mappingEntry.Value, cm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if mappingEntry.FootComment != nil {
|
||||
valueNode.FootComment = mappingEntry.FootComment.String()
|
||||
}
|
||||
o.AddKeyValueChild(keyNode, valueNode)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
type valueRepScenario struct {
|
||||
input string
|
||||
tag string
|
||||
expected interface{}
|
||||
}
|
||||
|
||||
var valueRepScenarios = []valueRepScenario{
|
||||
{
|
||||
input: `"cat"`,
|
||||
expected: `"cat"`,
|
||||
},
|
||||
{
|
||||
input: `3`,
|
||||
expected: int64(3),
|
||||
},
|
||||
{
|
||||
input: `3.1`,
|
||||
expected: float64(3.1),
|
||||
},
|
||||
{
|
||||
input: `true`,
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
input: `y`,
|
||||
tag: "!!bool",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
tag: "!!null",
|
||||
expected: nil,
|
||||
},
|
||||
}
|
||||
|
||||
func TestCandidateNodeGetValueRepScenarios(t *testing.T) {
|
||||
for _, tt := range valueRepScenarios {
|
||||
node := CandidateNode{Value: tt.input, Tag: tt.tag}
|
||||
actual, err := node.GetValueRep()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
test.AssertResult(t, tt.expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
|
||||
parent := CandidateNode{}
|
||||
child := parent.CreateChild()
|
||||
parent.SetDocument(1)
|
||||
parent.SetFileIndex(2)
|
||||
parent.SetFilename("meow")
|
||||
test.AssertResultWithContext(t, "meow", child.GetFilename(), "filename")
|
||||
test.AssertResultWithContext(t, 2, child.GetFileIndex(), "file index")
|
||||
test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
|
||||
}
|
||||
|
||||
type CreateScalarNodeScenario struct {
|
||||
value interface{}
|
||||
stringValue string
|
||||
expectedTag string
|
||||
}
|
||||
|
||||
var createScalarScenarios = []CreateScalarNodeScenario{
|
||||
{
|
||||
value: "mike",
|
||||
stringValue: "mike",
|
||||
expectedTag: "!!str",
|
||||
},
|
||||
{
|
||||
value: 3,
|
||||
stringValue: "3",
|
||||
expectedTag: "!!int",
|
||||
},
|
||||
{
|
||||
value: 3.1,
|
||||
stringValue: "3.1",
|
||||
expectedTag: "!!float",
|
||||
},
|
||||
{
|
||||
value: true,
|
||||
stringValue: "true",
|
||||
expectedTag: "!!bool",
|
||||
},
|
||||
{
|
||||
value: nil,
|
||||
stringValue: "~",
|
||||
expectedTag: "!!null",
|
||||
},
|
||||
}
|
||||
|
||||
func TestCreateScalarNodeScenarios(t *testing.T) {
|
||||
for _, tt := range createScalarScenarios {
|
||||
actual := CreateScalarNode(tt.value, tt.stringValue)
|
||||
test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
|
||||
test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetKeyForMapValue(t *testing.T) {
|
||||
key := CreateStringScalarNode("yourKey")
|
||||
n := CandidateNode{Key: key, Value: "meow", document: 3}
|
||||
test.AssertResult(t, "3 - yourKey", n.GetKey())
|
||||
}
|
||||
|
||||
func TestGetKeyForMapKey(t *testing.T) {
|
||||
key := CreateStringScalarNode("yourKey")
|
||||
key.IsMapKey = true
|
||||
key.document = 3
|
||||
test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
|
||||
}
|
||||
|
||||
func TestGetKeyForValue(t *testing.T) {
|
||||
n := CandidateNode{Value: "meow", document: 3}
|
||||
test.AssertResult(t, "3 - ", n.GetKey())
|
||||
}
|
||||
|
||||
func TestGetParsedKeyForMapKey(t *testing.T) {
|
||||
key := CreateStringScalarNode("yourKey")
|
||||
key.IsMapKey = true
|
||||
key.document = 3
|
||||
test.AssertResult(t, "yourKey", key.getParsedKey())
|
||||
}
|
||||
|
||||
func TestGetParsedKeyForLooseValue(t *testing.T) {
|
||||
n := CandidateNode{Value: "meow", document: 3}
|
||||
test.AssertResult(t, nil, n.getParsedKey())
|
||||
}
|
||||
|
||||
func TestGetParsedKeyForMapValue(t *testing.T) {
|
||||
key := CreateStringScalarNode("yourKey")
|
||||
n := CandidateNode{Key: key, Value: "meow", document: 3}
|
||||
test.AssertResult(t, "yourKey", n.getParsedKey())
|
||||
}
|
||||
|
||||
func TestGetParsedKeyForArrayValue(t *testing.T) {
|
||||
key := CreateScalarNode(4, "4")
|
||||
n := CandidateNode{Key: key, Value: "meow", document: 3}
|
||||
test.AssertResult(t, 4, n.getParsedKey())
|
||||
}
|
||||
|
||||
func TestCandidateNodeAddKeyValueChild(t *testing.T) {
|
||||
key := CandidateNode{Value: "cool", IsMapKey: true}
|
||||
node := CandidateNode{}
|
||||
|
||||
// if we use a key in a new node as a value, it should no longer be marked as a key
|
||||
|
||||
_, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key)
|
||||
|
||||
test.AssertResult(t, keyIsValueNow.IsMapKey, false)
|
||||
test.AssertResult(t, key.IsMapKey, true)
|
||||
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func MapYamlStyle(original yaml.Style) Style {
|
||||
switch original {
|
||||
case yaml.TaggedStyle:
|
||||
return TaggedStyle
|
||||
case yaml.DoubleQuotedStyle:
|
||||
return DoubleQuotedStyle
|
||||
case yaml.SingleQuotedStyle:
|
||||
return SingleQuotedStyle
|
||||
case yaml.LiteralStyle:
|
||||
return LiteralStyle
|
||||
case yaml.FoldedStyle:
|
||||
return FoldedStyle
|
||||
case yaml.FlowStyle:
|
||||
return FlowStyle
|
||||
case 0:
|
||||
return 0
|
||||
}
|
||||
return Style(original)
|
||||
}
|
||||
|
||||
func MapToYamlStyle(original Style) yaml.Style {
|
||||
switch original {
|
||||
case TaggedStyle:
|
||||
return yaml.TaggedStyle
|
||||
case DoubleQuotedStyle:
|
||||
return yaml.DoubleQuotedStyle
|
||||
case SingleQuotedStyle:
|
||||
return yaml.SingleQuotedStyle
|
||||
case LiteralStyle:
|
||||
return yaml.LiteralStyle
|
||||
case FoldedStyle:
|
||||
return yaml.FoldedStyle
|
||||
case FlowStyle:
|
||||
return yaml.FlowStyle
|
||||
case 0:
|
||||
return 0
|
||||
}
|
||||
return yaml.Style(original)
|
||||
}
|
||||
|
||||
func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) {
|
||||
o.Style = MapYamlStyle(node.Style)
|
||||
|
||||
o.Tag = node.Tag
|
||||
o.Value = node.Value
|
||||
o.Anchor = node.Anchor
|
||||
|
||||
if o.Anchor != "" {
|
||||
anchorMap[o.Anchor] = o
|
||||
log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
|
||||
}
|
||||
|
||||
// its a single alias
|
||||
if node.Alias != nil && node.Alias.Anchor != "" {
|
||||
o.Alias = anchorMap[node.Alias.Anchor]
|
||||
log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
|
||||
}
|
||||
o.HeadComment = node.HeadComment
|
||||
o.LineComment = node.LineComment
|
||||
o.FootComment = node.FootComment
|
||||
|
||||
o.Line = node.Line
|
||||
o.Column = node.Column
|
||||
}
|
||||
|
||||
func (o *CandidateNode) copyToYamlNode(node *yaml.Node) {
|
||||
node.Style = MapToYamlStyle(o.Style)
|
||||
|
||||
node.Tag = o.Tag
|
||||
node.Value = o.Value
|
||||
node.Anchor = o.Anchor
|
||||
|
||||
// node.Alias = TODO - find Alias in our own structure
|
||||
// might need to be a post process thing
|
||||
|
||||
node.HeadComment = o.HeadComment
|
||||
|
||||
node.LineComment = o.LineComment
|
||||
node.FootComment = o.FootComment
|
||||
|
||||
node.Line = o.Line
|
||||
node.Column = o.Column
|
||||
}
|
||||
|
||||
func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {
|
||||
newChild := o.CreateChild()
|
||||
|
||||
// null yaml.Nodes to not end up calling UnmarshalYAML
|
||||
// so we call it explicitly
|
||||
if childNode.Tag == "!!null" {
|
||||
newChild.Kind = ScalarNode
|
||||
newChild.copyFromYamlNode(childNode, anchorMap)
|
||||
return newChild, nil
|
||||
}
|
||||
|
||||
err := newChild.UnmarshalYAML(childNode, anchorMap)
|
||||
return newChild, err
|
||||
}
|
||||
|
||||
func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error {
|
||||
log.Debugf("UnmarshalYAML %v", node.Tag)
|
||||
switch node.Kind {
|
||||
case yaml.AliasNode:
|
||||
log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
|
||||
o.Kind = AliasNode
|
||||
o.copyFromYamlNode(node, anchorMap)
|
||||
return nil
|
||||
case yaml.ScalarNode:
|
||||
log.Debugf("UnmarshalYAML - a scalar")
|
||||
o.Kind = ScalarNode
|
||||
o.copyFromYamlNode(node, anchorMap)
|
||||
return nil
|
||||
case yaml.MappingNode:
|
||||
log.Debugf("UnmarshalYAML - a mapping node")
|
||||
o.Kind = MappingNode
|
||||
o.copyFromYamlNode(node, anchorMap)
|
||||
o.Content = make([]*CandidateNode, len(node.Content))
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
|
||||
keyNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
keyNode.IsMapKey = true
|
||||
|
||||
valueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valueNode.Key = keyNode
|
||||
|
||||
o.Content[i] = keyNode
|
||||
o.Content[i+1] = valueNode
|
||||
}
|
||||
log.Debugf("UnmarshalYAML - finished mapping node")
|
||||
return nil
|
||||
case yaml.SequenceNode:
|
||||
log.Debugf("UnmarshalYAML - a sequence: %v", len(node.Content))
|
||||
o.Kind = SequenceNode
|
||||
|
||||
o.copyFromYamlNode(node, anchorMap)
|
||||
log.Debugf("node Style: %v", node.Style)
|
||||
log.Debugf("o Style: %v", o.Style)
|
||||
o.Content = make([]*CandidateNode, len(node.Content))
|
||||
for i := 0; i < len(node.Content); i++ {
|
||||
keyNode := o.CreateChild()
|
||||
keyNode.IsMapKey = true
|
||||
keyNode.Tag = "!!int"
|
||||
keyNode.Kind = ScalarNode
|
||||
keyNode.Value = fmt.Sprintf("%v", i)
|
||||
|
||||
valueNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valueNode.Key = keyNode
|
||||
o.Content[i] = valueNode
|
||||
}
|
||||
return nil
|
||||
case 0:
|
||||
// not sure when this happens
|
||||
o.copyFromYamlNode(node, anchorMap)
|
||||
log.Debugf("UnmarshalYAML - err.. %v", NodeToString(o))
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("orderedMap: invalid yaml node")
|
||||
}
|
||||
}
|
||||
|
||||
func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {
|
||||
log.Debug("MarshalYAML to yaml: %v", o.Tag)
|
||||
switch o.Kind {
|
||||
case AliasNode:
|
||||
log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
|
||||
target := &yaml.Node{Kind: yaml.AliasNode}
|
||||
o.copyToYamlNode(target)
|
||||
return target, nil
|
||||
case ScalarNode:
|
||||
log.Debug("MarshalYAML - scalar: %v", o.Value)
|
||||
target := &yaml.Node{Kind: yaml.ScalarNode}
|
||||
o.copyToYamlNode(target)
|
||||
return target, nil
|
||||
case MappingNode, SequenceNode:
|
||||
targetKind := yaml.MappingNode
|
||||
if o.Kind == SequenceNode {
|
||||
targetKind = yaml.SequenceNode
|
||||
}
|
||||
target := &yaml.Node{Kind: targetKind}
|
||||
o.copyToYamlNode(target)
|
||||
log.Debugf("original style: %v", o.Style)
|
||||
log.Debugf("original: %v, tag: %v, style: %v, kind: %v", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode)
|
||||
target.Content = make([]*yaml.Node, len(o.Content))
|
||||
for i := 0; i < len(o.Content); i++ {
|
||||
|
||||
child, err := o.Content[i].MarshalYAML()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target.Content[i] = child
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
target := &yaml.Node{}
|
||||
o.copyToYamlNode(target)
|
||||
return target, nil
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/goccy/go-json"
|
||||
)
|
||||
|
||||
func (o *CandidateNode) setScalarFromJson(value interface{}) error {
|
||||
o.Kind = ScalarNode
|
||||
switch rawData := value.(type) {
|
||||
case nil:
|
||||
o.Tag = "!!null"
|
||||
o.Value = "null"
|
||||
case float64, float32:
|
||||
o.Value = fmt.Sprintf("%v", value)
|
||||
o.Tag = "!!float"
|
||||
// json decoder returns ints as float.
|
||||
if value == float64(int64(rawData.(float64))) {
|
||||
// aha it's an int disguised as a float
|
||||
o.Tag = "!!int"
|
||||
o.Value = fmt.Sprintf("%v", int64(value.(float64)))
|
||||
}
|
||||
case int, int64, int32:
|
||||
o.Value = fmt.Sprintf("%v", value)
|
||||
o.Tag = "!!int"
|
||||
case bool:
|
||||
o.Value = fmt.Sprintf("%v", value)
|
||||
o.Tag = "!!bool"
|
||||
case string:
|
||||
o.Value = rawData
|
||||
o.Tag = "!!str"
|
||||
default:
|
||||
return fmt.Errorf("unrecognised type :( %v", rawData)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *CandidateNode) UnmarshalJSON(data []byte) error {
|
||||
log.Debug("UnmarshalJSON")
|
||||
switch data[0] {
|
||||
case '{':
|
||||
log.Debug("UnmarshalJSON - its a map!")
|
||||
// its a map
|
||||
o.Kind = MappingNode
|
||||
o.Tag = "!!map"
|
||||
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
_, err := dec.Token() // open object
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// cycle through k/v
|
||||
var tok json.Token
|
||||
for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
|
||||
// we can expect two types: string or Delim. Delim automatically means
|
||||
// that it is the closing bracket of the object, whereas string means
|
||||
// that there is another key.
|
||||
if _, ok := tok.(json.Delim); ok {
|
||||
break
|
||||
}
|
||||
|
||||
childKey := o.CreateChild()
|
||||
childKey.IsMapKey = true
|
||||
childKey.Value = tok.(string)
|
||||
childKey.Kind = ScalarNode
|
||||
childKey.Tag = "!!str"
|
||||
|
||||
childValue := o.CreateChild()
|
||||
childValue.Key = childKey
|
||||
|
||||
if err := dec.Decode(childValue); err != nil {
|
||||
return err
|
||||
}
|
||||
o.Content = append(o.Content, childKey, childValue)
|
||||
}
|
||||
// unexpected error
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
case '[':
|
||||
o.Kind = SequenceNode
|
||||
o.Tag = "!!seq"
|
||||
log.Debug("UnmarshalJSON - its an array!")
|
||||
var children []*CandidateNode
|
||||
if err := json.Unmarshal(data, &children); err != nil {
|
||||
return err
|
||||
}
|
||||
// now we put the children into the content, and set a key value for them
|
||||
for i, child := range children {
|
||||
|
||||
if child == nil {
|
||||
// need to represent it as a null scalar
|
||||
child = CreateScalarNode(nil, "null")
|
||||
}
|
||||
childKey := o.CreateChild()
|
||||
childKey.Kind = ScalarNode
|
||||
childKey.Tag = "!!int"
|
||||
childKey.Value = fmt.Sprintf("%v", i)
|
||||
childKey.IsMapKey = true
|
||||
|
||||
child.Parent = o
|
||||
child.Key = childKey
|
||||
o.Content = append(o.Content, child)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
log.Debug("UnmarshalJSON - its a scalar!")
|
||||
// otherwise, must be a scalar
|
||||
var scalar interface{}
|
||||
err := json.Unmarshal(data, &scalar)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("UnmarshalJSON - scalar is %v", scalar)
|
||||
|
||||
return o.setScalarFromJson(scalar)
|
||||
|
||||
}
|
||||
|
||||
func (o *CandidateNode) MarshalJSON() ([]byte, error) {
|
||||
log.Debugf("MarshalJSON %v", NodeToString(o))
|
||||
buf := new(bytes.Buffer)
|
||||
enc := json.NewEncoder(buf)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
|
||||
|
||||
switch o.Kind {
|
||||
case AliasNode:
|
||||
log.Debugf("MarshalJSON AliasNode")
|
||||
err := enc.Encode(o.Alias)
|
||||
return buf.Bytes(), err
|
||||
case ScalarNode:
|
||||
log.Debugf("MarshalJSON ScalarNode")
|
||||
value, err := o.GetValueRep()
|
||||
if err != nil {
|
||||
return buf.Bytes(), err
|
||||
}
|
||||
err = enc.Encode(value)
|
||||
return buf.Bytes(), err
|
||||
case MappingNode:
|
||||
log.Debugf("MarshalJSON MappingNode")
|
||||
buf.WriteByte('{')
|
||||
for i := 0; i < len(o.Content); i += 2 {
|
||||
if err := enc.Encode(o.Content[i].Value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf.WriteByte(':')
|
||||
if err := enc.Encode(o.Content[i+1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i != len(o.Content)-2 {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
}
|
||||
buf.WriteByte('}')
|
||||
return buf.Bytes(), nil
|
||||
case SequenceNode:
|
||||
log.Debugf("MarshalJSON SequenceNode, %v, len: %v", o.Content, len(o.Content))
|
||||
var err error
|
||||
if len(o.Content) == 0 {
|
||||
buf.WriteString("[]")
|
||||
} else {
|
||||
err = enc.Encode(o.Content)
|
||||
}
|
||||
return buf.Bytes(), err
|
||||
default:
|
||||
err := enc.Encode(nil)
|
||||
return buf.Bytes(), err
|
||||
}
|
||||
}
|
||||
@@ -12,14 +12,7 @@ func changeOwner(info fs.FileInfo, file *os.File) error {
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
uid := int(stat.Uid)
|
||||
gid := int(stat.Gid)
|
||||
|
||||
err := os.Chown(file.Name(), uid, gid)
|
||||
if err != nil {
|
||||
// this happens with snap confinement
|
||||
// not really a big issue as users can chown
|
||||
// the file themselves if required.
|
||||
log.Info("Skipping chown: %v", err)
|
||||
}
|
||||
return os.Chown(file.Name(), uid, gid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,9 @@ package yqlib
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type compareTypePref struct {
|
||||
@@ -11,14 +14,14 @@ type compareTypePref struct {
|
||||
}
|
||||
|
||||
func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
log.Debugf("compareOperator")
|
||||
log.Debugf("-- compareOperator")
|
||||
prefs := expressionNode.Operation.Preferences.(compareTypePref)
|
||||
return crossFunction(d, context, expressionNode, compare(prefs), true)
|
||||
}
|
||||
|
||||
func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
|
||||
return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
|
||||
log.Debugf("compare cross function")
|
||||
log.Debugf("-- compare cross function")
|
||||
if lhs == nil && rhs == nil {
|
||||
owner := &CandidateNode{}
|
||||
return createBooleanCandidate(owner, prefs.OrEqual), nil
|
||||
@@ -30,24 +33,27 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
|
||||
return createBooleanCandidate(lhs, false), nil
|
||||
}
|
||||
|
||||
switch lhs.Kind {
|
||||
case MappingNode:
|
||||
lhs.Node = unwrapDoc(lhs.Node)
|
||||
rhs.Node = unwrapDoc(rhs.Node)
|
||||
|
||||
switch lhs.Node.Kind {
|
||||
case yaml.MappingNode:
|
||||
return nil, fmt.Errorf("maps not yet supported for comparison")
|
||||
case SequenceNode:
|
||||
case yaml.SequenceNode:
|
||||
return nil, fmt.Errorf("arrays not yet supported for comparison")
|
||||
default:
|
||||
if rhs.Kind != ScalarNode {
|
||||
return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Tag, rhs.GetNicePath(), lhs.Tag)
|
||||
if rhs.Node.Kind != yaml.ScalarNode {
|
||||
return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhs.Node.Tag)
|
||||
}
|
||||
target := lhs.CopyWithoutContent()
|
||||
boolV, err := compareScalars(context, prefs, lhs, rhs)
|
||||
target := lhs.CreateReplacement(&yaml.Node{})
|
||||
boolV, err := compareScalars(context, prefs, lhs.Node, rhs.Node)
|
||||
|
||||
return createBooleanCandidate(target, boolV), err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
|
||||
func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
|
||||
lhsTime, err := parseDateTime(layout, lhs.Value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -68,13 +74,13 @@ func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, r
|
||||
|
||||
}
|
||||
|
||||
func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
|
||||
lhsTag := lhs.GuessTagFromCustomType()
|
||||
rhsTag := rhs.GuessTagFromCustomType()
|
||||
func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
|
||||
lhsTag := guessTagFromCustomType(lhs)
|
||||
rhsTag := guessTagFromCustomType(rhs)
|
||||
|
||||
isDateTime := lhs.Tag == "!!timestamp"
|
||||
// if the lhs is a string, it might be a timestamp in a custom format.
|
||||
if lhsTag == "!!str" {
|
||||
if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
|
||||
_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
|
||||
isDateTime = err == nil
|
||||
}
|
||||
@@ -21,13 +21,6 @@ var compareOperatorScenarios = []expressionScenario{
|
||||
"D0, P[k], (!!bool)::true\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
expression: `"2022-01-30T15:53:09Z" > "2020-01-30T15:53:09Z"`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!bool)::true\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: "a: 5\nb: 4",
|
||||
+29
-17
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/copier"
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
@@ -57,19 +58,13 @@ func (n *Context) SetVariable(name string, value *list.List) {
|
||||
func (n *Context) ChildContext(results *list.List) Context {
|
||||
clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
|
||||
clone.Variables = make(map[string]*list.List)
|
||||
for variableKey, originalValueList := range n.Variables {
|
||||
|
||||
variableCopyList := list.New()
|
||||
for el := originalValueList.Front(); el != nil; el = el.Next() {
|
||||
// note that we dont make a copy of the candidate node
|
||||
// this is so the 'ref' operator can work correctly.
|
||||
clonedNode := el.Value.(*CandidateNode)
|
||||
variableCopyList.PushBack(clonedNode)
|
||||
if len(n.Variables) > 0 {
|
||||
err := copier.Copy(&clone.Variables, n.Variables)
|
||||
if err != nil {
|
||||
log.Error("Error cloning context :(")
|
||||
panic(err)
|
||||
}
|
||||
|
||||
clone.Variables[variableKey] = variableCopyList
|
||||
}
|
||||
|
||||
clone.MatchingNodes = results
|
||||
return clone
|
||||
}
|
||||
@@ -83,18 +78,35 @@ func (n *Context) ToString() string {
|
||||
}
|
||||
|
||||
func (n *Context) DeepClone() Context {
|
||||
|
||||
clonedContent := list.New()
|
||||
clone := Context{}
|
||||
err := copier.Copy(&clone, n)
|
||||
// copier doesn't do lists properly for some reason
|
||||
clone.MatchingNodes = list.New()
|
||||
for el := n.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
clonedNode := el.Value.(*CandidateNode).Copy()
|
||||
clonedContent.PushBack(clonedNode)
|
||||
clonedNode, err := el.Value.(*CandidateNode).Copy()
|
||||
if err != nil {
|
||||
log.Error("Error cloning context :(")
|
||||
panic(err)
|
||||
}
|
||||
clone.MatchingNodes.PushBack(clonedNode)
|
||||
}
|
||||
|
||||
return n.ChildContext(clonedContent)
|
||||
if err != nil {
|
||||
log.Error("Error cloning context :(")
|
||||
panic(err)
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func (n *Context) Clone() Context {
|
||||
return n.ChildContext(n.MatchingNodes)
|
||||
clone := Context{}
|
||||
err := copier.Copy(&clone, n)
|
||||
|
||||
if err != nil {
|
||||
log.Error("Error cloning context :(")
|
||||
panic(err)
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func (n *Context) ReadOnlyClone() Context {
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
func TestChildContext(t *testing.T) {
|
||||
|
||||
expectedOriginal := make(map[string]*list.List)
|
||||
expectedOriginal["dog"] = list.New()
|
||||
expectedOriginal["dog"].PushBack(&CandidateNode{Value: "woof"})
|
||||
|
||||
originalVariables := make(map[string]*list.List)
|
||||
originalVariables["dog"] = list.New()
|
||||
originalVariables["dog"].PushBack(&CandidateNode{Value: "woof"})
|
||||
|
||||
original := Context{
|
||||
DontAutoCreate: true,
|
||||
datetimeLayout: "cat",
|
||||
Variables: originalVariables,
|
||||
}
|
||||
|
||||
newResults := list.New()
|
||||
newResults.PushBack(&CandidateNode{Value: "bar"})
|
||||
|
||||
clone := original.ChildContext(newResults)
|
||||
test.AssertResultComplex(t, originalVariables, clone.Variables)
|
||||
|
||||
clone.Variables["dog"].PushBack("bark")
|
||||
// ensure this is a separate copy
|
||||
test.AssertResultComplex(t, 1, originalVariables["dog"].Len())
|
||||
|
||||
}
|
||||
|
||||
func TestChildContextNoVariables(t *testing.T) {
|
||||
|
||||
original := Context{
|
||||
DontAutoCreate: true,
|
||||
datetimeLayout: "cat",
|
||||
}
|
||||
|
||||
newResults := list.New()
|
||||
newResults.PushBack(&CandidateNode{Value: "bar"})
|
||||
|
||||
clone := original.ChildContext(newResults)
|
||||
test.AssertResultComplex(t, make(map[string]*list.List), clone.Variables)
|
||||
|
||||
}
|
||||
-3111
File diff suppressed because it is too large
Load Diff
@@ -1,23 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
type CsvPreferences struct {
|
||||
Separator rune
|
||||
AutoParse bool
|
||||
}
|
||||
|
||||
func NewDefaultCsvPreferences() CsvPreferences {
|
||||
return CsvPreferences{
|
||||
Separator: ',',
|
||||
AutoParse: true,
|
||||
}
|
||||
}
|
||||
|
||||
func NewDefaultTsvPreferences() CsvPreferences {
|
||||
return CsvPreferences{
|
||||
Separator: '\t',
|
||||
AutoParse: true,
|
||||
}
|
||||
}
|
||||
|
||||
var ConfiguredCsvPreferences = NewDefaultCsvPreferences()
|
||||
var ConfiguredTsvPreferences = NewDefaultTsvPreferences()
|
||||
+22
-123
@@ -12,11 +12,6 @@ const csvSimple = `name,numberOfCats,likesApples,height
|
||||
Gary,1,true,168.8
|
||||
Samantha's Rabbit,2,false,-188.8
|
||||
`
|
||||
|
||||
const csvSimpleWithObject = `name,numberOfCats,likesApples,height,facts
|
||||
Gary,1,true,168.8,cool: true
|
||||
Samantha's Rabbit,2,false,-188.8,tall: indeed
|
||||
`
|
||||
const csvMissing = `name,numberOfCats,likesApples,height
|
||||
,null,,168.8
|
||||
`
|
||||
@@ -44,31 +39,6 @@ const expectedYamlFromCSV = `- name: Gary
|
||||
likesApples: false
|
||||
height: -188.8
|
||||
`
|
||||
const expectedYamlFromCSVWithObject = `- name: Gary
|
||||
numberOfCats: 1
|
||||
likesApples: true
|
||||
height: 168.8
|
||||
facts:
|
||||
cool: true
|
||||
- name: Samantha's Rabbit
|
||||
numberOfCats: 2
|
||||
likesApples: false
|
||||
height: -188.8
|
||||
facts:
|
||||
tall: indeed
|
||||
`
|
||||
|
||||
const expectedYamlFromCSVNoParsing = `- name: Gary
|
||||
numberOfCats: 1
|
||||
likesApples: true
|
||||
height: 168.8
|
||||
facts: 'cool: true'
|
||||
- name: Samantha's Rabbit
|
||||
numberOfCats: 2
|
||||
likesApples: false
|
||||
height: -188.8
|
||||
facts: 'tall: indeed'
|
||||
`
|
||||
|
||||
const expectedYamlFromCSVMissingData = `- name: Gary
|
||||
numberOfCats: 1
|
||||
@@ -94,7 +64,7 @@ const tsvTestExpectedSimpleCsv = `i like csv
|
||||
because excel is cool
|
||||
`
|
||||
|
||||
var csvScenarios = []FormatScenario{
|
||||
var csvScenarios = []formatScenario{
|
||||
{
|
||||
description: "Encode CSV simple",
|
||||
input: csvTestSimpleYaml,
|
||||
@@ -149,50 +119,12 @@ var csvScenarios = []FormatScenario{
|
||||
expected: csvMissing,
|
||||
scenarioType: "roundtrip-csv",
|
||||
},
|
||||
{
|
||||
description: "decode csv key",
|
||||
skipDoc: true,
|
||||
input: csvSimple,
|
||||
expression: ".[0].name | key",
|
||||
expected: "name\n",
|
||||
scenarioType: "decode-csv",
|
||||
},
|
||||
{
|
||||
description: "decode csv parent",
|
||||
skipDoc: true,
|
||||
input: csvSimple,
|
||||
expression: ".[0].name | parent | .height",
|
||||
expected: "168.8\n",
|
||||
scenarioType: "decode-csv",
|
||||
},
|
||||
{
|
||||
description: "Parse CSV into an array of objects",
|
||||
subdescription: "First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!",
|
||||
input: csvSimpleWithObject,
|
||||
expected: expectedYamlFromCSVWithObject,
|
||||
scenarioType: "decode-csv",
|
||||
},
|
||||
{
|
||||
description: "Decode CSV line breaks",
|
||||
skipDoc: true,
|
||||
input: "heading1\n\"some data\nwith a line break\"\n",
|
||||
expected: "- heading1: |-\n some data\n with a line break\n",
|
||||
scenarioType: "decode-csv",
|
||||
},
|
||||
{
|
||||
description: "Parse CSV into an array of objects, no auto-parsing",
|
||||
subdescription: "First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.",
|
||||
input: csvSimpleWithObject,
|
||||
expected: expectedYamlFromCSVNoParsing,
|
||||
scenarioType: "decode-csv-no-auto",
|
||||
},
|
||||
{
|
||||
description: "Scalar roundtrip",
|
||||
skipDoc: true,
|
||||
input: "mike\ncat",
|
||||
expression: ".[0].mike",
|
||||
expected: "cat\n",
|
||||
scenarioType: "roundtrip-csv",
|
||||
subdescription: "First row is assumed to be the header row.",
|
||||
input: csvSimple,
|
||||
expected: expectedYamlFromCSV,
|
||||
scenarioType: "decode-csv-object",
|
||||
},
|
||||
{
|
||||
description: "Parse TSV into an array of objects",
|
||||
@@ -210,26 +142,24 @@ var csvScenarios = []FormatScenario{
|
||||
},
|
||||
}
|
||||
|
||||
func testCSVScenario(t *testing.T, s FormatScenario) {
|
||||
func testCSVScenario(t *testing.T, s formatScenario) {
|
||||
switch s.scenarioType {
|
||||
case "encode-csv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
|
||||
case "encode-tsv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)
|
||||
case "decode-csv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-csv-no-auto":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
|
||||
case "decode-csv-object":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "decode-tsv-object":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
|
||||
case "roundtrip-csv":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
}
|
||||
|
||||
func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatType string) {
|
||||
func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
@@ -250,36 +180,11 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatTy
|
||||
}
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))),
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
|
||||
)
|
||||
}
|
||||
|
||||
func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, formatType string) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
|
||||
writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v --csv-auto-parse=f sample.%v\n```\n", formatType, formatType))
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
separator := ','
|
||||
if formatType == "tsv" {
|
||||
separator = '\t'
|
||||
}
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))),
|
||||
)
|
||||
}
|
||||
|
||||
func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType string) {
|
||||
func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
@@ -305,14 +210,13 @@ func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType str
|
||||
if formatType == "tsv" {
|
||||
separator = '\t'
|
||||
}
|
||||
csvPrefs := NewDefaultCsvPreferences()
|
||||
csvPrefs.Separator = separator
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
|
||||
mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),
|
||||
mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
|
||||
)
|
||||
}
|
||||
|
||||
func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType string) {
|
||||
func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
@@ -339,16 +243,13 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType
|
||||
separator = '\t'
|
||||
}
|
||||
|
||||
csvPrefs := NewDefaultCsvPreferences()
|
||||
csvPrefs.Separator = separator
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),
|
||||
mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
|
||||
)
|
||||
}
|
||||
|
||||
func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
|
||||
s := i.(FormatScenario)
|
||||
func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
|
||||
s := i.(formatScenario)
|
||||
if s.skipDoc {
|
||||
return
|
||||
}
|
||||
@@ -357,10 +258,8 @@ func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
|
||||
documentCSVEncodeScenario(w, s, "csv")
|
||||
case "encode-tsv":
|
||||
documentCSVEncodeScenario(w, s, "tsv")
|
||||
case "decode-csv":
|
||||
case "decode-csv-object":
|
||||
documentCSVDecodeObjectScenario(w, s, "csv")
|
||||
case "decode-csv-no-auto":
|
||||
documentCSVDecodeObjectNoAutoScenario(w, s, "csv")
|
||||
case "decode-tsv-object":
|
||||
documentCSVDecodeObjectScenario(w, s, "tsv")
|
||||
case "roundtrip-csv":
|
||||
|
||||
@@ -7,13 +7,10 @@ import (
|
||||
)
|
||||
|
||||
type DataTreeNavigator interface {
|
||||
// given the context and an expressionNode,
|
||||
// given the context and a expressionNode,
|
||||
// this will process the against the given expressionNode and return
|
||||
// a new context of matching candidates
|
||||
GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
|
||||
|
||||
DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error
|
||||
DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error
|
||||
}
|
||||
|
||||
type dataTreeNavigator struct {
|
||||
@@ -23,41 +20,6 @@ func NewDataTreeNavigator() DataTreeNavigator {
|
||||
return &dataTreeNavigator{}
|
||||
}
|
||||
|
||||
func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
|
||||
return d.deeplyAssign(context, path, rhsCandidateNode, false)
|
||||
}
|
||||
|
||||
func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
|
||||
return d.deeplyAssign(context, path, rhsCandidateNode, true)
|
||||
}
|
||||
|
||||
func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error {
|
||||
|
||||
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
|
||||
|
||||
if rhsCandidateNode.Kind == MappingNode {
|
||||
log.Debug("DeeplyAssign: deeply merging object")
|
||||
// if the rhs is a map, we need to deeply merge it in.
|
||||
// otherwise we'll clobber any existing fields
|
||||
assignmentOp = &Operation{OperationType: multiplyAssignOpType, Preferences: multiplyPreferences{
|
||||
AppendArrays: true,
|
||||
TraversePrefs: traversePreferences{DontFollowAlias: true},
|
||||
AssignPrefs: assignPreferences{},
|
||||
}}
|
||||
}
|
||||
|
||||
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, targetKey),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
if expressionNode == nil {
|
||||
log.Debugf("getMatchingNodes - nothing to do")
|
||||
@@ -69,6 +31,7 @@ func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *Ex
|
||||
log.Debug(NodeToString(el.Value.(*CandidateNode)))
|
||||
}
|
||||
}
|
||||
log.Debug(">>")
|
||||
handler := expressionNode.Operation.OperationType.Handler
|
||||
if handler != nil {
|
||||
return handler(d, context, expressionNode)
|
||||
|
||||
@@ -1,10 +1,42 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
type InputFormat uint
|
||||
|
||||
const (
|
||||
YamlInputFormat = 1 << iota
|
||||
XMLInputFormat
|
||||
PropertiesInputFormat
|
||||
Base64InputFormat
|
||||
JsonInputFormat
|
||||
CSVObjectInputFormat
|
||||
TSVObjectInputFormat
|
||||
)
|
||||
|
||||
type Decoder interface {
|
||||
Init(reader io.Reader) error
|
||||
Decode() (*CandidateNode, error)
|
||||
}
|
||||
|
||||
func InputFormatFromString(format string) (InputFormat, error) {
|
||||
switch format {
|
||||
case "yaml", "y":
|
||||
return YamlInputFormat, nil
|
||||
case "xml", "x":
|
||||
return XMLInputFormat, nil
|
||||
case "props", "p":
|
||||
return PropertiesInputFormat, nil
|
||||
case "json", "ndjson", "j":
|
||||
return JsonInputFormat, nil
|
||||
case "csv", "c":
|
||||
return CSVObjectInputFormat, nil
|
||||
case "tsv", "t":
|
||||
return TSVObjectInputFormat, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
|
||||
}
|
||||
}
|
||||
|
||||
+10
-25
@@ -4,31 +4,10 @@ import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type base64Padder struct {
|
||||
count uint64
|
||||
io.Reader
|
||||
}
|
||||
|
||||
func (c *base64Padder) pad(buf []byte) (int, error) {
|
||||
pad := strings.Repeat("=", int(4-c.count%4))
|
||||
n, err := strings.NewReader(pad).Read(buf)
|
||||
c.count += uint64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (c *base64Padder) Read(buf []byte) (int, error) {
|
||||
n, err := c.Reader.Read(buf)
|
||||
c.count += uint64(n)
|
||||
|
||||
if err == io.EOF && c.count%4 != 0 {
|
||||
return c.pad(buf)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
type base64Decoder struct {
|
||||
reader io.Reader
|
||||
finished bool
|
||||
@@ -41,7 +20,7 @@ func NewBase64Decoder() Decoder {
|
||||
}
|
||||
|
||||
func (dec *base64Decoder) Init(reader io.Reader) error {
|
||||
dec.reader = &base64Padder{Reader: reader}
|
||||
dec.reader = reader
|
||||
dec.readAnything = false
|
||||
dec.finished = false
|
||||
return nil
|
||||
@@ -68,5 +47,11 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
|
||||
}
|
||||
}
|
||||
dec.readAnything = true
|
||||
return CreateStringScalarNode(buf.String()), nil
|
||||
return &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.ScalarNode,
|
||||
Tag: "!!str",
|
||||
Value: buf.String(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -6,42 +6,44 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/dimchansky/utfbom"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type csvObjectDecoder struct {
|
||||
prefs CsvPreferences
|
||||
reader csv.Reader
|
||||
finished bool
|
||||
separator rune
|
||||
reader csv.Reader
|
||||
finished bool
|
||||
}
|
||||
|
||||
func NewCSVObjectDecoder(prefs CsvPreferences) Decoder {
|
||||
return &csvObjectDecoder{prefs: prefs}
|
||||
func NewCSVObjectDecoder(separator rune) Decoder {
|
||||
return &csvObjectDecoder{separator: separator}
|
||||
}
|
||||
|
||||
func (dec *csvObjectDecoder) Init(reader io.Reader) error {
|
||||
cleanReader, enc := utfbom.Skip(reader)
|
||||
log.Debugf("Detected encoding: %s\n", enc)
|
||||
dec.reader = *csv.NewReader(cleanReader)
|
||||
dec.reader.Comma = dec.prefs.Separator
|
||||
dec.reader.Comma = dec.separator
|
||||
dec.finished = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {
|
||||
func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
|
||||
node, err := parseSnippet(content)
|
||||
// if we're not auto-parsing, then we wont put in parsed objects or arrays
|
||||
// but we still parse scalars
|
||||
if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) {
|
||||
return CreateScalarNode(content, content)
|
||||
if err != nil {
|
||||
return createScalarNode(content, content)
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *CandidateNode {
|
||||
objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
|
||||
func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node {
|
||||
objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
|
||||
for i, header := range headerRow {
|
||||
objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i]))
|
||||
objectNode.Content = append(
|
||||
objectNode.Content,
|
||||
createScalarNode(header, header),
|
||||
dec.convertToYamlNode(contentRow[i]))
|
||||
}
|
||||
return objectNode
|
||||
}
|
||||
@@ -56,13 +58,13 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rootArray := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
|
||||
rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
|
||||
contentRow, err := dec.reader.Read()
|
||||
|
||||
for err == nil && len(contentRow) > 0 {
|
||||
log.Debugf("Adding contentRow: %v", contentRow)
|
||||
rootArray.AddChild(dec.createObject(headerRow, contentRow))
|
||||
rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow))
|
||||
contentRow, err = dec.reader.Read()
|
||||
log.Debugf("Read next contentRow: %v, %v", contentRow, err)
|
||||
}
|
||||
@@ -70,5 +72,10 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rootArray, nil
|
||||
return &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{rootArray},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
//go:build !yq_noyaml
|
||||
|
||||
//
|
||||
// NOTE this is still a WIP - not yet ready.
|
||||
//
|
||||
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
yaml "github.com/goccy/go-yaml"
|
||||
"github.com/goccy/go-yaml/ast"
|
||||
)
|
||||
|
||||
type goccyYamlDecoder struct {
|
||||
decoder yaml.Decoder
|
||||
cm yaml.CommentMap
|
||||
}
|
||||
|
||||
func NewGoccyYAMLDecoder() Decoder {
|
||||
return &goccyYamlDecoder{}
|
||||
}
|
||||
|
||||
func (dec *goccyYamlDecoder) Init(reader io.Reader) error {
|
||||
dec.cm = yaml.CommentMap{}
|
||||
dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) {
|
||||
|
||||
var ast ast.Node
|
||||
|
||||
err := dec.decoder.Decode(&ast)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
candidateNode := &CandidateNode{}
|
||||
if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return candidateNode, nil
|
||||
}
|
||||
@@ -1,11 +1,11 @@
|
||||
//go:build !yq_nojson
|
||||
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/goccy/go-json"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type jsonDecoder struct {
|
||||
@@ -23,11 +23,65 @@ func (dec *jsonDecoder) Init(reader io.Reader) error {
|
||||
|
||||
func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
|
||||
|
||||
var dataBucket CandidateNode
|
||||
var dataBucket orderedMap
|
||||
log.Debug("going to decode")
|
||||
err := dec.decoder.Decode(&dataBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, err := dec.convertToYamlNode(&dataBucket)
|
||||
|
||||
return &dataBucket, nil
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{node},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
|
||||
if data.kv == nil {
|
||||
switch rawData := data.altVal.(type) {
|
||||
case nil:
|
||||
return createScalarNode(nil, "null"), nil
|
||||
case float64, float32:
|
||||
// json decoder returns ints as float.
|
||||
return parseSnippet(fmt.Sprintf("%v", rawData))
|
||||
case int, int64, int32, string, bool:
|
||||
return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
|
||||
case []*orderedMap:
|
||||
return dec.parseArray(rawData)
|
||||
default:
|
||||
return nil, fmt.Errorf("unrecognised type :( %v", rawData)
|
||||
}
|
||||
}
|
||||
|
||||
var yamlMap = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
for _, keyValuePair := range data.kv {
|
||||
yamlValue, err := dec.convertToYamlNode(&keyValuePair.V)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
|
||||
}
|
||||
return yamlMap, nil
|
||||
|
||||
}
|
||||
|
||||
func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) {
|
||||
|
||||
var yamlMap = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
|
||||
for _, value := range dataArray {
|
||||
yamlValue, err := dec.convertToYamlNode(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
yamlMap.Content = append(yamlMap.Content, yamlValue)
|
||||
}
|
||||
return yamlMap, nil
|
||||
}
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
//go:build !yq_nolua
|
||||
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
|
||||
lua "github.com/yuin/gopher-lua"
|
||||
)
|
||||
|
||||
type luaDecoder struct {
|
||||
reader io.Reader
|
||||
finished bool
|
||||
prefs LuaPreferences
|
||||
}
|
||||
|
||||
func NewLuaDecoder(prefs LuaPreferences) Decoder {
|
||||
return &luaDecoder{
|
||||
prefs: prefs,
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *luaDecoder) Init(reader io.Reader) error {
|
||||
dec.reader = reader
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *CandidateNode {
|
||||
switch lv.Type() {
|
||||
case lua.LTNil:
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!null",
|
||||
Value: "",
|
||||
}
|
||||
case lua.LTBool:
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!bool",
|
||||
Value: lv.String(),
|
||||
}
|
||||
case lua.LTNumber:
|
||||
n := float64(lua.LVAsNumber(lv))
|
||||
// various special case floats
|
||||
if math.IsNaN(n) {
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!float",
|
||||
Value: ".nan",
|
||||
}
|
||||
}
|
||||
if math.IsInf(n, 1) {
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!float",
|
||||
Value: ".inf",
|
||||
}
|
||||
}
|
||||
if math.IsInf(n, -1) {
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!float",
|
||||
Value: "-.inf",
|
||||
}
|
||||
}
|
||||
|
||||
// does it look like an integer?
|
||||
if n == float64(int(n)) {
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!int",
|
||||
Value: lv.String(),
|
||||
}
|
||||
}
|
||||
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!float",
|
||||
Value: lv.String(),
|
||||
}
|
||||
case lua.LTString:
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!str",
|
||||
Value: lv.String(),
|
||||
}
|
||||
case lua.LTFunction:
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "tag:lua.org,2006,function",
|
||||
Value: lv.String(),
|
||||
}
|
||||
case lua.LTTable:
|
||||
// Simultaneously create a sequence and a map, pick which one to return
|
||||
// based on whether all keys were consecutive integers
|
||||
i := 1
|
||||
yaml_sequence := &CandidateNode{
|
||||
Kind: SequenceNode,
|
||||
Tag: "!!seq",
|
||||
}
|
||||
yaml_map := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
}
|
||||
t := lv.(*lua.LTable)
|
||||
k, v := ls.Next(t, lua.LNil)
|
||||
for k != lua.LNil {
|
||||
if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i {
|
||||
i++
|
||||
} else {
|
||||
i = 0
|
||||
}
|
||||
newKey := dec.convertToYamlNode(ls, k)
|
||||
|
||||
yv := dec.convertToYamlNode(ls, v)
|
||||
yaml_map.AddKeyValueChild(newKey, yv)
|
||||
|
||||
if i != 0 {
|
||||
yaml_sequence.AddChild(yv)
|
||||
}
|
||||
k, v = ls.Next(t, k)
|
||||
}
|
||||
if i != 0 {
|
||||
return yaml_sequence
|
||||
}
|
||||
return yaml_map
|
||||
default:
|
||||
return &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()),
|
||||
Tag: "!!null",
|
||||
Value: lv.String(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *CandidateNode {
|
||||
if ls.GetTop() == 0 {
|
||||
// no items were explicitly returned, encode the globals table instead
|
||||
return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex))
|
||||
}
|
||||
return dec.convertToYamlNode(ls, ls.Get(1))
|
||||
}
|
||||
|
||||
func (dec *luaDecoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
ls := lua.NewState(lua.Options{SkipOpenLibs: true})
|
||||
defer ls.Close()
|
||||
fn, err := ls.Load(dec.reader, "@input")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ls.Push(fn)
|
||||
err = ls.PCall(0, lua.MultRet, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
firstNode := dec.decideTopLevelNode(ls)
|
||||
dec.finished = true
|
||||
return firstNode, nil
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/magiconair/properties"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type propertiesDecoder struct {
|
||||
reader io.Reader
|
||||
finished bool
|
||||
d DataTreeNavigator
|
||||
}
|
||||
|
||||
func NewPropertiesDecoder() Decoder {
|
||||
return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) Init(reader io.Reader) error {
|
||||
dec.reader = reader
|
||||
dec.finished = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func parsePropKey(key string) []interface{} {
|
||||
pathStrArray := strings.Split(key, ".")
|
||||
path := make([]interface{}, len(pathStrArray))
|
||||
for i, pathStr := range pathStrArray {
|
||||
num, err := strconv.ParseInt(pathStr, 10, 32)
|
||||
if err == nil {
|
||||
path[i] = num
|
||||
} else {
|
||||
path[i] = pathStr
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) processComment(c string) string {
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
return "# " + c
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
|
||||
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
|
||||
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Path: path,
|
||||
Node: &yaml.Node{
|
||||
Tag: "!!str",
|
||||
Value: fmt.Sprintf("%v", path[len(path)-1]),
|
||||
HeadComment: dec.processComment(strings.Join(comments, "\n")),
|
||||
Kind: yaml.ScalarNode,
|
||||
},
|
||||
}
|
||||
|
||||
rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node)
|
||||
|
||||
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, true),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
|
||||
value, _ := properties.Get(key)
|
||||
path := parsePropKey(key)
|
||||
|
||||
propertyComments := properties.GetComments(key)
|
||||
if len(propertyComments) > 0 {
|
||||
err := dec.applyPropertyComments(context, path, propertyComments)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
rhsNode := &yaml.Node{
|
||||
Value: value,
|
||||
Tag: "!!str",
|
||||
Kind: yaml.ScalarNode,
|
||||
}
|
||||
|
||||
rhsNode.Tag = guessTagFromCustomType(rhsNode)
|
||||
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Path: path,
|
||||
Node: rhsNode,
|
||||
}
|
||||
|
||||
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
|
||||
|
||||
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, false),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
if _, err := buf.ReadFrom(dec.reader); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if buf.Len() == 0 {
|
||||
dec.finished = true
|
||||
return nil, io.EOF
|
||||
}
|
||||
properties, err := properties.LoadString(buf.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
properties.DisableExpansion = true
|
||||
|
||||
rootMap := &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.MappingNode,
|
||||
Tag: "!!map",
|
||||
},
|
||||
}
|
||||
|
||||
context := Context{}
|
||||
context = context.SingleChildContext(rootMap)
|
||||
|
||||
for _, key := range properties.Keys() {
|
||||
if err := dec.applyProperty(context, properties, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
}
|
||||
dec.finished = true
|
||||
|
||||
return &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{rootMap.Node},
|
||||
},
|
||||
}, nil
|
||||
|
||||
}
|
||||
@@ -3,11 +3,10 @@ package yqlib
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type FormatScenario struct {
|
||||
type formatScenario struct {
|
||||
input string
|
||||
indent int
|
||||
expression string
|
||||
@@ -19,7 +18,7 @@ type FormatScenario struct {
|
||||
expectedError string
|
||||
}
|
||||
|
||||
func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (string, error) {
|
||||
func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
|
||||
var output bytes.Buffer
|
||||
writer := bufio.NewWriter(&output)
|
||||
|
||||
@@ -27,14 +26,11 @@ func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (
|
||||
decoder = NewYamlDecoder(ConfiguredYamlPreferences)
|
||||
}
|
||||
|
||||
log.Debugf("reading docs")
|
||||
inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
log.Debugf("done reading the documents")
|
||||
|
||||
expression := s.expression
|
||||
if expression == "" {
|
||||
expression = "."
|
||||
@@ -48,8 +44,6 @@ func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (
|
||||
|
||||
context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)
|
||||
|
||||
log.Debugf("Going to print: %v", NodesToString(context.MatchingNodes))
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -64,11 +58,11 @@ func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (
|
||||
return output.String(), nil
|
||||
}
|
||||
|
||||
func mustProcessFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) string {
|
||||
func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
|
||||
|
||||
result, err := processFormatScenario(s, decoder, encoder)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
|
||||
panic(err)
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
@@ -1,357 +0,0 @@
|
||||
//go:build !yq_notoml
|
||||
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
toml "github.com/pelletier/go-toml/v2/unstable"
|
||||
)
|
||||
|
||||
type tomlDecoder struct {
|
||||
parser toml.Parser
|
||||
finished bool
|
||||
d DataTreeNavigator
|
||||
rootMap *CandidateNode
|
||||
}
|
||||
|
||||
func NewTomlDecoder() Decoder {
|
||||
return &tomlDecoder{
|
||||
finished: false,
|
||||
d: NewDataTreeNavigator(),
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) Init(reader io.Reader) error {
|
||||
dec.parser = toml.Parser{}
|
||||
buf := new(bytes.Buffer)
|
||||
_, err := buf.ReadFrom(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dec.parser.Reset(buf.Bytes())
|
||||
dec.rootMap = &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
|
||||
path := make([]interface{}, 0)
|
||||
for {
|
||||
path = append(path, string(tomlNode.Data))
|
||||
tomlNode = tomlNode.Next()
|
||||
if tomlNode == nil {
|
||||
return path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
|
||||
value := tomlNode.Value()
|
||||
path := dec.getFullPath(value.Next())
|
||||
log.Debug("processKeyValueIntoMap: %v", path)
|
||||
|
||||
valueNode, err := dec.decodeNode(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
context := Context{}
|
||||
context = context.SingleChildContext(rootMap)
|
||||
|
||||
return dec.d.DeeplyAssign(context, path, valueNode)
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
|
||||
log.Debug("decodeKeyValuesIntoMap -- processing first (current) entry")
|
||||
if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for dec.parser.NextExpression() {
|
||||
nextItem := dec.parser.Expression()
|
||||
log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
|
||||
|
||||
if nextItem.Kind == toml.KeyValue {
|
||||
if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
|
||||
return false, err
|
||||
}
|
||||
} else {
|
||||
// run out of key values
|
||||
log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
log.Debug("no more things to read in")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := make([]*CandidateNode, 0)
|
||||
log.Debug("createInlineTableMap")
|
||||
|
||||
iterator := tomlNode.Children()
|
||||
for iterator.Next() {
|
||||
child := iterator.Node()
|
||||
if child.Kind != toml.KeyValue {
|
||||
return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind)
|
||||
}
|
||||
|
||||
keyValues := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
}
|
||||
|
||||
if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
content = append(content, keyValues.Content...)
|
||||
}
|
||||
|
||||
return &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: content,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := make([]*CandidateNode, 0)
|
||||
iterator := tomlNode.Children()
|
||||
for iterator.Next() {
|
||||
child := iterator.Node()
|
||||
yamlNode, err := dec.decodeNode(child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = append(content, yamlNode)
|
||||
}
|
||||
|
||||
return &CandidateNode{
|
||||
Kind: SequenceNode,
|
||||
Tag: "!!seq",
|
||||
Content: content,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := string(tomlNode.Data)
|
||||
return CreateScalarNode(content, content), nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := string(tomlNode.Data)
|
||||
return CreateScalarNode(content == "true", content), nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := string(tomlNode.Data)
|
||||
_, num, err := parseInt64(content)
|
||||
return CreateScalarNode(num, content), err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := string(tomlNode.Data)
|
||||
val, err := parseDateTime(time.RFC3339, content)
|
||||
return CreateScalarNode(val, content), err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
content := string(tomlNode.Data)
|
||||
num, err := strconv.ParseFloat(content, 64)
|
||||
return CreateScalarNode(num, content), err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {
|
||||
switch tomlNode.Kind {
|
||||
case toml.Key, toml.String:
|
||||
return dec.createStringScalar(tomlNode)
|
||||
case toml.Bool:
|
||||
return dec.createBoolScalar(tomlNode)
|
||||
case toml.Integer:
|
||||
return dec.createIntegerScalar(tomlNode)
|
||||
case toml.DateTime:
|
||||
return dec.createDateTimeScalar(tomlNode)
|
||||
case toml.Float:
|
||||
return dec.createFloatScalar(tomlNode)
|
||||
case toml.Array:
|
||||
return dec.createArray(tomlNode)
|
||||
case toml.InlineTable:
|
||||
return dec.createInlineTableMap(tomlNode)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
//
|
||||
// toml library likes to panic
|
||||
var deferredError error
|
||||
defer func() { //catch or finally
|
||||
if r := recover(); r != nil {
|
||||
var ok bool
|
||||
deferredError, ok = r.(error)
|
||||
if !ok {
|
||||
deferredError = fmt.Errorf("pkg: %v", r)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
log.Debug("ok here we go")
|
||||
var runAgainstCurrentExp = false
|
||||
var err error
|
||||
for runAgainstCurrentExp || dec.parser.NextExpression() {
|
||||
|
||||
if runAgainstCurrentExp {
|
||||
log.Debug("running against current exp")
|
||||
}
|
||||
|
||||
currentNode := dec.parser.Expression()
|
||||
|
||||
log.Debug("currentNode: %v ", currentNode.Kind)
|
||||
runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
|
||||
if err != nil {
|
||||
return dec.rootMap, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
err = dec.parser.Error()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// must have finished
|
||||
dec.finished = true
|
||||
|
||||
if len(dec.rootMap.Content) == 0 {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return dec.rootMap, deferredError
|
||||
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
|
||||
var runAgainstCurrentExp bool
|
||||
var err error
|
||||
log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
|
||||
if currentNode.Kind == toml.Table {
|
||||
runAgainstCurrentExp, err = dec.processTable(currentNode)
|
||||
} else if currentNode.Kind == toml.ArrayTable {
|
||||
runAgainstCurrentExp, err = dec.processArrayTable(currentNode)
|
||||
} else {
|
||||
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
|
||||
}
|
||||
|
||||
log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
|
||||
return runAgainstCurrentExp, err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("Enter processTable")
|
||||
fullPath := dec.getFullPath(currentNode.Child())
|
||||
log.Debug("fullpath: %v", fullPath)
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: make([]*CandidateNode, 0),
|
||||
}
|
||||
|
||||
var tableValue *toml.Node
|
||||
runAgainstCurrentExp := false
|
||||
var err error
|
||||
hasValue := dec.parser.NextExpression()
|
||||
// check to see if there is any table data
|
||||
if hasValue {
|
||||
tableValue = dec.parser.Expression()
|
||||
// next expression is not table data, so we are done
|
||||
if tableValue.Kind != toml.KeyValue {
|
||||
log.Debug("got an empty table, returning")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
|
||||
if err != nil && !errors.Is(io.EOF, err) {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
c := Context{}
|
||||
c = c.SingleChildContext(dec.rootMap)
|
||||
err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return runAgainstCurrentExp, nil
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {
|
||||
log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Kind: SequenceNode,
|
||||
Tag: "!!seq",
|
||||
Content: []*CandidateNode{rhsNode},
|
||||
}
|
||||
|
||||
assignmentOp := &Operation{OperationType: addAssignOpType}
|
||||
|
||||
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
|
||||
|
||||
assignmentOpNode := &ExpressionNode{
|
||||
Operation: assignmentOp,
|
||||
LHS: createTraversalTree(path, traversePreferences{}, false),
|
||||
RHS: &ExpressionNode{Operation: rhsOp},
|
||||
}
|
||||
|
||||
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("Entering processArrayTable")
|
||||
fullPath := dec.getFullPath(currentNode.Child())
|
||||
log.Debug("Fullpath: %v", fullPath)
|
||||
|
||||
// need to use the array append exp to add another entry to
|
||||
// this array: fullpath += [ thing ]
|
||||
|
||||
hasValue := dec.parser.NextExpression()
|
||||
if !hasValue {
|
||||
return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
|
||||
}
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
}
|
||||
|
||||
tableValue := dec.parser.Expression()
|
||||
runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
|
||||
log.Debugf("table node err: %w", err)
|
||||
if err != nil && !errors.Is(io.EOF, err) {
|
||||
return false, err
|
||||
}
|
||||
c := Context{}
|
||||
|
||||
c = c.SingleChildContext(dec.rootMap)
|
||||
|
||||
// += function
|
||||
err = dec.arrayAppend(c, fullPath, tableNodeValue)
|
||||
|
||||
return runAgainstCurrentExp, err
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type uriDecoder struct {
|
||||
reader io.Reader
|
||||
finished bool
|
||||
readAnything bool
|
||||
}
|
||||
|
||||
func NewUriDecoder() Decoder {
|
||||
return &uriDecoder{finished: false}
|
||||
}
|
||||
|
||||
func (dec *uriDecoder) Init(reader io.Reader) error {
|
||||
dec.reader = reader
|
||||
dec.readAnything = false
|
||||
dec.finished = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *uriDecoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
if _, err := buf.ReadFrom(dec.reader); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if buf.Len() == 0 {
|
||||
dec.finished = true
|
||||
|
||||
// if we've read _only_ an empty string, lets return that
|
||||
// otherwise if we've already read some bytes, and now we get
|
||||
// an empty string, then we are done.
|
||||
if dec.readAnything {
|
||||
return nil, io.EOF
|
||||
}
|
||||
}
|
||||
newValue, err := url.QueryUnescape(buf.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dec.readAnything = true
|
||||
return CreateStringScalarNode(newValue), nil
|
||||
}
|
||||
@@ -1,18 +1,15 @@
|
||||
//go:build !yq_noxml
|
||||
|
||||
package xml
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"golang.org/x/net/html/charset"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type xmlDecoder struct {
|
||||
@@ -22,7 +19,7 @@ type xmlDecoder struct {
|
||||
prefs XmlPreferences
|
||||
}
|
||||
|
||||
func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
|
||||
func NewXMLDecoder(prefs XmlPreferences) Decoder {
|
||||
return &xmlDecoder{
|
||||
finished: false,
|
||||
prefs: prefs,
|
||||
@@ -36,64 +33,55 @@ func (dec *xmlDecoder) Init(reader io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) {
|
||||
yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"}
|
||||
func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
|
||||
yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
for _, child := range nodes {
|
||||
yamlChild, err := dec.convertToYamlNode(child)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
yamlNode.AddChild(yamlChild)
|
||||
yamlNode.Content = append(yamlNode.Content, yamlChild)
|
||||
}
|
||||
|
||||
return yamlNode, nil
|
||||
}
|
||||
|
||||
var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
|
||||
|
||||
func (dec *xmlDecoder) processComment(c string) string {
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
//need to replace "cat " with "# cat"
|
||||
// "\ncat\n" with "\n cat\n"
|
||||
// ensure non-empty comments starting with newline have a space in front
|
||||
|
||||
replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
|
||||
replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
|
||||
return replacement
|
||||
return "#" + strings.TrimRight(c, " ")
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
|
||||
yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
|
||||
yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"}
|
||||
func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
|
||||
log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
|
||||
yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
|
||||
if len(n.Data) > 0 {
|
||||
yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName)
|
||||
log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
|
||||
label := dec.prefs.ContentName
|
||||
labelNode := yqlib.CreateScalarNode(label, label)
|
||||
labelNode := createScalarNode(label, label)
|
||||
labelNode.HeadComment = dec.processComment(n.HeadComment)
|
||||
labelNode.LineComment = dec.processComment(n.LineComment)
|
||||
labelNode.FootComment = dec.processComment(n.FootComment)
|
||||
yamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data))
|
||||
yamlNode.Content = append(yamlNode.Content, labelNode, dec.createValueNodeFromData(n.Data))
|
||||
}
|
||||
|
||||
for i, keyValuePair := range n.Children {
|
||||
label := keyValuePair.K
|
||||
children := keyValuePair.V
|
||||
labelNode := yqlib.CreateScalarNode(label, label)
|
||||
var valueNode *yqlib.CandidateNode
|
||||
labelNode := createScalarNode(label, label)
|
||||
var valueNode *yaml.Node
|
||||
var err error
|
||||
|
||||
if i == 0 {
|
||||
yqlib.GetLogger().Debugf("head comment here")
|
||||
labelNode.HeadComment = dec.processComment(n.HeadComment)
|
||||
|
||||
}
|
||||
yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
|
||||
log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
|
||||
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
|
||||
|
||||
yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children))
|
||||
log.Debug("len of children in %v is %v", label, len(children))
|
||||
if len(children) > 1 {
|
||||
valueNode, err = dec.createSequence(children)
|
||||
if err != nil {
|
||||
@@ -106,7 +94,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
|
||||
if len(children[0].Children) == 0 && children[0].HeadComment != "" {
|
||||
if len(children[0].Data) > 0 {
|
||||
|
||||
yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
|
||||
log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
|
||||
labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
|
||||
children[0].HeadComment = ""
|
||||
} else {
|
||||
@@ -120,39 +108,39 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
yamlNode.AddKeyValueChild(labelNode, valueNode)
|
||||
yamlNode.Content = append(yamlNode.Content, labelNode, valueNode)
|
||||
}
|
||||
|
||||
return yamlNode, nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode {
|
||||
func (dec *xmlDecoder) createValueNodeFromData(values []string) *yaml.Node {
|
||||
switch len(values) {
|
||||
case 0:
|
||||
return yqlib.CreateScalarNode(nil, "")
|
||||
return createScalarNode(nil, "")
|
||||
case 1:
|
||||
return yqlib.CreateScalarNode(values[0], values[0])
|
||||
return createScalarNode(values[0], values[0])
|
||||
default:
|
||||
content := make([]*yqlib.CandidateNode, 0)
|
||||
content := make([]*yaml.Node, 0)
|
||||
for _, value := range values {
|
||||
content = append(content, yqlib.CreateScalarNode(value, value))
|
||||
content = append(content, createScalarNode(value, value))
|
||||
}
|
||||
return &yqlib.CandidateNode{
|
||||
Kind: yqlib.SequenceNode,
|
||||
return &yaml.Node{
|
||||
Kind: yaml.SequenceNode,
|
||||
Tag: "!!seq",
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) {
|
||||
func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
|
||||
if len(n.Children) > 0 {
|
||||
return dec.createMap(n)
|
||||
}
|
||||
|
||||
scalar := dec.createValueNodeFromData(n.Data)
|
||||
|
||||
yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
|
||||
log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
|
||||
scalar.HeadComment = dec.processComment(n.HeadComment)
|
||||
scalar.LineComment = dec.processComment(n.LineComment)
|
||||
if scalar.Tag == "!!seq" {
|
||||
@@ -165,7 +153,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, erro
|
||||
return scalar, nil
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
|
||||
func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
@@ -189,7 +177,12 @@ func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
|
||||
dec.readAnything = true
|
||||
dec.finished = true
|
||||
|
||||
return firstNode, nil
|
||||
return &CandidateNode{
|
||||
Node: &yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{firstNode},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type xmlNode struct {
|
||||
@@ -212,17 +205,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
|
||||
if n.Children == nil {
|
||||
n.Children = make([]*xmlChildrenKv, 0)
|
||||
}
|
||||
yqlib.GetLogger().Debug("looking for %s", s)
|
||||
log.Debug("looking for %s", s)
|
||||
// see if we can find an existing entry to add to
|
||||
for _, childEntry := range n.Children {
|
||||
if childEntry.K == s {
|
||||
yqlib.GetLogger().Debug("found it, appending an entry%s", s)
|
||||
log.Debug("found it, appending an entry%s", s)
|
||||
childEntry.V = append(childEntry.V, c)
|
||||
yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V))
|
||||
log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
|
||||
return
|
||||
}
|
||||
}
|
||||
yqlib.GetLogger().Debug("not there, making a new one %s", s)
|
||||
log.Debug("not there, making a new one %s", s)
|
||||
n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
|
||||
}
|
||||
|
||||
@@ -268,7 +261,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
yqlib.GetLogger().Debug("start element %v", se.Name.Local)
|
||||
log.Debug("start element %v", se.Name.Local)
|
||||
elem.state = "started"
|
||||
// Build new a new current element and link it to its parent
|
||||
elem = &element{
|
||||
@@ -297,14 +290,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
if len(newBit) > 0 {
|
||||
elem.n.Data = append(elem.n.Data, newBit)
|
||||
elem.state = "chardata"
|
||||
yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label)
|
||||
log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem == nil {
|
||||
yqlib.GetLogger().Debug("no element, probably bad xml")
|
||||
continue
|
||||
}
|
||||
yqlib.GetLogger().Debug("end element %v", elem.label)
|
||||
log.Debug("end element %v", elem.label)
|
||||
elem.state = "finished"
|
||||
// And add it to its parent list
|
||||
if elem.parent != nil {
|
||||
@@ -320,10 +309,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
applyFootComment(elem, commentStr)
|
||||
|
||||
} else if elem.state == "chardata" {
|
||||
yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
|
||||
} else {
|
||||
yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
|
||||
}
|
||||
|
||||
@@ -348,7 +337,7 @@ func applyFootComment(elem *element, commentStr string) {
|
||||
if len(elem.n.Children) > 0 {
|
||||
lastChildIndex := len(elem.n.Children) - 1
|
||||
childKv := elem.n.Children[lastChildIndex]
|
||||
yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
|
||||
log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
|
||||
// if it's an array of scalars, put the foot comment on the scalar itself
|
||||
if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
|
||||
nodeToUpdate := childKv.V[len(childKv.V)-1]
|
||||
@@ -357,7 +346,7 @@ func applyFootComment(elem *element, commentStr string) {
|
||||
childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
|
||||
}
|
||||
} else {
|
||||
yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
|
||||
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
|
||||
elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
|
||||
}
|
||||
}
|
||||
+21
-35
@@ -20,12 +20,8 @@ type yamlDecoder struct {
|
||||
leadingContent string
|
||||
bufferRead bytes.Buffer
|
||||
|
||||
// anchor map persists over multiple documents for convenience.
|
||||
anchorMap map[string]*CandidateNode
|
||||
|
||||
readAnything bool
|
||||
firstFile bool
|
||||
documentIndex uint
|
||||
readAnything bool
|
||||
firstFile bool
|
||||
}
|
||||
|
||||
func NewYamlDecoder(prefs YamlPreferences) Decoder {
|
||||
@@ -37,7 +33,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
|
||||
var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
|
||||
var sb strings.Builder
|
||||
for {
|
||||
peekBytes, err := reader.Peek(4)
|
||||
peekBytes, err := reader.Peek(3)
|
||||
if errors.Is(err, io.EOF) {
|
||||
// EOF are handled else where..
|
||||
return reader, sb.String(), nil
|
||||
@@ -51,17 +47,9 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
|
||||
} else if err != nil {
|
||||
return reader, sb.String(), err
|
||||
}
|
||||
} else if string(peekBytes) == "--- " {
|
||||
_, err := reader.ReadString(' ')
|
||||
sb.WriteString("$yqDocSeparator$\n")
|
||||
if errors.Is(err, io.EOF) {
|
||||
return reader, sb.String(), nil
|
||||
} else if err != nil {
|
||||
return reader, sb.String(), err
|
||||
}
|
||||
} else if string(peekBytes) == "---\n" {
|
||||
} else if string(peekBytes) == "---" {
|
||||
_, err := reader.ReadString('\n')
|
||||
sb.WriteString("$yqDocSeparator$\n")
|
||||
sb.WriteString("$yqDocSeperator$\n")
|
||||
if errors.Is(err, io.EOF) {
|
||||
return reader, sb.String(), nil
|
||||
} else if err != nil {
|
||||
@@ -105,15 +93,12 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
|
||||
dec.readAnything = false
|
||||
dec.decoder = *yaml.NewDecoder(readerToUse)
|
||||
dec.firstFile = false
|
||||
dec.documentIndex = 0
|
||||
dec.anchorMap = make(map[string]*CandidateNode)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
|
||||
var yamlNode yaml.Node
|
||||
err := dec.decoder.Decode(&yamlNode)
|
||||
|
||||
var dataBucket yaml.Node
|
||||
err := dec.decoder.Decode(&dataBucket)
|
||||
if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
|
||||
// force returning an empty node with a comment.
|
||||
dec.readAnything = true
|
||||
@@ -131,27 +116,28 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
candidateNode := CandidateNode{document: dec.documentIndex}
|
||||
// don't bother with the DocumentNode
|
||||
err = candidateNode.UnmarshalYAML(yamlNode.Content[0], dec.anchorMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
candidateNode := &CandidateNode{
|
||||
Node: &dataBucket,
|
||||
}
|
||||
|
||||
candidateNode.HeadComment = yamlNode.HeadComment + candidateNode.HeadComment
|
||||
candidateNode.FootComment = yamlNode.FootComment + candidateNode.FootComment
|
||||
|
||||
if dec.leadingContent != "" {
|
||||
candidateNode.LeadingContent = dec.leadingContent
|
||||
dec.leadingContent = ""
|
||||
}
|
||||
dec.readAnything = true
|
||||
dec.documentIndex++
|
||||
return &candidateNode, nil
|
||||
// move document comments into candidate node
|
||||
// otherwise unwrap drops them.
|
||||
candidateNode.TrailingContent = dataBucket.FootComment
|
||||
dataBucket.FootComment = ""
|
||||
return candidateNode, nil
|
||||
}
|
||||
|
||||
func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
|
||||
node := CreateScalarNode(nil, "")
|
||||
node.LeadingContent = dec.leadingContent
|
||||
return node
|
||||
return &CandidateNode{
|
||||
Document: 0,
|
||||
Filename: "",
|
||||
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
|
||||
FileIndex: 0,
|
||||
LeadingContent: dec.leadingContent,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,23 +86,6 @@ will output
|
||||
a: ['dog', 'cat']
|
||||
```
|
||||
|
||||
## Prepend to existing array
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a:
|
||||
- dog
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.a = ["cat"] + .a' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a:
|
||||
- cat
|
||||
- dog
|
||||
```
|
||||
|
||||
## Add new object to array
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
|
||||
@@ -302,7 +302,7 @@ foobar:
|
||||
```
|
||||
|
||||
## Dereference and update a field
|
||||
Use explode with multiply to dereference an object
|
||||
`Use explode with multiply to dereference an object
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
This operator is used to update node values. It can be used in either the:
|
||||
|
||||
### plain form: `=`
|
||||
Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
|
||||
Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
|
||||
|
||||
### relative form: `|=`
|
||||
This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.
|
||||
This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
|
||||
|
||||
|
||||
### Flags
|
||||
@@ -203,7 +203,7 @@ a:
|
||||
```
|
||||
|
||||
## Update node value that has an anchor
|
||||
Anchor will remain
|
||||
Anchor will remaple
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
@@ -250,7 +250,7 @@ a: !cat woof
|
||||
b: !dog woof
|
||||
```
|
||||
|
||||
## Custom types: clobber
|
||||
## Custom types: clovver
|
||||
Use the `c` option to clobber custom tags
|
||||
|
||||
Given a sample.yml file of:
|
||||
|
||||
@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.
|
||||
|
||||
`not` flips a boolean from true to false, or vice versa.
|
||||
|
||||
`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
|
||||
`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
|
||||
|
||||
`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
|
||||
|
||||
@@ -26,24 +26,6 @@ will output
|
||||
true
|
||||
```
|
||||
|
||||
## "yes" and "no" are strings
|
||||
In the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
- yes
|
||||
- no
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.[] | tag' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
!!str
|
||||
!!str
|
||||
```
|
||||
|
||||
## `and` example
|
||||
Running
|
||||
```bash
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
|
||||
|
||||
Column is the number of characters that precede that node on the line it starts.
|
||||
|
||||
## Returns column of _value_ node
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
|
||||
@@ -5,10 +5,10 @@ Use these comment operators to set or retrieve comments. Note that line comments
|
||||
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
|
||||
|
||||
### plain form: `=`
|
||||
This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
|
||||
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
|
||||
|
||||
### relative form: `|=`
|
||||
This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
|
||||
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
|
||||
|
||||
## Set line comment
|
||||
Set the comment on the key node for more reliability (see below).
|
||||
@@ -187,6 +187,7 @@ yq '. head_comment="single"' sample.yml
|
||||
will output
|
||||
```yaml
|
||||
# single
|
||||
|
||||
a: cat
|
||||
```
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ This returns `true` if the context contains the passed in parameter, and false o
|
||||
|
||||
{% hint style="warning" %}
|
||||
|
||||
_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains(["cat"])` will return true for an array `["cats"]`.
|
||||
_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`.
|
||||
|
||||
See the "Array has a subset array" example below on how to check for a subset.
|
||||
|
||||
|
||||
@@ -66,7 +66,6 @@ will output
|
||||
```yaml
|
||||
Mike: cat
|
||||
Mike: dog
|
||||
---
|
||||
Rosey: monkey
|
||||
Rosey: sheep
|
||||
```
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Various operators for parsing and manipulating dates.
|
||||
|
||||
## Date time formattings
|
||||
This uses Golang's built in time library for parsing and formatting date times.
|
||||
This uses the golangs built in time library for parsing and formatting date times.
|
||||
|
||||
When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
|
||||
|
||||
@@ -17,13 +17,13 @@ See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of fo
|
||||
|
||||
|
||||
## Timezones
|
||||
This uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
|
||||
This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
|
||||
|
||||
|
||||
## Durations
|
||||
Durations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
|
||||
Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
|
||||
|
||||
You can add durations to time using the `+` operator.
|
||||
You can durations to time using the `+` operator.
|
||||
|
||||
## Format: from standard RFC3339 format
|
||||
Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
|
||||
@@ -86,30 +86,6 @@ a: cool
|
||||
updated: 2021-05-19T01:02:03Z
|
||||
```
|
||||
|
||||
## From Unix
|
||||
Converts from unix time. Note, you don't have to pipe through the tz operator :)
|
||||
|
||||
Running
|
||||
```bash
|
||||
yq --null-input '1675301929 | from_unix | tz("UTC")'
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
2023-02-02T01:38:49Z
|
||||
```
|
||||
|
||||
## To Unix
|
||||
Converts to unix time
|
||||
|
||||
Running
|
||||
```bash
|
||||
yq --null-input 'now | to_unix'
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
1621386123
|
||||
```
|
||||
|
||||
## Timezone: from standard RFC3339 format
|
||||
Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
|
||||
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
# Divide
|
||||
|
||||
Divide behaves differently according to the type of the LHS:
|
||||
* strings: split by the divider
|
||||
* number: arithmetic division
|
||||
|
||||
## String split
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a: cat_meow
|
||||
b: _
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.c = .a / .b' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: cat_meow
|
||||
b: _
|
||||
c:
|
||||
- cat
|
||||
- meow
|
||||
```
|
||||
|
||||
## Number division
|
||||
The result during division is calculated as a float
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a: 12
|
||||
b: 2.5
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.a = .a / .b' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: 4.8
|
||||
b: 2.5
|
||||
```
|
||||
|
||||
## Number division by zero
|
||||
Dividing by zero results in +Inf or -Inf
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a: 1
|
||||
b: -1
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.a = .a / 0 | .b = .b / 0' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: !!float +Inf
|
||||
b: !!float -Inf
|
||||
```
|
||||
|
||||
@@ -85,7 +85,6 @@ will output
|
||||
```yaml
|
||||
match: cat
|
||||
doc: 0
|
||||
---
|
||||
match: frog
|
||||
doc: 1
|
||||
```
|
||||
|
||||
@@ -16,8 +16,6 @@ These operators are useful to process yaml documents that have stringified embed
|
||||
| TSV | from_tsv/@tsvd | to_tsv/@tsv |
|
||||
| XML | from_xml/@xmld | to_xml(i)/@xml |
|
||||
| Base64 | @base64d | @base64 |
|
||||
| URI | @urid | @uri |
|
||||
| Shell | | @sh |
|
||||
|
||||
|
||||
See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
|
||||
@@ -25,7 +23,7 @@ See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) f
|
||||
XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
|
||||
|
||||
|
||||
Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content.
|
||||
Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string.
|
||||
|
||||
## Encode value as json string
|
||||
Given a sample.yml file of:
|
||||
@@ -309,7 +307,7 @@ cat,"thing1,thing2",true,3.40
|
||||
dog,thing3,false,12
|
||||
```
|
||||
|
||||
## Encode array of arrays as tsv string
|
||||
## Encode array of array scalars as tsv string
|
||||
Scalars are strings, numbers and booleans.
|
||||
|
||||
Given a sample.yml file of:
|
||||
@@ -437,50 +435,6 @@ will output
|
||||
YTogYXBwbGUK
|
||||
```
|
||||
|
||||
## Encode a string to uri
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
coolData: this has & special () characters *
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.coolData | @uri' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
this+has+%26+special+%28%29+characters+%2A
|
||||
```
|
||||
|
||||
## Decode a URI to a string
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
this+has+%26+special+%28%29+characters+%2A
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '@urid' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
this has & special () characters *
|
||||
```
|
||||
|
||||
## Encode a string to sh
|
||||
Sh/Bash friendly string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
coolData: strings with spaces and a 'quote'
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.coolData | @sh' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
strings' with spaces and a '\'quote\'
|
||||
```
|
||||
|
||||
## Decode a base64 encoded string
|
||||
Decoded data is assumed to be a string.
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ a: 1
|
||||
b: 2
|
||||
```
|
||||
|
||||
## from_entries with numeric key indices
|
||||
## from_entries with numeric key indexes
|
||||
from_entries always creates a map, even for numeric keys
|
||||
|
||||
Given a sample.yml file of:
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
Use `eval` to dynamically process an expression - for instance from an environment variable.
|
||||
|
||||
`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
|
||||
`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
|
||||
|
||||
Tip: This can be a useful way to parameterise complex scripts.
|
||||
Tip: This can be useful way parameterise complex scripts.
|
||||
|
||||
## Dynamically evaluate a path
|
||||
Given a sample.yml file of:
|
||||
|
||||
@@ -54,6 +54,7 @@ yq eval-all 'file_index' sample.yml another.yml
|
||||
will output
|
||||
```yaml
|
||||
0
|
||||
---
|
||||
1
|
||||
```
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# Filter
|
||||
|
||||
Filters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`.
|
||||
|
||||
|
||||
## Filter array
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq 'filter(. < 3)' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
- 1
|
||||
- 2
|
||||
```
|
||||
|
||||
## Filter map values
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
c:
|
||||
things: cool
|
||||
frog: yes
|
||||
d:
|
||||
things: hot
|
||||
frog: false
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq 'filter(.things == "cool")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
- things: cool
|
||||
frog: yes
|
||||
```
|
||||
|
||||
@@ -26,7 +26,7 @@ will output
|
||||
bar: 100
|
||||
```
|
||||
|
||||
## Group by field, with nulls
|
||||
## Group by field, with nuls
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
- cat: dog
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Has
|
||||
|
||||
This operation returns true if the key exists in a map (or index in an array), false otherwise.
|
||||
This is operation that returns true if the key exists in a map (or index in an array), false otherwise.
|
||||
|
||||
## Has map key
|
||||
Given a sample.yml file of:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# NAME
|
||||
*yq* is a portable command-line data file processor
|
||||
*yq* is a portable command-line YAML processor
|
||||
|
||||
# SYNOPSIS
|
||||
|
||||
@@ -11,7 +11,7 @@ eval-all/ea - Loads all yaml documents of all yaml files and runs expression onc
|
||||
|
||||
# DESCRIPTION
|
||||
|
||||
a lightweight and portable command-line data file processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
|
||||
a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
|
||||
|
||||
This documentation is also available at https://mikefarah.gitbook.io/yq/
|
||||
# QUICK GUIDE
|
||||
@@ -26,7 +26,7 @@ yq '.a.b[0].c' file.yaml
|
||||
cat file.yaml | yq '.a.b[0].c'
|
||||
```
|
||||
|
||||
## Update a yaml file, in place
|
||||
## Update a yaml file, inplace
|
||||
```bash
|
||||
yq -i '.a.b[0].c = "cool"' file.yaml
|
||||
```
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
This operator is used to update node values. It can be used in either the:
|
||||
|
||||
### plain form: `=`
|
||||
Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
|
||||
Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
|
||||
|
||||
### relative form: `|=`
|
||||
This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.
|
||||
This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
|
||||
|
||||
|
||||
### Flags
|
||||
|
||||
@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.
|
||||
|
||||
`not` flips a boolean from true to false, or vice versa.
|
||||
|
||||
`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
|
||||
`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
|
||||
|
||||
`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# Column
|
||||
|
||||
Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.
|
||||
|
||||
Column is the number of characters that precede that node on the line it starts.
|
||||
|
||||
@@ -5,7 +5,7 @@ Use these comment operators to set or retrieve comments. Note that line comments
|
||||
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
|
||||
|
||||
### plain form: `=`
|
||||
This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
|
||||
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
|
||||
|
||||
### relative form: `|=`
|
||||
This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
|
||||
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user