diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index cd95c3d2..45951e8a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -42,7 +42,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -53,7 +53,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -67,4 +67,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.gitignore b/.gitignore index 15b884fe..8bedd020 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ _testmain.go cover.out coverage.out coverage.html +coverage_sorted.txt *.exe *.test *.prof diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 94cd04c8..39b883de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,42 +1,214 @@ -.# Development +# Before you begin +Not all new PRs will be merged in -1. Install (Golang)[https://golang.org/] -1. Run `scripts/devtools.sh` to install the required devtools -2. Run `make [local] vendor` to install the vendor dependencies -2. Run `make [local] test` to ensure you can run the existing tests -3. Write unit tests - (see existing examples). Changes will not be accepted without corresponding unit tests. -4. Make the code changes. -5. `make [local] test` to lint code and run tests -6. Profit! ok no profit, but raise a PR and get kudos :) +It's recommended to check with the owner first (e.g. raise an issue) to discuss a new feature before developing, to ensure your hard efforts don't go to waste. +PRs to fix bugs and issues are almost always welcome :pray: please ensure you write tests as well. + +The following types of PRs will _not_ be accepted: +- **Significant refactors** take a lot of time to understand and can have all sorts of unintended side effects. If you think there's a better way to do things (that requires significant changes) raise an issue for discussion first :) +- **Release pipeline PRs** are a security risk - it's too easy for a serious vulnerability to sneak in (either intended or not). If there is a new cool way of releasing things, raise an issue for discussion first - it will need to be gone over with a fine tooth comb. +- **Version bumps** are handled by dependabot, the bot will auto-raise PRs and they will be regularly merged in. +- **New release platforms** At this stage, yq is not going to maintain any other release platforms other than GitHub and Docker - that said, I'm more than happy to put in other community maintained methods in the README for visibility :heart: + + +# Development + +## Initial Setup + +1. Install [Golang](https://golang.org/) (version 1.24.0 or later) +2. Run `scripts/devtools.sh` to install required development tools: + - golangci-lint for code linting + - gosec for security analysis +3. Run `make [local] vendor` to install vendor dependencies +4. Run `make [local] test` to ensure you can run the existing tests + +## Development Workflow + +1. **Write unit tests first** - Changes will not be accepted without corresponding unit tests (see Testing section below) +2. **Make your code changes** +3. **Run tests and linting**: `make [local] test` (this runs formatting, linting, security checks, and tests) +4. **Create your PR** and get kudos! :) + +## Make Commands + +- Use `make [local] ` for local development (runs in Docker container) +- Use `make ` for CI/CD environments +- Common commands: + - `make [local] vendor` - Install dependencies + - `make [local] test` - Run all checks and tests + - `make [local] build` - Build the yq binary + - `make [local] format` - Format code + - `make [local] check` - Run linting and security checks + +# Code Quality + +## Linting and Formatting + +The project uses strict linting rules defined in `.golangci.yml`. All code must pass: + +- **Code formatting**: gofmt, goimports, gci +- **Linting**: revive, errorlint, gosec, misspell, and others +- **Security checks**: gosec security analysis +- **Spelling checks**: misspell detection + +Run `make [local] check` to verify your code meets all quality standards. + +## Code Style Guidelines + +- Follow standard Go conventions +- Use meaningful variable names +- Add comments for public functions and complex logic +- Keep functions focused and reasonably sized +- Use the project's existing patterns and conventions + +# Testing + +## Test Structure + +Tests in yq use the `expressionScenario` pattern. Each test scenario includes: +- `expression`: The yq expression to test +- `document`: Input YAML/JSON (optional) +- `expected`: Expected output +- `skipDoc`: Whether to skip documentation generation + +## Writing Tests + +1. **Find the appropriate test file** (e.g., `operator_add_test.go` for addition operations) +2. **Add your test scenario** to the `*OperatorScenarios` slice +3. **Run the specific test**: `go test -run TestAddOperatorScenarios` (replace with appropriate test name) +4. **Verify documentation generation** (see Documentation section) + +## Test Examples + +```go +var addOperatorScenarios = []expressionScenario{ + { + skipDoc: true, + expression: `"foo" + "bar"`, + expected: []string{ + "D0, P[], (!!str)::foobar\n", + }, + }, + { + document: "apples: 3", + expression: `.apples + 3`, + expected: []string{ + "D0, P[apples], (!!int)::6\n", + }, + }, +} +``` + +## Running Tests + +- **All tests**: `make [local] test` +- **Specific test**: `go test -run TestName` +- **With coverage**: `make [local] cover` # Documentation -The documentation is a bit of a mixed bag (sorry in advance, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined. +## Documentation Generation -Documentation is written in markdown, and is published in the 'gitbook' branch. +The project uses a documentation system that combines static headers with dynamically generated content from tests. -The various operator documentation (e.g. 'strings') are generated from the 'master' branch, and have a statically defined header (e.g. `pkg/yqlib/doc/operators/headers/add.md`) and the bulk of the docs are generated from the unit tests e.g. `pkg/yqlib/operator_add_test.go`. +### How It Works -The pipeline will run the tests and automatically concatenate the files together, and put them under -`pkg/qylib/doc/add.md`. These files are checked in the master branch (and are copied to the gitbook branch as part of the release process). +1. **Static headers** are defined in `pkg/yqlib/doc/operators/headers/*.md` +2. **Dynamic content** is generated from test scenarios in `*_test.go` files +3. **Generated docs** are created in `pkg/yqlib/doc/*.md` by concatenating headers with test-generated content +4. **Documentation is synced** to the gitbook branch for the website -## How to contribute +### Updating Operator Documentation -The first step is to find if what you want is automatically generated or not - start by looking in the master branch. +#### For Test-Generated Documentation -Note that PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama). +Most operator documentation is generated from tests. To update: -### Updating dynamic documentation from master -- Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file -- Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run: -`go test -run TestAddOperatorScenarios` which will run that test defined in the `operator_add_test.go` file. -- Ensure the tests still pass, and check the generated documentation have your update. -- Note: If the documentation is only in a `headers/*.md` file, then just update that directly -- Raise a PR to merge the changes into master! +1. **Find the test file** (e.g., `operator_add_test.go`) +2. **Update test scenarios** - each `expressionScenario` with `skipDoc: false` becomes documentation +3. **Run the test** to regenerate docs: + ```bash + cd pkg/yqlib + go test -run TestAddOperatorScenarios + ``` +4. **Verify the generated documentation** in `pkg/yqlib/doc/add.md` +5. **Create a PR** with your changes -### Updating static documentation from the gitbook branch -If you haven't found what you want to update in the master branch, then check the gitbook branch directly as there are a few pages in there that are not in master. +#### For Header-Only Documentation -- Update the `*.md` files -- Raise a PR to merge the changes into gitbook. \ No newline at end of file +If documentation exists only in `headers/*.md` files: +1. **Update the header file directly** (e.g., `pkg/yqlib/doc/operators/headers/add.md`) +2. **Create a PR** with your changes + +### Updating Static Documentation + +For documentation not in the master branch: + +1. **Check the gitbook branch** for additional pages +2. **Update the `*.md` files** directly +3. **Create a PR** to the gitbook branch + +### Documentation Best Practices + +- **Write clear, concise examples** in test scenarios +- **Use meaningful variable names** in examples +- **Include edge cases** and error conditions +- **Test your documentation changes** by running the specific test +- **Verify generated output** matches expectations + +Note: PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama). + +# Troubleshooting + +## Common Setup Issues + +### Docker/Podman Issues +- **Problem**: `make` commands fail with Docker errors +- **Solution**: Ensure Docker or Podman is running and accessible +- **Alternative**: Use `make local ` to run in containers + +### Go Version Issues +- **Problem**: Build fails with Go version errors +- **Solution**: Ensure you have Go 1.24.0 or later installed +- **Check**: Run `go version` to verify + +### Vendor Dependencies +- **Problem**: `make vendor` fails or dependencies are outdated +- **Solution**: + ```bash + go mod tidy + make [local] vendor + ``` + +### Linting Failures +- **Problem**: `make check` fails with linting errors +- **Solution**: + ```bash + make [local] format # Auto-fix formatting + # Manually fix remaining linting issues + make [local] check # Verify fixes + ``` + +### Test Failures +- **Problem**: Tests fail locally but pass in CI +- **Solution**: + ```bash + make [local] test # Run in Docker container + ``` + +### Documentation Generation Issues +- **Problem**: Generated docs don't update after test changes +- **Solution**: + ```bash + cd pkg/yqlib + go test -run TestSpecificOperatorScenarios + # Check if generated file updated in pkg/yqlib/doc/ + ``` + +## Getting Help + +- **Check existing issues**: Search GitHub issues for similar problems +- **Create an issue**: If you can't find a solution, create a detailed issue +- **Ask questions**: Use GitHub Discussions for general questions +- **Join the community**: Check the project's community channels \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 02d95af0..8c2f0d02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.25.0 AS builder +FROM golang:1.25.4 AS builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index e509d3d1..4f6986bb 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.25.0 +FROM golang:1.25.4 RUN apt-get update && \ apt-get install -y npm && \ diff --git a/README.md b/README.md index 21604ddb..6ebf614f 100644 --- a/README.md +++ b/README.md @@ -3,44 +3,46 @@ ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg) -a lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. +A lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) (a popular JSON processor) like syntax but works with yaml files as well as json, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. -yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below. +yq is written in Go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below. ## Quick Usage Guide -Read a value: +### Basic Operations + +**Read a value:** ```bash yq '.a.b[0].c' file.yaml ``` -Pipe from STDIN: +**Pipe from STDIN:** ```bash yq '.a.b[0].c' < file.yaml ``` -Update a yaml file, in place +**Update a yaml file in place:** ```bash yq -i '.a.b[0].c = "cool"' file.yaml ``` -Update using environment variables +**Update using environment variables:** ```bash NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml ``` -Merge multiple files +### Advanced Operations + +**Merge multiple files:** ```bash # merge two files yq -n 'load("file1.yaml") * load("file2.yaml")' -# merge using globs: -# note the use of `ea` to evaluate all the files at once -# instead of in sequence +# merge using globs (note: `ea` evaluates all files at once instead of in sequence) yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml ``` -Multiple updates to a yaml file +**Multiple updates to a yaml file:** ```bash yq -i ' .a.b[0].c = "cool" | @@ -49,14 +51,22 @@ yq -i ' ' file.yaml ``` -Find and update an item in an array: +**Find and update an item in an array:** ```bash -yq '(.[] | select(.name == "foo") | .address) = "12 cat st"' +# Note: requires input file - add your file at the end +yq -i '(.[] | select(.name == "foo") | .address) = "12 cat st"' data.yaml ``` -Convert JSON to YAML +**Convert between formats:** ```bash +# Convert JSON to YAML (pretty print) yq -Poy sample.json + +# Convert YAML to JSON +yq -o json file.yaml + +# Convert XML to YAML +yq -o yaml file.xml ``` See [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information. @@ -68,31 +78,31 @@ Take a look at the discussions for [common questions](https://github.com/mikefar ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest) ### wget -Use wget to download, gzipped pre-compiled binaries: +Use wget to download pre-compiled binaries. Choose your platform and architecture: - -For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64 - -#### Compressed via tar.gz +**For Linux (example):** ```bash -wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\ - tar xz && mv ${BINARY} /usr/local/bin/yq -``` +# Set your platform variables (adjust as needed) +VERSION=v4.2.0 +PLATFORM=linux_amd64 -#### Plain binary +# Download compressed binary +wget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM}.tar.gz -O - |\ + tar xz && sudo mv yq_${PLATFORM} /usr/local/bin/yq -```bash -wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/local/bin/yq &&\ +# Or download plain binary +wget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM} -O /usr/local/bin/yq &&\ chmod +x /usr/local/bin/yq ``` -#### Latest version - +**Latest version (Linux AMD64):** ```bash wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq &&\ chmod +x /usr/local/bin/yq ``` +**Available platforms:** `linux_amd64`, `linux_arm64`, `linux_arm`, `linux_386`, `darwin_amd64`, `darwin_arm64`, `windows_amd64`, `windows_386`, etc. + ### MacOS / Linux via Homebrew: Using [Homebrew](https://brew.sh/) ``` @@ -123,28 +133,31 @@ rm /etc/myfile.tmp ``` ### Run with Docker or Podman -#### Oneshot use: +#### One-time use: ```bash -docker run --rm -v "${PWD}":/workdir mikefarah/yq [command] [flags] [expression ]FILE... +# Docker - process files in current directory +docker run --rm -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml + +# Podman - same usage as Docker +podman run --rm -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml ``` -Note that you can run `yq` in docker without network access and other privileges if you desire, -namely `--security-opt=no-new-privileges --cap-drop all --network none`. - +**Security note:** You can run `yq` in Docker with restricted privileges: ```bash -podman run --rm -v "${PWD}":/workdir mikefarah/yq [command] [flags] [expression ]FILE... +docker run --rm --security-opt=no-new-privileges --cap-drop all --network none \ + -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml ``` -#### Pipe in via STDIN: +#### Pipe data via STDIN: -You'll need to pass the `-i\--interactive` flag to docker: +You'll need to pass the `-i --interactive` flag to Docker/Podman: ```bash +# Process piped data docker run -i --rm mikefarah/yq '.this.thing' < myfile.yml -``` -```bash +# Same with Podman podman run -i --rm mikefarah/yq '.this.thing' < myfile.yml ``` @@ -340,7 +353,7 @@ gah install yq - Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble) - Colorized yaml output - [Date/Time manipulation and formatting with TZ](https://mikefarah.gitbook.io/yq/operators/datetime) -- [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read) +- [Deep data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read) - [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys) - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators). - [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) @@ -423,6 +436,27 @@ Flags: Use "yq [command] --help" for more information about a command. ``` + +## Troubleshooting + +### Common Issues + +**PowerShell quoting issues:** +```powershell +# Use single quotes for expressions +yq '.a.b[0].c' file.yaml + +# Or escape double quotes +yq ".a.b[0].c = \"value\"" file.yaml +``` + +### Getting Help + +- **Check existing issues**: [GitHub Issues](https://github.com/mikefarah/yq/issues) +- **Ask questions**: [GitHub Discussions](https://github.com/mikefarah/yq/discussions) +- **Documentation**: [Complete documentation](https://mikefarah.gitbook.io/yq/) +- **Examples**: [Recipes and examples](https://mikefarah.gitbook.io/yq/recipes) + ## Known Issues / Missing Features - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details) - Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell) diff --git a/cmd/evaluate_all_command_test.go b/cmd/evaluate_all_command_test.go new file mode 100644 index 00000000..36147245 --- /dev/null +++ b/cmd/evaluate_all_command_test.go @@ -0,0 +1,328 @@ +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCreateEvaluateAllCommand(t *testing.T) { + cmd := createEvaluateAllCommand() + + if cmd == nil { + t.Fatal("createEvaluateAllCommand returned nil") + } + + // Test basic command properties + if cmd.Use != "eval-all [expression] [yaml_file1]..." { + t.Errorf("Expected Use to be 'eval-all [expression] [yaml_file1]...', got %q", cmd.Use) + } + + if cmd.Short == "" { + t.Error("Expected Short description to be non-empty") + } + + if cmd.Long == "" { + t.Error("Expected Long description to be non-empty") + } + + // Test aliases + expectedAliases := []string{"ea"} + if len(cmd.Aliases) != len(expectedAliases) { + t.Errorf("Expected %d aliases, got %d", len(expectedAliases), len(cmd.Aliases)) + } + + for i, expected := range expectedAliases { + if i >= len(cmd.Aliases) || cmd.Aliases[i] != expected { + t.Errorf("Expected alias %d to be %q, got %q", i, expected, cmd.Aliases[i]) + } + } +} + +func TestEvaluateAll_NoArgs(t *testing.T) { + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with no arguments and no null input + nullInput = false + defer func() { nullInput = false }() + + err := evaluateAll(cmd, []string{}) + + // Should not error, but should print usage + if err != nil { + t.Errorf("evaluateAll with no args should not error, got: %v", err) + } + + // Should have printed usage information + if output.Len() == 0 { + t.Error("Expected usage information to be printed") + } +} + +func TestEvaluateAll_NullInput(t *testing.T) { + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with null input + nullInput = true + defer func() { nullInput = false }() + + err := evaluateAll(cmd, []string{}) + + // Should not error when using null input + if err != nil { + t.Errorf("evaluateAll with null input should not error, got: %v", err) + } +} + +func TestEvaluateAll_WithSingleFile(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with a single file + err = evaluateAll(cmd, []string{yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateAll with single file should not error, got: %v", err) + } + + // Should have some output + if output.Len() == 0 { + t.Error("Expected output from evaluateAll with single file") + } +} + +func TestEvaluateAll_WithMultipleFiles(t *testing.T) { + // Create temporary YAML files + tempDir := t.TempDir() + + yamlFile1 := filepath.Join(tempDir, "test1.yaml") + yamlContent1 := []byte("name: test1\nage: 25\n") + err := os.WriteFile(yamlFile1, yamlContent1, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file 1: %v", err) + } + + yamlFile2 := filepath.Join(tempDir, "test2.yaml") + yamlContent2 := []byte("name: test2\nage: 30\n") + err = os.WriteFile(yamlFile2, yamlContent2, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file 2: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with multiple files + err = evaluateAll(cmd, []string{yamlFile1, yamlFile2}) + + // Should not error + if err != nil { + t.Errorf("evaluateAll with multiple files should not error, got: %v", err) + } + + // Should have output + if output.Len() == 0 { + t.Error("Expected output from evaluateAll with multiple files") + } +} + +func TestEvaluateAll_WithExpression(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with expression + err = evaluateAll(cmd, []string{".name", yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateAll with expression should not error, got: %v", err) + } + + // Should have output + if output.Len() == 0 { + t.Error("Expected output from evaluateAll with expression") + } +} + +func TestEvaluateAll_WriteInPlace(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Enable write in place + originalWriteInplace := writeInplace + writeInplace = true + defer func() { writeInplace = originalWriteInplace }() + + // Test with write in place + err = evaluateAll(cmd, []string{".name = \"updated\"", yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateAll with write in place should not error, got: %v", err) + } + + // Verify the file was updated + updatedContent, err := os.ReadFile(yamlFile) + if err != nil { + t.Fatalf("Failed to read updated file: %v", err) + } + + // Should contain the updated content + if !strings.Contains(string(updatedContent), "updated") { + t.Errorf("Expected file to contain 'updated', got: %s", string(updatedContent)) + } +} + +func TestEvaluateAll_ExitStatus(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Enable exit status + originalExitStatus := exitStatus + exitStatus = true + defer func() { exitStatus = originalExitStatus }() + + // Test with expression that should find no matches + err = evaluateAll(cmd, []string{".nonexistent", yamlFile}) + + // Should error when no matches found and exit status is enabled + if err == nil { + t.Error("Expected error when no matches found and exit status is enabled") + } +} + +func TestEvaluateAll_WithMultipleDocuments(t *testing.T) { + // Create a temporary YAML file with multiple documents + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("---\nname: doc1\nage: 25\n---\nname: doc2\nage: 30\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with multiple documents + err = evaluateAll(cmd, []string{".", yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateAll with multiple documents should not error, got: %v", err) + } + + // Should have output + if output.Len() == 0 { + t.Error("Expected output from evaluateAll with multiple documents") + } +} + +func TestEvaluateAll_NulSepOutput(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateAllCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Enable nul separator output + originalNulSepOutput := nulSepOutput + nulSepOutput = true + defer func() { nulSepOutput = originalNulSepOutput }() + + // Test with nul separator output + err = evaluateAll(cmd, []string{".name", yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateAll with nul separator output should not error, got: %v", err) + } + + // Should have output + if output.Len() == 0 { + t.Error("Expected output from evaluateAll with nul separator output") + } +} diff --git a/cmd/evaluate_sequence_command_test.go b/cmd/evaluate_sequence_command_test.go new file mode 100644 index 00000000..539b4a20 --- /dev/null +++ b/cmd/evaluate_sequence_command_test.go @@ -0,0 +1,276 @@ +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestCreateEvaluateSequenceCommand(t *testing.T) { + cmd := createEvaluateSequenceCommand() + + if cmd == nil { + t.Fatal("createEvaluateSequenceCommand returned nil") + } + + // Test basic command properties + if cmd.Use != "eval [expression] [yaml_file1]..." { + t.Errorf("Expected Use to be 'eval [expression] [yaml_file1]...', got %q", cmd.Use) + } + + if cmd.Short == "" { + t.Error("Expected Short description to be non-empty") + } + + if cmd.Long == "" { + t.Error("Expected Long description to be non-empty") + } + + // Test aliases + expectedAliases := []string{"e"} + if len(cmd.Aliases) != len(expectedAliases) { + t.Errorf("Expected %d aliases, got %d", len(expectedAliases), len(cmd.Aliases)) + } + + for i, expected := range expectedAliases { + if i >= len(cmd.Aliases) || cmd.Aliases[i] != expected { + t.Errorf("Expected alias %d to be %q, got %q", i, expected, cmd.Aliases[i]) + } + } +} + +func TestProcessExpression(t *testing.T) { + // Reset global variables + originalPrettyPrint := prettyPrint + defer func() { prettyPrint = originalPrettyPrint }() + + tests := []struct { + name string + prettyPrint bool + expression string + expected string + }{ + { + name: "empty expression without pretty print", + prettyPrint: false, + expression: "", + expected: "", + }, + { + name: "empty expression with pretty print", + prettyPrint: true, + expression: "", + expected: `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`, + }, + { + name: "simple expression without pretty print", + prettyPrint: false, + expression: ".a.b", + expected: ".a.b", + }, + { + name: "simple expression with pretty print", + prettyPrint: true, + expression: ".a.b", + expected: `.a.b | (... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`, + }, + { + name: "complex expression with pretty print", + prettyPrint: true, + expression: ".items[] | select(.active == true)", + expected: `.items[] | select(.active == true) | (... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prettyPrint = tt.prettyPrint + result := processExpression(tt.expression) + if result != tt.expected { + t.Errorf("processExpression(%q) = %q, want %q", tt.expression, result, tt.expected) + } + }) + } +} + +func TestEvaluateSequence_NoArgs(t *testing.T) { + // Create a temporary command + cmd := createEvaluateSequenceCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with no arguments and no null input + nullInput = false + defer func() { nullInput = false }() + + err := evaluateSequence(cmd, []string{}) + + // Should not error, but should print usage + if err != nil { + t.Errorf("evaluateSequence with no args should not error, got: %v", err) + } + + // Should have printed usage information + if output.Len() == 0 { + t.Error("Expected usage information to be printed") + } +} + +func TestEvaluateSequence_NullInput(t *testing.T) { + // Create a temporary command + cmd := createEvaluateSequenceCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with null input + nullInput = true + defer func() { nullInput = false }() + + err := evaluateSequence(cmd, []string{}) + + // Should not error when using null input + if err != nil { + t.Errorf("evaluateSequence with null input should not error, got: %v", err) + } +} + +func TestEvaluateSequence_WithFile(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateSequenceCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with a file + err = evaluateSequence(cmd, []string{yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateSequence with file should not error, got: %v", err) + } + + // Should have some output + if output.Len() == 0 { + t.Error("Expected output from evaluateSequence with file") + } +} + +func TestEvaluateSequence_WithExpressionAndFile(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateSequenceCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Test with expression and file + err = evaluateSequence(cmd, []string{".name", yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateSequence with expression and file should not error, got: %v", err) + } + + // Should have output + if output.Len() == 0 { + t.Error("Expected output from evaluateSequence with expression and file") + } +} + +func TestEvaluateSequence_WriteInPlace(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateSequenceCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Enable write in place + originalWriteInplace := writeInplace + writeInplace = true + defer func() { writeInplace = originalWriteInplace }() + + // Test with write in place + err = evaluateSequence(cmd, []string{".name = \"updated\"", yamlFile}) + + // Should not error + if err != nil { + t.Errorf("evaluateSequence with write in place should not error, got: %v", err) + } + + // Verify the file was updated + updatedContent, err := os.ReadFile(yamlFile) + if err != nil { + t.Fatalf("Failed to read updated file: %v", err) + } + + // Should contain the updated content + if !strings.Contains(string(updatedContent), "updated") { + t.Errorf("Expected file to contain 'updated', got: %s", string(updatedContent)) + } +} + +func TestEvaluateSequence_ExitStatus(t *testing.T) { + // Create a temporary YAML file + tempDir := t.TempDir() + yamlFile := filepath.Join(tempDir, "test.yaml") + yamlContent := []byte("name: test\nage: 25\n") + err := os.WriteFile(yamlFile, yamlContent, 0600) + if err != nil { + t.Fatalf("Failed to create test YAML file: %v", err) + } + + // Create a temporary command + cmd := createEvaluateSequenceCommand() + + // Set up command to capture output + var output bytes.Buffer + cmd.SetOut(&output) + + // Enable exit status + originalExitStatus := exitStatus + exitStatus = true + defer func() { exitStatus = originalExitStatus }() + + // Test with expression that should find no matches + err = evaluateSequence(cmd, []string{".nonexistent", yamlFile}) + + // Should error when no matches found and exit status is enabled + if err == nil { + t.Error("Expected error when no matches found and exit status is enabled") + } +} diff --git a/cmd/root.go b/cmd/root.go index 268ff169..7a669dad 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -168,6 +168,11 @@ yq -P -oy sample.json rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "shell-key-separator", yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "separator for shell variable key paths") + if err = rootCmd.RegisterFlagCompletionFunc("shell-key-separator", cobra.NoFileCompletions); err != nil { + panic(err) + } + rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)") rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") diff --git a/cmd/root_test.go b/cmd/root_test.go new file mode 100644 index 00000000..c141f800 --- /dev/null +++ b/cmd/root_test.go @@ -0,0 +1,264 @@ +package cmd + +import ( + "strings" + "testing" +) + +func TestNewRuneVar(t *testing.T) { + var r rune + runeVar := newRuneVar(&r) + + if runeVar == nil { + t.Fatal("newRuneVar returned nil") + } +} + +func TestRuneValue_String(t *testing.T) { + tests := []struct { + name string + runeVal rune + expected string + }{ + { + name: "simple character", + runeVal: 'a', + expected: "a", + }, + { + name: "special character", + runeVal: '\n', + expected: "\n", + }, + { + name: "unicode character", + runeVal: 'ñ', + expected: "ñ", + }, + { + name: "zero rune", + runeVal: 0, + expected: string(rune(0)), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runeVal := runeValue(tt.runeVal) + result := runeVal.String() + if result != tt.expected { + t.Errorf("runeValue.String() = %q, want %q", result, tt.expected) + } + }) + } +} + +func TestRuneValue_Set(t *testing.T) { + tests := []struct { + name string + input string + expected rune + expectError bool + }{ + { + name: "simple character", + input: "a", + expected: 'a', + expectError: false, + }, + { + name: "newline escape", + input: "\\n", + expected: '\n', + expectError: false, + }, + { + name: "tab escape", + input: "\\t", + expected: '\t', + expectError: false, + }, + { + name: "carriage return escape", + input: "\\r", + expected: '\r', + expectError: false, + }, + { + name: "form feed escape", + input: "\\f", + expected: '\f', + expectError: false, + }, + { + name: "vertical tab escape", + input: "\\v", + expected: '\v', + expectError: false, + }, + { + name: "empty string", + input: "", + expected: 0, + expectError: true, + }, + { + name: "multiple characters", + input: "ab", + expected: 0, + expectError: true, + }, + { + name: "special character", + input: "ñ", + expected: 'ñ', + expectError: true, // This will fail because the Set function checks len(val) != 1 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var r rune + runeVal := newRuneVar(&r) + + err := runeVal.Set(tt.input) + + if tt.expectError { + if err == nil { + t.Errorf("Expected error for input %q, but got none", tt.input) + } + } else { + if err != nil { + t.Errorf("Unexpected error for input %q: %v", tt.input, err) + } + if r != tt.expected { + t.Errorf("Expected rune %q (%d), got %q (%d)", + string(tt.expected), tt.expected, string(r), r) + } + } + }) + } +} + +func TestRuneValue_Set_ErrorMessages(t *testing.T) { + tests := []struct { + name string + input string + expectedError string + }{ + { + name: "empty string error", + input: "", + expectedError: "[] is not a valid character. Must be length 1 was 0", + }, + { + name: "multiple characters error", + input: "abc", + expectedError: "[abc] is not a valid character. Must be length 1 was 3", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var r rune + runeVal := newRuneVar(&r) + + err := runeVal.Set(tt.input) + + if err == nil { + t.Errorf("Expected error for input %q, but got none", tt.input) + return + } + + if !strings.Contains(err.Error(), tt.expectedError) { + t.Errorf("Expected error message to contain %q, got %q", + tt.expectedError, err.Error()) + } + }) + } +} + +func TestRuneValue_Type(t *testing.T) { + var r rune + runeVal := newRuneVar(&r) + + result := runeVal.Type() + expected := "char" + + if result != expected { + t.Errorf("runeValue.Type() = %q, want %q", result, expected) + } +} + +func TestNew(t *testing.T) { + rootCmd := New() + + if rootCmd == nil { + t.Fatal("New() returned nil") + } + + // Test basic command properties + if rootCmd.Use != "yq" { + t.Errorf("Expected Use to be 'yq', got %q", rootCmd.Use) + } + + if rootCmd.Short == "" { + t.Error("Expected Short description to be non-empty") + } + + if rootCmd.Long == "" { + t.Error("Expected Long description to be non-empty") + } + + // Test that the command has the expected subcommands + expectedCommands := []string{"eval", "eval-all", "completion"} + actualCommands := make([]string, 0, len(rootCmd.Commands())) + + for _, cmd := range rootCmd.Commands() { + actualCommands = append(actualCommands, cmd.Name()) + } + + for _, expected := range expectedCommands { + found := false + for _, actual := range actualCommands { + if actual == expected { + found = true + break + } + } + if !found { + t.Errorf("Expected command %q not found in actual commands: %v", + expected, actualCommands) + } + } +} + +func TestNew_FlagCompletions(t *testing.T) { + rootCmd := New() + + // Test that flag completion functions are registered + // This is a basic smoke test - we can't easily test the actual completion logic + // without more complex setup + flags := []string{ + "output-format", + "input-format", + "xml-attribute-prefix", + "xml-content-name", + "xml-proc-inst-prefix", + "xml-directive-name", + "lua-prefix", + "lua-suffix", + "properties-separator", + "indent", + "front-matter", + "expression", + "split-exp", + } + + for _, flagName := range flags { + flag := rootCmd.PersistentFlags().Lookup(flagName) + if flag == nil { + t.Errorf("Expected flag %q to exist", flagName) + } + } +} diff --git a/cmd/version.go b/cmd/version.go index b98f3432..1fe987ef 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.47.2" + Version = "v4.48.1" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/go.mod b/go.mod index a98b661c..8a9ff3c7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4 require ( github.com/a8m/envsubst v1.4.3 github.com/alecthomas/participle/v2 v2.1.4 - github.com/alecthomas/repr v0.5.1 + github.com/alecthomas/repr v0.5.2 github.com/dimchansky/utfbom v1.1.1 github.com/elliotchance/orderedmap v1.8.0 github.com/fatih/color v1.18.0 @@ -18,8 +18,8 @@ require ( github.com/spf13/pflag v1.0.10 github.com/yuin/gopher-lua v1.1.1 go.yaml.in/yaml/v4 v4.0.0-rc.3 - golang.org/x/net v0.43.0 - golang.org/x/text v0.28.0 + golang.org/x/net v0.46.0 + golang.org/x/text v0.30.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 ) @@ -27,9 +27,9 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - golang.org/x/sys v0.35.0 // indirect + golang.org/x/sys v0.37.0 // indirect ) -go 1.24 +go 1.24.0 toolchain go1.24.1 diff --git a/go.sum b/go.sum index 9b6c8833..6afb12f5 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= -github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= -github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -52,13 +52,14 @@ github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= diff --git a/pkg/yqlib/candidate_node_test.go b/pkg/yqlib/candidate_node_test.go index ceaa4052..58f436d7 100644 --- a/pkg/yqlib/candidate_node_test.go +++ b/pkg/yqlib/candidate_node_test.go @@ -152,8 +152,6 @@ func TestCandidateNodeAddKeyValueChild(t *testing.T) { key := CandidateNode{Value: "cool", IsMapKey: true} node := CandidateNode{} - // if we use a key in a new node as a value, it should no longer be marked as a key - _, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key) test.AssertResult(t, keyIsValueNow.IsMapKey, false) @@ -204,3 +202,193 @@ func TestConvertToNodeInfo(t *testing.T) { test.AssertResult(t, 2, childInfo.Line) test.AssertResult(t, 3, childInfo.Column) } + +func TestCandidateNodeGetPath(t *testing.T) { + // Test root node with no parent + root := CandidateNode{Value: "root"} + path := root.GetPath() + test.AssertResult(t, 0, len(path)) + + // Test node with key + key := createStringScalarNode("myKey") + node := CandidateNode{Key: key, Value: "myValue"} + path = node.GetPath() + test.AssertResult(t, 1, len(path)) + test.AssertResult(t, "myKey", path[0]) + + // Test nested path + parent := CandidateNode{} + parentKey := createStringScalarNode("parent") + parent.Key = parentKey + node.Parent = &parent + path = node.GetPath() + test.AssertResult(t, 2, len(path)) + test.AssertResult(t, "parent", path[0]) + test.AssertResult(t, "myKey", path[1]) +} + +func TestCandidateNodeGetNicePath(t *testing.T) { + // Test simple key + key := createStringScalarNode("simple") + node := CandidateNode{Key: key} + nicePath := node.GetNicePath() + test.AssertResult(t, "simple", nicePath) + + // Test array index + arrayKey := createScalarNode(0, "0") + arrayNode := CandidateNode{Key: arrayKey} + nicePath = arrayNode.GetNicePath() + test.AssertResult(t, "[0]", nicePath) + + dotKey := createStringScalarNode("key.with.dots") + dotNode := CandidateNode{Key: dotKey} + nicePath = dotNode.GetNicePath() + test.AssertResult(t, "key.with.dots", nicePath) + + // Test nested path + parentKey := createStringScalarNode("parent") + parent := CandidateNode{Key: parentKey} + childKey := createStringScalarNode("child") + child := CandidateNode{Key: childKey, Parent: &parent} + nicePath = child.GetNicePath() + test.AssertResult(t, "parent.child", nicePath) +} + +func TestCandidateNodeFilterMapContentByKey(t *testing.T) { + // Create a map with multiple key-value pairs + key1 := createStringScalarNode("key1") + value1 := createStringScalarNode("value1") + key2 := createStringScalarNode("key2") + value2 := createStringScalarNode("value2") + key3 := createStringScalarNode("key3") + value3 := createStringScalarNode("value3") + + mapNode := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value1, key2, value2, key3, value3}, + } + + // Filter by key predicate that matches key1 and key3 + filtered := mapNode.FilterMapContentByKey(func(key *CandidateNode) bool { + return key.Value == "key1" || key.Value == "key3" + }) + + // Should return key1, value1, key3, value3 + test.AssertResult(t, 4, len(filtered)) + test.AssertResult(t, "key1", filtered[0].Value) + test.AssertResult(t, "value1", filtered[1].Value) + test.AssertResult(t, "key3", filtered[2].Value) + test.AssertResult(t, "value3", filtered[3].Value) +} + +func TestCandidateNodeVisitValues(t *testing.T) { + // Test mapping node + key1 := createStringScalarNode("key1") + value1 := createStringScalarNode("value1") + key2 := createStringScalarNode("key2") + value2 := createStringScalarNode("value2") + + mapNode := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value1, key2, value2}, + } + + var visited []string + err := mapNode.VisitValues(func(node *CandidateNode) error { + visited = append(visited, node.Value) + return nil + }) + + test.AssertResult(t, nil, err) + test.AssertResult(t, 2, len(visited)) + test.AssertResult(t, "value1", visited[0]) + test.AssertResult(t, "value2", visited[1]) + + // Test sequence node + item1 := createStringScalarNode("item1") + item2 := createStringScalarNode("item2") + + seqNode := &CandidateNode{ + Kind: SequenceNode, + Content: []*CandidateNode{item1, item2}, + } + + visited = []string{} + err = seqNode.VisitValues(func(node *CandidateNode) error { + visited = append(visited, node.Value) + return nil + }) + + test.AssertResult(t, nil, err) + test.AssertResult(t, 2, len(visited)) + test.AssertResult(t, "item1", visited[0]) + test.AssertResult(t, "item2", visited[1]) + + // Test scalar node (should not visit anything) + scalarNode := &CandidateNode{ + Kind: ScalarNode, + Value: "scalar", + } + + visited = []string{} + err = scalarNode.VisitValues(func(node *CandidateNode) error { + visited = append(visited, node.Value) + return nil + }) + + test.AssertResult(t, nil, err) + test.AssertResult(t, 0, len(visited)) +} + +func TestCandidateNodeCanVisitValues(t *testing.T) { + mapNode := &CandidateNode{Kind: MappingNode} + seqNode := &CandidateNode{Kind: SequenceNode} + scalarNode := &CandidateNode{Kind: ScalarNode} + + test.AssertResult(t, true, mapNode.CanVisitValues()) + test.AssertResult(t, true, seqNode.CanVisitValues()) + test.AssertResult(t, false, scalarNode.CanVisitValues()) +} + +func TestCandidateNodeAddChild(t *testing.T) { + parent := &CandidateNode{Kind: SequenceNode} + child := createStringScalarNode("child") + + parent.AddChild(child) + + test.AssertResult(t, 1, len(parent.Content)) + test.AssertResult(t, false, parent.Content[0].IsMapKey) + test.AssertResult(t, "0", parent.Content[0].Key.Value) + // Check that parent is set correctly + if parent.Content[0].Parent != parent { + t.Errorf("Expected parent to be set correctly") + } +} + +func TestCandidateNodeAddChildren(t *testing.T) { + // Test sequence node + parent := &CandidateNode{Kind: SequenceNode} + child1 := createStringScalarNode("child1") + child2 := createStringScalarNode("child2") + + parent.AddChildren([]*CandidateNode{child1, child2}) + + test.AssertResult(t, 2, len(parent.Content)) + test.AssertResult(t, "child1", parent.Content[0].Value) + test.AssertResult(t, "child2", parent.Content[1].Value) + + // Test mapping node + mapParent := &CandidateNode{Kind: MappingNode} + key1 := createStringScalarNode("key1") + value1 := createStringScalarNode("value1") + key2 := createStringScalarNode("key2") + value2 := createStringScalarNode("value2") + + mapParent.AddChildren([]*CandidateNode{key1, value1, key2, value2}) + + test.AssertResult(t, 4, len(mapParent.Content)) + test.AssertResult(t, true, mapParent.Content[0].IsMapKey) // key1 + test.AssertResult(t, false, mapParent.Content[1].IsMapKey) // value1 + test.AssertResult(t, true, mapParent.Content[2].IsMapKey) // key2 + test.AssertResult(t, false, mapParent.Content[3].IsMapKey) // value2 +} diff --git a/pkg/yqlib/chown_linux_test.go b/pkg/yqlib/chown_linux_test.go new file mode 100644 index 00000000..674b3bd9 --- /dev/null +++ b/pkg/yqlib/chown_linux_test.go @@ -0,0 +1,139 @@ +//go:build linux + +package yqlib + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestChangeOwner(t *testing.T) { + // Create a temporary file for testing + tempDir := t.TempDir() + testFile := filepath.Join(tempDir, "testfile.txt") + + // Create a test file + err := os.WriteFile(testFile, []byte("test content"), 0600) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + // Get file info + info, err := os.Stat(testFile) + if err != nil { + t.Fatalf("Failed to stat test file: %v", err) + } + + // Create another temporary file to change ownership of + tempFile, err := os.CreateTemp(tempDir, "chown_test_*.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + tempFile.Close() + + // Test changeOwner function + err = changeOwner(info, tempFile) + if err != nil { + t.Errorf("changeOwner failed: %v", err) + } + + // Verify that the function doesn't panic with valid input + tempFile2, err := os.CreateTemp(tempDir, "chown_test2_*.txt") + if err != nil { + t.Fatalf("Failed to create second temp file: %v", err) + } + defer os.Remove(tempFile2.Name()) + tempFile2.Close() + + // Test with the second file + err = changeOwner(info, tempFile2) + if err != nil { + t.Errorf("changeOwner failed on second file: %v", err) + } +} + +func TestChangeOwnerWithInvalidFileInfo(t *testing.T) { + // Create a mock file info that doesn't have syscall.Stat_t + mockInfo := &mockFileInfo{ + name: "mock", + size: 0, + mode: 0600, + } + + // Create a temporary file + tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + tempFile.Close() + + // Test changeOwner with mock file info (should not panic) + err = changeOwner(mockInfo, tempFile) + if err != nil { + t.Errorf("changeOwner failed with mock file info: %v", err) + } +} + +func TestChangeOwnerWithNonExistentFile(t *testing.T) { + // Create a temporary file + tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + tempFile.Close() + + // Get file info + info, err := os.Stat(tempFile.Name()) + if err != nil { + t.Fatalf("Failed to stat temp file: %v", err) + } + + // Remove the file + os.Remove(tempFile.Name()) + + err = changeOwner(info, tempFile) + // The function should not panic even if the file doesn't exist + if err != nil { + t.Logf("Expected error when changing owner of non-existent file: %v", err) + } +} + +// mockFileInfo implements fs.FileInfo but doesn't have syscall.Stat_t +type mockFileInfo struct { + name string + size int64 + mode os.FileMode +} + +func (m *mockFileInfo) Name() string { return m.name } +func (m *mockFileInfo) Size() int64 { return m.size } +func (m *mockFileInfo) Mode() os.FileMode { return m.mode } +func (m *mockFileInfo) ModTime() time.Time { return time.Time{} } +func (m *mockFileInfo) IsDir() bool { return false } +func (m *mockFileInfo) Sys() interface{} { return nil } // This will cause the type assertion to fail + +func TestChangeOwnerWithSyscallStatT(t *testing.T) { + // Create a temporary file + tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + tempFile.Close() + + // Get file info + info, err := os.Stat(tempFile.Name()) + if err != nil { + t.Fatalf("Failed to stat temp file: %v", err) + } + + err = changeOwner(info, tempFile) + if err != nil { + t.Logf("changeOwner returned error (this might be expected in some environments): %v", err) + } +} diff --git a/pkg/yqlib/color_print_test.go b/pkg/yqlib/color_print_test.go new file mode 100644 index 00000000..904c3d59 --- /dev/null +++ b/pkg/yqlib/color_print_test.go @@ -0,0 +1,153 @@ +package yqlib + +import ( + "bytes" + "strings" + "testing" + + "github.com/fatih/color" +) + +func TestFormat(t *testing.T) { + tests := []struct { + name string + attr color.Attribute + expected string + }{ + { + name: "reset color", + attr: color.Reset, + expected: "\x1b[0m", + }, + { + name: "red color", + attr: color.FgRed, + expected: "\x1b[31m", + }, + { + name: "green color", + attr: color.FgGreen, + expected: "\x1b[32m", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := format(tt.attr) + if result != tt.expected { + t.Errorf("format(%d) = %q, want %q", tt.attr, result, tt.expected) + } + }) + } +} + +func TestColorizeAndPrint(t *testing.T) { + tests := []struct { + name string + yamlBytes []byte + expectErr bool + }{ + { + name: "simple yaml", + yamlBytes: []byte("name: test\nage: 25\n"), + expectErr: false, + }, + { + name: "yaml with strings", + yamlBytes: []byte("name: \"hello world\"\nactive: true\ncount: 42\n"), + expectErr: false, + }, + { + name: "yaml with anchors and aliases", + yamlBytes: []byte("default: &default\n name: test\nuser: *default\n"), + expectErr: false, + }, + { + name: "yaml with comments", + yamlBytes: []byte("# This is a comment\nname: test\n"), + expectErr: false, + }, + { + name: "empty yaml", + yamlBytes: []byte(""), + expectErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + err := colorizeAndPrint(tt.yamlBytes, &buf) + + if tt.expectErr && err == nil { + t.Error("Expected error but got none") + } + if !tt.expectErr && err != nil { + t.Errorf("Unexpected error: %v", err) + } + + // Check that output contains escape sequences (color codes) + if !tt.expectErr && len(tt.yamlBytes) > 0 { + output := buf.String() + if !strings.Contains(output, "\x1b[") { + t.Error("Expected output to contain color escape sequences") + } + } + }) + } +} + +func TestColorizeAndPrintWithDifferentYamlTypes(t *testing.T) { + testCases := []struct { + name string + yaml string + expectErr bool + }{ + { + name: "boolean values", + yaml: "active: true\ninactive: false\n", + }, + { + name: "numeric values", + yaml: "integer: 42\nfloat: 3.14\nnegative: -10\n", + }, + { + name: "map keys", + yaml: "user:\n name: john\n age: 30\n", + }, + { + name: "string values", + yaml: "message: \"hello world\"\ndescription: 'single quotes'\n", + }, + { + name: "mixed types", + yaml: "config:\n debug: true\n port: 8080\n host: \"localhost\"\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + err := colorizeAndPrint([]byte(tc.yaml), &buf) + + if tc.expectErr && err == nil { + t.Error("Expected error but got none") + } + if !tc.expectErr && err != nil { + t.Errorf("Unexpected error: %v", err) + } + + // Verify output contains color codes + if !tc.expectErr { + output := buf.String() + if !strings.Contains(output, "\x1b[") { + t.Error("Expected output to contain color escape sequences") + } + // Should end with newline + if !strings.HasSuffix(output, "\n") { + t.Error("Expected output to end with newline") + } + } + }) + } +} diff --git a/pkg/yqlib/data_tree_navigator_test.go b/pkg/yqlib/data_tree_navigator_test.go index a0e2e59f..f0ac37b9 100644 --- a/pkg/yqlib/data_tree_navigator_test.go +++ b/pkg/yqlib/data_tree_navigator_test.go @@ -312,7 +312,6 @@ func TestDeeplyAssign_ErrorHandling(t *testing.T) { Value: "value", } - // Try to assign to a path on a scalar (should fail) path := []interface{}{"key"} err := navigator.DeeplyAssign(context, path, assignNode) @@ -321,7 +320,6 @@ func TestDeeplyAssign_ErrorHandling(t *testing.T) { t.Logf("Actual error: %v", err) } - // This should fail because we can't assign to a scalar test.AssertResult(t, nil, err) } diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go index 6759c9cd..febf112f 100644 --- a/pkg/yqlib/decoder_xml.go +++ b/pkg/yqlib/decoder_xml.go @@ -270,10 +270,16 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error { log.Debug("start element %v", se.Name.Local) elem.state = "started" // Build new a new current element and link it to its parent + var label = se.Name.Local + if dec.prefs.KeepNamespace { + if se.Name.Space != "" { + label = se.Name.Space + ":" + se.Name.Local + } + } elem = &element{ parent: elem, n: &xmlNode{}, - label: se.Name.Local, + label: label, } // Extract attributes as children diff --git a/pkg/yqlib/doc/operators/headers/first.md b/pkg/yqlib/doc/operators/headers/first.md new file mode 100644 index 00000000..c3afc64b --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/first.md @@ -0,0 +1,5 @@ +# First + +Returns the first matching element in an array, or first matching value in a map. + +Can be given an expression to match with, otherwise will just return the first. diff --git a/pkg/yqlib/doc/usage/shellvariables.md b/pkg/yqlib/doc/usage/shellvariables.md index ba50315a..455e669b 100644 --- a/pkg/yqlib/doc/usage/shellvariables.md +++ b/pkg/yqlib/doc/usage/shellvariables.md @@ -84,3 +84,23 @@ will output name='Miles O'"'"'Brien' ``` +## Encode shell variables: custom separator +Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores. + +Given a sample.yml file of: +```yaml +my_app: + db_config: + host: localhost + port: 5432 +``` +then +```bash +yq -o=shell --shell-key-separator="__" sample.yml +``` +will output +```sh +my_app__db_config__host=localhost +my_app__db_config__port=5432 +``` + diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index 4238e971..c5bb6bc5 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -319,7 +319,10 @@ Defaults to true Given a sample.xml file of: ```xml - + + baz + foobar + ``` then @@ -329,13 +332,19 @@ yq --xml-keep-namespace=false '.' sample.xml will output ```xml - + + baz + foobar + ``` instead of ```xml - + + baz + foobar + ``` ## Parse xml: keep raw attribute namespace @@ -344,7 +353,10 @@ Defaults to true Given a sample.xml file of: ```xml - + + baz + foobar + ``` then @@ -354,13 +366,19 @@ yq --xml-raw-token=false '.' sample.xml will output ```xml - + + baz + foobar + ``` instead of ```xml - + + baz + foobar + ``` ## Encode xml: simple diff --git a/pkg/yqlib/encoder_shellvariables.go b/pkg/yqlib/encoder_shellvariables.go index be54efc6..39236703 100644 --- a/pkg/yqlib/encoder_shellvariables.go +++ b/pkg/yqlib/encoder_shellvariables.go @@ -12,10 +12,13 @@ import ( ) type shellVariablesEncoder struct { + prefs ShellVariablesPreferences } func NewShellVariablesEncoder() Encoder { - return &shellVariablesEncoder{} + return &shellVariablesEncoder{ + prefs: ConfiguredShellVariablesPreferences, + } } func (pe *shellVariablesEncoder) CanHandleAliases() bool { @@ -58,7 +61,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, pat return err case SequenceNode: for index, child := range node.Content { - err := pe.doEncode(w, child, appendPath(path, index)) + err := pe.doEncode(w, child, pe.appendPath(path, index)) if err != nil { return err } @@ -68,7 +71,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, pat for index := 0; index < len(node.Content); index = index + 2 { key := node.Content[index] value := node.Content[index+1] - err := pe.doEncode(w, value, appendPath(path, key.Value)) + err := pe.doEncode(w, value, pe.appendPath(path, key.Value)) if err != nil { return err } @@ -81,7 +84,7 @@ func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, pat } } -func appendPath(cookedPath string, rawKey interface{}) string { +func (pe *shellVariablesEncoder) appendPath(cookedPath string, rawKey interface{}) string { // Shell variable names must match // [a-zA-Z_]+[a-zA-Z0-9_]* @@ -126,7 +129,7 @@ func appendPath(cookedPath string, rawKey interface{}) string { } return key } - return cookedPath + "_" + key + return cookedPath + pe.prefs.KeySeparator + key } func quoteValue(value string) string { diff --git a/pkg/yqlib/encoder_shellvariables_test.go b/pkg/yqlib/encoder_shellvariables_test.go index 3d3d905f..d41ad663 100644 --- a/pkg/yqlib/encoder_shellvariables_test.go +++ b/pkg/yqlib/encoder_shellvariables_test.go @@ -91,3 +91,47 @@ func TestShellVariablesEncoderEmptyMap(t *testing.T) { func TestShellVariablesEncoderScalarNode(t *testing.T) { assertEncodesTo(t, "some string", "value='some string'") } + +func assertEncodesToWithSeparator(t *testing.T, yaml string, shellvars string, separator string) { + var output bytes.Buffer + writer := bufio.NewWriter(&output) + + // Save the original separator + originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator + defer func() { + ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator + }() + + // Set the custom separator + ConfiguredShellVariablesPreferences.KeySeparator = separator + + var encoder = NewShellVariablesEncoder() + inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) + if err != nil { + panic(err) + } + node := inputs.Front().Value.(*CandidateNode) + err = encoder.Encode(writer, node) + if err != nil { + panic(err) + } + writer.Flush() + + test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n")) +} + +func TestShellVariablesEncoderCustomSeparator(t *testing.T) { + assertEncodesToWithSeparator(t, "a:\n b: Lewis\n c: Carroll", "a__b=Lewis\na__c=Carroll", "__") +} + +func TestShellVariablesEncoderCustomSeparatorNested(t *testing.T) { + assertEncodesToWithSeparator(t, "my_app:\n db_config:\n host: localhost", "my_app__db_config__host=localhost", "__") +} + +func TestShellVariablesEncoderCustomSeparatorArray(t *testing.T) { + assertEncodesToWithSeparator(t, "a: [{n: Alice}, {n: Bob}]", "a__0__n=Alice\na__1__n=Bob", "__") +} + +func TestShellVariablesEncoderCustomSeparatorSingleChar(t *testing.T) { + assertEncodesToWithSeparator(t, "a:\n b: value", "aXb=value", "X") +} diff --git a/pkg/yqlib/expression_parser_test.go b/pkg/yqlib/expression_parser_test.go index 7356762a..5f970ff9 100644 --- a/pkg/yqlib/expression_parser_test.go +++ b/pkg/yqlib/expression_parser_test.go @@ -84,3 +84,42 @@ func TestParserExtraArgs(t *testing.T) { _, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)") test.AssertResultComplex(t, "bad expression, please check expression syntax", err.Error()) } + +func TestParserEmptyExpression(t *testing.T) { + _, err := getExpressionParser().ParseExpression("") + test.AssertResultComplex(t, nil, err) +} + +func TestParserSingleOperation(t *testing.T) { + result, err := getExpressionParser().ParseExpression(".") + test.AssertResultComplex(t, nil, err) + if result == nil { + t.Fatal("Expected non-nil result for single operation") + } + if result.Operation == nil { + t.Fatal("Expected operation to be set") + } +} + +func TestParserFirstOpWithZeroArgs(t *testing.T) { + // Test the special case where firstOpType can accept zero args + result, err := getExpressionParser().ParseExpression("first") + test.AssertResultComplex(t, nil, err) + if result == nil { + t.Fatal("Expected non-nil result for first operation with zero args") + } +} + +func TestParserInvalidExpressionTree(t *testing.T) { + // This tests the createExpressionTree function with malformed postfix + parser := getExpressionParser().(*expressionParserImpl) + + // Create invalid postfix operations that would leave more than one item on stack + invalidOps := []*Operation{ + {OperationType: &operationType{NumArgs: 0}}, + {OperationType: &operationType{NumArgs: 0}}, + } + + _, err := parser.createExpressionTree(invalidOps) + test.AssertResultComplex(t, "bad expression, please check expression syntax", err.Error()) +} diff --git a/pkg/yqlib/lib_test.go b/pkg/yqlib/lib_test.go index 60fdee01..a9aada14 100644 --- a/pkg/yqlib/lib_test.go +++ b/pkg/yqlib/lib_test.go @@ -2,6 +2,7 @@ package yqlib import ( "fmt" + "strings" "testing" "github.com/mikefarah/yq/v4/test" @@ -160,3 +161,250 @@ func TestParseInt64(t *testing.T) { test.AssertResultComplexWithContext(t, tt.expectedFormatString, fmt.Sprintf(format, actualNumber), fmt.Sprintf("Formatting of: %v", tt.numberString)) } } + +func TestGetContentValueByKey(t *testing.T) { + // Create content with key-value pairs + key1 := createStringScalarNode("key1") + value1 := createStringScalarNode("value1") + key2 := createStringScalarNode("key2") + value2 := createStringScalarNode("value2") + + content := []*CandidateNode{key1, value1, key2, value2} + + // Test finding existing key + result := getContentValueByKey(content, "key1") + test.AssertResult(t, value1, result) + + // Test finding another existing key + result = getContentValueByKey(content, "key2") + test.AssertResult(t, value2, result) + + // Test finding non-existing key + result = getContentValueByKey(content, "nonexistent") + test.AssertResult(t, (*CandidateNode)(nil), result) + + // Test with empty content + result = getContentValueByKey([]*CandidateNode{}, "key1") + test.AssertResult(t, (*CandidateNode)(nil), result) +} + +func TestRecurseNodeArrayEqual(t *testing.T) { + // Create two arrays with same content + array1 := &CandidateNode{ + Kind: SequenceNode, + Content: []*CandidateNode{ + createStringScalarNode("item1"), + createStringScalarNode("item2"), + }, + } + + array2 := &CandidateNode{ + Kind: SequenceNode, + Content: []*CandidateNode{ + createStringScalarNode("item1"), + createStringScalarNode("item2"), + }, + } + + array3 := &CandidateNode{ + Kind: SequenceNode, + Content: []*CandidateNode{ + createStringScalarNode("item1"), + createStringScalarNode("different"), + }, + } + + array4 := &CandidateNode{ + Kind: SequenceNode, + Content: []*CandidateNode{ + createStringScalarNode("item1"), + }, + } + + test.AssertResult(t, true, recurseNodeArrayEqual(array1, array2)) + test.AssertResult(t, false, recurseNodeArrayEqual(array1, array3)) + test.AssertResult(t, false, recurseNodeArrayEqual(array1, array4)) +} + +func TestFindInArray(t *testing.T) { + item1 := createStringScalarNode("item1") + item2 := createStringScalarNode("item2") + item3 := createStringScalarNode("item3") + + array := &CandidateNode{ + Kind: SequenceNode, + Content: []*CandidateNode{item1, item2, item3}, + } + + // Test finding existing items + test.AssertResult(t, 0, findInArray(array, item1)) + test.AssertResult(t, 1, findInArray(array, item2)) + test.AssertResult(t, 2, findInArray(array, item3)) + + // Test finding non-existing item + nonExistent := createStringScalarNode("nonexistent") + test.AssertResult(t, -1, findInArray(array, nonExistent)) +} + +func TestFindKeyInMap(t *testing.T) { + key1 := createStringScalarNode("key1") + value1 := createStringScalarNode("value1") + key2 := createStringScalarNode("key2") + value2 := createStringScalarNode("value2") + + mapNode := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value1, key2, value2}, + } + + // Test finding existing keys + test.AssertResult(t, 0, findKeyInMap(mapNode, key1)) + test.AssertResult(t, 2, findKeyInMap(mapNode, key2)) + + // Test finding non-existing key + nonExistent := createStringScalarNode("nonexistent") + test.AssertResult(t, -1, findKeyInMap(mapNode, nonExistent)) +} + +func TestRecurseNodeObjectEqual(t *testing.T) { + // Create two objects with same content + key1 := createStringScalarNode("key1") + value1 := createStringScalarNode("value1") + key2 := createStringScalarNode("key2") + value2 := createStringScalarNode("value2") + + obj1 := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value1, key2, value2}, + } + + obj2 := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value1, key2, value2}, + } + + // Create object with different values + value3 := createStringScalarNode("value3") + obj3 := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value3, key2, value2}, + } + + // Create object with different keys + key3 := createStringScalarNode("key3") + obj4 := &CandidateNode{ + Kind: MappingNode, + Content: []*CandidateNode{key1, value1, key3, value2}, + } + + test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2)) + test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3)) + test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4)) +} + +func TestParseInt(t *testing.T) { + type parseIntScenario struct { + numberString string + expectedParsedNumber int + expectedError string + } + + scenarios := []parseIntScenario{ + { + numberString: "34", + expectedParsedNumber: 34, + }, + { + numberString: "10_000", + expectedParsedNumber: 10000, + }, + { + numberString: "0x10", + expectedParsedNumber: 16, + }, + { + numberString: "0o10", + expectedParsedNumber: 8, + }, + { + numberString: "invalid", + expectedError: "strconv.ParseInt", + }, + } + + for _, tt := range scenarios { + actualNumber, err := parseInt(tt.numberString) + if tt.expectedError != "" { + if err == nil { + t.Errorf("Expected error for '%s' but got none", tt.numberString) + } else if !strings.Contains(err.Error(), tt.expectedError) { + t.Errorf("Expected error containing '%s' for '%s', got '%s'", tt.expectedError, tt.numberString, err.Error()) + } + continue + } + if err != nil { + t.Errorf("Unexpected error for '%s': %v", tt.numberString, err) + } + test.AssertResultComplexWithContext(t, tt.expectedParsedNumber, actualNumber, tt.numberString) + } +} + +func TestHeadAndLineComment(t *testing.T) { + node := &CandidateNode{ + HeadComment: "# head comment", + LineComment: "# line comment", + } + + result := headAndLineComment(node) + test.AssertResult(t, " head comment line comment", result) +} + +func TestHeadComment(t *testing.T) { + node := &CandidateNode{ + HeadComment: "# head comment", + } + + result := headComment(node) + test.AssertResult(t, " head comment", result) + + // Test without # + node.HeadComment = "no hash comment" + result = headComment(node) + test.AssertResult(t, "no hash comment", result) +} + +func TestLineComment(t *testing.T) { + node := &CandidateNode{ + LineComment: "# line comment", + } + + result := lineComment(node) + test.AssertResult(t, " line comment", result) + + // Test without # + node.LineComment = "no hash comment" + result = lineComment(node) + test.AssertResult(t, "no hash comment", result) +} + +func TestFootComment(t *testing.T) { + node := &CandidateNode{ + FootComment: "# foot comment", + } + + result := footComment(node) + test.AssertResult(t, " foot comment", result) + + // Test without # + node.FootComment = "no hash comment" + result = footComment(node) + test.AssertResult(t, "no hash comment", result) +} + +func TestKindString(t *testing.T) { + test.AssertResult(t, "ScalarNode", KindString(ScalarNode)) + test.AssertResult(t, "SequenceNode", KindString(SequenceNode)) + test.AssertResult(t, "MappingNode", KindString(MappingNode)) + test.AssertResult(t, "AliasNode", KindString(AliasNode)) + test.AssertResult(t, "unknown!", KindString(Kind(999))) // Invalid kind +} diff --git a/pkg/yqlib/operator_first_test.go b/pkg/yqlib/operator_first_test.go index 44ad9c0e..a83af4f6 100644 --- a/pkg/yqlib/operator_first_test.go +++ b/pkg/yqlib/operator_first_test.go @@ -13,15 +13,15 @@ var firstOperatorScenarios = []expressionScenario{ }, { description: "First matching element from array with multiple matches", - document: "[{a: banana},{a: cat},{a: apple},{a: cat}]", + document: "[{a: banana},{a: cat, b: firstCat},{a: apple},{a: cat, b: secondCat}]", expression: `first(.a == "cat")`, expected: []string{ - "D0, P[1], (!!map)::{a: cat}\n", + "D0, P[1], (!!map)::{a: cat, b: firstCat}\n", }, }, { description: "First matching element from array with numeric condition", - document: "[{a: 10},{a: 100},{a: 1}]", + document: "[{a: 10},{a: 100},{a: 1},{a: 101}]", expression: `first(.a > 50)`, expected: []string{ "D0, P[1], (!!map)::{a: 100}\n", @@ -29,10 +29,10 @@ var firstOperatorScenarios = []expressionScenario{ }, { description: "First matching element from array with boolean condition", - document: "[{a: false},{a: true},{a: false}]", + document: "[{a: false},{a: true, b: firstTrue},{a: false}, {a: true, b: secondTrue}]", expression: `first(.a == true)`, expected: []string{ - "D0, P[1], (!!map)::{a: true}\n", + "D0, P[1], (!!map)::{a: true, b: firstTrue}\n", }, }, { @@ -45,10 +45,10 @@ var firstOperatorScenarios = []expressionScenario{ }, { description: "First matching element from array with complex condition", - document: "[{a: dog, b: 5},{a: cat, b: 3},{a: apple, b: 7}]", - expression: `first(.b > 4)`, + document: "[{a: dog, b: 7},{a: cat, b: 3},{a: apple, b: 5}]", + expression: `first(.b > 4 and .b < 6)`, expected: []string{ - "D0, P[0], (!!map)::{a: dog, b: 5}\n", + "D0, P[2], (!!map)::{a: apple, b: 5}\n", }, }, { @@ -61,7 +61,7 @@ var firstOperatorScenarios = []expressionScenario{ }, { description: "First matching element from map with numeric condition", - document: "x: {a: 10}\ny: {a: 100}\nz: {a: 1}", + document: "x: {a: 10}\ny: {a: 100}\nz: {a: 101}", expression: `first(.a > 50)`, expected: []string{ "D0, P[y], (!!map)::{a: 100}\n", diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 19ddd5df..d55c9b41 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -414,3 +414,100 @@ func TestPrinterRootUnwrap(t *testing.T) { ` test.AssertResult(t, expected, output.String()) } + +func TestRemoveLastEOL(t *testing.T) { + // Test with \r\n + buffer := bytes.NewBufferString("test\r\n") + removeLastEOL(buffer) + test.AssertResult(t, "test", buffer.String()) + + // Test with \n only + buffer = bytes.NewBufferString("test\n") + removeLastEOL(buffer) + test.AssertResult(t, "test", buffer.String()) + + // Test with \r only + buffer = bytes.NewBufferString("test\r") + removeLastEOL(buffer) + test.AssertResult(t, "test", buffer.String()) + + // Test with no EOL + buffer = bytes.NewBufferString("test") + removeLastEOL(buffer) + test.AssertResult(t, "test", buffer.String()) + + // Test with empty buffer + buffer = bytes.NewBufferString("") + removeLastEOL(buffer) + test.AssertResult(t, "", buffer.String()) + + // Test with multiple \r\n + buffer = bytes.NewBufferString("line1\r\nline2\r\n") + removeLastEOL(buffer) + test.AssertResult(t, "line1\r\nline2", buffer.String()) +} + +func TestPrinterPrintedAnything(t *testing.T) { + var output bytes.Buffer + var writer = bufio.NewWriter(&output) + printer := NewSimpleYamlPrinter(writer, true, 2, true) + + test.AssertResult(t, false, printer.PrintedAnything()) + + // Print a scalar value + node := createStringScalarNode("test") + nodeList := nodeToList(node) + err := printer.PrintResults(nodeList) + if err != nil { + t.Fatal(err) + } + + // Should now be true + test.AssertResult(t, true, printer.PrintedAnything()) +} + +func TestPrinterNulSeparatorWithNullChar(t *testing.T) { + var output bytes.Buffer + var writer = bufio.NewWriter(&output) + printer := NewSimpleYamlPrinter(writer, true, 2, false) + printer.SetNulSepOutput(true) + + // Create a node with null character + node := createStringScalarNode("test\x00value") + nodeList := nodeToList(node) + + err := printer.PrintResults(nodeList) + if err == nil { + t.Fatal("Expected error for null character in NUL separated output") + } + + expectedError := "can't serialize value because it contains NUL char and you are using NUL separated output" + if err.Error() != expectedError { + t.Fatalf("Expected error '%s', got '%s'", expectedError, err.Error()) + } +} + +func TestPrinterSetNulSepOutput(t *testing.T) { + var output bytes.Buffer + var writer = bufio.NewWriter(&output) + printer := NewSimpleYamlPrinter(writer, true, 2, false) + + // Test setting NUL separator output + printer.SetNulSepOutput(true) + test.AssertResult(t, true, true) // Placeholder assertion + + printer.SetNulSepOutput(false) + // Should also not cause errors + test.AssertResult(t, false, false) // Placeholder assertion +} + +func TestPrinterSetAppendix(t *testing.T) { + var output bytes.Buffer + var writer = bufio.NewWriter(&output) + printer := NewSimpleYamlPrinter(writer, true, 2, true) + + // Test setting appendix + appendix := strings.NewReader("appendix content") + printer.SetAppendix(appendix) + test.AssertResult(t, true, true) // Placeholder assertion +} diff --git a/pkg/yqlib/shellvariables.go b/pkg/yqlib/shellvariables.go new file mode 100644 index 00000000..634fc6fa --- /dev/null +++ b/pkg/yqlib/shellvariables.go @@ -0,0 +1,14 @@ +package yqlib + +type ShellVariablesPreferences struct { + KeySeparator string +} + +func NewDefaultShellVariablesPreferences() ShellVariablesPreferences { + return ShellVariablesPreferences{ + KeySeparator: "_", + } +} + +var ConfiguredShellVariablesPreferences = NewDefaultShellVariablesPreferences() + diff --git a/pkg/yqlib/shellvariables_test.go b/pkg/yqlib/shellvariables_test.go index 5a7f56f4..d123f4bd 100644 --- a/pkg/yqlib/shellvariables_test.go +++ b/pkg/yqlib/shellvariables_test.go @@ -54,12 +54,33 @@ var shellVariablesScenarios = []formatScenario{ input: "name: Miles O'Brien", expected: `name='Miles O'"'"'Brien'` + "\n", }, + { + description: "Encode shell variables: custom separator", + subdescription: "Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores.", + input: "" + + "my_app:" + "\n" + + " db_config:" + "\n" + + " host: localhost" + "\n" + + " port: 5432", + expected: "" + + "my_app__db_config__host=localhost" + "\n" + + "my_app__db_config__port=5432" + "\n", + scenarioType: "shell-separator", + }, } func TestShellVariableScenarios(t *testing.T) { for _, s := range shellVariablesScenarios { //fmt.Printf("\t<%s> <%s>\n", s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())) - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description) + if s.scenarioType == "shell-separator" { + // Save and restore the original separator + originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator + ConfiguredShellVariablesPreferences.KeySeparator = "__" + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description) + ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator + } else { + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description) + } } genericScenarios := make([]interface{}, len(shellVariablesScenarios)) for i, s := range shellVariablesScenarios { @@ -87,12 +108,22 @@ func documentShellVariableScenario(_ *testing.T, w *bufio.Writer, i interface{}) expression := s.expression - if expression != "" { + if s.scenarioType == "shell-separator" { + writeOrPanic(w, "```bash\nyq -o=shell --shell-key-separator=\"__\" sample.yml\n```\n") + } else if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=shell '%v' sample.yml\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -o=shell sample.yml\n```\n") } writeOrPanic(w, "will output\n") - writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))) + if s.scenarioType == "shell-separator" { + // Save and restore the original separator + originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator + ConfiguredShellVariablesPreferences.KeySeparator = "__" + writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))) + ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator + } else { + writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))) + } } diff --git a/pkg/yqlib/write_in_place_handler_test.go b/pkg/yqlib/write_in_place_handler_test.go new file mode 100644 index 00000000..a15bdf32 --- /dev/null +++ b/pkg/yqlib/write_in_place_handler_test.go @@ -0,0 +1,222 @@ +package yqlib + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWriteInPlaceHandlerImpl_CreateTempFile(t *testing.T) { + // Create a temporary directory and file for testing + tempDir := t.TempDir() + inputFile := filepath.Join(tempDir, "input.yaml") + + // Create input file with some content + content := []byte("test: value\n") + err := os.WriteFile(inputFile, content, 0600) + if err != nil { + t.Fatalf("Failed to create input file: %v", err) + } + + handler := NewWriteInPlaceHandler(inputFile) + tempFile, err := handler.CreateTempFile() + + if err != nil { + t.Fatalf("CreateTempFile failed: %v", err) + } + + if tempFile == nil { + t.Fatal("CreateTempFile returned nil file") + } + + // Clean up + tempFile.Close() + os.Remove(tempFile.Name()) +} + +func TestWriteInPlaceHandlerImpl_CreateTempFile_NonExistentInput(t *testing.T) { + // Test with non-existent input file + handler := NewWriteInPlaceHandler("/non/existent/file.yaml") + tempFile, err := handler.CreateTempFile() + + if err == nil { + t.Error("Expected error for non-existent input file, got nil") + } + + if tempFile != nil { + t.Error("Expected nil temp file for non-existent input file") + tempFile.Close() + } +} + +func TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Success(t *testing.T) { + // Create a temporary directory and file for testing + tempDir := t.TempDir() + inputFile := filepath.Join(tempDir, "input.yaml") + + // Create input file with some content + content := []byte("test: value\n") + err := os.WriteFile(inputFile, content, 0600) + if err != nil { + t.Fatalf("Failed to create input file: %v", err) + } + + handler := NewWriteInPlaceHandler(inputFile) + tempFile, err := handler.CreateTempFile() + if err != nil { + t.Fatalf("CreateTempFile failed: %v", err) + } + defer tempFile.Close() + + // Write some content to temp file + tempContent := []byte("updated: content\n") + _, err = tempFile.Write(tempContent) + if err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + // Test successful finish + err = handler.FinishWriteInPlace(true) + if err != nil { + t.Fatalf("FinishWriteInPlace failed: %v", err) + } + + // Verify the original file was updated + updatedContent, err := os.ReadFile(inputFile) + if err != nil { + t.Fatalf("Failed to read updated file: %v", err) + } + + if string(updatedContent) != string(tempContent) { + t.Errorf("File content not updated correctly. Expected %q, got %q", + string(tempContent), string(updatedContent)) + } +} + +func TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Failure(t *testing.T) { + // Create a temporary directory and file for testing + tempDir := t.TempDir() + inputFile := filepath.Join(tempDir, "input.yaml") + + // Create input file with some content + content := []byte("test: value\n") + err := os.WriteFile(inputFile, content, 0600) + if err != nil { + t.Fatalf("Failed to create input file: %v", err) + } + + handler := NewWriteInPlaceHandler(inputFile) + tempFile, err := handler.CreateTempFile() + if err != nil { + t.Fatalf("CreateTempFile failed: %v", err) + } + defer tempFile.Close() + + // Write some content to temp file + tempContent := []byte("updated: content\n") + _, err = tempFile.Write(tempContent) + if err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + // Test failure finish (should not update the original file) + err = handler.FinishWriteInPlace(false) + if err != nil { + t.Fatalf("FinishWriteInPlace failed: %v", err) + } + + // Verify the original file was NOT updated + originalContent, err := os.ReadFile(inputFile) + if err != nil { + t.Fatalf("Failed to read original file: %v", err) + } + + if string(originalContent) != string(content) { + t.Errorf("File content should not have been updated. Expected %q, got %q", + string(content), string(originalContent)) + } +} + +func TestWriteInPlaceHandlerImpl_CreateTempFile_Permissions(t *testing.T) { + // Create a temporary directory and file for testing + tempDir := t.TempDir() + inputFile := filepath.Join(tempDir, "input.yaml") + + // Create input file with specific permissions + content := []byte("test: value\n") + err := os.WriteFile(inputFile, content, 0600) + if err != nil { + t.Fatalf("Failed to create input file: %v", err) + } + + handler := NewWriteInPlaceHandler(inputFile) + tempFile, err := handler.CreateTempFile() + if err != nil { + t.Fatalf("CreateTempFile failed: %v", err) + } + defer tempFile.Close() + + // Check that temp file has same permissions as input file + tempFileInfo, err := os.Stat(tempFile.Name()) + if err != nil { + t.Fatalf("Failed to stat temp file: %v", err) + } + + inputFileInfo, err := os.Stat(inputFile) + if err != nil { + t.Fatalf("Failed to stat input file: %v", err) + } + + if tempFileInfo.Mode() != inputFileInfo.Mode() { + t.Errorf("Temp file permissions don't match input file. Expected %v, got %v", + inputFileInfo.Mode(), tempFileInfo.Mode()) + } +} + +func TestWriteInPlaceHandlerImpl_Integration(t *testing.T) { + // Create a temporary directory and file for testing + tempDir := t.TempDir() + inputFile := filepath.Join(tempDir, "integration_test.yaml") + + // Create input file with some content + originalContent := []byte("original: content\n") + err := os.WriteFile(inputFile, originalContent, 0600) + if err != nil { + t.Fatalf("Failed to create input file: %v", err) + } + + handler := NewWriteInPlaceHandler(inputFile) + + // Create temp file + tempFile, err := handler.CreateTempFile() + if err != nil { + t.Fatalf("CreateTempFile failed: %v", err) + } + + // Write new content to temp file + newContent := []byte("new: content\n") + _, err = tempFile.Write(newContent) + if err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + // Finish with success + err = handler.FinishWriteInPlace(true) + if err != nil { + t.Fatalf("FinishWriteInPlace failed: %v", err) + } + + // Verify the file was updated + finalContent, err := os.ReadFile(inputFile) + if err != nil { + t.Fatalf("Failed to read final file: %v", err) + } + + if string(finalContent) != string(newContent) { + t.Errorf("File not updated correctly. Expected %q, got %q", + string(newContent), string(finalContent)) + } +} diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 08e1f0ae..87ad4301 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -188,7 +188,10 @@ above_cat ` const inputXMLWithNamespacedAttr = ` - + + baz + foobar + ` const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0" @@ -196,6 +199,10 @@ map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@xsi:schemaLocation: some-url + item: + +content: baz + +@foo: bar + xsi:item: foobar ` const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0" @@ -203,13 +210,21 @@ map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@xsi:schemaLocation: some-url + item: + +content: baz + +@foo: bar + xsi:item: foobar ` const expectedYAMLWithoutRawNamespacedAttr = `+p_xml: version="1.0" -map: +some-namespace:map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@some-instance:schemaLocation: some-url + some-namespace:item: + +content: baz + +@foo: bar + some-instance:item: foobar ` const xmlWithCustomDtd = ` diff --git a/release_notes.txt b/release_notes.txt index 5de83a21..54b67bed 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,10 @@ +4.48.1: + - Added 'parents' operator, to return a list of all the hierarchical parents of a node + - Added 'first(exp)' operator, to return the first entry matching an expression in an array + - Fixed xml namespace prefixes #1730 (thanks @baodrate) + - Fixed out of range panic in yaml decoder #2460 (thanks @n471d) + - Bumped dependencies + 4.47.2: - Conversion from TOML to JSON no longer omits empty tables #2459 (thanks @louislouislouislouis) - Bumped dependencies diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 88b4b974..a64e16b5 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -2,5 +2,71 @@ set -e +echo "Running tests and generating coverage..." go test -coverprofile=coverage.out -v $(go list ./... | grep -v -E 'examples' | grep -v -E 'test') + +echo "Generating HTML coverage report..." go tool cover -html=coverage.out -o coverage.html + +echo "" +echo "Generating sorted coverage table..." + +# Create a simple approach using grep and sed to extract file coverage +# First, get the total coverage +total_coverage=$(go tool cover -func=coverage.out | grep "^total:" | sed 's/.*([^)]*)[[:space:]]*\([0-9.]*\)%.*/\1/') + +# Extract file-level coverage by finding the last occurrence of each file +go tool cover -func=coverage.out | grep -E "\.go:[0-9]+:" | \ +sed 's/^\([^:]*\.go\):.*[[:space:]]\([0-9.]*\)%.*/\2 \1/' | \ +sort -k2 | \ +awk '{file_coverage[$2] = $1} END {for (file in file_coverage) printf "%.2f %s\n", file_coverage[file], file}' | \ +sort -nr > coverage_sorted.txt + +# Add total coverage to the file +if [[ -n "$total_coverage" && "$total_coverage" != "0" ]]; then + echo "TOTAL: $total_coverage" >> coverage_sorted.txt +fi + +echo "" +echo "Coverage Summary (sorted by percentage - lowest coverage first):" +echo "=================================================================" +printf "%-60s %10s %12s\n" "FILE" "COVERAGE" "STATUS" +echo "=================================================================" + +# Display results with status indicators +tail -n +1 coverage_sorted.txt | while read percent file; do + if [[ "$file" == "TOTAL:" ]]; then + echo "" + printf "%-60s %8s%% %12s\n" "OVERALL PROJECT COVERAGE" "$percent" "📊 TOTAL" + echo "=================================================================" + continue + fi + + filename=$(basename "$file") + status="" + if (( $(echo "$percent < 50" | bc -l 2>/dev/null || echo "0") )); then + status="🔴 CRITICAL" + elif (( $(echo "$percent < 70" | bc -l 2>/dev/null || echo "0") )); then + status="🟡 LOW" + elif (( $(echo "$percent < 90" | bc -l 2>/dev/null || echo "0") )); then + status="🟢 GOOD" + else + status="✅ EXCELLENT" + fi + + printf "%-60s %8s%% %12s\n" "$filename" "$percent" "$status" +done + +echo "" +echo "Top 10 files needing attention (lowest coverage):" +echo "=================================================" +grep -v "TOTAL:" coverage_sorted.txt | tail -10 | while read percent file; do + filename=$(basename "$file") + printf "%-60s %8.1f%%\n" "$filename" "$percent" +done + +echo "" +echo "Coverage reports generated:" +echo "- HTML report: coverage.html (detailed line-by-line coverage)" +echo "- Sorted table: coverage_sorted.txt" +echo "- Use 'go tool cover -func=coverage.out' for function-level details" diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 36c46f53..9d7a93fd 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.47.2' +version: 'v4.48.1' summary: A lightweight and portable command-line data file processor description: | `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. @@ -27,6 +27,6 @@ parts: build-environment: - CGO_ENABLED: 0 source: https://github.com/mikefarah/yq.git - source-tag: v4.47.2 + source-tag: v4.48.1 build-snaps: - go/latest/stable diff --git a/yq_test.go b/yq_test.go new file mode 100644 index 00000000..beb25ef5 --- /dev/null +++ b/yq_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "testing" + + command "github.com/mikefarah/yq/v4/cmd" +) + +func TestMainFunction(t *testing.T) { + // This is a basic smoke test for the main function + // We can't easily test the main function directly since it calls os.Exit + // But we can test the logic that would be executed + + cmd := command.New() + if cmd == nil { + t.Fatal("command.New() returned nil") + } + + if cmd.Use != "yq" { + t.Errorf("Expected command Use to be 'yq', got %q", cmd.Use) + } +} + +func TestMainFunctionLogic(t *testing.T) { + // Test the logic that would be executed in main() + cmd := command.New() + + args := []string{} + _, _, err := cmd.Find(args) + if err != nil { + t.Errorf("Expected no error with empty args, but got: %v", err) + } + + args = []string{"invalid-command"} + _, _, err = cmd.Find(args) + if err == nil { + t.Error("Expected error when invalid command found, but got nil") + } + + args = []string{"eval"} + _, _, err = cmd.Find(args) + if err != nil { + t.Errorf("Expected no error with valid command 'eval', got: %v", err) + } + + args = []string{"__complete"} + _, _, err = cmd.Find(args) + if err == nil { + t.Error("Expected error when no command found for '__complete', but got nil") + } +} + +func TestMainFunctionWithArgs(t *testing.T) { + // Test the argument processing logic + cmd := command.New() + + args := []string{} + _, _, err := cmd.Find(args) + if err != nil { + t.Errorf("Expected no error with empty args, but got: %v", err) + } + + // When Find fails and args[0] is not "__complete", main would set args to ["eval"] + original args + // This is the logic: newArgs := []string{"eval"} + // cmd.SetArgs(append(newArgs, os.Args[1:]...)) + + args = []string{"invalid"} + _, _, err = cmd.Find(args) + if err == nil { + t.Error("Expected error with invalid command") + } + + args = []string{"__complete"} + _, _, err = cmd.Find(args) + if err == nil { + t.Error("Expected error with __complete command") + } +} + +func TestMainFunctionExecution(t *testing.T) { + // Test that the command can be executed without crashing + cmd := command.New() + + cmd.SetArgs([]string{"--version"}) + + // We can't easily test os.Exit(1) behaviour, but we can test that + // the command structure is correct and can be configured + if cmd == nil { + t.Fatal("Command should not be nil") + } + + if cmd.Use != "yq" { + t.Errorf("Expected command Use to be 'yq', got %q", cmd.Use) + } +} + +func TestMainFunctionErrorHandling(t *testing.T) { + // Test the error handling logic that would be in main() + cmd := command.New() + + args := []string{"nonexistent-command"} + _, _, err := cmd.Find(args) + if err == nil { + t.Error("Expected error with nonexistent command") + } + + // The main function logic would be: + // if err != nil && args[0] != "__complete" { + // newArgs := []string{"eval"} + // cmd.SetArgs(append(newArgs, os.Args[1:]...)) + // } + + // Test that this logic would work + if args[0] != "__complete" { + // This is what main() would do + newArgs := []string{"eval"} + cmd.SetArgs(append(newArgs, args...)) + + // We can't easily verify the args were set correctly since cmd.Args is a function + // But we can test that SetArgs doesn't crash and the command is still valid + if cmd == nil { + t.Error("Command should not be nil after SetArgs") + } + + _, _, err := cmd.Find([]string{"eval"}) + if err != nil { + t.Errorf("Should be able to find eval command after SetArgs: %v", err) + } + } +} + +func TestMainFunctionWithCompletionCommand(t *testing.T) { + // Test that __complete command doesn't trigger default command logic + cmd := command.New() + + args := []string{"__complete"} + _, _, err := cmd.Find(args) + if err == nil { + t.Error("Expected error with __complete command") + } + + // The main function logic would be: + // if err != nil && args[0] != "__complete" { + // // This should NOT execute for __complete + // } + + // Verify that __complete doesn't trigger the default command logic + if args[0] == "__complete" { + // This means the default command logic should NOT execute + t.Log("__complete command correctly identified, default command logic should not execute") + } +} + +func TestMainFunctionIntegration(t *testing.T) { + // Integration test to verify the main function logic works end-to-end + + cmd := command.New() + cmd.SetArgs([]string{"eval", "--help"}) + + // This should not crash (we can't test the actual execution due to os.Exit) + if cmd == nil { + t.Fatal("Command should not be nil") + } + + cmd2 := command.New() + cmd2.SetArgs([]string{"invalid-command"}) + + // Simulate the main function logic + args := []string{"invalid-command"} + _, _, err := cmd2.Find(args) + if err != nil { + // This is what main() would do + newArgs := []string{"eval"} + cmd2.SetArgs(append(newArgs, args...)) + } + + // We can't directly access cmd.Args since it's a function, but we can test + // that SetArgs worked by ensuring the command is still functional + if cmd2 == nil { + t.Error("Command should not be nil after SetArgs") + } + + _, _, err = cmd2.Find([]string{"eval"}) + if err != nil { + t.Errorf("Should be able to find eval command after SetArgs: %v", err) + } +}