mirror of
https://github.com/mikefarah/yq.git
synced 2026-07-07 14:25:38 +00:00
fix(json): preserve floats with trailing zero when encoding YAML to JSON (#2701)
YAML scalars tagged `!!float` were round-tripped through `float64` and re-serialized by Go's JSON encoder, which strips the decimal part of whole-number floats. As a result, `50.0` came out as `50` and a sequence like `[50.0, 95.0, 99.0, 99.9]` became `[50,95,99,99.9]`, turning a uniform array of floats into a mixed int/float array that downstream consumers (Horreum, JSON Schema validators, jq, etc.) reject. The JSON spec does not distinguish ints from floats, but every common JSON library (Go's `encoding/json`, Python's `json`, jq) preserves the fractional form of values that came in as floats. yq's YAML decoder already parses these as `!!float` with the original text intact, so we can emit them verbatim instead of round-tripping. `MarshalJSON` for `ScalarNode` now special-cases `!!float`: - if `Value` is already a JSON-shaped number literal containing a `.` or exponent, emit it verbatim (e.g. `50.0`, `99.9`, `1.5e-3`, `-7.0`); - if `Value` is an integer-shaped string tagged `!!float` (e.g. `!!float 5`), format the parsed float and append `.0` so it stays a JSON number with a fractional part; - otherwise (empty value, parse error, or non-finite result), fall back to the existing encoding path so behaviour for `.inf` / `.nan` and anything unusual is unchanged. `!!int` nodes still encode as JSON integers. Closes #2683 Signed-off-by: ChrisJr404 <chris@hacknow.com>
This commit is contained in:
parent
fcb79822dd
commit
2861815f71
@ -7,6 +7,9 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/goccy/go-json"
|
"github.com/goccy/go-json"
|
||||||
)
|
)
|
||||||
@ -140,6 +143,12 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
|
|||||||
return buf.Bytes(), err
|
return buf.Bytes(), err
|
||||||
case ScalarNode:
|
case ScalarNode:
|
||||||
log.Debugf("MarshalJSON ScalarNode")
|
log.Debugf("MarshalJSON ScalarNode")
|
||||||
|
if o.guessTagFromCustomType() == "!!float" {
|
||||||
|
if raw, ok := jsonFloatLiteral(o.Value); ok {
|
||||||
|
buf.WriteString(raw)
|
||||||
|
return buf.Bytes(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
value, err := o.GetValueRep()
|
value, err := o.GetValueRep()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return buf.Bytes(), err
|
return buf.Bytes(), err
|
||||||
@ -177,3 +186,85 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
|
|||||||
return buf.Bytes(), err
|
return buf.Bytes(), err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// jsonFloatLiteral returns a JSON-shaped representation of a YAML !!float scalar
|
||||||
|
// value, preserving the original textual form (e.g. "50.0" stays "50.0") whenever
|
||||||
|
// possible. The second return value is false when the value cannot be safely
|
||||||
|
// rendered as a JSON number (e.g. ".inf", ".nan", or anything that parses to a
|
||||||
|
// non-finite float); callers should fall back to the normal encoding path in
|
||||||
|
// that case, which preserves the existing behaviour for those inputs.
|
||||||
|
func jsonFloatLiteral(raw string) (string, bool) {
|
||||||
|
if raw == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
f, err := strconv.ParseFloat(raw, 64)
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if math.IsInf(f, 0) || math.IsNaN(f) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if isJSONNumberLiteral(raw) {
|
||||||
|
return raw, true
|
||||||
|
}
|
||||||
|
formatted := strconv.FormatFloat(f, 'f', -1, 64)
|
||||||
|
if !strings.ContainsAny(formatted, ".eE") {
|
||||||
|
formatted += ".0"
|
||||||
|
}
|
||||||
|
return formatted, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// isJSONNumberLiteral reports whether s is already a valid JSON number literal
|
||||||
|
// representing a fractional value (i.e. contains a "." or an exponent), so it
|
||||||
|
// can be emitted verbatim without round-tripping through a float64.
|
||||||
|
func isJSONNumberLiteral(s string) bool {
|
||||||
|
if s == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
i := 0
|
||||||
|
if s[i] == '-' {
|
||||||
|
i++
|
||||||
|
if i == len(s) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// integer part: 0 or [1-9][0-9]*
|
||||||
|
if s[i] == '0' {
|
||||||
|
i++
|
||||||
|
} else if s[i] >= '1' && s[i] <= '9' {
|
||||||
|
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
hasFraction := false
|
||||||
|
if i < len(s) && s[i] == '.' {
|
||||||
|
hasFraction = true
|
||||||
|
i++
|
||||||
|
if i == len(s) || s[i] < '0' || s[i] > '9' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hasExponent := false
|
||||||
|
if i < len(s) && (s[i] == 'e' || s[i] == 'E') {
|
||||||
|
hasExponent = true
|
||||||
|
i++
|
||||||
|
if i < len(s) && (s[i] == '+' || s[i] == '-') {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
if i == len(s) || s[i] < '0' || s[i] > '9' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if i != len(s) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return hasFraction || hasExponent
|
||||||
|
}
|
||||||
|
|||||||
@ -125,6 +125,22 @@ will output
|
|||||||
{"whatever":"cat"}
|
{"whatever":"cat"}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Encode json: preserve floats with trailing zero
|
||||||
|
Whole-number floats keep their decimal point so downstream consumers see a JSON number with a fractional part (matches Go's encoding/json, Python's json, and jq).
|
||||||
|
|
||||||
|
Given a sample.yml file of:
|
||||||
|
```yaml
|
||||||
|
percentiles: [50.0, 95.0, 99.0, 99.9]
|
||||||
|
```
|
||||||
|
then
|
||||||
|
```bash
|
||||||
|
yq -o=json -I=0 '.' sample.yml
|
||||||
|
```
|
||||||
|
will output
|
||||||
|
```json
|
||||||
|
{"percentiles":[50.0,95.0,99.0,99.9]}
|
||||||
|
```
|
||||||
|
|
||||||
## Roundtrip JSON Lines / NDJSON
|
## Roundtrip JSON Lines / NDJSON
|
||||||
Given a sample.json file of:
|
Given a sample.json file of:
|
||||||
```json
|
```json
|
||||||
|
|||||||
@ -220,6 +220,54 @@ var jsonScenarios = []formatScenario{
|
|||||||
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
|
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
|
||||||
scenarioType: "encode",
|
scenarioType: "encode",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
description: "Encode json: preserve floats with trailing zero",
|
||||||
|
subdescription: "Whole-number floats keep their decimal point so downstream consumers see a JSON number with a fractional part (matches Go's encoding/json, Python's json, and jq).",
|
||||||
|
input: `percentiles: [50.0, 95.0, 99.0, 99.9]`,
|
||||||
|
indent: 0,
|
||||||
|
expected: "{\"percentiles\":[50.0,95.0,99.0,99.9]}\n",
|
||||||
|
scenarioType: "encode",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Encode json: ints stay ints",
|
||||||
|
skipDoc: true,
|
||||||
|
input: `a: 50`,
|
||||||
|
indent: 0,
|
||||||
|
expected: "{\"a\":50}\n",
|
||||||
|
scenarioType: "encode",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Encode json: !!float tagged whole number gets .0",
|
||||||
|
skipDoc: true,
|
||||||
|
input: `a: !!float 5`,
|
||||||
|
indent: 0,
|
||||||
|
expected: "{\"a\":5.0}\n",
|
||||||
|
scenarioType: "encode",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Encode json: scientific notation float preserved",
|
||||||
|
skipDoc: true,
|
||||||
|
input: `a: 1.5e-3`,
|
||||||
|
indent: 0,
|
||||||
|
expected: "{\"a\":1.5e-3}\n",
|
||||||
|
scenarioType: "encode",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Encode json: negative float preserved",
|
||||||
|
skipDoc: true,
|
||||||
|
input: `a: -7.0`,
|
||||||
|
indent: 0,
|
||||||
|
expected: "{\"a\":-7.0}\n",
|
||||||
|
scenarioType: "encode",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Encode json: mixed int and float array",
|
||||||
|
skipDoc: true,
|
||||||
|
input: `a: [1, 2.0, 3, 4.5]`,
|
||||||
|
indent: 0,
|
||||||
|
expected: "{\"a\":[1,2.0,3,4.5]}\n",
|
||||||
|
scenarioType: "encode",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
description: "Roundtrip JSON Lines / NDJSON",
|
description: "Roundtrip JSON Lines / NDJSON",
|
||||||
input: sampleNdJson,
|
input: sampleNdJson,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user