From d113344abfea8c08447111f9c362eba49a55c53a Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 5 Oct 2023 15:13:46 +1100 Subject: [PATCH] Added tonumber support #71 --- pkg/yqlib/doc/operators/headers/to_number.md | 2 + pkg/yqlib/doc/operators/to_number.md | 49 +++++++++++++++++ pkg/yqlib/lexer_participle.go | 1 + pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_to_number.go | 56 ++++++++++++++++++++ pkg/yqlib/operator_to_number_test.go | 51 ++++++++++++++++++ project-words.txt | 1 + 7 files changed, 161 insertions(+) create mode 100644 pkg/yqlib/doc/operators/headers/to_number.md create mode 100644 pkg/yqlib/doc/operators/to_number.md create mode 100644 pkg/yqlib/operator_to_number.go create mode 100644 pkg/yqlib/operator_to_number_test.go diff --git a/pkg/yqlib/doc/operators/headers/to_number.md b/pkg/yqlib/doc/operators/headers/to_number.md new file mode 100644 index 00000000..edcb9bd1 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/to_number.md @@ -0,0 +1,2 @@ +# To Number +Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone. diff --git a/pkg/yqlib/doc/operators/to_number.md b/pkg/yqlib/doc/operators/to_number.md new file mode 100644 index 00000000..2e329033 --- /dev/null +++ b/pkg/yqlib/doc/operators/to_number.md @@ -0,0 +1,49 @@ +# To Number +Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone. + +## Converts strings to numbers +Given a sample.yml file of: +```yaml +- "3" +- "3.1" +- "-1e3" +``` +then +```bash +yq '.[] | to_number' sample.yml +``` +will output +```yaml +3 +3.1 +-1e3 +``` + +## Doesn't change numbers +Given a sample.yml file of: +```yaml +- 3 +- 3.1 +- -1e3 +``` +then +```bash +yq '.[] | to_number' sample.yml +``` +will output +```yaml +3 +3.1 +-1e3 +``` + +## Cannot convert null +Running +```bash +yq --null-input '.a.b | to_number' +``` +will output +```bash +Error: cannot convert node value [null] at path a.b of tag !!null to number +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 4388752b..f2184a43 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -34,6 +34,7 @@ var participleYqRules = []*participleYqRule{ simpleOp("line", lineOpType), simpleOp("column", columnOpType), simpleOp("eval", evalOpType), + simpleOp("to_?number", toNumberOpType), {"MapValues", `map_?values`, opToken(mapValuesOpType), 0}, simpleOp("map", mapOpType), diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 756c161e..b88b9e2d 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -167,6 +167,7 @@ var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Hand var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator} var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator} var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator} +var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator} var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator} var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator} diff --git a/pkg/yqlib/operator_to_number.go b/pkg/yqlib/operator_to_number.go new file mode 100644 index 00000000..6c97d4ed --- /dev/null +++ b/pkg/yqlib/operator_to_number.go @@ -0,0 +1,56 @@ +package yqlib + +import ( + "container/list" + "fmt" + "strconv" + + yaml "gopkg.in/yaml.v3" +) + +func tryConvertToNumber(value string) (string, bool) { + // try a int first + _, _, err := parseInt64(value) + if err == nil { + return "!!int", true + } + // try float + _, floatErr := strconv.ParseFloat(value, 64) + + if floatErr == nil { + return "!!float", true + } + return "", false + +} + +func toNumberOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + log.Debugf("ToNumberOperator") + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + if candidate.Node.Kind != yaml.ScalarNode { + return Context{}, fmt.Errorf("cannot convert node at path %v of tag %v to number", candidate.GetNicePath(), candidate.GetNiceTag()) + } + + if candidate.Node.Tag == "!!int" || candidate.Node.Tag == "!!float" { + // it already is a number! + results.PushBack(candidate) + } else { + tag, converted := tryConvertToNumber(candidate.Node.Value) + if converted { + node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Value, Tag: tag} + + result := candidate.CreateReplacement(node) + results.PushBack(result) + } else { + return Context{}, fmt.Errorf("cannot convert node value [%v] at path %v of tag %v to number", candidate.Node.Value, candidate.GetNicePath(), candidate.GetNiceTag()) + } + + } + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_to_number_test.go b/pkg/yqlib/operator_to_number_test.go new file mode 100644 index 00000000..00fe576b --- /dev/null +++ b/pkg/yqlib/operator_to_number_test.go @@ -0,0 +1,51 @@ +package yqlib + +import ( + "testing" +) + +var toNumberScenarios = []expressionScenario{ + { + description: "Converts strings to numbers", + document: `["3", "3.1", "-1e3"]`, + expression: `.[] | to_number`, + expected: []string{ + "D0, P[0], (!!int)::3\n", + "D0, P[1], (!!float)::3.1\n", + "D0, P[2], (!!float)::-1e3\n", + }, + }, + { + skipDoc: true, + description: "Converts strings to numbers, with tonumber because jq", + document: `["3", "3.1", "-1e3"]`, + expression: `.[] | tonumber`, + expected: []string{ + "D0, P[0], (!!int)::3\n", + "D0, P[1], (!!float)::3.1\n", + "D0, P[2], (!!float)::-1e3\n", + }, + }, + { + description: "Doesn't change numbers", + document: `[3, 3.1, -1e3]`, + expression: `.[] | to_number`, + expected: []string{ + "D0, P[0], (!!int)::3\n", + "D0, P[1], (!!float)::3.1\n", + "D0, P[2], (!!float)::-1e3\n", + }, + }, + { + description: "Cannot convert null", + expression: `.a.b | to_number`, + expectedError: "cannot convert node value [null] at path a.b of tag !!null to number", + }, +} + +func TestToNumberOperatorScenarios(t *testing.T) { + for _, tt := range toNumberScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "to_number", toNumberScenarios) +} diff --git a/project-words.txt b/project-words.txt index ab50a416..2371c1fb 100644 --- a/project-words.txt +++ b/project-words.txt @@ -249,3 +249,4 @@ yamld yqlib yuin zabbix +tonumber \ No newline at end of file