From d374ff777ef62466dbff7dc85719fc503ffbfdf8 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Thu, 27 Feb 2025 23:37:26 +0100 Subject: [PATCH] yaml: Support multi document decode/encode Related to #1087 --- doc/usage.md | 5 ++- format/format.go | 4 ++ format/yaml/testdata/multi.fqtest | 44 +++++++++++++++++++ format/yaml/testdata/multi.yaml | 5 +++ format/yaml/testdata/single.yaml | 1 + format/yaml/testdata/trailing.fqtest | 2 +- format/yaml/yaml.go | 64 +++++++++++++++++++++------- 7 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 format/yaml/testdata/multi.fqtest create mode 100644 format/yaml/testdata/multi.yaml create mode 100644 format/yaml/testdata/single.yaml diff --git a/doc/usage.md b/doc/usage.md index e1e079a9c7..20b8781ba0 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -814,9 +814,12 @@ JSON and jq-flavoured JSON Note that `fromjson` and `tojson` use different naming conventions as they originate from jq's standard library. YAML -- `from_yaml` Parse YAML into jq value. +- `from_yaml`/`from_yaml($opts)` Parse YAML into jq value.
+ `$opts` are: + - `{multi_document: boolean}` Force multi document mode.
- `to_yaml`/`to_yaml($opts)` Serialize jq value into YAML. `$opts` are: - `{indent: number}` Indent depth. + - `{multi_document: boolean}` Force multi document mode.
TOML - `from_toml` Parse TOML into jq value. diff --git a/format/format.go b/format/format.go index 94e14bada0..a1116ea237 100644 --- a/format/format.go +++ b/format/format.go @@ -408,3 +408,7 @@ type Pg_Heap_In struct { type Pg_BTree_In struct { Page int `doc:"First page number in file, default is 0"` } + +type YAML_In struct { + MultiDocument bool `doc:"Force multi document"` +} diff --git a/format/yaml/testdata/multi.fqtest b/format/yaml/testdata/multi.fqtest new file mode 100644 index 0000000000..9b970b1907 --- /dev/null +++ b/format/yaml/testdata/multi.fqtest @@ -0,0 +1,44 @@ +$ fq -d yaml . single.yaml +[] +$ fq -d yaml . multi.yaml +[ + 1, + 2, + 3 +] +$ fq -o multi_document=true -d yaml . single.yaml +[ + [] +] +$ fq -o multi_document=true -d yaml . multi.yaml +[ + 1, + 2, + 3 +] +$ fq -o multi_document=false -d yaml . single.yaml +[] +$ fq -o multi_document=false -d yaml . multi.yaml +[ + 1, + 2, + 3 +] +$ fq -r -o multi_document=true -d yaml '. as $i | to_yaml({multi_document: true}), $i == from_yaml' multi.yaml +1 +--- +2 +--- +3 + +true +$ fq -nr '"[]" | from_yaml({multi_document: true})' +[ + [] +] +$ fq -nr '"[]" | from_yaml({multi_document: false})' +[] +$ fq -n '123 | to_yaml({multi_document: true})' +exitcode: 5 +stderr: +error: to_yaml cannot be applied to: number (123) diff --git a/format/yaml/testdata/multi.yaml b/format/yaml/testdata/multi.yaml new file mode 100644 index 0000000000..5fd5cb2ad6 --- /dev/null +++ b/format/yaml/testdata/multi.yaml @@ -0,0 +1,5 @@ +1 +--- +2 +--- +3 diff --git a/format/yaml/testdata/single.yaml b/format/yaml/testdata/single.yaml new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/format/yaml/testdata/single.yaml @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/format/yaml/testdata/trailing.fqtest b/format/yaml/testdata/trailing.fqtest index c0a818b986..d378255d28 100644 --- a/format/yaml/testdata/trailing.fqtest +++ b/format/yaml/testdata/trailing.fqtest @@ -5,4 +5,4 @@ error: error at position 0xc: yaml: line 2: could not find expected ':' $ fq -n '`{"a":123}{"b":444}` | from_yaml' exitcode: 5 stderr: -error: error at position 0x12: trialing data after top-level value +error: error at position 0x12: trialing data after document diff --git a/format/yaml/yaml.go b/format/yaml/yaml.go index 107fad4d09..9108550f0e 100644 --- a/format/yaml/yaml.go +++ b/format/yaml/yaml.go @@ -28,26 +28,46 @@ func init() { ProbeOrder: format.ProbeOrderTextFuzzy, Groups: []*decode.Group{format.Probe}, DecodeFn: decodeYAML, - Functions: []string{"_todisplay"}, + DefaultInArg: format.YAML_In{ + MultiDocument: false, + }, + Functions: []string{"_todisplay"}, }) interp.RegisterFS(yamlFS) interp.RegisterFunc1("_to_yaml", toYAML) } func decodeYAML(d *decode.D) any { + var yi format.YAML_In + d.ArgAs(&yi) + br := d.RawLen(d.Len()) - var r any + + var vs []any yd := yaml.NewDecoder(bitio.NewIOReader(br)) - if err := yd.Decode(&r); err != nil { - d.Fatalf("%s", err) - } - if err := yd.Decode(new(any)); !errors.Is(err, io.EOF) { - d.Fatalf("trialing data after top-level value") + for { + var v any + err := yd.Decode(&v) + if err != nil { + if len(vs) == 0 { + d.Fatalf("%s", err) + } else if errors.Is(err, io.EOF) { + break + } else { + d.Fatalf("trialing data after document") + } + } + + vs = append(vs, v) } var s scalar.Any - s.Actual = gojqx.Normalize(r) + if !yi.MultiDocument && len(vs) == 1 { + s.Actual = gojqx.Normalize(vs[0]) + } else { + s.Actual = gojqx.Normalize(vs) + } switch s.Actual.(type) { case map[string]any, @@ -63,18 +83,32 @@ func decodeYAML(d *decode.D) any { } type ToYAMLOpts struct { - Indent int `default:"4"` // 4 is default for gopkg.in/yaml.v3 + Indent int `default:"4"` // 4 is default for gopkg.in/yaml.v3 + MultiDocument bool `default:"false"` } func toYAML(_ *interp.Interp, c any, opts ToYAMLOpts) any { + c = gojqx.Normalize(c) + + cs, isArray := c.([]any) + if opts.MultiDocument { + if !isArray { + return gojqx.FuncTypeError{Name: "to_yaml", V: c} + } + } else { + cs = []any{c} + } + b := &bytes.Buffer{} e := yaml.NewEncoder(b) - // yaml.SetIndent panics if < 0 - if opts.Indent >= 0 { - e.SetIndent(opts.Indent) - } - if err := e.Encode(gojqx.Normalize(c)); err != nil { - return err + for _, c := range cs { + // yaml.SetIndent panics if < 0 + if opts.Indent >= 0 { + e.SetIndent(opts.Indent) + } + if err := e.Encode(gojqx.Normalize(c)); err != nil { + return err + } } return b.String()