Skip to content

Commit 94834c0

Browse files
waderwader
authored
Merge pull request #649 from wader/tovalue-skip-gaps
interp: Add skip_gaps option for tovalue/-V
2 parents 1bdf1df + c512713 commit 94834c0

File tree

9 files changed

+96
-31
lines changed

9 files changed

+96
-31
lines changed

‎doc/usage.md

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -796,16 +796,36 @@ fq has some general options in addition to decode and decoders specific options.
796796

797797
`<value>` is fuzzily parsed based on the type of the option. Ex: a string can be specified as `-o name=string` or `-o name="string"`.
798798

799-
### `bits_format`
799+
### `-o bits_format=<string>`
800800

801-
How to represent raw bits as JSON.
801+
How to represent raw binary as JSON.
802802

803-
- `-o bits_foramt=string` String with raw bytes (zero bit padded). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
803+
- `-o bits_foramt=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
804804
- `-o bits_format=md5` MD5 hex string (zero bit padded).
805805
- `-o bits_format=base64` Base64 string.
806806
- `-p bits_foramt=truncate` Truncated string.
807807
- `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.
808808

809+
```sh
810+
$ fq -V -o bits_format=base64 . file`
811+
```
812+
In query
813+
```jq
814+
tovalue({bits_format: "md5"})
815+
```
816+
817+
### `-o skip_gaps=<boolean>`
818+
819+
Skip gaps fields (`gap0` etc) when using `tovalue` or `-V`. Note that this might affect array indexes if one more more gaps fields are skipped in an array.
820+
821+
```sh
822+
$ fq -V -o skip_gaps=true . file`
823+
```
824+
In query
825+
```jq
826+
tovalue({skip_gaps: true})
827+
```
828+
809829
## Color and unicode output
810830
811831
fq by default tries to use colors if possible, this can be disabled with `-M`. You can also

‎pkg/interp/decode.go

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -163,9 +163,12 @@ func (i *Interp) _registry(c any) any {
163163
}
164164
}
165165

166-
func (i *Interp) _toValue(c any, opts map[string]any) any {
166+
func (i *Interp) _toValue(c any, om map[string]any) any {
167167
return toValue(
168-
func() Options { return OptionsFromValue(opts) },
168+
func() *Options {
169+
opts := OptionsFromValue(om)
170+
return &opts
171+
},
169172
c,
170173
)
171174
}
@@ -307,7 +310,7 @@ func valueHas(key any, a func(name string) any, b func(key any) any) any {
307310

308311
// TODO: make more efficient somehow? shallow values but might be hard
309312
// when things like tovalue.key should behave like a jq value and not a decode value etc
310-
func toValue(optsFn func() Options, v any) any {
313+
func toValue(optsFn func() *Options, v any) any {
311314
nv, _ := gojqex.ToGoJQValueFn(v, func(v any) (any, bool) {
312315
switch v := v.(type) {
313316
case JQValueEx:
@@ -540,7 +543,7 @@ func (dvb decodeValueBase) JQValueKey(name string) any {
540543
case "_gap":
541544
switch vv := dv.V.(type) {
542545
case Scalarable:
543-
return vv.ScalarGap()
546+
return vv.ScalarIsGap()
544547
default:
545548
return false
546549
}
@@ -606,7 +609,7 @@ func (v decodeValue) JQValueKey(name string) any {
606609
func (v decodeValue) JQValueHas(key any) any {
607610
return valueHas(key, v.decodeValueBase.JQValueKey, v.JQValue.JQValueHas)
608611
}
609-
func (v decodeValue) JQValueToGoJQEx(optsFn func() Options) any {
612+
func (v decodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
610613
if !v.bitsFormat {
611614
return v.JQValueToGoJQ()
612615
}
@@ -695,13 +698,26 @@ func (v ArrayDecodeValue) JQValueHas(key any) any {
695698
return intKey >= 0 && intKey < len(v.Compound.Children)
696699
})
697700
}
698-
func (v ArrayDecodeValue) JQValueToGoJQ() any {
699-
vs := make([]any, len(v.Compound.Children))
700-
for i, f := range v.Compound.Children {
701-
vs[i] = makeDecodeValue(f, decodeValueValue)
701+
func (v ArrayDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
702+
opts := optsFn()
703+
704+
vs := make([]any, 0, len(v.Compound.Children))
705+
for _, f := range v.Compound.Children {
706+
switch s := f.V.(type) {
707+
case Scalarable:
708+
if s.ScalarIsGap() && opts.SkipGaps {
709+
// skip, note for arrays this will affect indexes
710+
continue
711+
}
712+
}
713+
714+
vs = append(vs, makeDecodeValue(f, decodeValueValue))
702715
}
703716
return vs
704717
}
718+
func (v ArrayDecodeValue) JQValueToGoJQ() any {
719+
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
720+
}
705721

706722
// decode value struct
707723

@@ -767,10 +783,22 @@ func (v StructDecodeValue) JQValueHas(key any) any {
767783
},
768784
)
769785
}
770-
func (v StructDecodeValue) JQValueToGoJQ() any {
786+
func (v StructDecodeValue) JQValueToGoJQEx(optsFn func() *Options) any {
787+
opts := optsFn()
788+
771789
vm := make(map[string]any, len(v.Compound.Children))
772790
for _, f := range v.Compound.Children {
791+
switch s := f.V.(type) {
792+
case Scalarable:
793+
if s.ScalarIsGap() && opts.SkipGaps {
794+
continue
795+
}
796+
}
797+
773798
vm[f.Name] = makeDecodeValue(f, decodeValueValue)
774799
}
775800
return vm
776801
}
802+
func (v StructDecodeValue) JQValueToGoJQ() any {
803+
return v.JQValueToGoJQEx(func() *Options { return &Options{} })
804+
}

‎pkg/interp/interp.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ type Scalarable interface {
8282
ScalarValue() any
8383
ScalarSym() any
8484
ScalarDescription() string
85-
ScalarGap() bool
85+
ScalarIsGap() bool
8686
ScalarDisplayFormat() scalar.DisplayFormat
8787
}
8888

@@ -210,7 +210,7 @@ type Display interface {
210210

211211
type JQValueEx interface {
212212
gojq.JQValue
213-
JQValueToGoJQEx(optsFn func() Options) any
213+
JQValueToGoJQEx(optsFn func() *Options) any
214214
}
215215

216216
func valuePath(v *decode.Value) []any {
@@ -682,7 +682,7 @@ func (i *Interp) _printColorJSON(c any, v any) gojq.Iter {
682682
Color: opts.Color,
683683
Tab: false,
684684
Indent: indent,
685-
ValueFn: func(v any) any { return toValue(func() Options { return opts }, v) },
685+
ValueFn: func(v any) any { return toValue(func() *Options { return &opts }, v) },
686686
Colors: colorjson.Colors{
687687
Reset: []byte(ansi.Reset.SetString),
688688
Null: []byte(opts.Decorator.Null.SetString),
@@ -1034,6 +1034,7 @@ type Options struct {
10341034
DisplayBytes int
10351035
Addrbase int
10361036
Sizebase int
1037+
SkipGaps bool
10371038

10381039
Decorator Decorator
10391040
BitsFormatFn func(br bitio.ReaderAtSeeker) (any, error)

‎pkg/interp/options.jq

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ def _opt_build_default_fixed:
5959
raw_output: ($stdout.is_terminal | not),
6060
raw_string: false,
6161
repl: false,
62+
skip_gaps: false,
6263
sizebase: 10,
6364
show_formats: false,
6465
show_help: false,
@@ -101,9 +102,10 @@ def _opt_options:
101102
raw_output: "boolean",
102103
raw_string: "boolean",
103104
repl: "boolean",
104-
sizebase: "number",
105105
show_formats: "boolean",
106106
show_help: "boolean",
107+
sizebase: "number",
108+
skip_gaps: "boolean",
107109
slurp: "boolean",
108110
string_input: "boolean",
109111
unicode: "boolean",

‎pkg/interp/testdata/args.fqtest

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ repl false
9999
show_formats false
100100
show_help options
101101
sizebase 10
102+
skip_gaps false
102103
slurp false
103104
string_input false
104105
unicode false

‎pkg/interp/testdata/options.fqtest

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ $ fq -n options
8383
"show_formats": false,
8484
"show_help": false,
8585
"sizebase": 10,
86+
"skip_gaps": false,
8687
"slurp": false,
8788
"string_input": false,
8889
"unicode": false,

‎pkg/interp/testdata/tovalue.fqtest

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,21 @@
22
$ fq -i . test.mp3
33
mp3> .headers[0] | tovalue.header.magic
44
"ID3"
5+
mp3> [tobytes, "abc"] | mp3 | tovalue, tovalue({skip_gaps: true}) | keys
6+
[
7+
"footers",
8+
"frames",
9+
"gap0",
10+
"headers"
11+
]
12+
[
13+
"footers",
14+
"frames",
15+
"headers"
16+
]
17+
mp3> "abc" | mpeg_ts | tovalue, tovalue({skip_gaps: true}) | keys
18+
[
19+
"gap0"
20+
]
21+
[]
522
mp3> ^D
6-
$ fq -i
7-
null> "aaa" | mp3_frame | .gap0 | tovalue, tovalue({sizebase: 2})
8-
"aaa"
9-
"aaa"
10-
null> ^D

‎pkg/scalar/scalar_gen.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func (s Any) ScalarValue() any {
2727
}
2828
func (s Any) ScalarSym() any { return s.Sym }
2929
func (s Any) ScalarDescription() string { return s.Description }
30-
func (s Any) ScalarGap() bool { return s.Gap }
30+
func (s Any) ScalarIsGap() bool { return s.Gap }
3131
func (s Any) ScalarDisplayFormat() DisplayFormat { return 0 }
3232

3333
func AnyActual(v any) AnyMapper {
@@ -224,7 +224,7 @@ func (s BigInt) ScalarValue() any {
224224
}
225225
func (s BigInt) ScalarSym() any { return s.Sym }
226226
func (s BigInt) ScalarDescription() string { return s.Description }
227-
func (s BigInt) ScalarGap() bool { return s.Gap }
227+
func (s BigInt) ScalarIsGap() bool { return s.Gap }
228228
func (s BigInt) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
229229

230230
func BigIntActual(v *big.Int) BigIntMapper {
@@ -420,7 +420,7 @@ func (s BitBuf) ScalarValue() any {
420420
}
421421
func (s BitBuf) ScalarSym() any { return s.Sym }
422422
func (s BitBuf) ScalarDescription() string { return s.Description }
423-
func (s BitBuf) ScalarGap() bool { return s.Gap }
423+
func (s BitBuf) ScalarIsGap() bool { return s.Gap }
424424
func (s BitBuf) ScalarDisplayFormat() DisplayFormat { return 0 }
425425

426426
func BitBufActual(v bitio.ReaderAtSeeker) BitBufMapper {
@@ -616,7 +616,7 @@ func (s Bool) ScalarValue() any {
616616
}
617617
func (s Bool) ScalarSym() any { return s.Sym }
618618
func (s Bool) ScalarDescription() string { return s.Description }
619-
func (s Bool) ScalarGap() bool { return s.Gap }
619+
func (s Bool) ScalarIsGap() bool { return s.Gap }
620620
func (s Bool) ScalarDisplayFormat() DisplayFormat { return 0 }
621621

622622
func BoolActual(v bool) BoolMapper {
@@ -812,7 +812,7 @@ func (s Flt) ScalarValue() any {
812812
}
813813
func (s Flt) ScalarSym() any { return s.Sym }
814814
func (s Flt) ScalarDescription() string { return s.Description }
815-
func (s Flt) ScalarGap() bool { return s.Gap }
815+
func (s Flt) ScalarIsGap() bool { return s.Gap }
816816
func (s Flt) ScalarDisplayFormat() DisplayFormat { return 0 }
817817

818818
func FltActual(v float64) FltMapper {
@@ -1009,7 +1009,7 @@ func (s Sint) ScalarValue() any {
10091009
}
10101010
func (s Sint) ScalarSym() any { return s.Sym }
10111011
func (s Sint) ScalarDescription() string { return s.Description }
1012-
func (s Sint) ScalarGap() bool { return s.Gap }
1012+
func (s Sint) ScalarIsGap() bool { return s.Gap }
10131013
func (s Sint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
10141014

10151015
func SintActual(v int64) SintMapper {
@@ -1205,7 +1205,7 @@ func (s Str) ScalarValue() any {
12051205
}
12061206
func (s Str) ScalarSym() any { return s.Sym }
12071207
func (s Str) ScalarDescription() string { return s.Description }
1208-
func (s Str) ScalarGap() bool { return s.Gap }
1208+
func (s Str) ScalarIsGap() bool { return s.Gap }
12091209
func (s Str) ScalarDisplayFormat() DisplayFormat { return 0 }
12101210

12111211
func StrActual(v string) StrMapper {
@@ -1402,7 +1402,7 @@ func (s Uint) ScalarValue() any {
14021402
}
14031403
func (s Uint) ScalarSym() any { return s.Sym }
14041404
func (s Uint) ScalarDescription() string { return s.Description }
1405-
func (s Uint) ScalarGap() bool { return s.Gap }
1405+
func (s Uint) ScalarIsGap() bool { return s.Gap }
14061406
func (s Uint) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
14071407

14081408
func UintActual(v uint64) UintMapper {

‎pkg/scalar/scalar_gen.go.tmpl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import (
3131
}
3232
func (s {{$name}}) ScalarSym() any { return s.Sym }
3333
func (s {{$name}}) ScalarDescription() string { return s.Description }
34-
func (s {{$name}}) ScalarGap() bool { return s.Gap }
34+
func (s {{$name}}) ScalarIsGap() bool { return s.Gap }
3535
{{- if $t.display_format}}
3636
func (s {{$name}}) ScalarDisplayFormat() DisplayFormat { return s.DisplayFormat }
3737
{{- else }}

0 commit comments

Comments
 (0)