Skip to content

Commit fae7f23

Browse files
williamthomewilliamthome
authored
Update and bump to v0.2 (#14)
* Add a formatter to json erlang/otp#8511 * Let json:decode/3 keep whitespaces erlang/otp#8809 * Add json format functions for key-value lists erlang/otp#8889 * Fix spec for json:format/3 erlang/otp#8914 * Optimize json object encoding erlang/otp#9251 * Use simple binaries * Update README.md * Bump to v0.2 * Act on CI errors * Always run tests
1 parent fcc444f commit fae7f23

File tree

4 files changed

+720
-31
lines changed

4 files changed

+720
-31
lines changed

‎README.md‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,15 @@ The module and function names are exactly the same. Then, when your app supports
1212

1313
```erlang
1414
% rebar.config
15-
{deps, [{json_polyfill, "0.1.4"}]}.
15+
{deps, [{json_polyfill, "0.2"}]}.
1616
```
1717

1818
### Elixir
1919

2020
```elixir
2121
# mix.exs
2222
def deps do
23-
[{:json_polyfill, "~> 0.1.4"}]
23+
[{:json_polyfill, "~> 0.2"}]
2424
end
2525
```
2626

@@ -73,6 +73,78 @@ An encoder that uses a heuristic to differentiate object-like lists of key-value
7373
<<"{\"a\":[],\"b\":1}">>
7474
```
7575

76+
## Format
77+
78+
### format/1
79+
80+
Generates formatted JSON corresponding to `Term`.
81+
Similiar to `encode/1` but with added whitespaces for formatting.
82+
83+
```erlang
84+
> io:put_chars(json:format(#{foo => <<"bar">>, baz => 52})).
85+
{
86+
"baz": 52,
87+
"foo": "bar"
88+
}
89+
ok
90+
```
91+
92+
### format/2
93+
94+
Generates formatted JSON corresponding to `Term`.
95+
Equivalent to `format(Term, fun json:format_value/3, Options)` or `format(Term, Encoder, #{})`.
96+
97+
### format/3
98+
99+
Generates formatted JSON corresponding to `Term`.
100+
Similar to `encode/2`, can be customised with the `Encoder` callback and `Options`.
101+
`Options` can include 'indent' to specify number of spaces per level and 'max' which loosely limits
102+
the width of lists.
103+
The `Encoder` will get a 'State' argument which contains the 'Options' maps merged with other data
104+
when recursing through 'Term'.
105+
`format_value/3` or various `encode_*` functions in this module can be used
106+
to help in constructing such callbacks.
107+
108+
```erlang
109+
> formatter({posix_time, SysTimeSecs}, Encode, State) ->
110+
TimeStr = calendar:system_time_to_rfc3339(SysTimeSecs, [{offset, "Z"}]),
111+
json:format_value(unicode:characters_to_binary(TimeStr), Encode, State);
112+
> formatter(Other, Encode, State) -> json:format_value(Other, Encode, State).
113+
>
114+
> Fun = fun(Value, Encode, State) -> formatter(Value, Encode, State) end.
115+
> Options = #{indent => 4}.
116+
> Term = #{id => 1, time => {posix_time, erlang:system_time(seconds)}}.
117+
>
118+
> io:put_chars(json:format(Term, Fun, Options)).
119+
{
120+
"id": 1,
121+
"time": "2024-05-23T16:07:48Z"
122+
}
123+
ok
124+
```
125+
126+
### format_value/3
127+
128+
Default format function used by `json:format/1`.
129+
Recursively calls `Encode` on all the values in `Value`,
130+
and indents objects and lists.
131+
132+
### format_key_value_list/3
133+
134+
Format function for lists of key-value pairs as JSON objects.
135+
Accepts lists with atom, binary, integer, or float keys.
136+
137+
### format_key_value_list_checked/3
138+
139+
@doc Format function for lists of key-value pairs as JSON objects.
140+
Accepts lists with atom, binary, integer, or float keys.
141+
Verifies that no duplicate keys will be produced in the
142+
resulting JSON object.
143+
144+
#### Errors
145+
146+
Raises `error({duplicate_key, Key})` if there are duplicates.
147+
76148
## Decode
77149

78150
### decode/1

‎src/json.erl‎

Lines changed: 260 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,14 @@
4747
]).
4848
-export_type([encoder/0, encode_value/0]).
4949

50+
-export([
51+
format/1, format/2, format/3,
52+
format_value/3,
53+
format_key_value_list/3,
54+
format_key_value_list_checked/3
55+
]).
56+
-export_type([formatter/0]).
57+
5058
-export([
5159
decode/1, decode/3, decode_start/3, decode_continue/2
5260
]).
@@ -313,7 +321,7 @@ key(Key, _Encode) when is_integer(Key) -> [$", encode_integer(Key), $"];
313321
key(Key, _Encode) when is_float(Key) -> [$", encode_float(Key), $"].
314322

315323
encode_object([]) -> <<"{}">>;
316-
encode_object([[_Comma | Entry] | Rest]) -> ["{", Entry, Rest, "}"].
324+
encode_object([[_Comma | Entry] | Rest]) -> [${, Entry, Rest, $}].
317325

318326
%% @doc
319327
%% Default encoder for binaries as JSON strings used by `json:encode/1'.
@@ -529,6 +537,252 @@ invalid_byte(Bin, Skip) ->
529537
error_info(Skip) ->
530538
[{error_info, #{cause => #{position => Skip}}}].
531539

540+
%%
541+
%% Format implementation
542+
%%
543+
544+
-if(?OTP_RELEASE >= 26).
545+
-type formatter() :: fun((Term :: dynamic(), Encoder :: formatter(), State :: map()) -> iodata()).
546+
-else.
547+
-type formatter() :: fun((Term :: term(), Encoder :: formatter(), State :: map()) -> iodata()).
548+
-endif.
549+
550+
%% @doc Generates formatted JSON corresponding to `Term`.
551+
%% Similiar to `encode/1` but with added whitespaces for formatting.
552+
%% ```erlang
553+
%% > io:put_chars(json:format(#{foo => <<"bar">>, baz => 52})).
554+
%% {
555+
%% "baz": 52,
556+
%% "foo": "bar"
557+
%% }
558+
%% ok
559+
%% ```
560+
-if(?OTP_RELEASE >= 26).
561+
-spec format(Term :: dynamic()) -> iodata().
562+
-else.
563+
-spec format(Term :: term()) -> iodata().
564+
-endif.
565+
format(Term) ->
566+
Enc = fun format_value/3,
567+
format(Term, Enc, #{}).
568+
569+
%% @doc Generates formatted JSON corresponding to `Term`.
570+
%% Equivalent to `format(Term, fun json:format_value/3, Options)` or `format(Term, Encoder, #{})`
571+
-if(?OTP_RELEASE >= 26).
572+
-spec format(Term :: encode_value(), Opts :: map()) -> iodata();
573+
(Term :: dynamic(), Encoder::formatter()) -> iodata().
574+
-else.
575+
-spec format(Term :: encode_value(), Opts :: map()) -> iodata();
576+
(Term :: term(), Encoder::formatter()) -> iodata().
577+
-endif.
578+
format(Term, Options) when is_map(Options) ->
579+
Enc = fun format_value/3,
580+
format(Term, Enc, Options);
581+
format(Term, Encoder) when is_function(Encoder, 3) ->
582+
format(Term, Encoder, #{}).
583+
584+
%% @doc Generates formatted JSON corresponding to `Term`.
585+
%% Similar to `encode/2`, can be customised with the `Encoder` callback and `Options`.
586+
%% `Options` can include 'indent' to specify number of spaces per level and 'max' which loosely limits
587+
%% the width of lists.
588+
%% The `Encoder` will get a 'State' argument which contains the 'Options' maps merged with other data
589+
%% when recursing through 'Term'.
590+
%% `format_value/3` or various `encode_*` functions in this module can be used
591+
%% to help in constructing such callbacks.
592+
%% ```erlang
593+
%% > formatter({posix_time, SysTimeSecs}, Encode, State) ->
594+
%% TimeStr = calendar:system_time_to_rfc3339(SysTimeSecs, [{offset, "Z"}]),
595+
%% json:format_value(unicode:characters_to_binary(TimeStr), Encode, State);
596+
%% > formatter(Other, Encode, State) -> json:format_value(Other, Encode, State).
597+
%% >
598+
%% > Fun = fun(Value, Encode, State) -> formatter(Value, Encode, State) end.
599+
%% > Options = #{indent => 4}.
600+
%% > Term = #{id => 1, time => {posix_time, erlang:system_time(seconds)}}.
601+
%% >
602+
%% > io:put_chars(json:format(Term, Fun, Options)).
603+
%% {
604+
%% "id": 1,
605+
%% "time": "2024-05-23T16:07:48Z"
606+
%% }
607+
%% ok
608+
%% ```
609+
-spec format(Term :: encode_value(), Encoder::formatter(), Options :: map()) -> iodata().
610+
format(Term, Encoder, Options) when is_function(Encoder, 3) ->
611+
Def = #{level => 0,
612+
col => 0,
613+
indent => 2,
614+
max => 100
615+
},
616+
[Encoder(Term, Encoder, maps:merge(Def, Options)),$\n].
617+
618+
%% @doc Default format function used by `json:format/1`.
619+
%% Recursively calls `Encode` on all the values in `Value`,
620+
%% and indents objects and lists.
621+
-if(?OTP_RELEASE >= 26).
622+
-spec format_value(Value::dynamic(), Encode::formatter(), State::map()) -> iodata().
623+
-else.
624+
-spec format_value(Value::term(), Encode::formatter(), State::map()) -> iodata().
625+
-endif.
626+
-if(?OTP_RELEASE >= 26).
627+
format_value(Atom, UserEnc, State) when is_atom(Atom) ->
628+
json:encode_atom(Atom, fun(Value, Enc) -> UserEnc(Value, Enc, State) end);
629+
format_value(Bin, _Enc, _State) when is_binary(Bin) ->
630+
json:encode_binary(Bin);
631+
format_value(Int, _Enc, _State) when is_integer(Int) ->
632+
json:encode_integer(Int);
633+
format_value(Float, _Enc, _State) when is_float(Float) ->
634+
json:encode_float(Float);
635+
format_value(List, UserEnc, State) when is_list(List) ->
636+
format_list(List, UserEnc, State);
637+
format_value(Map, UserEnc, State) when is_map(Map) ->
638+
%% Ensure order of maps are the same in each export
639+
OrderedKV = maps:to_list(maps:iterator(Map, ordered)),
640+
format_key_value_list(OrderedKV, UserEnc, State);
641+
format_value(Other, _Enc, _State) ->
642+
error({unsupported_type, Other}).
643+
-else.
644+
format_value(Atom, UserEnc, State) when is_atom(Atom) ->
645+
json:encode_atom(Atom, fun(Value, Enc) -> UserEnc(Value, Enc, State) end);
646+
format_value(Bin, _Enc, _State) when is_binary(Bin) ->
647+
json:encode_binary(Bin);
648+
format_value(Int, _Enc, _State) when is_integer(Int) ->
649+
json:encode_integer(Int);
650+
format_value(Float, _Enc, _State) when is_float(Float) ->
651+
json:encode_float(Float);
652+
format_value(List, UserEnc, State) when is_list(List) ->
653+
format_list(List, UserEnc, State);
654+
format_value(Map, UserEnc, State) when is_map(Map) ->
655+
%% Ensure order of maps are the same in each export
656+
OrderedKV = lists:keysort(1, maps:to_list(Map)),
657+
format_key_value_list(OrderedKV, UserEnc, State);
658+
format_value(Other, _Enc, _State) ->
659+
error({unsupported_type, Other}).
660+
-endif.
661+
662+
format_list([Head|Rest], UserEnc, #{level := Level, col := Col0, max := Max} = State0) ->
663+
State1 = State0#{level := Level+1},
664+
{Len, IndentElement} = indent(State1),
665+
if is_list(Head); %% Indent list in lists
666+
is_map(Head); %% Indent maps
667+
is_binary(Head); %% Indent Strings
668+
Col0 > Max -> %% Throw in the towel
669+
State = State1#{col := Len},
670+
First = UserEnc(Head, UserEnc, State),
671+
{_, IndLast} = indent(State0),
672+
[$[, IndentElement, First,
673+
format_tail(Rest, UserEnc, State, IndentElement, IndentElement),
674+
IndLast, $] ];
675+
true ->
676+
First = UserEnc(Head, UserEnc, State1),
677+
Col = Col0 + 1 + erlang:iolist_size(First),
678+
[$[, First,
679+
format_tail(Rest, UserEnc, State1#{col := Col}, [], IndentElement),
680+
$] ]
681+
end;
682+
format_list([], _, _) ->
683+
<<"[]">>.
684+
685+
format_tail([Head|Tail], Enc, #{max := Max, col := Col0} = State, [], IndentRow)
686+
when Col0 < Max ->
687+
EncHead = Enc(Head, Enc, State),
688+
String = [$,|EncHead],
689+
Col = Col0 + 1 + erlang:iolist_size(EncHead),
690+
[String|format_tail(Tail, Enc, State#{col := Col}, [], IndentRow)];
691+
format_tail([Head|Tail], Enc, State, [], IndentRow) ->
692+
EncHead = Enc(Head, Enc, State),
693+
String = [[$,|IndentRow]|EncHead],
694+
Col = erlang:iolist_size(String)-2,
695+
[String|format_tail(Tail, Enc, State#{col := Col}, [], IndentRow)];
696+
format_tail([Head|Tail], Enc, State, IndentAll, IndentRow) ->
697+
%% These are handling their own indentation, so optimize away size calculation
698+
EncHead = Enc(Head, Enc, State),
699+
String = [[$,|IndentAll]|EncHead],
700+
[String|format_tail(Tail, Enc, State, IndentAll, IndentRow)];
701+
format_tail([], _, _, _, _) ->
702+
[].
703+
704+
%% @doc Format function for lists of key-value pairs as JSON objects.
705+
%% Accepts lists with atom, binary, integer, or float keys.
706+
-spec format_key_value_list([{term(), term()}], Encode::formatter(), State::map()) -> iodata().
707+
format_key_value_list(KVList, UserEnc, #{level := Level} = State) ->
708+
{_,Indent} = indent(State),
709+
NextState = State#{level := Level+1},
710+
{KISize, KeyIndent} = indent(NextState),
711+
EncKeyFun = fun(KeyVal, _Fun) -> UserEnc(KeyVal, UserEnc, NextState) end,
712+
EntryFun = fun({Key, Value}) ->
713+
EncKey = key(Key, EncKeyFun),
714+
ValState = NextState#{col := KISize + 2 + erlang:iolist_size(EncKey)},
715+
[$, , KeyIndent, EncKey, ": " | UserEnc(Value, UserEnc, ValState)]
716+
end,
717+
format_object(lists:map(EntryFun, KVList), Indent).
718+
719+
%% @doc Format function for lists of key-value pairs as JSON objects.
720+
%% Accepts lists with atom, binary, integer, or float keys.
721+
%% Verifies that no duplicate keys will be produced in the
722+
%% resulting JSON object.
723+
%% ## Errors
724+
%% Raises `error({duplicate_key, Key})` if there are duplicates.
725+
-spec format_key_value_list_checked([{term(), term()}], Encoder::formatter(), State::map()) -> iodata().
726+
format_key_value_list_checked(KVList, UserEnc, State) when is_function(UserEnc, 3) ->
727+
{_,Indent} = indent(State),
728+
format_object(do_format_checked(KVList, UserEnc, State), Indent).
729+
730+
do_format_checked([], _, _) ->
731+
[];
732+
733+
do_format_checked(KVList, UserEnc, #{level := Level} = State) ->
734+
NextState = State#{level := Level + 1},
735+
{KISize, KeyIndent} = indent(NextState),
736+
EncKeyFun = fun(KeyVal, _Fun) -> UserEnc(KeyVal, UserEnc, NextState) end,
737+
EncListFun =
738+
fun({Key, Value}, {Acc, Visited0}) ->
739+
EncKey = iolist_to_binary(key(Key, EncKeyFun)),
740+
case is_map_key(EncKey, Visited0) of
741+
true ->
742+
error({duplicate_key, Key});
743+
false ->
744+
Visited1 = Visited0#{EncKey => true},
745+
ValState = NextState#{col := KISize + 2 + erlang:iolist_size(EncKey)},
746+
EncEntry = [$, , KeyIndent, EncKey, ": "
747+
| UserEnc(Value, UserEnc, ValState)],
748+
{[EncEntry | Acc], Visited1}
749+
end
750+
end,
751+
{EncKVList, _} = lists:foldl(EncListFun, {[], #{}}, KVList),
752+
lists:reverse(EncKVList).
753+
754+
format_object([], _) -> <<"{}">>;
755+
format_object([[_Comma,KeyIndent|Entry]], Indent) ->
756+
[_Key,_Colon|Value] = Entry,
757+
{_, Rest} = string:take(Value, [$\s,$\n]),
758+
[CP|_] = string:next_codepoint(Rest),
759+
if CP =:= ${ ->
760+
[${, KeyIndent, Entry, Indent, $}];
761+
CP =:= $[ ->
762+
[${, KeyIndent, Entry, Indent, $}];
763+
true ->
764+
["{ ", Entry, " }"]
765+
end;
766+
format_object([[_Comma,KeyIndent|Entry] | Rest], Indent) ->
767+
[${, KeyIndent, Entry, Rest, Indent, $}].
768+
769+
indent(#{level := Level, indent := Indent}) ->
770+
Steps = Level * Indent,
771+
{Steps, steps(Steps)}.
772+
773+
steps(0) -> <<"\n"/utf8>>;
774+
steps(2) -> <<"\n "/utf8>>;
775+
steps(4) -> <<"\n "/utf8>>;
776+
steps(6) -> <<"\n "/utf8>>;
777+
steps(8) -> <<"\n "/utf8>>;
778+
steps(10) -> <<"\n "/utf8>>;
779+
steps(12) -> <<"\n "/utf8>>;
780+
steps(14) -> <<"\n "/utf8>>;
781+
steps(16) -> <<"\n "/utf8>>;
782+
steps(18) -> <<"\n "/utf8>>;
783+
steps(20) -> <<"\n "/utf8>>;
784+
steps(N) -> ["\n", lists:duplicate(N, " ")].
785+
532786
%%
533787
%% Decoding implementation
534788
%%
@@ -1181,8 +1435,11 @@ continue(<<Rest/bits>>, Original, Skip, Acc, Stack0, Decode, Value) ->
11811435
end.
11821436

11831437
terminate(<<Byte, Rest/bits>>, Original, Skip, Acc, Value) when ?is_ws(Byte) ->
1184-
terminate(Rest, Original, Skip + 1, Acc, Value);
1185-
terminate(<<Rest/bits>>, _Original, _Skip, Acc, Value) ->
1438+
terminate(Rest, Original, Skip, Acc, Value);
1439+
terminate(<<>>, _, _Skip, Acc, Value) ->
1440+
{Value, Acc, <<>>};
1441+
terminate(<<_/bits>>, Original, Skip, Acc, Value) ->
1442+
<<_:Skip/binary, Rest/binary>> = Original,
11861443
{Value, Acc, Rest}.
11871444

11881445
-spec unexpected_utf8(binary(), non_neg_integer()) -> no_return().

‎src/json_polyfill.app.src‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{application, json_polyfill, [
22
{description, "A polyfill for the OTP json module (EEP68)"},
3-
{vsn, "0.1.4"},
3+
{vsn, "0.2"},
44
{registered, []},
55
{applications, [
66
kernel,

0 commit comments

Comments
 (0)