defmodule ExSQL.Json do
@moduledoc """
The json1 function family's data layer: a JSON parser/renderer over an
order-preserving representation, plus path navigation (`$.a.b[0]`,
`$[#-1]`).
JSON values map to:
* `:null`, `true`, `false`
* numbers (integer, float, or preserved real literal token)
* strings (binaries)
* `{:array, [value]}`
* `{:object, [{key, value}]}` — pairs in source/insertion order, as
SQLite preserves object member order in `json_object` and rewrites
Hand-rolled rather than `JSON`/`:json` because object member order is
observable in SQLite's output.
"""
@type object_key ::
binary()
| {:jsonb_path_key, binary()}
| {:jsonb_integer_literal, 4, binary(), integer()}
| {:jsonb_escaped_text, 9, binary(), binary()}
| {:jsonb_escaped_key, 8 | 9, binary(), binary()}
@type t ::
:null
| boolean()
| number()
| {:real_literal, binary()}
| {:jsonb_integer_literal, 4, binary(), integer()}
| {:jsonb_real_literal, 5 | 6, binary()}
| {:jsonb_text, 10, binary()}
| {:jsonb_escaped_text, 8 | 9, binary(), binary()}
| binary()
| {:array, [t()]}
| {:object, [{object_key(), t()}]}
@type path_step :: {:key, binary()} | {:index, non_neg_integer()} | {:last_offset, integer()}
# -- parsing -----------------------------------------------------------------
@spec parse(binary()) :: {:ok, t()} | :error
def parse(text), do: parse(text, :canonical)
@spec parse_json5(binary()) :: {:ok, t()} | :error
def parse_json5(text), do: parse(text, :json5)
defp parse(text, mode) when is_binary(text) do
case value(skip_ws(text, mode), mode) do
{:ok, jv, rest} ->
case skip_ws(rest, mode) do
"" -> {:ok, jv}
_trailing -> :error
end
:error ->
:error
end
end
@spec parse_sqlite_jsonb(binary()) :: {:ok, t()} | :error
def parse_sqlite_jsonb(blob) when is_binary(blob) do
case sqlite_jsonb_value(blob) do
{:ok, jv, ""} -> {:ok, jv}
_other -> :error
end
end
@spec sqlite_jsonb_strict?(binary()) :: boolean()
def sqlite_jsonb_strict?(blob) when is_binary(blob) do
case sqlite_jsonb_payload(blob) do
{:ok, type, payload, ""} -> strict_jsonb_payload?(type, payload)
_other -> false
end
end
@spec sqlite_jsonb_superficial?(binary()) :: boolean()
def sqlite_jsonb_superficial?(<<header, _rest::binary>> = blob) do
case sqlite_jsonb_payload(blob) do
{:ok, type, payload, ""} -> superficial_jsonb_payload?(header, type, payload)
_other -> false
end
end
def sqlite_jsonb_superficial?(blob) when is_binary(blob), do: false
defp superficial_jsonb_payload?(_header, type, payload) when type in [0, 1, 2],
do: payload == ""
defp superficial_jsonb_payload?(header, _type, payload) do
if byte_size(payload) > 7 or not jsonb_text_json_ambiguous?(header) do
true
else
case sqlite_jsonb_payload(Bitwise.band(header, 0x0F), payload, "") do
{:ok, value, ""} -> strict_jsonb_value?(value)
:error -> false
end
end
end
defp jsonb_text_json_ambiguous?(header), do: header in [?{, ?[] or header in ?0..?9
defp skip_ws(<<c, rest::binary>>, mode) when c in [?\s, ?\t, ?\n, ?\r], do: skip_ws(rest, mode)
defp skip_ws(<<c, rest::binary>>, :json5) when c in [?\f, ?\v],
do: skip_ws(rest, :json5)
defp skip_ws(<<"\u00A0", rest::binary>>, :json5), do: skip_ws(rest, :json5)
defp skip_ws(<<"//", rest::binary>>, :json5) do
rest
|> skip_line_comment()
|> skip_ws(:json5)
end
defp skip_ws(<<"/*", rest::binary>>, :json5) do
case skip_block_comment(rest) do
{:ok, rest} -> skip_ws(rest, :json5)
:error -> <<0>>
end
end
defp skip_ws(text, _mode), do: text
defp skip_line_comment(<<"\n", rest::binary>>), do: rest
defp skip_line_comment(<<"\r", rest::binary>>), do: rest
defp skip_line_comment(<<_c, rest::binary>>), do: skip_line_comment(rest)
defp skip_line_comment(<<>>), do: ""
defp skip_block_comment(<<"*/", rest::binary>>), do: {:ok, rest}
defp skip_block_comment(<<_c, rest::binary>>), do: skip_block_comment(rest)
defp skip_block_comment(<<>>), do: :error
defp value(<<"null", rest::binary>>, _mode), do: {:ok, :null, rest}
defp value(<<"true", rest::binary>>, _mode), do: {:ok, true, rest}
defp value(<<"false", rest::binary>>, _mode), do: {:ok, false, rest}
defp value(<<?", rest::binary>>, mode), do: string(rest, ?", mode, [])
defp value(<<?', rest::binary>>, :json5), do: string(rest, ?', :json5, [])
defp value(<<?{, rest::binary>>, mode), do: object(skip_ws(rest, mode), mode, [])
defp value(<<?[, rest::binary>>, mode), do: array(skip_ws(rest, mode), mode, [])
defp value(text, :json5) do
case json5_nonfinite(text) do
{:ok, value, rest} -> {:ok, value, rest}
:error -> json5_value(text)
end
end
defp value(<<c, _::binary>> = text, _mode) when c in [?-, ?+] or c in ?0..?9,
do: number(text, :canonical)
defp value(_, _mode), do: :error
defp json5_value(<<c, _::binary>> = text) when c in [?-, ?+] or c in ?0..?9,
do: number(text, :json5)
defp json5_value(<<?., rest::binary>> = text) do
case rest do
<<c, _::binary>> when c in ?0..?9 -> number(text, :json5)
_ -> :error
end
end
defp json5_value(_text), do: :error
defp json5_nonfinite(text) do
downcased = String.downcase(text)
[
{"+infinity", :json_pos_inf},
{"-infinity", :json_neg_inf},
{"infinity", :json_pos_inf},
{"+inf", :json_pos_inf},
{"-inf", :json_neg_inf},
{"inf", :json_pos_inf},
{"qnan", :json_nan},
{"snan", :json_nan},
{"nan", :json_nan}
]
|> Enum.find_value(:error, fn {token, value} ->
size = byte_size(token)
case downcased do
<<^token::binary-size(^size), _::binary>> ->
<<_matched::binary-size(^size), rest::binary>> = text
if json5_token_boundary?(rest) do
{:ok, value, rest}
end
_other ->
nil
end
end)
end
defp json5_token_boundary?(<<c, _rest::binary>>)
when c == ?_ or c == ?$ or c in ?A..?Z or c in ?a..?z or c in ?0..?9,
do: false
defp json5_token_boundary?(_rest), do: true
defp sqlite_jsonb_value(<<header, rest::binary>>) do
with {:ok, type, payload, rest} <- sqlite_jsonb_payload(<<header, rest::binary>>) do
sqlite_jsonb_payload(type, payload, rest)
end
end
defp sqlite_jsonb_value(_blob), do: :error
defp sqlite_jsonb_payload(<<header, rest::binary>>) do
type = Bitwise.band(header, 0x0F)
size_code = Bitwise.bsr(header, 4)
with {:ok, size, rest} <- sqlite_jsonb_payload_size(size_code, rest),
true <- byte_size(rest) >= size do
<<payload::binary-size(^size), rest::binary>> = rest
{:ok, type, payload, rest}
else
_other -> :error
end
end
defp sqlite_jsonb_payload(_blob), do: :error
defp sqlite_jsonb_payload_size(size, rest) when size <= 11, do: {:ok, size, rest}
defp sqlite_jsonb_payload_size(12, <<size, rest::binary>>), do: {:ok, size, rest}
defp sqlite_jsonb_payload_size(13, <<size::16, rest::binary>>), do: {:ok, size, rest}
defp sqlite_jsonb_payload_size(14, <<size::32, rest::binary>>), do: {:ok, size, rest}
defp sqlite_jsonb_payload_size(15, <<size::64, rest::binary>>), do: {:ok, size, rest}
defp sqlite_jsonb_payload_size(_size, _rest), do: :error
defp sqlite_jsonb_payload(0, "", rest), do: {:ok, :null, rest}
defp sqlite_jsonb_payload(1, "", rest), do: {:ok, true, rest}
defp sqlite_jsonb_payload(2, "", rest), do: {:ok, false, rest}
defp sqlite_jsonb_payload(3, payload, rest) do
case Integer.parse(payload) do
{value, ""} -> {:ok, value, rest}
_other -> :error
end
end
defp sqlite_jsonb_payload(4, payload, rest) do
case Integer.parse(payload) do
{value, ""} -> {:ok, {:jsonb_integer_literal, 4, payload, value}, rest}
_other -> :error
end
end
defp sqlite_jsonb_payload(type, payload, rest) when type in [5, 6] do
case parse_jsonb_real_literal(type, payload) do
{:ok, _value} -> {:ok, {:jsonb_real_literal, type, payload}, rest}
:error -> :error
end
end
defp sqlite_jsonb_payload(7, payload, rest), do: {:ok, payload, rest}
defp sqlite_jsonb_payload(10, payload, rest), do: {:ok, {:jsonb_text, 10, payload}, rest}
defp sqlite_jsonb_payload(type, payload, rest) when type in [8, 9] do
case sqlite_jsonb_decode_escaped_text(type, payload) do
{:ok, value} -> {:ok, {:jsonb_escaped_text, type, payload, value}, rest}
_other -> :error
end
end
defp sqlite_jsonb_payload(11, payload, rest) do
case sqlite_jsonb_items(payload, []) do
{:ok, items} -> {:ok, {:array, items}, rest}
:error -> :error
end
end
defp sqlite_jsonb_payload(12, payload, rest) do
case sqlite_jsonb_pairs(payload, []) do
{:ok, pairs} -> {:ok, {:object, pairs}, rest}
:error -> :error
end
end
defp sqlite_jsonb_payload(_type, _payload, _rest), do: :error
defp sqlite_jsonb_items("", acc), do: {:ok, Enum.reverse(acc)}
defp sqlite_jsonb_items(payload, acc) do
case sqlite_jsonb_value(payload) do
{:ok, item, rest} -> sqlite_jsonb_items(rest, [item | acc])
:error -> :error
end
end
defp sqlite_jsonb_pairs("", acc), do: {:ok, Enum.reverse(acc)}
defp sqlite_jsonb_pairs(payload, acc) do
with {:ok, key, rest} <- sqlite_jsonb_object_key(payload),
{:ok, value, rest} <- sqlite_jsonb_value(rest) do
sqlite_jsonb_pairs(rest, [{key, value} | acc])
else
_other -> :error
end
end
defp sqlite_jsonb_object_key(blob) do
with {:ok, type, payload, rest} <- sqlite_jsonb_payload(blob) do
sqlite_jsonb_object_key(type, payload, rest)
end
end
defp sqlite_jsonb_object_key(type, payload, rest) when type in [7, 10],
do: {:ok, if(type == 10, do: {:jsonb_path_key, payload}, else: payload), rest}
defp sqlite_jsonb_object_key(type, payload, rest) when type in [8, 9] do
case sqlite_jsonb_decode_escaped_text(type, payload) do
{:ok, value} -> {:ok, {:jsonb_escaped_key, type, payload, value}, rest}
_other -> :error
end
end
defp sqlite_jsonb_object_key(_type, _payload, _rest), do: :error
defp strict_jsonb_payload?(type, payload) when type in [0, 1, 2], do: payload == ""
defp strict_jsonb_payload?(3, payload) do
match?({_value, ""}, Integer.parse(payload))
end
defp strict_jsonb_payload?(4, _payload), do: false
defp strict_jsonb_payload?(5, payload),
do: match?({:ok, _value}, parse_jsonb_real_literal(5, payload))
defp strict_jsonb_payload?(6, "+" <> _payload), do: false
defp strict_jsonb_payload?(6, payload),
do: match?({:ok, _value}, parse_jsonb_real_literal(6, payload))
defp strict_jsonb_payload?(7, payload), do: jsonb_text_payload_strict?(payload)
defp strict_jsonb_payload?(8, payload), do: jsonb_textj_payload_strict?(payload)
defp strict_jsonb_payload?(type, payload) when type in [9, 10] do
match?({:ok, _value}, sqlite_jsonb_decode_escaped_text(type, payload))
end
defp strict_jsonb_payload?(11, payload), do: strict_jsonb_items?(payload)
defp strict_jsonb_payload?(12, payload), do: strict_jsonb_pairs?(payload)
defp strict_jsonb_payload?(_type, _payload), do: false
defp strict_jsonb_items?(""), do: true
defp strict_jsonb_items?(payload) do
with {:ok, type, item, rest} <- sqlite_jsonb_payload(payload),
true <- strict_jsonb_payload?(type, item) do
strict_jsonb_items?(rest)
else
_other -> false
end
end
defp strict_jsonb_pairs?(""), do: true
defp strict_jsonb_pairs?(payload) do
with {:ok, type, key, rest} <- sqlite_jsonb_payload(payload),
true <- type in 7..10,
true <- strict_jsonb_payload?(type, key),
{:ok, value_type, value, rest} <- sqlite_jsonb_payload(rest),
true <- strict_jsonb_payload?(value_type, value) do
strict_jsonb_pairs?(rest)
else
_other -> false
end
end
defp jsonb_text_payload_strict?(payload) do
payload
|> :binary.bin_to_list()
|> Enum.all?(fn byte -> byte > 0x1F and byte not in [?", ?\\] end)
end
defp jsonb_textj_payload_strict?(payload), do: jsonb_textj_payload_strict?(payload, true)
defp jsonb_textj_payload_strict?(<<>>, ok?), do: ok?
defp jsonb_textj_payload_strict?(<<byte, rest::binary>>, true)
when byte > 0x1F and byte not in [?", ?\\],
do: jsonb_textj_payload_strict?(rest, true)
defp jsonb_textj_payload_strict?(<<?\\, escape, rest::binary>>, true)
when escape in [?\", ?\\, ?/, ?b, ?f, ?n, ?r, ?t],
do: jsonb_textj_payload_strict?(rest, true)
defp jsonb_textj_payload_strict?(<<?\\, ?u, hex::binary-size(4), rest::binary>>, true) do
case Integer.parse(hex, 16) do
{_code, ""} -> jsonb_textj_payload_strict?(rest, true)
_other -> false
end
end
defp jsonb_textj_payload_strict?(_payload, _ok?), do: false
defp strict_jsonb_value?({:jsonb_integer_literal, 4, _payload, _value}), do: false
defp strict_jsonb_value?({:jsonb_real_literal, 6, "+" <> _payload}), do: false
defp strict_jsonb_value?({:array, items}), do: Enum.all?(items, &strict_jsonb_value?/1)
defp strict_jsonb_value?({:object, pairs}) do
Enum.all?(pairs, fn {key, value} ->
strict_jsonb_value?(key) and strict_jsonb_value?(value)
end)
end
defp strict_jsonb_value?(_jv), do: true
defp sqlite_jsonb_decode_escaped_text(type, payload) do
case parse("\"" <> payload <> "\"", if(type == 9, do: :json5, else: :canonical)) do
{:ok, value} when is_binary(value) -> {:ok, value}
{:ok, {:jsonb_escaped_text, _type, _payload, value}} -> {:ok, value}
_other -> :error
end
end
defp object(<<?}, rest::binary>>, _mode, pairs), do: {:ok, {:object, Enum.reverse(pairs)}, rest}
defp object(text, mode, pairs) do
with {:ok, key, rest} <- object_key(text, mode),
<<?:, rest::binary>> <- skip_ws(rest, mode),
{:ok, jv, rest} <- value(skip_ws(rest, mode), mode) do
case skip_ws(rest, mode) do
<<?,, rest::binary>> ->
rest = skip_ws(rest, mode)
case {rest, mode} do
{<<?}, rest::binary>>, :json5} ->
{:ok, {:object, Enum.reverse([{key, jv} | pairs])}, rest}
_ ->
object(rest, mode, [{key, jv} | pairs])
end
<<?}, rest::binary>> ->
{:ok, {:object, Enum.reverse([{key, jv} | pairs])}, rest}
_ ->
:error
end
else
_ -> :error
end
end
defp object_key(<<?", rest::binary>>, mode), do: string(rest, ?", mode, [])
defp object_key(<<?', rest::binary>>, :json5), do: string(rest, ?', :json5, [])
defp object_key(text, :json5), do: bare_key(text, [])
defp object_key(_text, _mode), do: :error
defp bare_key(<<?\\, ?u, rest::binary>>, acc), do: bare_key_unicode_escape(rest, acc)
defp bare_key(<<c, rest::binary>>, acc) when c == ?_ or c == ?$ or c in ?A..?Z or c in ?a..?z,
do: bare_key_rest(rest, [acc, c])
defp bare_key(<<c::utf8, rest::binary>>, acc) when c >= 0x80,
do: bare_key_rest(rest, [acc, <<c::utf8>>])
defp bare_key(_text, _acc), do: :error
defp bare_key_rest(<<?\\, ?u, rest::binary>>, acc), do: bare_key_unicode_escape(rest, acc)
defp bare_key_rest(<<c, rest::binary>>, acc)
when c == ?_ or c == ?$ or c in ?A..?Z or c in ?a..?z or c in ?0..?9,
do: bare_key_rest(rest, [acc, c])
defp bare_key_rest(<<c::utf8, rest::binary>>, acc) when c >= 0x80,
do: bare_key_rest(rest, [acc, <<c::utf8>>])
defp bare_key_rest(rest, acc), do: {:ok, finish_bare_key(acc), rest}
defp bare_key_unicode_escape(<<hex::binary-size(4), rest::binary>>, acc) do
case Integer.parse(hex, 16) do
{code, ""} when code in 0xD800..0xDBFF ->
case rest do
<<?\\, ?u, hex2::binary-size(4), rest2::binary>> ->
case Integer.parse(hex2, 16) do
{low, ""} when low in 0xDC00..0xDFFF ->
code = 0x10000 + (code - 0xD800) * 0x400 + (low - 0xDC00)
payload = ["\\u", hex, "\\u", hex2]
bare_key_rest(rest2, [acc, {:bare_key_unicode_escape, payload, <<code::utf8>>}])
_other ->
:error
end
_other ->
:error
end
{code, ""} when code not in 0xDC00..0xDFFF ->
bare_key_rest(rest, [acc, {:bare_key_unicode_escape, ["\\u", hex], <<code::utf8>>}])
_other ->
:error
end
end
defp bare_key_unicode_escape(_rest, _acc), do: :error
defp finish_bare_key(acc) do
if bare_key_has_unicode_escape?(acc) do
{payload, value} = bare_key_parts(List.wrap(acc), [], [])
{:jsonb_escaped_key, 8, IO.iodata_to_binary(payload), IO.iodata_to_binary(value)}
else
IO.iodata_to_binary(acc)
end
end
defp bare_key_has_unicode_escape?({:bare_key_unicode_escape, _payload, _value}), do: true
defp bare_key_has_unicode_escape?(list) when is_list(list),
do: Enum.any?(list, &bare_key_has_unicode_escape?/1)
defp bare_key_has_unicode_escape?(_part), do: false
defp bare_key_parts([], payload, value), do: {Enum.reverse(payload), Enum.reverse(value)}
defp bare_key_parts([{:bare_key_unicode_escape, escape, decoded} | rest], payload, value) do
bare_key_parts(rest, [escape | payload], [decoded | value])
end
defp bare_key_parts([part | rest], payload, value) when is_list(part) do
{part_payload, part_value} = bare_key_parts(part, [], [])
bare_key_parts(rest, [part_payload | payload], [part_value | value])
end
defp bare_key_parts([part | rest], payload, value) do
bare_key_parts(rest, [part | payload], [part | value])
end
defp array(<<?], rest::binary>>, _mode, items), do: {:ok, {:array, Enum.reverse(items)}, rest}
defp array(text, mode, items) do
case value(text, mode) do
{:ok, jv, rest} ->
case skip_ws(rest, mode) do
<<?,, rest::binary>> ->
rest = skip_ws(rest, mode)
case {rest, mode} do
{<<?], rest::binary>>, :json5} -> {:ok, {:array, Enum.reverse([jv | items])}, rest}
_ -> array(rest, mode, [jv | items])
end
<<?], rest::binary>> ->
{:ok, {:array, Enum.reverse([jv | items])}, rest}
_ ->
:error
end
:error ->
:error
end
end
defp string(<<quote, rest::binary>>, quote, _mode, acc),
do: {:ok, finish_string(acc), rest}
defp string(<<?\\, escape, rest::binary>>, quote, mode, acc) do
case escape do
?" -> string(rest, quote, mode, [acc, ?"])
?' when mode == :json5 -> json5_string_escape(rest, quote, mode, acc, "\\'", "'")
?' -> string(rest, quote, mode, [acc, ?'])
?\\ -> string(rest, quote, mode, [acc, ?\\])
?/ -> string(rest, quote, mode, [acc, ?/])
?0 when mode == :json5 -> json5_nul_escape(rest, quote, mode, acc)
?b -> string(rest, quote, mode, [acc, ?\b])
?f -> string(rest, quote, mode, [acc, ?\f])
?n -> string(rest, quote, mode, [acc, ?\n])
?r -> string(rest, quote, mode, [acc, ?\r])
?t -> string(rest, quote, mode, [acc, ?\t])
?u -> unicode_escape(rest, quote, mode, acc)
?x when mode == :json5 -> json5_hex_escape(rest, quote, mode, acc)
?v when mode == :json5 -> json5_string_escape(rest, quote, mode, acc, "\\v", <<?\v>>)
0xE2 when mode == :json5 -> json5_line_separator_escape(rest, quote, mode, acc)
?\n when mode == :json5 -> json5_string_escape(rest, quote, mode, acc, "\\\n", "")
?\r when mode == :json5 -> json5_cr_escape(rest, quote, mode, acc)
_ -> :error
end
end
defp string(<<c, rest::binary>>, quote, mode, acc) when c != quote,
do: string(rest, quote, mode, [acc, c])
defp string(<<>>, _quote, _mode, _acc), do: :error
defp unicode_escape(<<hex::binary-size(4), rest::binary>>, quote, mode, acc) do
case Integer.parse(hex, 16) do
{code, ""} when code in 0xD800..0xDBFF ->
# A UTF-16 surrogate pair split across two \u escapes.
case rest do
<<?\\, ?u, hex2::binary-size(4), rest2::binary>> ->
case Integer.parse(hex2, 16) do
{low, ""} when low in 0xDC00..0xDFFF ->
code = 0x10000 + (code - 0xD800) * 0x400 + (low - 0xDC00)
json_string_escape(
rest2,
quote,
mode,
acc,
["\\u", hex, "\\u", hex2],
<<code::utf8>>
)
_ ->
json_string_escape(
rest,
quote,
mode,
acc,
["\\u", hex],
json_unicode_code_bytes(code)
)
end
_ ->
json_string_escape(
rest,
quote,
mode,
acc,
["\\u", hex],
json_unicode_code_bytes(code)
)
end
{code, ""} ->
json_string_escape(rest, quote, mode, acc, ["\\u", hex], json_unicode_code_bytes(code))
_ ->
:error
end
end
defp unicode_escape(_, _quote, _mode, _acc), do: :error
defp json_unicode_code_bytes(code) when code in 0xD800..0xDFFF do
<<0xE0 + Bitwise.bsr(code, 12), 0x80 + Bitwise.band(Bitwise.bsr(code, 6), 0x3F),
0x80 + Bitwise.band(code, 0x3F)>>
end
defp json_unicode_code_bytes(code), do: <<code::utf8>>
defp json5_hex_escape(<<hex::binary-size(2), rest::binary>>, quote, mode, acc) do
case Integer.parse(hex, 16) do
{code, ""} -> json5_string_escape(rest, quote, mode, acc, ["\\x", hex], <<code>>)
_other -> :error
end
end
defp json5_hex_escape(_rest, _quote, _mode, _acc), do: :error
defp json5_nul_escape(<<digit, _rest::binary>>, _quote, _mode, _acc) when digit in ?0..?9,
do: :error
defp json5_nul_escape(rest, quote, mode, acc),
do: json5_string_escape(rest, quote, mode, acc, "\\0", <<0>>)
defp json5_cr_escape(<<"\n", rest::binary>>, quote, mode, acc),
do: json5_string_escape(rest, quote, mode, acc, "\\\r\n", "")
defp json5_cr_escape(rest, quote, mode, acc),
do: json5_string_escape(rest, quote, mode, acc, "\\\r", "")
defp json5_line_separator_escape(<<0x80, sep, rest::binary>>, quote, mode, acc)
when sep in [0xA8, 0xA9],
do: json5_string_escape(rest, quote, mode, acc, <<?\\, 0xE2, 0x80, sep>>, "")
defp json5_line_separator_escape(_rest, _quote, _mode, _acc), do: :error
defp json_string_escape(rest, quote, mode, acc, payload, decoded),
do: string(rest, quote, mode, [acc, {:json_string_escape, payload, decoded}])
defp json5_string_escape(rest, quote, mode, acc, payload, decoded),
do: string(rest, quote, mode, [acc, {:json5_string_escape, payload, decoded}])
defp finish_string(acc) do
cond do
string_has_json5_escape?(acc) ->
{payload, value} = json_string_parts(List.wrap(acc), [], [])
{:jsonb_escaped_text, 9, IO.iodata_to_binary(payload), IO.iodata_to_binary(value)}
string_has_json_escape?(acc) ->
{payload, value} = json_string_parts(List.wrap(acc), [], [])
{:jsonb_escaped_text, 8, IO.iodata_to_binary(payload), IO.iodata_to_binary(value)}
true ->
IO.iodata_to_binary(acc)
end
end
defp string_has_json5_escape?({:json5_string_escape, _payload, _value}), do: true
defp string_has_json5_escape?(list) when is_list(list),
do: Enum.any?(list, &string_has_json5_escape?/1)
defp string_has_json5_escape?(_part), do: false
defp string_has_json_escape?({:json_string_escape, _payload, _value}), do: true
defp string_has_json_escape?({:json5_string_escape, _payload, _value}), do: true
defp string_has_json_escape?(list) when is_list(list),
do: Enum.any?(list, &string_has_json_escape?/1)
defp string_has_json_escape?(_part), do: false
defp json_string_parts([], payload, value),
do: {Enum.reverse(payload), Enum.reverse(value)}
defp json_string_parts([{:json_string_escape, escape, decoded} | rest], payload, value) do
json_string_parts(rest, [escape | payload], [decoded | value])
end
defp json_string_parts([{:json5_string_escape, escape, decoded} | rest], payload, value) do
json_string_parts(rest, [escape | payload], [decoded | value])
end
defp json_string_parts([part | rest], payload, value) when is_list(part) do
{part_payload, part_value} = json_string_parts(part, [], [])
json_string_parts(rest, [part_payload | payload], [part_value | value])
end
defp json_string_parts([part | rest], payload, value) do
json_string_parts(rest, [part | payload], [part | value])
end
defp number(text, mode) do
{token, rest} = take_number(text, [])
token = IO.iodata_to_binary(token)
parse_number_token(token, rest, mode)
end
defp parse_number_token(token, rest, :canonical) do
cond do
String.match?(token, ~r/^-?\d+$/) ->
{:ok, String.to_integer(token), rest}
String.match?(token, ~r/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/) ->
case parse_real_literal(token) do
{:ok, value} -> {:ok, value, rest}
:error -> :error
end
true ->
:error
end
end
defp parse_number_token(token, rest, :json5) do
cond do
String.match?(token, ~r/^[+-]?0[xX][0-9a-fA-F]+$/) ->
sign = if String.starts_with?(token, "-"), do: -1, else: 1
unsigned = token |> String.trim_leading("+") |> String.trim_leading("-")
digits = binary_part(unsigned, 2, byte_size(unsigned) - 2)
{value, ""} = Integer.parse(digits, 16)
{:ok, sign * value, rest}
String.match?(token, ~r/^[+-]?(?:0|[1-9]\d*)$/) ->
{:ok, String.to_integer(token), rest}
String.match?(token, ~r/^[+-]?(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?$/) or
String.match?(token, ~r/^[+-]?\d+[eE][+-]?\d+$/) ->
case token |> normalize_json5_number() |> parse_real_literal() do
{:ok, value} -> {:ok, value, rest}
:error -> :error
end
true ->
:error
end
end
defp normalize_json5_number(<<"-.", rest::binary>>), do: "-0." <> rest
defp normalize_json5_number(<<"+.", rest::binary>>), do: "0." <> rest
defp normalize_json5_number(<<?., rest::binary>>), do: "0." <> rest
defp normalize_json5_number(<<?+, rest::binary>>), do: normalize_json5_number(rest)
defp normalize_json5_number(token) do
if String.ends_with?(token, ".") do
token <> "0"
else
token
end
end
defp parse_real_literal(token) do
case Float.parse(token) do
{_value, ""} ->
{:ok, {:real_literal, token}}
:error ->
if String.match?(token, ~r/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)[eE][+-]?\d+$/) do
{:ok, {:real_literal, token}}
else
:error
end
_other ->
:error
end
end
defp parse_jsonb_real_literal(5, token), do: parse_real_literal(token)
defp parse_jsonb_real_literal(6, "+" <> token) do
if String.contains?(token, ".") do
:error
else
parse_real_literal(token)
end
end
defp parse_jsonb_real_literal(6, token) do
token
|> normalize_json5_number()
|> parse_real_literal()
end
defp take_number(<<c, rest::binary>>, acc)
when c in ?0..?9 or c in [?-, ?+, ?., ?e, ?E, ?x, ?X] or c in ?a..?f or c in ?A..?F,
do: take_number(rest, [acc, c])
defp take_number(rest, acc), do: {acc, rest}
# -- rendering ---------------------------------------------------------------
@spec render(t()) :: binary()
def render(jv), do: jv |> encode() |> IO.iodata_to_binary()
@spec to_sqlite_jsonb(t()) :: binary()
def to_sqlite_jsonb(jv), do: jv |> sqlite_jsonb_encode() |> IO.iodata_to_binary()
@spec object_key_text(object_key()) :: binary()
def object_key_text({:jsonb_path_key, key}), do: key
def object_key_text({:jsonb_escaped_text, _type, _payload, key}), do: key
def object_key_text({:jsonb_escaped_key, _type, _payload, key}), do: key
def object_key_text(key), do: key
@spec pretty(t(), binary()) :: binary()
def pretty(jv, indent \\ " "), do: jv |> pretty_encode(indent, 0) |> IO.iodata_to_binary()
defp encode(:null), do: "null"
defp encode(true), do: "true"
defp encode(false), do: "false"
defp encode(:json_pos_inf), do: "9e999"
defp encode(:json_neg_inf), do: "-9e999"
defp encode(:json_nan), do: "null"
defp encode({:real_literal, token}), do: token
defp encode({:jsonb_integer_literal, 4, payload, _value}), do: encode_jsonb_int5(payload)
defp encode({:jsonb_real_literal, 6, token}), do: encode_jsonb_float5(token)
defp encode({:jsonb_real_literal, _type, token}), do: token
defp encode({:jsonb_text, 10, value}), do: [?", escape(value), ?"]
defp encode({:jsonb_escaped_text, 9, payload, _value}),
do: [?", canonical_json5_payload(payload), ?"]
defp encode({:jsonb_escaped_text, _type, payload, _value}), do: [?", payload, ?"]
defp encode(n) when is_integer(n), do: Integer.to_string(n)
defp encode(n) when is_float(n), do: Float.to_string(n)
defp encode(s) when is_binary(s), do: [?", escape(s), ?"]
defp encode({:array, items}), do: [?[, Enum.map_intersperse(items, ?,, &encode/1), ?]]
defp encode({:object, pairs}) do
[
?{,
Enum.map_intersperse(pairs, ?,, fn {k, v} ->
[object_key_encode(k), ?:, encode(v)]
end),
?}
]
end
defp encode_jsonb_int5(<<"-", _a, _b, rest::binary>>),
do: ["-", jsonb_int5_tail_to_decimal(rest)]
defp encode_jsonb_int5(<<"+", _a, _b, rest::binary>>), do: jsonb_int5_tail_to_decimal(rest)
defp encode_jsonb_int5(<<_a, _b, rest::binary>>), do: jsonb_int5_tail_to_decimal(rest)
defp encode_jsonb_int5(_payload), do: "0"
defp jsonb_int5_tail_to_decimal(tail) do
tail
|> :binary.bin_to_list()
|> Enum.reduce_while({0, false}, fn byte, {value, overflow?} ->
cond do
not hex_digit?(byte) ->
{:halt, {value, overflow?}}
Bitwise.bsr(value, 60) != 0 ->
{:cont, {value, true}}
true ->
{:cont, {value * 16 + hex_digit_value(byte), overflow?}}
end
end)
|> case do
{_value, true} -> "9.0e999"
{value, false} -> Integer.to_string(value)
end
end
defp hex_digit?(byte), do: hex_digit_value(byte) != :error
defp hex_digit_value(byte) when byte in ?0..?9, do: byte - ?0
defp hex_digit_value(byte) when byte in ?a..?f, do: byte - ?a + 10
defp hex_digit_value(byte) when byte in ?A..?F, do: byte - ?A + 10
defp hex_digit_value(_byte), do: :error
defp encode_jsonb_float5(<<"-", rest::binary>>), do: ["-", encode_jsonb_float5_unsigned(rest)]
defp encode_jsonb_float5(token), do: encode_jsonb_float5_unsigned(token)
defp encode_jsonb_float5_unsigned(<<?., _rest::binary>> = token),
do: ["0", jsonb_float5_dot_tail(token)]
defp encode_jsonb_float5_unsigned(token), do: jsonb_float5_dot_tail(token)
defp jsonb_float5_dot_tail(token) do
if String.ends_with?(token, ".") do
[token, "0"]
else
token
end
end
defp object_key_encode({:jsonb_escaped_text, 9, payload, _key}),
do: [?", canonical_json5_payload(payload), ?"]
defp object_key_encode({:jsonb_escaped_key, 9, payload, _key}),
do: [?", canonical_json5_payload(payload), ?"]
defp object_key_encode({:jsonb_escaped_key, _type, payload, _key}), do: [?", payload, ?"]
defp object_key_encode(key), do: [?", key |> object_key_text() |> escape(), ?"]
defp sqlite_jsonb_encode(:null), do: <<0x00>>
defp sqlite_jsonb_encode(true), do: <<0x01>>
defp sqlite_jsonb_encode(false), do: <<0x02>>
defp sqlite_jsonb_encode(n) when is_integer(n),
do: sqlite_jsonb_node(3, Integer.to_string(n))
defp sqlite_jsonb_encode(n) when is_float(n),
do: sqlite_jsonb_node(5, Float.to_string(n))
defp sqlite_jsonb_encode({:real_literal, token}), do: sqlite_jsonb_node(5, token)
defp sqlite_jsonb_encode({:jsonb_integer_literal, 4, payload, _value}),
do: sqlite_jsonb_node(4, payload)
defp sqlite_jsonb_encode({:jsonb_real_literal, type, token}),
do: sqlite_jsonb_node(type, token)
defp sqlite_jsonb_encode({:jsonb_escaped_text, type, payload, _value}),
do: sqlite_jsonb_node(type, payload)
defp sqlite_jsonb_encode({:jsonb_text, 10, value}), do: sqlite_jsonb_node(10, value)
defp sqlite_jsonb_encode(s) when is_binary(s) do
payload = escape(s)
type = if payload == s, do: 7, else: 8
sqlite_jsonb_node(type, payload)
end
defp sqlite_jsonb_encode({:jsonb_path_key, key}), do: sqlite_jsonb_node(10, key)
defp sqlite_jsonb_encode({:jsonb_escaped_key, type, payload, _key}),
do: sqlite_jsonb_node(type, payload)
defp sqlite_jsonb_encode({:array, items}) do
payload = Enum.map(items, &sqlite_jsonb_encode/1)
sqlite_jsonb_node(11, payload)
end
defp sqlite_jsonb_encode({:object, pairs}) do
payload =
Enum.map(pairs, fn {key, value} ->
[sqlite_jsonb_encode(key), sqlite_jsonb_encode(value)]
end)
sqlite_jsonb_node(12, payload)
end
defp sqlite_jsonb_node(type, payload) do
payload = IO.iodata_to_binary(payload)
size = byte_size(payload)
cond do
size <= 11 ->
<<Bitwise.bsl(size, 4) + type, payload::binary>>
size <= 0xFF ->
<<Bitwise.bsl(12, 4) + type, size, payload::binary>>
size <= 0xFFFF ->
<<Bitwise.bsl(13, 4) + type, size::16, payload::binary>>
size <= 0xFFFFFFFF ->
<<Bitwise.bsl(14, 4) + type, size::32, payload::binary>>
true ->
<<Bitwise.bsl(15, 4) + type, size::64, payload::binary>>
end
end
defp pretty_encode({:array, []}, _indent, _level), do: "[]"
defp pretty_encode({:array, items}, indent, level) do
child_prefix = pretty_indent(indent, level + 1)
prefix = pretty_indent(indent, level)
[
?[,
?\n,
Enum.map_intersperse(items, [?,, ?\n], fn item ->
[child_prefix, pretty_encode(item, indent, level + 1)]
end),
?\n,
prefix,
?]
]
end
defp pretty_encode({:object, []}, _indent, _level), do: "{}"
defp pretty_encode({:object, pairs}, indent, level) do
child_prefix = pretty_indent(indent, level + 1)
prefix = pretty_indent(indent, level)
[
?{,
?\n,
Enum.map_intersperse(pairs, [?,, ?\n], fn {key, value} ->
[
child_prefix,
object_key_encode(key),
?:,
?\s,
pretty_encode(value, indent, level + 1)
]
end),
?\n,
prefix,
?}
]
end
defp pretty_encode(jv, _indent, _level), do: encode(jv)
defp pretty_indent(indent, level), do: String.duplicate(indent, level)
defp escape(s) do
for <<c <- s>>, into: "" do
case c do
?" -> "\\\""
?\\ -> "\\\\"
?\n -> "\\n"
?\r -> "\\r"
?\t -> "\\t"
c when c < 0x20 -> "\\u" <> String.pad_leading(Integer.to_string(c, 16), 4, "0")
c -> <<c>>
end
end
end
defp canonical_json5_payload(<<?\\, ?x, h1, h2, rest::binary>>) do
["\\u00", <<h1>>, <<h2>>, canonical_json5_payload(rest)]
end
defp canonical_json5_payload(<<?\\, ?v, rest::binary>>),
do: ["\\u0009", canonical_json5_payload(rest)]
defp canonical_json5_payload(<<?\\, ?0, rest::binary>>),
do: ["\\u0000", canonical_json5_payload(rest)]
defp canonical_json5_payload(<<?\\, ?', rest::binary>>), do: [?', canonical_json5_payload(rest)]
defp canonical_json5_payload(<<?\\, ?\r, ?\n, rest::binary>>), do: canonical_json5_payload(rest)
defp canonical_json5_payload(<<?\\, ?\r, rest::binary>>), do: canonical_json5_payload(rest)
defp canonical_json5_payload(<<?\\, ?\n, rest::binary>>), do: canonical_json5_payload(rest)
defp canonical_json5_payload(<<?\\, 0xE2, 0x80, sep, rest::binary>>) when sep in [0xA8, 0xA9],
do: canonical_json5_payload(rest)
defp canonical_json5_payload(<<c, rest::binary>>), do: [c, canonical_json5_payload(rest)]
defp canonical_json5_payload(<<>>), do: []
# -- JSON paths ---------------------------------------------------------------
@doc ~S|Parses `$`, `$.key`, `$."quoted key"`, `$[2]`, `$[#-1]` paths.|
@spec parse_path(binary()) :: {:ok, [path_step()]} | :error
def parse_path(<<?$, rest::binary>>), do: path_steps(rest, [])
def parse_path(_), do: :error
defp path_steps("", acc), do: {:ok, Enum.reverse(acc)}
defp path_steps(<<?., ?", rest::binary>>, acc) do
case String.split(rest, "\"", parts: 2) do
[key, rest] -> path_steps(rest, [{:key, key} | acc])
_ -> :error
end
end
defp path_steps(<<?., rest::binary>>, acc) do
{key, rest} = take_path_key(rest, [])
if key == "", do: :error, else: path_steps(rest, [{:key, key} | acc])
end
# Bare `$[#]` is the one-past-last index: a no-op read, and the append slot
# for json_insert/json_set (SQLite uses it as `# == array length`).
defp path_steps(<<?[, ?#, ?], rest::binary>>, acc) do
path_steps(rest, [{:last_offset, 0} | acc])
end
defp path_steps(<<?[, ?#, rest::binary>>, acc) do
case Integer.parse(rest) do
{offset, <<?], rest::binary>>} when offset <= 0 ->
path_steps(rest, [{:last_offset, offset} | acc])
_ ->
:error
end
end
defp path_steps(<<?[, rest::binary>>, acc) do
case Integer.parse(rest) do
{index, <<?], rest::binary>>} when index >= 0 -> path_steps(rest, [{:index, index} | acc])
_ -> :error
end
end
defp path_steps(_, _acc), do: :error
defp take_path_key(<<c, _::binary>> = rest, acc) when c in [?., ?[],
do: {IO.iodata_to_binary(acc), rest}
defp take_path_key(<<c, rest::binary>>, acc), do: take_path_key(rest, [acc, c])
defp take_path_key(<<>>, acc), do: {IO.iodata_to_binary(acc), ""}
# -- navigation ---------------------------------------------------------------
@spec get(t(), [path_step()]) :: {:ok, t()} | :missing
def get(jv, []), do: {:ok, jv}
def get({:object, pairs}, [{:key, key} | rest]) do
case object_keyfind(pairs, key) do
{_key, jv} -> get(jv, rest)
nil -> :missing
end
end
def get({:array, items}, [{:index, index} | rest]) do
case Enum.at(items, index) do
nil -> :missing
jv -> get(jv, rest)
end
end
def get({:array, items}, [{:last_offset, offset} | rest]) do
case Enum.at(items, length(items) + offset) do
nil -> :missing
jv -> get(jv, rest)
end
end
def get(_jv, _steps), do: :missing
@doc """
Writes `value` at the path. `:insert` only creates missing leaves,
`:replace` only overwrites existing ones, `:set` does both. Missing
intermediate containers leave the document unchanged, as in SQLite.
"""
@spec write(t(), [path_step()], t(), :insert | :replace | :set) :: t()
# The root always exists: replace/set overwrite it, insert is a no-op.
def write(jv, [], _value, :insert), do: jv
def write(_jv, [], value, _mode), do: value
def write({:object, pairs}, [{:key, key}], value, mode) do
case {object_keyfind(pairs, key), mode} do
{nil, mode} when mode in [:insert, :set] ->
{:object, pairs ++ [{{:jsonb_path_key, key}, value}]}
{{existing_key, _v}, mode} when mode in [:replace, :set] ->
{:object, object_keyreplace(pairs, existing_key, value)}
_ ->
{:object, pairs}
end
end
def write({:array, items}, [{:index, index}], value, mode) do
cond do
index < length(items) and mode in [:replace, :set] ->
{:array, List.replace_at(items, index, value)}
index >= length(items) and mode in [:insert, :set] ->
{:array, items ++ [value]}
true ->
{:array, items}
end
end
def write({:array, items}, [{:last_offset, offset}], value, mode) do
write({:array, items}, [{:index, length(items) + offset}], value, mode)
end
def write({:object, pairs}, [{:key, key} | rest], value, mode) do
case object_keyfind(pairs, key) do
{existing_key, jv} ->
{:object, object_keyreplace(pairs, existing_key, write(jv, rest, value, mode))}
nil ->
{:object, pairs}
end
end
def write({:array, items}, [{:index, index} | rest], value, mode) do
case Enum.at(items, index) do
nil -> {:array, items}
jv -> {:array, List.replace_at(items, index, write(jv, rest, value, mode))}
end
end
def write({:array, items}, [{:last_offset, offset} | rest], value, mode) do
write({:array, items}, [{:index, length(items) + offset} | rest], value, mode)
end
def write(jv, _steps, _value, _mode), do: jv
@spec remove(t(), [path_step()]) :: t()
def remove({:object, pairs}, [{:key, key}]), do: {:object, object_keydelete(pairs, key)}
def remove({:array, items}, [{:index, index}]) when index < length(items),
do: {:array, List.delete_at(items, index)}
def remove({:array, items}, [{:last_offset, offset}]),
do: {:array, List.delete_at(items, length(items) + offset)}
def remove({:object, pairs}, [{:key, key} | rest]) do
case object_keyfind(pairs, key) do
{existing_key, jv} -> {:object, object_keyreplace(pairs, existing_key, remove(jv, rest))}
nil -> {:object, pairs}
end
end
def remove({:array, items}, [{:index, index} | rest]) do
case Enum.at(items, index) do
nil -> {:array, items}
jv -> {:array, List.replace_at(items, index, remove(jv, rest))}
end
end
def remove({:array, items}, [{:last_offset, offset} | rest]),
do: remove({:array, items}, [{:index, length(items) + offset} | rest])
def remove(jv, _steps), do: jv
@doc """
RFC 7386 MergePatch, as `json_patch()` implements it: patch objects merge
recursively (null members delete), anything else replaces the target.
"""
@spec merge_patch(t(), t()) :: t()
def merge_patch(target, {:object, patch_pairs}) do
base =
case target do
{:object, pairs} -> pairs
_other -> []
end
Enum.reduce(patch_pairs, {:object, base}, fn {key, value}, {:object, pairs} ->
case {value, object_keyfind(pairs, object_key_text(key))} do
{:null, _found} ->
{:object, object_keydelete(pairs, object_key_text(key))}
{value, nil} ->
{:object, pairs ++ [{key, merge_patch(:null, value)}]}
{value, {existing_key, existing}} ->
{:object, object_keyreplace(pairs, existing_key, merge_patch(existing, value))}
end
end)
end
def merge_patch(_target, patch), do: patch
defp object_keyfind(pairs, key) do
Enum.find(pairs, fn {existing_key, _value} -> object_key_text(existing_key) == key end)
end
defp object_keyreplace(pairs, key, value) do
Enum.map(pairs, fn
{^key, _old_value} -> {key, value}
other -> other
end)
end
defp object_keydelete(pairs, key) do
Enum.reject(pairs, fn {existing_key, _value} -> object_key_text(existing_key) == key end)
end
# -- SQL bridging --------------------------------------------------------------
@doc "The `json_type()` name for a JSON value."
@spec type_name(t()) :: binary()
def type_name(:null), do: "null"
def type_name(true), do: "true"
def type_name(false), do: "false"
def type_name(:json_pos_inf), do: "real"
def type_name(:json_neg_inf), do: "real"
def type_name(:json_nan), do: "null"
def type_name({:real_literal, _token}), do: "real"
def type_name({:jsonb_integer_literal, 4, _payload, _value}), do: "integer"
def type_name({:jsonb_real_literal, _type, _token}), do: "real"
def type_name({:jsonb_text, 10, _value}), do: "text"
def type_name({:jsonb_escaped_text, _type, _payload, _value}), do: "text"
def type_name(n) when is_integer(n), do: "integer"
def type_name(n) when is_float(n), do: "real"
def type_name(s) when is_binary(s), do: "text"
def type_name({:array, _}), do: "array"
def type_name({:object, _}), do: "object"
@doc "Converts a JSON value to its SQL representation (leaves stay scalar, containers render)."
@spec to_sql(t()) :: ExSQL.Value.t()
def to_sql(:null), do: nil
def to_sql(true), do: 1
def to_sql(false), do: 0
def to_sql(:json_pos_inf), do: "9e999"
def to_sql(:json_neg_inf), do: "-9e999"
def to_sql(:json_nan), do: nil
def to_sql({:real_literal, token}), do: real_literal_to_sql(token)
def to_sql({:jsonb_integer_literal, 4, _payload, value}), do: value
def to_sql({:jsonb_real_literal, 6, token}),
do: token |> normalize_json5_number() |> real_literal_to_sql()
def to_sql({:jsonb_real_literal, _type, token}), do: real_literal_to_sql(token)
def to_sql({:jsonb_text, 10, value}), do: value
def to_sql({:jsonb_escaped_text, _type, _payload, value}), do: value
def to_sql(n) when is_number(n), do: n
def to_sql(s) when is_binary(s), do: s
def to_sql(container), do: render(container)
defp real_literal_to_sql(token) do
case Float.parse(token) do
{value, ""} ->
value
:error ->
real_literal_overflow_to_sql(token)
_other ->
real_literal_overflow_to_sql(token)
end
end
defp real_literal_overflow_to_sql("-" <> _rest), do: "-9e999"
defp real_literal_overflow_to_sql(_token), do: "9e999"
end