defmodule ExSQL.Executor do
@moduledoc """
Statement execution against an `ExSQL.Database`.
Where SQLite compiles statements to VDBE bytecode and runs them in a
register VM (`vdbe.c`, 190 opcodes), this first implementation walks the
AST directly — the natural functional starting point, and the layer a
bytecode compiler can replace later without touching parsing or storage.
## Environments
Expressions evaluate inside an environment holding the current row of each
table in scope:
* `frames` — one frame per FROM source (table or subquery), each with the
source's alias, its column order, and the current row. Joins produce
envs whose frame list has one entry per joined source.
* `outer` — the enclosing query's env, for correlated subqueries; column
resolution falls back outward like SQLite's `resolveExprStep`.
* `group` — in aggregate queries, the list of member envs of the current
group; aggregate functions consume it, bare columns delegate to the
first member.
All functions are pure: they take a database value and return
`{:ok, result, new_database}` or `{:error, %ExSQL.Error{}}`.
"""
alias ExSQL.AST.{
AlterTable,
ColumnDef,
Compound,
CreateIndex,
CreateTable,
CreateTrigger,
CreateView,
Delete,
DropIndex,
DropTable,
DropView,
Insert,
Pragma,
Select,
Update,
Values,
With
}
alias ExSQL.{Database, DateTime, Error, Json, Parser, Result, Table, Value}
@aggregate_functions ~w(count sum avg total min max group_concat string_agg json_group_array json_group_object jsonb_group_array jsonb_group_object)
@window_functions ~w(row_number rank dense_rank ntile lag lead first_value last_value nth_value percent_rank cume_dist)
@rowid_names ~w(rowid oid _rowid_)
@max_trigger_depth 1000
@sqlite_version "3.51.0"
@supported_pragmas [
"analysis_limit",
"application_id",
"auto_vacuum",
"automatic_index",
"busy_timeout",
"cache_size",
"cache_spill",
"case_sensitive_like",
"cell_size_check",
"checkpoint_fullfsync",
"collation_list",
"compile_options",
"count_changes",
"data_version",
"database_list",
"default_cache_size",
"defer_foreign_keys",
"empty_result_callbacks",
"encoding",
"foreign_key_check",
"foreign_key_list",
"foreign_keys",
"freelist_count",
"full_column_names",
"fullfsync",
"function_list",
"hard_heap_limit",
"ignore_check_constraints",
"incremental_vacuum",
"index_info",
"index_list",
"index_xinfo",
"integrity_check",
"journal_mode",
"journal_size_limit",
"locking_mode",
"max_page_count",
"mmap_size",
"module_list",
"optimize",
"page_count",
"page_size",
"pragma_list",
"query_only",
"quick_check",
"read_uncommitted",
"recursive_triggers",
"reverse_unordered_selects",
"schema_version",
"secure_delete",
"short_column_names",
"shrink_memory",
"soft_heap_limit",
"stats",
"synchronous",
"table_info",
"table_list",
"table_xinfo",
"temp_store",
"threads",
"trusted_schema",
"user_version",
"wal_autocheckpoint",
"wal_checkpoint"
]
@compile_options [
"DEFAULT_CACHE_SIZE=2000",
"DEFAULT_JOURNAL_SIZE_LIMIT=32768",
"DEFAULT_PAGE_SIZE=4096",
"DEFAULT_RECURSIVE_TRIGGERS=0",
"DEFAULT_SYNCHRONOUS=2",
"DEFAULT_WAL_AUTOCHECKPOINT=1000",
"ENABLE_JSON1",
"ENABLE_MATH_FUNCTIONS",
"ENABLE_WINDOW_FUNCTIONS",
"MAX_FUNCTION_ARG=127",
"MAX_VARIABLE_NUMBER=32766",
"THREADSAFE=0"
]
# Known scalar functions and the argument counts they accept, for SQLite's
# distinction between "no such function" and "wrong number of arguments".
@scalar_arity %{
"abs" => 1..1,
"acos" => 1..1,
"acosh" => 1..1,
"asin" => 1..1,
"asinh" => 1..1,
"atan" => 1..1,
"atan2" => 2..2,
"atanh" => 1..1,
"char" => 0..127,
"ceil" => 1..1,
"ceiling" => 1..1,
"changes" => 0..0,
"coalesce" => 2..127,
"concat" => 1..127,
"concat_ws" => 2..127,
"cos" => 1..1,
"cosh" => 1..1,
"degrees" => 1..1,
"exp" => 1..1,
"floor" => 1..1,
"glob" => 2..2,
"format" => 1..127,
"hex" => 1..1,
"ifnull" => 2..2,
"iif" => 3..3,
"instr" => 2..2,
"json" => 1..1,
"jsonb" => 1..1,
"jsonb_array" => 0..127,
"jsonb_extract" => 2..127,
"jsonb_insert" => 3..127,
"jsonb_object" => 0..127,
"jsonb_patch" => 2..2,
"jsonb_remove" => 2..127,
"jsonb_replace" => 3..127,
"jsonb_set" => 3..127,
"json_array" => 0..127,
"json_array_length" => 1..2,
"json_extract" => 2..127,
"json_insert" => 3..127,
"json_object" => 0..127,
"json_patch" => 2..2,
"json_pretty" => 1..2,
"json_quote" => 1..1,
"json_remove" => 2..127,
"json_replace" => 3..127,
"json_set" => 3..127,
"json_type" => 1..2,
"json_valid" => 1..2,
"length" => 1..1,
"last_insert_rowid" => 0..0,
"like" => 2..3,
"ln" => 1..1,
"log" => 1..2,
"log10" => 1..1,
"log2" => 1..1,
"lower" => 1..1,
"ltrim" => 1..2,
"match" => 2..2,
"max" => 1..127,
"min" => 1..127,
"mod" => 2..2,
"nullif" => 2..2,
"octet_length" => 1..1,
"pi" => 0..0,
"pow" => 2..2,
"power" => 2..2,
"printf" => 1..127,
"quote" => 1..1,
"radians" => 1..1,
"random" => 0..0,
"randomblob" => 1..1,
"regexp" => 2..2,
"replace" => 3..3,
"round" => 1..2,
"rtrim" => 1..2,
"sign" => 1..1,
"sin" => 1..1,
"sinh" => 1..1,
"sqlite_version" => 0..0,
"sqrt" => 1..1,
"substr" => 2..3,
"substring" => 2..3,
"tan" => 1..1,
"tanh" => 1..1,
"trim" => 1..2,
"total_changes" => 0..0,
"trunc" => 1..1,
"typeof" => 1..1,
"unhex" => 1..2,
"unicode" => 1..1,
"upper" => 1..1,
"zeroblob" => 1..1,
# Date/time functions (date.c)
"date" => 1..127,
"time" => 1..127,
"timediff" => 2..2,
"datetime" => 0..127,
"julianday" => 1..127,
"unixepoch" => 1..127,
"strftime" => 2..127
}
@explain_literal_scalar_functions ~w(abs acos acosh asin asinh atan atan2 atanh ceil ceiling char concat concat_ws cos cosh degrees exp floor format hex instr length ln log log10 log2 lower ltrim mod octet_length pi pow power printf quote radians replace round rtrim sign sin sinh sqrt substr substring tan tanh trim trunc typeof unicode upper zeroblob)
@doc """
Parses and executes every statement in `sql`, threading the database
through. Returns the results in statement order. A failing statement stops
execution; effects of prior statements in the same string are kept (as with
`sqlite3_exec`), so the error tuple carries the database too.
"""
@spec run(Database.t(), String.t(), [Value.t()] | map()) ::
{:ok, [Result.t()], Database.t()} | {:error, Error.t(), Database.t()}
def run(db, sql, params \\ []) do
case Parser.parse(sql) do
{:ok, statements} ->
statements
|> Enum.reduce_while({:ok, [], db}, fn stmt, {:ok, results, db} ->
case bind_and_execute(db, stmt, params) do
{:ok, result, db} ->
{:cont, {:ok, [result | results], db}}
# OR FAIL / OR ROLLBACK leave effects behind even though the
# statement errors; the error carries that database state.
{:error, %Error{db: %Database{} = error_db} = error} ->
{:halt, {:error, %{error | db: nil}, error_db}}
{:error, error} ->
{:halt, {:error, error, db}}
end
end)
|> case do
{:ok, results, db} -> {:ok, Enum.reverse(results), db}
error -> error
end
{:error, error} ->
{:error, error, db}
end
end
defp bind_and_execute(db, stmt, params) do
execute(db, bind_parameters(stmt, params))
rescue
e in Error -> {:error, e}
end
# Substitutes bound values for `{:param, index, name}` expressions across a
# statement. Indexes were assigned in source order by the tokenizer;
# parameters left unbound evaluate to NULL, as in the C API.
defp bind_parameters(stmt, params) when params == [] or params == %{}, do: stmt
defp bind_parameters(stmt, params), do: bind_walk(stmt, params)
defp bind_walk({:param, index, raw}, params), do: {:literal, param_value(params, index, raw)}
defp bind_walk(tuple, params) when is_tuple(tuple) do
tuple |> Tuple.to_list() |> Enum.map(&bind_walk(&1, params)) |> List.to_tuple()
end
defp bind_walk(list, params) when is_list(list), do: Enum.map(list, &bind_walk(&1, params))
defp bind_walk(%module{} = node, params) do
struct!(
module,
node
|> Map.from_struct()
|> Enum.map(fn {key, value} -> {key, bind_walk(value, params)} end)
)
end
defp bind_walk(%{} = map, params) do
Map.new(map, fn {key, value} -> {key, bind_walk(value, params)} end)
end
defp bind_walk(other, _params), do: other
defp param_value(params, index, _raw) when is_list(params) do
validate_param_value!(Enum.at(params, index - 1))
end
defp param_value(params, index, raw) when is_map(params) do
bare =
case raw do
<<sigil, name::binary>> when sigil in [?:, ?@, ?$] -> name
_ -> nil
end
atom_key = bare && safe_existing_atom(bare)
value =
cond do
Map.has_key?(params, raw) -> Map.get(params, raw)
bare != nil and Map.has_key?(params, bare) -> Map.get(params, bare)
atom_key != nil and Map.has_key?(params, atom_key) -> Map.get(params, atom_key)
Map.has_key?(params, index) -> Map.get(params, index)
true -> nil
end
validate_param_value!(value)
end
defp safe_existing_atom(name) do
String.to_existing_atom(name)
rescue
ArgumentError -> nil
end
defp validate_param_value!(value)
when is_nil(value) or is_integer(value) or is_float(value) or is_binary(value),
do: value
defp validate_param_value!({:blob, bin} = blob) when is_binary(bin), do: blob
defp validate_param_value!(other) do
fail("invalid bind value: #{inspect(other)}")
end
@doc "Executes a single parsed statement."
@spec execute(Database.t(), Parser.statement()) ::
{:ok, Result.t(), Database.t()} | {:error, Error.t()}
def execute(db, stmt) do
if db.query_only and query_only_write_statement?(stmt) do
fail("attempt to write a readonly database")
end
{result, db} = exec(db, stmt)
{:ok, result, db}
rescue
e in Error -> {:error, e}
end
defp query_only_write_statement?(%stmt{})
when stmt in [
AlterTable,
CreateIndex,
CreateTable,
CreateTrigger,
CreateView,
Delete,
DropIndex,
DropTable,
DropView,
Insert,
Update
],
do: true
defp query_only_write_statement?(%With{query: query}), do: query_only_write_statement?(query)
defp query_only_write_statement?(%Pragma{name: name, arg: arg}) do
arg != nil and name in ["schema_version", "user_version", "application_id"]
end
defp query_only_write_statement?({:drop_trigger, _schema, _name, _if_exists}), do: true
defp query_only_write_statement?({kind, _name}) when kind in [:analyze, :vacuum, :reindex],
do: true
defp query_only_write_statement?(_stmt), do: false
defp ctas_column_names(names) do
names
|> Enum.with_index(1)
|> Enum.reduce({[], %{}}, fn {name, index}, {acc, seen} ->
base = if is_binary(name) and name != "", do: name, else: "column#{index}"
{name, seen} = unique_ctas_column_name(base, seen)
{[name | acc], seen}
end)
|> elem(0)
|> Enum.reverse()
end
defp unique_ctas_column_name(base, seen) do
key = Table.key(base)
count = Map.get(seen, key, 0)
name = if count == 0, do: base, else: "#{base}:#{count}"
{name, Map.put(seen, key, count + 1)}
end
# -- CREATE TABLE ------------------------------------------------------------
defp exec(db, %Pragma{name: "database_list", arg: {:schema, schema, _arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "database_list", arg: nil})
end
defp exec(db, %Pragma{name: "database_list", arg: nil}) do
{%Result{
command: :select,
columns: ["seq", "name", "file"],
rows: database_list_rows(db),
rows_affected: 0
}, db}
end
defp exec(db, {:attach, filename_expr, name}) do
key = attached_database_key(name)
cond do
key in ["main", "temp"] or attached_database?(db, key) ->
fail("database #{name} is already in use")
true ->
filename =
filename_expr
|> eval(%{db: db, frames: [], outer: nil, group: nil})
|> attach_filename()
attached = %{seq: next_attached_database_seq(db), name: name, file: filename}
{%Result{command: :attach},
%{db | attached_databases: db.attached_databases ++ [attached]}}
end
end
defp exec(db, {:detach, name}) do
key = attached_database_key(name)
cond do
key in ["main", "temp"] ->
fail("cannot detach database #{name}")
not attached_database?(db, key) ->
fail("no such database: #{name}")
true ->
attached =
Enum.reject(db.attached_databases, fn attached ->
attached_database_key(attached.name) == key
end)
tables =
Map.reject(db.tables, fn {_table_key, table} ->
table.schema != nil and attached_database_key(table.schema) == key
end)
views =
Map.reject(db.views, fn {_view_key, view} ->
view.schema != nil and attached_database_key(view.schema) == key
end)
triggers =
Map.reject(db.triggers, fn {_trigger_key, trigger} ->
trigger.schema != nil and attached_database_key(trigger.schema) == key
end)
{%Result{command: :detach},
db
|> Map.merge(%{
attached_databases: attached,
tables: tables,
views: views,
triggers: triggers
})
|> Database.drop_schema_header(name)}
end
end
defp exec(db, %Pragma{name: name, arg: {:schema, schema, arg}} = pragma)
when name not in [
"application_id",
"foreign_key_check",
"foreign_key_list",
"index_info",
"index_list",
"index_xinfo",
"integrity_check",
"page_count",
"quick_check",
"schema_version",
"table_info",
"table_list",
"table_xinfo",
"user_version"
] do
ensure_schema_exists!(db, schema)
exec(db, %{pragma | arg: arg})
end
defp exec(db, %Pragma{name: "collation_list"}) do
rows =
db
|> collation_list_rows()
|> Enum.with_index()
|> Enum.map(fn {name, seq} -> [seq, name] end)
{%Result{
command: :select,
columns: ["seq", "name"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "compile_options"}) do
rows = Enum.map(@compile_options, &[&1])
{%Result{
command: :select,
columns: ["compile_options"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "function_list"}) do
{%Result{
command: :select,
columns: ["name", "builtin", "type", "enc", "narg", "flags"],
rows: function_list_rows(db),
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "module_list"}) do
{%Result{
command: :select,
columns: ["name"],
rows: [["json_each"], ["json_tree"]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "pragma_list"}) do
rows = Enum.map(@supported_pragmas, &[&1])
{%Result{
command: :select,
columns: ["name"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "encoding", arg: nil}) do
{%Result{
command: :select,
columns: ["encoding"],
rows: [["UTF-8"]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "encoding", arg: _arg}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "foreign_keys", arg: nil}) do
{%Result{
command: :select,
columns: ["foreign_keys"],
rows: [[bool_int(db.foreign_keys)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "foreign_keys", arg: arg}) do
# As in SQLite, foreign-key enforcement may only be toggled outside a
# transaction; within one the pragma is a no-op.
db =
if db.txn_stack == [] do
%{db | foreign_keys: pragma_enabled?(arg)}
else
db
end
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "defer_foreign_keys", arg: nil}) do
{%Result{
command: :select,
columns: ["defer_foreign_keys"],
rows: [[bool_int(db.defer_foreign_keys)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "defer_foreign_keys", arg: arg}) do
{%Result{command: :pragma}, %{db | defer_foreign_keys: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "recursive_triggers", arg: nil}) do
{%Result{
command: :select,
columns: ["recursive_triggers"],
rows: [[bool_int(db.recursive_triggers)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "recursive_triggers", arg: arg}) do
{%Result{command: :pragma}, %{db | recursive_triggers: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "ignore_check_constraints", arg: nil}) do
{%Result{
command: :select,
columns: ["ignore_check_constraints"],
rows: [[bool_int(db.ignore_check_constraints)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "ignore_check_constraints", arg: arg}) do
{%Result{command: :pragma}, %{db | ignore_check_constraints: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "count_changes", arg: nil}) do
{%Result{
command: :select,
columns: ["count_changes"],
rows: [[bool_int(db.count_changes)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "count_changes", arg: arg}) do
{%Result{command: :pragma}, %{db | count_changes: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "read_uncommitted", arg: nil}) do
{%Result{
command: :select,
columns: ["read_uncommitted"],
rows: [[bool_int(db.read_uncommitted)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "read_uncommitted", arg: arg}) do
{%Result{command: :pragma}, %{db | read_uncommitted: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "case_sensitive_like", arg: nil}) do
{%Result{
command: :select,
columns: ["case_sensitive_like"],
rows: [[bool_int(db.case_sensitive_like)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "case_sensitive_like", arg: arg}) do
{%Result{command: :pragma}, %{db | case_sensitive_like: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "short_column_names", arg: nil}) do
{%Result{
command: :select,
columns: ["short_column_names"],
rows: [[bool_int(db.short_column_names)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "short_column_names", arg: arg}) do
{%Result{command: :pragma}, %{db | short_column_names: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "full_column_names", arg: nil}) do
{%Result{
command: :select,
columns: ["full_column_names"],
rows: [[bool_int(db.full_column_names)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "full_column_names", arg: arg}) do
{%Result{command: :pragma}, %{db | full_column_names: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "reverse_unordered_selects", arg: nil}) do
{%Result{
command: :select,
columns: ["reverse_unordered_selects"],
rows: [[bool_int(db.reverse_unordered_selects)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "reverse_unordered_selects", arg: arg}) do
{%Result{command: :pragma}, %{db | reverse_unordered_selects: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "query_only", arg: nil}) do
{%Result{
command: :select,
columns: ["query_only"],
rows: [[bool_int(db.query_only)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "query_only", arg: arg}) do
{%Result{command: :pragma}, %{db | query_only: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "empty_result_callbacks", arg: nil}) do
pragma_bool_result("empty_result_callbacks", db.empty_result_callbacks, db)
end
defp exec(db, %Pragma{name: "empty_result_callbacks", arg: arg}) do
{%Result{command: :pragma}, %{db | empty_result_callbacks: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "automatic_index", arg: nil}) do
{%Result{
command: :select,
columns: ["automatic_index"],
rows: [[bool_int(db.automatic_index)]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "automatic_index", arg: arg}) do
{%Result{command: :pragma}, %{db | automatic_index: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "auto_vacuum", arg: nil}) do
pragma_integer_result("auto_vacuum", db.auto_vacuum, db)
end
defp exec(db, %Pragma{name: "auto_vacuum", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "auto_vacuum", arg: arg})
end
defp exec(db, %Pragma{name: "auto_vacuum", arg: arg}) do
db =
if db.page_size_locked do
db
else
%{db | auto_vacuum: pragma_auto_vacuum(arg, db.auto_vacuum)}
end
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "incremental_vacuum"}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "shrink_memory"}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: name, arg: {:schema, schema, arg}})
when name in [
"cache_spill",
"data_version",
"default_cache_size",
"mmap_size",
"secure_delete",
"wal_autocheckpoint"
] do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: name, arg: arg})
end
defp exec(db, %Pragma{name: "mmap_size"}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "analysis_limit", arg: nil}) do
pragma_integer_result("analysis_limit", db.analysis_limit, db)
end
defp exec(db, %Pragma{name: "analysis_limit", arg: arg}) do
value = pragma_analysis_limit(arg, db.analysis_limit)
db = %{db | analysis_limit: value}
pragma_integer_result("analysis_limit", value, db)
end
defp exec(db, %Pragma{name: "cell_size_check", arg: nil}) do
pragma_bool_result("cell_size_check", db.cell_size_check, db)
end
defp exec(db, %Pragma{name: "cell_size_check", arg: arg}) do
{%Result{command: :pragma}, %{db | cell_size_check: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "checkpoint_fullfsync", arg: nil}) do
pragma_bool_result("checkpoint_fullfsync", db.checkpoint_fullfsync, db)
end
defp exec(db, %Pragma{name: "checkpoint_fullfsync", arg: arg}) do
{%Result{command: :pragma}, %{db | checkpoint_fullfsync: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "fullfsync", arg: nil}) do
pragma_bool_result("fullfsync", db.fullfsync, db)
end
defp exec(db, %Pragma{name: "fullfsync", arg: arg}) do
{%Result{command: :pragma}, %{db | fullfsync: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "trusted_schema", arg: nil}) do
pragma_bool_result("trusted_schema", db.trusted_schema, db)
end
defp exec(db, %Pragma{name: "trusted_schema", arg: arg}) do
{%Result{command: :pragma}, %{db | trusted_schema: pragma_enabled?(arg)}}
end
defp exec(db, %Pragma{name: "busy_timeout", arg: nil}) do
pragma_integer_result("busy_timeout", db.busy_timeout, db)
end
defp exec(db, %Pragma{name: "busy_timeout", arg: arg}) do
value = non_negative_pragma_integer(arg)
db = %{db | busy_timeout: value}
pragma_integer_result("busy_timeout", value, db)
end
defp exec(db, %Pragma{name: "page_count", arg: {:schema, schema, nil}}) do
ensure_schema_exists!(db, schema)
pragma_integer_result("page_count", pragma_page_count(db, schema), db)
end
defp exec(db, %Pragma{name: "page_count", arg: {:schema, schema, _arg}}) do
ensure_schema_exists!(db, schema)
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "page_count", arg: nil}) do
pragma_integer_result("page_count", pragma_page_count(db, nil), db)
end
defp exec(db, %Pragma{name: "page_count", arg: _arg}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "page_size", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "page_size", arg: arg})
end
defp exec(db, %Pragma{name: "page_size", arg: nil}) do
{%Result{
command: :select,
columns: ["page_size"],
rows: [[db.page_size]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "page_size", arg: arg}) do
db =
if db.page_size_locked do
db
else
case pragma_page_size(arg) do
nil -> db
page_size -> %{db | page_size: page_size}
end
end
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "cache_size", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "cache_size", arg: arg})
end
defp exec(db, %Pragma{name: "cache_size", arg: nil}) do
{%Result{
command: :select,
columns: ["cache_size"],
rows: [[db.cache_size]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "cache_size", arg: arg}) do
{%Result{command: :pragma}, %{db | cache_size: pragma_cache_size(arg)}}
end
defp exec(db, %Pragma{name: "default_cache_size", arg: nil}) do
pragma_integer_result("default_cache_size", db.default_cache_size, db)
end
defp exec(db, %Pragma{name: "default_cache_size", arg: arg}) do
value = pragma_default_cache_size(arg)
db = %{db | default_cache_size: value}
pragma_integer_result("default_cache_size", value, db)
end
defp exec(db, %Pragma{name: "cache_spill", arg: nil}) do
pragma_integer_result("cache_spill", db.cache_spill, db)
end
defp exec(db, %Pragma{name: "cache_spill", arg: arg}) do
value = pragma_cache_spill(arg, db.cache_spill)
db = %{db | cache_spill: value}
pragma_integer_result("cache_spill", value, db)
end
defp exec(db, %Pragma{name: "max_page_count", arg: nil}) do
pragma_integer_result("max_page_count", db.max_page_count, db)
end
defp exec(db, %Pragma{name: "max_page_count", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "max_page_count", arg: arg})
end
defp exec(db, %Pragma{name: "max_page_count", arg: arg}) do
value = pragma_max_page_count(arg, db.max_page_count)
db = %{db | max_page_count: value}
pragma_integer_result("max_page_count", value, db)
end
defp exec(db, %Pragma{name: "journal_mode", arg: nil}) do
journal_mode_result(db.journal_mode, db)
end
defp exec(db, %Pragma{name: "journal_mode", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "journal_mode", arg: arg})
end
defp exec(db, %Pragma{name: "journal_mode", arg: arg}) do
mode = pragma_journal_mode(arg, db.journal_mode)
db = %{db | journal_mode: mode}
journal_mode_result(mode, db)
end
defp exec(db, %Pragma{name: "journal_size_limit", arg: nil}) do
pragma_integer_result("journal_size_limit", db.journal_size_limit, db)
end
defp exec(db, %Pragma{name: "journal_size_limit", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "journal_size_limit", arg: arg})
end
defp exec(db, %Pragma{name: "journal_size_limit", arg: arg}) do
value = pragma_journal_size_limit(arg)
db = %{db | journal_size_limit: value}
pragma_integer_result("journal_size_limit", value, db)
end
defp exec(db, %Pragma{name: "locking_mode", arg: nil}) do
locking_mode_result(db.locking_mode, db)
end
defp exec(db, %Pragma{name: "locking_mode", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "locking_mode", arg: arg})
end
defp exec(db, %Pragma{name: "locking_mode", arg: arg}) do
mode = pragma_locking_mode(arg, db.locking_mode)
db = %{db | locking_mode: mode}
locking_mode_result(mode, db)
end
defp exec(db, %Pragma{name: "synchronous", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "synchronous", arg: arg})
end
defp exec(db, %Pragma{name: "synchronous", arg: nil}) do
{%Result{
command: :select,
columns: ["synchronous"],
rows: [[db.synchronous]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "synchronous", arg: arg}) do
{%Result{command: :pragma}, %{db | synchronous: pragma_synchronous(arg)}}
end
defp exec(db, %Pragma{name: "temp_store", arg: nil}) do
{%Result{
command: :select,
columns: ["temp_store"],
rows: [[db.temp_store]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "temp_store", arg: {:schema, schema, arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "temp_store", arg: arg})
end
defp exec(db, %Pragma{name: "temp_store", arg: arg}) do
{%Result{command: :pragma}, %{db | temp_store: pragma_temp_store(arg)}}
end
defp exec(db, %Pragma{name: "soft_heap_limit", arg: nil}) do
pragma_integer_result("soft_heap_limit", db.soft_heap_limit, db)
end
defp exec(db, %Pragma{name: "soft_heap_limit", arg: arg}) do
value = non_negative_pragma_integer(arg)
db = %{db | soft_heap_limit: value}
pragma_integer_result("soft_heap_limit", value, db)
end
defp exec(db, %Pragma{name: "hard_heap_limit"}) do
pragma_integer_result("hard_heap_limit", 0, db)
end
defp exec(db, %Pragma{name: "secure_delete", arg: nil}) do
pragma_integer_result("secure_delete", db.secure_delete, db)
end
defp exec(db, %Pragma{name: "secure_delete", arg: arg}) do
value = pragma_secure_delete(arg)
db = %{db | secure_delete: value}
pragma_integer_result("secure_delete", value, db)
end
defp exec(db, %Pragma{name: "threads", arg: nil}) do
pragma_integer_result("threads", db.threads, db)
end
defp exec(db, %Pragma{name: "threads", arg: arg}) do
value = pragma_threads(arg, db.threads)
db = %{db | threads: value}
pragma_integer_result("threads", value, db)
end
defp exec(db, %Pragma{name: "wal_autocheckpoint", arg: nil}) do
pragma_integer_result("wal_autocheckpoint", db.wal_autocheckpoint, db)
end
defp exec(db, %Pragma{name: "wal_autocheckpoint", arg: arg}) do
value = non_negative_pragma_integer(arg)
db = %{db | wal_autocheckpoint: value}
pragma_integer_result("wal_autocheckpoint", value, db)
end
defp exec(db, %Pragma{name: "wal_checkpoint"}) do
{%Result{
command: :select,
columns: ["busy", "log", "checkpointed"],
rows: [[0, -1, -1]],
rows_affected: 0,
affinities: [:integer, :integer, :integer]
}, db}
end
defp exec(db, %Pragma{name: "optimize"}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: "stats"}) do
{%Result{command: :pragma}, db}
end
defp exec(db, %Pragma{name: name, arg: {:schema, schema, nil}})
when name in ["schema_version", "user_version", "application_id"] do
ensure_schema_exists!(db, schema)
field = pragma_header_field(name)
pragma_integer_result(name, Database.schema_header_value(db, schema, field), db)
end
defp exec(db, %Pragma{name: name, arg: {:schema, schema, arg}})
when name in ["schema_version", "user_version", "application_id"] do
ensure_schema_exists!(db, schema)
field = pragma_header_field(name)
value = pragma_header_value(arg)
{%Result{command: :pragma}, Database.put_schema_header_value(db, schema, field, value)}
end
defp exec(db, %Pragma{name: "schema_version", arg: nil}) do
{%Result{
command: :select,
columns: ["schema_version"],
rows: [[db.schema_version]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "schema_version", arg: arg}) do
{%Result{command: :pragma}, %{db | schema_version: pragma_header_value(arg)}}
end
defp exec(db, %Pragma{name: "user_version", arg: nil}) do
{%Result{
command: :select,
columns: ["user_version"],
rows: [[db.user_version]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "user_version", arg: arg}) do
{%Result{command: :pragma}, %{db | user_version: pragma_header_value(arg)}}
end
defp exec(db, %Pragma{name: "application_id", arg: nil}) do
{%Result{
command: :select,
columns: ["application_id"],
rows: [[db.application_id]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "application_id", arg: arg}) do
{%Result{command: :pragma}, %{db | application_id: pragma_header_value(arg)}}
end
defp exec(db, %Pragma{name: "data_version"}) do
pragma_integer_result("data_version", 2, db)
end
defp exec(db, %Pragma{name: "freelist_count", arg: {:schema, schema, _arg}}) do
ensure_schema_exists!(db, schema)
exec(db, %Pragma{name: "freelist_count", arg: nil})
end
defp exec(db, %Pragma{name: "freelist_count"}) do
{%Result{
command: :select,
columns: ["freelist_count"],
rows: [[0]],
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "table_list", arg: arg}) do
rows =
db
|> table_list_rows()
|> filter_pragma_table_list(db, arg)
{%Result{
command: :select,
columns: ["schema", "name", "type", "ncol", "wr", "strict"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "integrity_check", arg: arg}) do
rows = integrity_check_rows(db, arg)
{%Result{
command: :select,
columns: ["integrity_check"],
rows: if(rows == [], do: [["ok"]], else: rows),
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "quick_check", arg: arg}) do
rows = integrity_check_rows(db, arg)
{%Result{
command: :select,
columns: ["quick_check"],
rows: if(rows == [], do: [["ok"]], else: rows),
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "table_info", arg: table_name}) do
rows =
case pragma_fetch_table(db, table_name) do
{:ok, table} -> table_info_rows(table, false)
:error -> []
end
{%Result{
command: :select,
columns: ["cid", "name", "type", "notnull", "dflt_value", "pk"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "table_xinfo", arg: table_name}) do
rows =
case pragma_fetch_table(db, table_name) do
{:ok, table} -> table_info_rows(table, true)
:error -> []
end
{%Result{
command: :select,
columns: ["cid", "name", "type", "notnull", "dflt_value", "pk", "hidden"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "foreign_key_list", arg: table_name}) do
rows =
case pragma_fetch_table(db, table_name) do
{:ok, table} -> foreign_key_list_rows(table)
:error -> []
end
{%Result{
command: :select,
columns: ["id", "seq", "table", "from", "to", "on_update", "on_delete", "match"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "foreign_key_check", arg: table_name}) do
rows = foreign_key_check_rows(db, table_name)
{%Result{
command: :select,
columns: ["table", "rowid", "parent", "fkid"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "index_list", arg: table_name}) do
rows =
case pragma_fetch_table(db, table_name) do
{:ok, table} -> index_list_rows(table)
:error -> []
end
{%Result{
command: :select,
columns: ["seq", "name", "unique", "origin", "partial"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "index_info", arg: index_name}) do
rows =
case pragma_find_index_owner(db, index_name) do
{table, index} -> index_info_rows(table, index)
nil -> []
end
{%Result{
command: :select,
columns: ["seqno", "cid", "name"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %Pragma{name: "index_xinfo", arg: index_name}) do
rows =
case pragma_find_index_owner(db, index_name) do
{table, index} -> index_xinfo_rows(table, index)
nil -> []
end
{%Result{
command: :select,
columns: ["seqno", "cid", "name", "desc", "coll", "key"],
rows: rows,
rows_affected: 0
}, db}
end
defp exec(db, %CreateTable{query: query} = stmt) when query != nil do
ensure_schema_exists!(db, stmt.schema)
key = Database.table_storage_key(stmt.schema, stmt.name)
if stmt.if_not_exists and (Map.has_key?(db.tables, key) or Map.has_key?(db.views, key)) do
{%Result{command: :create_table}, db}
else
result = query_result(db, query, nil)
column_names = ctas_column_names(result.columns)
columns = Enum.map(column_names, &%ColumnDef{name: &1, affinity: :blob})
table =
result.rows
|> Enum.reduce(Table.new(stmt.name, columns, schema: stmt.schema), fn row, table ->
values =
column_names
|> Enum.map(&Table.key/1)
|> Enum.zip(row)
|> Map.new()
case Table.insert(table, values) do
{:ok, table, _rowid} -> table
{:error, message} -> fail(message)
:ignore -> table
end
end)
case Database.create_table(db, table) do
{:ok, db} -> {%Result{command: :create_table}, db}
{:error, "there is already an index named " <> _ = message} -> fail(message)
{:error, _message} when stmt.if_not_exists -> {%Result{command: :create_table}, db}
{:error, message} -> fail(message)
end
end
end
defp exec(db, %CreateTable{} = stmt) do
ensure_schema_exists!(db, stmt.schema)
ensure_unique_names(stmt)
ensure_valid_autoincrement!(stmt)
ensure_without_rowid_primary_key!(stmt)
ensure_valid_strict_types!(stmt)
ensure_valid_check_constraints!(stmt.name, stmt.columns, stmt.constraints)
{composite_keys, composite_uniques} = partition_table_constraints(stmt.constraints)
table_checks = table_check_constraints(stmt.constraints)
foreign_keys = table_foreign_keys(stmt.constraints)
# Collect column-level CHECK constraints too
column_checks =
for col <- stmt.columns, col.check != nil do
{col.check_name, col.check}
end
table =
Table.new(stmt.name, stmt.columns,
schema: stmt.schema,
composite_keys: composite_keys,
composite_uniques: composite_uniques,
foreign_keys: foreign_keys,
checks: column_checks ++ table_checks,
without_rowid: stmt.without_rowid,
strict: stmt.strict
)
|> put_autoindexes(stmt.constraints)
case Database.create_table(db, table) do
{:ok, db} ->
{%Result{command: :create_table}, db}
{:error, "there is already an index named " <> _ = message} ->
fail(message)
{:error, _message} when stmt.if_not_exists ->
{%Result{command: :create_table}, db}
{:error, message} ->
fail(message)
end
end
defp exec(db, %DropTable{} = stmt) do
ensure_schema_exists!(db, stmt.schema)
schema = drop_object_schema(db, stmt.schema, stmt.name)
db = drop_table_fk_cleanup(db, schema, stmt.name)
case Database.drop_table(db, schema, stmt.name) do
{:ok, db} -> {%Result{command: :drop_table}, drop_triggers_on(db, schema, stmt.name)}
{:error, _} when stmt.if_exists -> {%Result{command: :drop_table}, db}
{:error, message} -> fail(message)
end
end
# -- CREATE INDEX ------------------------------------------------------------
defp exec(db, %CreateIndex{} = stmt) do
ensure_schema_exists!(db, stmt.schema)
table = fetch_table!(db, stmt.schema, stmt.table)
index_schema = table.schema
index_table_key = Database.table_storage_key(index_schema, stmt.name)
if internal_sqlite_object_name?(stmt.name) do
fail("object name reserved for internal use: #{stmt.name}")
end
# Index names are scoped to their schema and must not collide with other
# indexes in that schema.
if Database.index_exists?(db, index_schema, stmt.name) do
if stmt.if_not_exists do
{%Result{command: :create_index}, db}
else
fail("index #{stmt.name} already exists")
end
else
# Index name must not collide with an existing table name
if Map.has_key?(db.tables, index_table_key) or Map.has_key?(db.views, index_table_key) do
fail("there is already a table named #{stmt.name}")
else
# Validate columns and build the member list; an index with any
# expression member is enforced executor-side (it needs eval).
members =
Enum.map(stmt.columns, fn col ->
case col do
%{name: name} when name != nil ->
unless Table.column(table, name) do
fail("no such column: #{name}")
end
{:column, Table.key(name)}
%{expr: expr} ->
validate_index_expression!(table, expr)
{:expr, expr}
end
end)
collations =
stmt.columns
|> Enum.zip(members)
|> Enum.map(fn {col, member} -> index_member_collation(table, col, member) end)
directions = Enum.map(stmt.columns, & &1.direction)
index =
if Enum.any?(members, &match?({:expr, _}, &1)) do
%{
name: stmt.name,
columns: [],
members: members,
collations: collations,
directions: directions,
unique: stmt.unique,
where: stmt.where
}
else
%{
name: stmt.name,
columns: Enum.map(members, fn {:column, key} -> key end),
members: members,
collations: collations,
directions: directions,
unique: stmt.unique,
where: stmt.where
}
end
# For UNIQUE indexes, check existing data for duplicates
if stmt.unique do
check_unique_index_data!(db, table, index)
end
updated_table = %{table | indexes: table.indexes ++ [index]}
{%Result{command: :create_index},
db |> put_table(updated_table) |> Database.schema_changed(table.schema)}
end
end
end
# -- DROP INDEX ------------------------------------------------------------
defp exec(db, %DropIndex{} = stmt) do
ensure_schema_exists!(db, stmt.schema)
owner =
if stmt.schema do
drop_index_owner(db, stmt.schema, stmt.name)
else
drop_index_owner(db, :any, stmt.name)
end
case owner do
nil ->
if stmt.if_exists do
{%Result{command: :drop_index}, db}
else
fail("no such index: #{stmt.name}")
end
{_table, %{autoindex: true}} ->
fail("index associated with UNIQUE or PRIMARY KEY constraint cannot be dropped")
{table, _index} ->
index_key = Table.key(stmt.name)
updated_table = %{
table
| indexes: Enum.reject(table.indexes, &(Table.key(&1.name) == index_key))
}
{%Result{command: :drop_index},
db |> put_table(updated_table) |> Database.schema_changed(table.schema)}
end
end
# -- CREATE VIEW / DROP VIEW ------------------------------------------------
defp exec(db, %CreateView{} = stmt) do
ensure_schema_exists!(db, stmt.schema)
view = %{
name: stmt.name,
schema: stmt.schema,
columns: stmt.columns,
query: qualify_view_query(stmt.query, stmt.schema)
}
case Database.create_view(db, view) do
{:ok, db} ->
{%Result{command: :create_view}, db}
{:error, "there is already an index named " <> _ = message} ->
fail(message)
{:error, _message} when stmt.if_not_exists ->
{%Result{command: :create_view}, db}
{:error, message} ->
fail(message)
end
end
defp exec(db, %DropView{} = stmt) do
ensure_schema_exists!(db, stmt.schema)
schema = drop_object_schema(db, stmt.schema, stmt.name)
case Database.drop_view(db, schema, stmt.name) do
{:ok, db} -> {%Result{command: :drop_view}, drop_triggers_on(db, schema, stmt.name)}
{:error, _} when stmt.if_exists -> {%Result{command: :drop_view}, db}
{:error, message} -> fail(message)
end
end
# -- CREATE TRIGGER / DROP TRIGGER ---------------------------------------------
defp exec(db, %CreateTrigger{} = stmt) do
ensure_trigger_schema_exists!(db, stmt.schema)
target_schema = trigger_target_schema!(db, stmt)
key = Database.table_storage_key(stmt.schema, stmt.name)
target_key = Database.table_storage_key(target_schema, stmt.table)
view? = match?({:ok, _}, Database.fetch_view(db, target_schema, stmt.table))
table? = Map.has_key?(db.tables, target_key)
target_label = trigger_target_label(stmt, target_schema)
cond do
not view? and not table? ->
fail("no such table: #{target_label}")
Map.has_key?(db.triggers, key) and stmt.if_not_exists ->
{%Result{command: :create_trigger}, db}
Map.has_key?(db.triggers, key) ->
fail("trigger #{stmt.name} already exists")
stmt.timing == :instead_of and not view? ->
fail("cannot create INSTEAD OF trigger on table: #{target_label}")
stmt.timing != :instead_of and view? ->
fail(
"cannot create #{String.upcase(Atom.to_string(stmt.timing))} " <>
"trigger on view: #{target_label}"
)
true ->
validate_trigger_definition!(stmt)
trigger = %{
key: key,
name: stmt.name,
schema: stmt.schema,
table_schema: target_schema,
table_key: target_key,
table_name: stmt.table,
timing: stmt.timing,
event: stmt.event,
update_columns: stmt.update_columns,
when: stmt.when,
body: stmt.body,
seq: map_size(db.triggers)
}
{%Result{command: :create_trigger},
%{db | triggers: Map.put(db.triggers, key, trigger)}
|> Database.schema_changed(stmt.schema)}
end
end
defp exec(db, {:drop_trigger, schema, name, if_exists}) do
ensure_trigger_schema_exists!(db, schema)
key = drop_trigger_key(db, schema, name)
cond do
key != nil ->
trigger = Map.fetch!(db.triggers, key)
{%Result{command: :drop_trigger},
%{db | triggers: Map.delete(db.triggers, key)} |> Database.schema_changed(trigger.schema)}
if_exists ->
{%Result{command: :drop_trigger}, db}
true ->
fail("no such trigger: #{name}")
end
end
# -- WITH (CTEs) ------------------------------------------------------------
defp exec(db, %With{} = stmt) do
# Extract outer LIMIT to use as a cap for recursive CTE expansion.
outer_limit = extract_query_limit(stmt.query, db)
db_with_ctes = resolve_ctes(db, stmt.ctes, stmt.recursive, outer_limit)
case stmt.query do
%Select{} = q ->
{select_result(db_with_ctes, q, nil), db}
%Compound{} = q ->
{compound_result(db_with_ctes, q, nil), db}
%Values{} = q ->
{values_result(db_with_ctes, q, nil), db}
%With{} = q ->
# Nested WITH: chain CTEs
{result, _} = exec(db_with_ctes, q)
{result, db}
%Insert{} = q ->
{result, updated_db} = exec(db_with_ctes, q)
# Write DML changes back to the real db but without CTEs
{result, %{updated_db | ctes: db.ctes}}
%Update{} = q ->
{result, updated_db} = exec(db_with_ctes, q)
{result, %{updated_db | ctes: db.ctes}}
%Delete{} = q ->
{result, updated_db} = exec(db_with_ctes, q)
{result, %{updated_db | ctes: db.ctes}}
end
end
# -- INSERT --------------------------------------------------------------------
defp exec(db, %Insert{} = stmt) do
cond do
main_schema?(stmt.schema) and Table.key(stmt.table) == "sqlite_sequence" ->
exec_sqlite_sequence_insert(db, stmt)
match?({:ok, _}, dml_view(db, stmt.schema, stmt.table)) ->
{:ok, view} = dml_view(db, stmt.schema, stmt.table)
exec_view_dml(db, view, stmt, :insert)
true ->
with_fk_statement_check(db, stmt, fn -> exec_insert(db, stmt) end)
end
end
# -- UPDATE --------------------------------------------------------------------
defp exec(db, %Update{} = stmt) do
cond do
main_schema?(stmt.schema) and Table.key(stmt.table) == "sqlite_sequence" ->
exec_sqlite_sequence_update(db, stmt)
match?({:ok, _}, dml_view(db, stmt.schema, stmt.table)) ->
{:ok, view} = dml_view(db, stmt.schema, stmt.table)
exec_view_dml(db, view, stmt, :update)
true ->
with_fk_statement_check(db, stmt, fn -> exec_update(db, stmt) end)
end
end
# -- DELETE --------------------------------------------------------------------
defp exec(db, %Delete{} = stmt) do
cond do
main_schema?(stmt.schema) and Table.key(stmt.table) == "sqlite_sequence" ->
exec_sqlite_sequence_delete(db, stmt)
match?({:ok, _}, dml_view(db, stmt.schema, stmt.table)) ->
{:ok, view} = dml_view(db, stmt.schema, stmt.table)
exec_view_dml(db, view, stmt, :delete)
true ->
with_fk_statement_check(db, stmt, fn -> exec_delete(db, stmt) end)
end
end
# -- ALTER TABLE -----------------------------------------------------------------
defp exec(db, %AlterTable{} = stmt) do
table = fetch_table!(db, stmt.schema, stmt.name)
case stmt.op do
{:rename_table, new_name} ->
new_key = Database.table_storage_key(table.schema, new_name)
if Map.has_key?(db.tables, new_key) or Map.has_key?(db.views, new_key) or
Database.index_exists?(db, table.schema, new_name) do
fail("there is already another table or index with this name: #{new_name}")
end
renamed = %{table | name: new_name} |> rename_autoindexes()
db = %{
db
| tables:
db.tables
|> Map.delete(Database.table_storage_key(table.schema, table.name))
|> Map.put(new_key, renamed)
}
{%Result{command: :alter_table}, Database.schema_changed(db, table.schema)}
{:rename_column, old_name, new_name} ->
old_key = Table.key(old_name)
col = Table.column(table, old_name)
unless col do
fail(~s(no such column: "#{old_name}"))
end
new_key = Table.key(new_name)
if Enum.any?(table.columns, &(Table.key(&1.name) == new_key)) do
fail("error in table #{table.name} after rename: duplicate column name: #{new_name}")
end
new_columns =
Enum.map(table.columns, fn c ->
if Table.key(c.name) == old_key, do: %{c | name: new_name}, else: c
end)
# Update rowid_alias if needed
new_rowid_alias =
if table.rowid_alias == old_key, do: new_key, else: table.rowid_alias
# Update composite_keys, composite_uniques, and index column lists
new_composite_keys = rename_in_composites(table.composite_keys, old_key, new_key)
new_composite_uniques = rename_in_composites(table.composite_uniques, old_key, new_key)
new_indexes =
Enum.map(table.indexes, fn index ->
%{index | columns: Enum.map(index.columns, &if(&1 == old_key, do: new_key, else: &1))}
end)
new_autoindexes =
Enum.map(table.autoindexes, fn index ->
rename_index_column(index, old_key, new_key)
end)
# Update row maps: rename the column key in all rows
new_rows =
Map.new(Table.scan(table), fn {rowid, row} ->
{value, rest} = Map.pop(row, old_key, nil)
{rowid, Map.put(rest, new_key, value)}
end)
new_table =
Table.narrow_all_rows(%{
table
| columns: new_columns,
rowid_alias: new_rowid_alias,
composite_keys: new_composite_keys,
composite_uniques: new_composite_uniques,
indexes: new_indexes,
autoindexes: new_autoindexes,
rows: new_rows,
frame_columns: nil,
column_index: nil
})
{%Result{command: :alter_table}, db |> put_table(new_table) |> Database.schema_changed()}
{:add_column, col_def} ->
alter_add_column(db, table, col_def)
{:drop_column, col_name} ->
alter_drop_column(db, table, col_name)
end
end
# -- operational no-ops ------------------------------------------------------------
defp exec(db, {:analyze, name}) do
validate_analyze_target!(db, name)
{%Result{command: :analyze}, db}
end
defp exec(db, {:vacuum, _name}), do: {%Result{command: :vacuum}, db}
defp exec(db, {:reindex, name}) do
validate_reindex_target!(db, name)
{%Result{command: :reindex}, db}
end
# -- transactions ----------------------------------------------------------------
#
# On an immutable database value a transaction is just a snapshot of the
# tables: BEGIN/SAVEPOINT push one, ROLLBACK restores one, COMMIT/RELEASE
# discard entries. SAVEPOINT outside a transaction starts an implicit one,
# as in SQLite.
defp exec(db, {:begin}) do
if db.txn_stack != [], do: fail("cannot start a transaction within a transaction")
{%Result{command: :begin}, %{db | txn_stack: [{:begin, Database.schema_snapshot(db)}]}}
end
defp exec(db, {:commit}) do
if db.txn_stack == [], do: fail("cannot commit - no transaction is active")
check_deferred_foreign_keys!(db)
{%Result{command: :commit}, %{db | txn_stack: [], defer_foreign_keys: false}}
end
defp exec(db, {:rollback}) do
case List.last(db.txn_stack) do
nil ->
fail("cannot rollback - no transaction is active")
{_kind, snapshot} ->
db = %{db | txn_stack: [], defer_foreign_keys: false}
{%Result{command: :rollback}, Database.restore_schema(db, snapshot)}
end
end
defp exec(db, {:savepoint, name}) do
stack = [{{:savepoint, Table.key(name)}, Database.schema_snapshot(db)} | db.txn_stack]
{%Result{command: :savepoint}, %{db | txn_stack: stack}}
end
defp exec(db, {:release, name}) do
case savepoint_index(db.txn_stack, name) do
nil ->
fail("no such savepoint: #{name}")
index ->
remaining = Enum.drop(db.txn_stack, index + 1)
# Releasing the outermost savepoint commits the implicit transaction,
# which is when deferred foreign keys are checked.
db =
if remaining == [] do
check_deferred_foreign_keys!(db)
%{db | txn_stack: remaining, defer_foreign_keys: false}
else
%{db | txn_stack: remaining}
end
{%Result{command: :release}, db}
end
end
defp exec(db, {:rollback_to, name}) do
case savepoint_index(db.txn_stack, name) do
nil ->
fail("no such savepoint: #{name}")
index ->
{_kind, snapshot} = Enum.at(db.txn_stack, index)
{%Result{command: :rollback},
Database.restore_schema(%{db | txn_stack: Enum.drop(db.txn_stack, index)}, snapshot)}
end
end
# -- SELECT --------------------------------------------------------------------
defp exec(db, %Select{} = stmt), do: {select_result(db, stmt, nil), db}
defp exec(db, %Compound{} = stmt), do: {compound_result(db, stmt, nil), db}
defp exec(db, %Values{} = stmt), do: {values_result(db, stmt, nil), db}
defp exec(db, {:explain, stmt}) do
case explain_bytecode_rows(db, stmt) do
{:ok, rows} ->
{%Result{
command: :select,
columns: ["addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment"],
rows: rows,
rows_affected: 0,
affinities: [:integer, :text, :integer, :integer, :integer, :text, :integer, :text]
}, db}
:error ->
fail("unsupported EXPLAIN statement")
end
end
defp exec(db, {:explain_query_plan, stmt}) do
{%Result{
command: :select,
columns: ["id", "parent", "notused", "detail"],
rows: explain_query_plan_rows(db, stmt),
rows_affected: 0,
affinities: [:integer, :integer, :integer, :text]
}, db}
end
defp explain_bytecode_rows(_db, %Select{
columns: columns,
from: nil,
where: nil,
group_by: [],
having: nil,
windows: %{},
order_by: [],
limit: limit,
offset: offset,
distinct: false
}) do
with {:ok, limit} <- explain_limit(limit, offset) do
columns
|> Enum.map(fn
{expr, _alias} -> explain_literal_expr(expr)
_other -> :error
end)
|> explain_literal_program(limit)
end
end
defp explain_bytecode_rows(
db,
%Select{
columns: columns,
from: {:table, name, alias_name},
where: where,
group_by: [],
having: nil,
windows: %{},
order_by: [],
limit: limit,
offset: offset,
distinct: false
}
) do
with table_key when not is_nil(table_key) <- relation_unqualified_table_key(db, name),
%Table{} = table <- plain_table(db, table_key),
{:ok, projection} <- explain_table_projection(table, name, alias_name, columns),
{:ok, filter} <- explain_table_filter(table, name, alias_name, where),
{:ok, limit} <- explain_limit(limit, offset) do
{:ok, explain_table_scan_program(db, table, projection, filter, limit)}
else
_other -> :error
end
end
defp explain_bytecode_rows(_db, %Values{rows: [row], order_by: [], limit: limit, offset: offset}) do
with {:ok, limit} <- explain_limit(limit, offset) do
row
|> Enum.map(&explain_literal_expr/1)
|> explain_literal_program(limit)
end
end
defp explain_bytecode_rows(_db, %Values{
rows: [_, _ | _] = rows,
order_by: [],
limit: limit,
offset: offset
}) do
with {:ok, limit} <- explain_limit(limit, offset) do
rows
|> Enum.map(&Enum.map(&1, fn expr -> explain_literal_expr(expr) end))
|> explain_values_program(limit)
end
end
defp explain_bytecode_rows(_db, _stmt), do: :error
defp explain_literal_program(items, limit) do
if Enum.any?(items, &(&1 == :error)) do
:error
else
width = length(items)
setup_width = explain_limit_setup_width(limit)
register_width = explain_limit_register_width(limit)
offset_width = explain_limit_offset_width(limit)
output_start = 1 + register_width
offset_addr = 1 + setup_width
literal_start_addr = offset_addr + offset_width
{literal_rows, state, result_addr} =
explain_literal_expression_rows(
items,
literal_start_addr,
output_start,
output_start + width
)
decr_addr = result_addr + 1
halt_addr = result_addr + 1 + explain_limit_decr_width(limit)
start_addr = halt_addr + 1
post_rows = explain_literal_post_rows(state, start_addr, state.next_reg)
goto_addr = start_addr + length(post_rows)
rows =
[[0, "Init", 0, start_addr, 0, nil, 0, "Start at #{start_addr}"]] ++
explain_limit_rows(limit, halt_addr) ++
explain_limit_offset_rows(limit, offset_addr, halt_addr) ++
literal_rows ++
[
[
result_addr,
"ResultRow",
output_start,
width,
0,
nil,
0,
explain_result_comment(output_start, width)
]
] ++
explain_limit_decr_rows(limit, decr_addr, halt_addr) ++
[
[halt_addr, "Halt", 0, 0, 0, nil, 0, nil]
] ++
post_rows ++
[
[goto_addr, "Goto", 0, 1, 0, nil, 0, nil]
]
{:ok, rows}
end
end
defp explain_literal_expression_rows(items, start_addr, output_start, operand_start_reg)
when is_integer(operand_start_reg) do
explain_literal_expression_rows(
items,
start_addr,
output_start,
explain_operand_state(operand_start_reg)
)
end
defp explain_literal_expression_rows(items, start_addr, output_start, state) do
items
|> Enum.with_index()
|> Enum.reduce({[], state, start_addr}, fn {item, index}, {rows, state, addr} ->
output_reg = output_start + index
{item_rows, state, next_addr} =
case item do
{:literal, opcode} ->
{[explain_literal_row(addr, opcode, output_reg)], state, addr + 1}
{:cast, opcode, affinity} ->
item_rows = [
explain_literal_row(addr, opcode, output_reg),
[
addr + 1,
"Cast",
output_reg,
explain_cast_affinity_code(affinity),
0,
nil,
0,
"affinity(r[#{output_reg}])"
]
]
{item_rows, state, addr + 2}
{:negate, opcode} ->
{zero_reg, state} = explain_operand_register(explain_zero_opcode(), state)
{operand_reg, state} = explain_operand_register(opcode, state)
row = [
addr,
"Subtract",
operand_reg,
zero_reg,
output_reg,
nil,
0,
"r[#{output_reg}]=r[#{zero_reg}]-r[#{operand_reg}]"
]
{[row], state, addr + 1}
{:bitnot, opcode} ->
{operand_reg, state} = explain_operand_register(opcode, state)
row = [
addr,
"BitNot",
operand_reg,
output_reg,
0,
nil,
0,
"r[#{output_reg}]= ~r[#{operand_reg}]"
]
{[row], state, addr + 1}
{:not, opcode} ->
{operand_reg, state} = explain_operand_register(opcode, state)
row = [
addr,
"Not",
operand_reg,
output_reg,
0,
nil,
0,
"r[#{output_reg}]= !r[#{operand_reg}]"
]
{[row], state, addr + 1}
{:binary, opcode, left_opcode, right_opcode} ->
{left_reg, state} = explain_operand_register(left_opcode, state)
{right_reg, state} = explain_operand_register(right_opcode, state)
row = [
addr,
opcode,
right_reg,
left_reg,
output_reg,
nil,
0,
explain_binary_comment(opcode, output_reg, left_reg, right_reg)
]
{[row], state, addr + 1}
{:compare, opcode, left_opcode, right_opcode, collation_p4} ->
{left_reg, state} = explain_operand_register(left_opcode, state)
{right_reg, state} = explain_operand_register(right_opcode, state)
item_rows = [
[addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"],
[
addr + 1,
opcode,
right_reg,
addr + 3,
left_reg,
collation_p4,
64,
explain_compare_comment(opcode, left_reg, right_reg, addr + 3)
],
[
addr + 2,
"ZeroOrNull",
left_reg,
output_reg,
right_reg,
nil,
0,
"r[#{output_reg}] = 0 OR NULL"
]
]
{item_rows, state, addr + 3}
{:is_compare, opcode, left_opcode, right_opcode, collation_p4} ->
{left_reg, state} = explain_operand_register(left_opcode, state)
{right_reg, state} = explain_operand_register(right_opcode, state)
item_rows = [
[addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"],
[
addr + 1,
opcode,
right_reg,
addr + 3,
left_reg,
collation_p4,
192,
explain_compare_comment(opcode, left_reg, right_reg, addr + 3)
],
[addr + 2, "Integer", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=0"]
]
{item_rows, state, addr + 3}
{:null_test, opcode, operand_opcode} ->
{operand_reg, state} = explain_operand_register(operand_opcode, state)
item_rows = [
[addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"],
[
addr + 1,
opcode,
operand_reg,
addr + 3,
0,
nil,
0,
explain_null_test_comment(opcode, operand_reg, addr + 3)
],
[addr + 2, "Integer", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=0"]
]
{item_rows, state, addr + 3}
{:in_list, value_opcode, list_opcodes} ->
{{value_reg, bitand_reg, list_regs}, state} =
explain_in_registers(value_opcode, list_opcodes, state)
list_width = length(list_regs)
eq_start_addr = addr + 4
match_addr = addr + list_width + 6
addimm_addr = addr + list_width + 7
next_addr = addr + list_width + 8
eq_rows =
list_regs
|> Enum.with_index()
|> Enum.map(fn {list_reg, eq_index} ->
[
eq_start_addr + eq_index,
"Eq",
value_reg,
match_addr,
list_reg,
nil,
0,
explain_compare_comment("Eq", list_reg, value_reg, match_addr)
]
end)
item_rows =
[
[addr, "Null", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=NULL"],
[addr + 1, "Noop", 0, 0, 0, nil, 0, "begin IN expr"],
explain_literal_row(addr + 2, value_opcode, value_reg),
[
addr + 3,
"BitAnd",
value_reg,
value_reg,
bitand_reg,
nil,
0,
"r[#{bitand_reg}]=r[#{value_reg}]&r[#{value_reg}]"
]
] ++
eq_rows ++
[
[
addr + list_width + 4,
"IsNull",
bitand_reg,
next_addr,
0,
nil,
0,
explain_null_test_comment("IsNull", bitand_reg, next_addr)
],
[addr + list_width + 5, "Goto", 0, addimm_addr, 0, nil, 0, "end IN expr"],
[match_addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"],
[
addimm_addr,
"AddImm",
output_reg,
0,
0,
nil,
0,
"r[#{output_reg}]=r[#{output_reg}]+0"
]
]
{item_rows, state, next_addr}
{:not_in_list, value_opcode, list_opcodes} ->
{in_reg, state} =
explain_post_in_register(value_opcode, list_opcodes, state)
row = [
addr,
"Not",
in_reg,
output_reg,
0,
nil,
0,
"r[#{output_reg}]= !r[#{in_reg}]"
]
{[row], state, addr + 1}
{:case, nil, branches, else_opcode} ->
{item_rows, state, next_addr} =
explain_searched_case_rows(addr, output_reg, branches, else_opcode, state)
{item_rows, state, next_addr}
{:case, operand_opcode, branches, else_opcode} ->
{item_rows, state, next_addr} =
explain_simple_case_rows(
addr,
output_reg,
operand_opcode,
branches,
else_opcode,
state
)
{item_rows, state, next_addr}
{:literal_function, name, arg_opcodes, negated} ->
{item_rows, state, next_addr} =
explain_literal_function_rows(addr, output_reg, name, arg_opcodes, negated, state)
{item_rows, state, next_addr}
{:coalesce_function, arg_opcodes} ->
{item_rows, state, next_addr} =
explain_literal_coalesce_rows(addr, output_reg, arg_opcodes, state)
{item_rows, state, next_addr}
{:nullif_function, left_opcode, right_opcode, collation_p4} ->
{item_rows, state, next_addr} =
explain_literal_nullif_rows(
addr,
output_reg,
left_opcode,
right_opcode,
collation_p4,
state
)
{item_rows, state, next_addr}
{:collated_function, name, arg_opcodes, collation_p4} ->
{item_rows, state, next_addr} =
explain_literal_collated_function_rows(
addr,
output_reg,
name,
arg_opcodes,
collation_p4,
state
)
{item_rows, state, next_addr}
{:iif_function, condition_opcode, true_opcode, false_opcode} ->
{item_rows, state, next_addr} =
explain_literal_iif_rows(
addr,
output_reg,
condition_opcode,
true_opcode,
false_opcode,
state
)
{item_rows, state, next_addr}
{:between, value_opcode, low_opcode, high_opcode} ->
{{value_reg, lower_temp, upper_temp, low_reg, high_reg}, state} =
explain_between_registers(value_opcode, low_opcode, high_opcode, state)
item_rows = [
[addr, "Integer", 1, lower_temp, 0, nil, 0, "r[#{lower_temp}]=1"],
[
addr + 1,
"Ge",
low_reg,
addr + 3,
value_reg,
nil,
64,
explain_compare_comment("Ge", value_reg, low_reg, addr + 3)
],
[
addr + 2,
"ZeroOrNull",
value_reg,
lower_temp,
low_reg,
nil,
0,
"r[#{lower_temp}] = 0 OR NULL"
],
[addr + 3, "Integer", 1, upper_temp, 0, nil, 0, "r[#{upper_temp}]=1"],
[
addr + 4,
"Le",
high_reg,
addr + 6,
value_reg,
nil,
64,
explain_compare_comment("Le", value_reg, high_reg, addr + 6)
],
[
addr + 5,
"ZeroOrNull",
value_reg,
upper_temp,
high_reg,
nil,
0,
"r[#{upper_temp}] = 0 OR NULL"
],
[
addr + 6,
"And",
upper_temp,
lower_temp,
output_reg,
nil,
0,
"r[#{output_reg}]=(r[#{upper_temp}] && r[#{lower_temp}])"
]
]
{item_rows, state, addr + 7}
{:not_between, value_opcode, low_opcode, high_opcode} ->
{between_reg, state} =
explain_post_between_register(value_opcode, low_opcode, high_opcode, state)
row = [
addr,
"Not",
between_reg,
output_reg,
0,
nil,
0,
"r[#{output_reg}]= !r[#{between_reg}]"
]
{[row], state, addr + 1}
end
{rows ++ item_rows, state, next_addr}
end)
end
defp explain_literal_post_rows(%{operands: [], post_exprs: []}, _addr, _scratch_start), do: []
defp explain_literal_post_rows(state, addr, scratch_start) do
{post_expression_rows, next_addr} =
state.post_exprs
|> Enum.with_index()
|> Enum.reduce({[], addr}, fn {expr, index}, {rows, row_addr} ->
expr_rows = explain_post_expression_rows(expr, row_addr, scratch_start, index)
{rows ++ expr_rows, row_addr + length(expr_rows)}
end)
operand_rows =
state.operands
|> Enum.with_index()
|> Enum.map(fn {{opcode, reg}, index} ->
explain_literal_row(next_addr + index, opcode, reg)
end)
post_expression_rows ++ operand_rows
end
defp explain_post_expression_rows(
{:in_list, value_opcode, list_opcodes, result_reg},
addr,
scratch_start,
index
) do
value_reg = scratch_start + index
bitand_reg = value_reg + 1
list_reg = value_reg + 2
list_width = length(list_opcodes)
match_addr = addr + 2 * list_width + 6
addimm_addr = addr + 2 * list_width + 7
next_addr = addr + 2 * list_width + 8
list_rows =
list_opcodes
|> Enum.with_index()
|> Enum.flat_map(fn {list_opcode, list_index} ->
literal_addr = addr + 4 + 2 * list_index
eq_addr = literal_addr + 1
[
explain_literal_row(literal_addr, list_opcode, list_reg),
[
eq_addr,
"Eq",
value_reg,
match_addr,
list_reg,
nil,
0,
explain_compare_comment("Eq", list_reg, value_reg, match_addr)
]
]
end)
[
[addr, "Null", 0, result_reg, 0, nil, 0, "r[#{result_reg}]=NULL"],
[addr + 1, "Noop", 0, 0, 0, nil, 0, "begin IN expr"],
explain_literal_row(addr + 2, value_opcode, value_reg),
[
addr + 3,
"BitAnd",
value_reg,
value_reg,
bitand_reg,
nil,
0,
"r[#{bitand_reg}]=r[#{value_reg}]&r[#{value_reg}]"
]
] ++
list_rows ++
[
[
addr + 2 * list_width + 4,
"IsNull",
bitand_reg,
next_addr,
0,
nil,
0,
explain_null_test_comment("IsNull", bitand_reg, next_addr)
],
[addr + 2 * list_width + 5, "Goto", 0, addimm_addr, 0, nil, 0, "end IN expr"],
[match_addr, "Integer", 1, result_reg, 0, nil, 0, "r[#{result_reg}]=1"],
[
addimm_addr,
"AddImm",
result_reg,
0,
0,
nil,
0,
"r[#{result_reg}]=r[#{result_reg}]+0"
]
]
end
defp explain_post_expression_rows(
{:between, value_opcode, low_opcode, high_opcode, result_reg},
addr,
scratch_start,
index
) do
value_reg = scratch_start
high_reg = scratch_start + 3
{low_reg, lower_temp, upper_temp} =
if rem(index, 2) == 0 do
{scratch_start + 2, scratch_start + 1, scratch_start + 2}
else
{scratch_start + 1, scratch_start + 2, scratch_start + 1}
end
[
explain_literal_row(addr, value_opcode, value_reg),
explain_literal_row(addr + 1, low_opcode, low_reg),
[addr + 2, "Integer", 1, lower_temp, 0, nil, 0, "r[#{lower_temp}]=1"],
[
addr + 3,
"Ge",
low_reg,
addr + 5,
value_reg,
nil,
64,
explain_compare_comment("Ge", value_reg, low_reg, addr + 5)
],
[
addr + 4,
"ZeroOrNull",
value_reg,
lower_temp,
low_reg,
nil,
0,
"r[#{lower_temp}] = 0 OR NULL"
],
explain_literal_row(addr + 5, high_opcode, high_reg),
[addr + 6, "Integer", 1, upper_temp, 0, nil, 0, "r[#{upper_temp}]=1"],
[
addr + 7,
"Le",
high_reg,
addr + 9,
value_reg,
nil,
64,
explain_compare_comment("Le", value_reg, high_reg, addr + 9)
],
[
addr + 8,
"ZeroOrNull",
value_reg,
upper_temp,
high_reg,
nil,
0,
"r[#{upper_temp}] = 0 OR NULL"
],
[
addr + 9,
"And",
upper_temp,
lower_temp,
result_reg,
nil,
0,
"r[#{result_reg}]=(r[#{upper_temp}] && r[#{lower_temp}])"
]
]
end
defp explain_post_in_register(value_opcode, list_opcodes, state) do
result_reg = state.next_reg
state = %{
state
| next_reg: result_reg + 1,
post_exprs: state.post_exprs ++ [{:in_list, value_opcode, list_opcodes, result_reg}]
}
{result_reg, state}
end
defp explain_post_between_register(value_opcode, low_opcode, high_opcode, state) do
result_reg = state.next_reg
state = %{
state
| next_reg: result_reg + 1,
post_exprs:
state.post_exprs ++ [{:between, value_opcode, low_opcode, high_opcode, result_reg}]
}
{result_reg, state}
end
defp explain_operand_state(next_reg) do
%{
operands: [],
operand_regs: %{},
next_reg: next_reg,
between_temps: nil,
between_count: 0,
between_scratch_reserved: false,
in_value_reg: nil,
iif_condition_reg: nil,
function_output_regs: MapSet.new(),
post_exprs: []
}
end
defp explain_operand_register(opcode, state) do
case state.operand_regs do
%{^opcode => reg} ->
{reg, state}
_other ->
reg = state.next_reg
state = %{
state
| operands: state.operands ++ [{opcode, reg}],
operand_regs: Map.put(state.operand_regs, opcode, reg),
next_reg: reg + 1
}
{reg, state}
end
end
defp explain_in_registers(_value_opcode, list_opcodes, %{in_value_reg: nil} = state) do
value_reg = state.next_reg
bitand_reg = value_reg + 1
state = %{state | next_reg: bitand_reg + 1, in_value_reg: bitand_reg}
{list_regs, state} = explain_operand_registers(list_opcodes, state)
{{value_reg, bitand_reg, list_regs}, state}
end
defp explain_in_registers(_value_opcode, list_opcodes, state) do
value_reg = state.in_value_reg
bitand_reg = state.next_reg
state = %{state | next_reg: bitand_reg + 1, in_value_reg: bitand_reg}
{list_regs, state} = explain_operand_registers(list_opcodes, state)
{{value_reg, bitand_reg, list_regs}, state}
end
defp explain_operand_registers(opcodes, state) do
Enum.map_reduce(opcodes, state, fn opcode, state ->
explain_operand_register(opcode, state)
end)
end
defp explain_case_operand_registers(operand_opcode, when_opcodes, state) do
{operand_reg, state} = explain_operand_register(operand_opcode, state)
state = %{state | next_reg: state.next_reg + 1}
{when_regs, state} = explain_operand_registers(when_opcodes, state)
{{operand_reg, when_regs}, state}
end
defp explain_function_registers(arg_opcodes, state) do
result_reg = state.next_reg
arg_start_reg = result_reg + 1
arg_regs = Enum.to_list(arg_start_reg..(arg_start_reg + length(arg_opcodes) - 1)//1)
state = %{state | next_reg: result_reg + length(arg_opcodes) + 1}
{{result_reg, arg_start_reg, arg_regs}, state}
end
defp explain_between_registers(value_opcode, low_opcode, high_opcode, state) do
{value_reg, state} = explain_operand_register(value_opcode, state)
{lower_base, upper_base, state} = explain_between_temp_registers(state)
{lower_temp, upper_temp} =
if rem(state.between_count, 2) == 0 do
{lower_base, upper_base}
else
{upper_base, lower_base}
end
{low_reg, state} = explain_operand_register(low_opcode, state)
state = explain_reserve_between_scratch(state)
{high_reg, state} = explain_operand_register(high_opcode, state)
state = %{state | between_count: state.between_count + 1}
{{value_reg, lower_temp, upper_temp, low_reg, high_reg}, state}
end
defp explain_between_temp_registers(%{between_temps: {lower_reg, upper_reg}} = state),
do: {lower_reg, upper_reg, state}
defp explain_between_temp_registers(state) do
lower_reg = state.next_reg
upper_reg = lower_reg + 1
state = %{state | between_temps: {lower_reg, upper_reg}, next_reg: upper_reg + 1}
{lower_reg, upper_reg, state}
end
defp explain_reserve_between_scratch(%{between_scratch_reserved: true} = state), do: state
defp explain_reserve_between_scratch(state),
do: %{state | next_reg: state.next_reg + 1, between_scratch_reserved: true}
defp explain_searched_case_rows(addr, output_reg, branches, else_opcode, state) do
{branch_specs, state} =
Enum.map_reduce(branches, state, fn {when_opcode, then_opcode}, state ->
case explain_case_truth_opcode(when_opcode) do
true ->
{{true, then_opcode}, state}
false ->
{{false, then_opcode}, state}
:dynamic ->
{when_reg, state} = explain_operand_register(when_opcode, state)
{{:dynamic, when_reg, then_opcode}, state}
end
end)
end_addr =
addr +
Enum.reduce(branch_specs, 1, fn spec, width -> width + explain_case_branch_width(spec) end)
{branch_rows, else_addr} =
Enum.reduce(branch_specs, {[], addr}, fn spec, {rows, branch_addr} ->
next_addr = branch_addr + explain_case_branch_width(spec)
{rows ++
explain_searched_case_branch_rows(spec, branch_addr, next_addr, end_addr, output_reg),
next_addr}
end)
else_rows = [explain_case_else_row(else_addr, else_opcode, output_reg)]
{branch_rows ++ else_rows, state, end_addr}
end
defp explain_simple_case_rows(addr, output_reg, operand_opcode, branches, else_opcode, state) do
when_opcodes = Enum.map(branches, &elem(&1, 0))
{{operand_reg, when_regs}, state} =
explain_case_operand_registers(operand_opcode, when_opcodes, state)
branch_specs =
branches
|> Enum.map(&elem(&1, 1))
|> Enum.zip(when_regs)
end_addr = addr + 1 + length(branch_specs) * 3
{branch_rows, else_addr} =
Enum.reduce(branch_specs, {[], addr}, fn {then_opcode, when_reg}, {rows, branch_addr} ->
next_addr = branch_addr + 3
branch_rows = [
[
branch_addr,
"Ne",
when_reg,
next_addr,
operand_reg,
nil,
80,
explain_compare_comment("Ne", operand_reg, when_reg, next_addr)
],
explain_literal_row(branch_addr + 1, then_opcode, output_reg),
[branch_addr + 2, "Goto", 0, end_addr, 0, nil, 0, nil]
]
{rows ++ branch_rows, next_addr}
end)
else_rows = [explain_case_else_row(else_addr, else_opcode, output_reg)]
{branch_rows ++ else_rows, state, end_addr}
end
defp explain_case_truth_opcode({"Integer", value, nil}) when value != 0, do: true
defp explain_case_truth_opcode({"Integer", 0, nil}), do: false
defp explain_case_truth_opcode(_opcode), do: :dynamic
defp explain_case_branch_width({true, _then_opcode}), do: 2
defp explain_case_branch_width({false, _then_opcode}), do: 3
defp explain_case_branch_width({:dynamic, _when_reg, _then_opcode}), do: 3
defp explain_searched_case_branch_rows(
{true, then_opcode},
addr,
_next_addr,
end_addr,
output_reg
) do
[
explain_literal_row(addr, then_opcode, output_reg),
[addr + 1, "Goto", 0, end_addr, 0, nil, 0, nil]
]
end
defp explain_searched_case_branch_rows(
{false, then_opcode},
addr,
next_addr,
end_addr,
output_reg
) do
[
[addr, "Goto", 0, next_addr, 0, nil, 0, nil],
explain_literal_row(addr + 1, then_opcode, output_reg),
[addr + 2, "Goto", 0, end_addr, 0, nil, 0, nil]
]
end
defp explain_searched_case_branch_rows(
{:dynamic, when_reg, then_opcode},
addr,
next_addr,
end_addr,
output_reg
) do
[
[addr, "IfNot", when_reg, next_addr, 1, nil, 0, nil],
explain_literal_row(addr + 1, then_opcode, output_reg),
[addr + 2, "Goto", 0, end_addr, 0, nil, 0, nil]
]
end
defp explain_case_else_row(addr, nil, output_reg),
do: [addr, "Null", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=NULL"]
defp explain_case_else_row(addr, opcode, output_reg),
do: explain_literal_row(addr, opcode, output_reg)
defp explain_literal_function_rows(addr, output_reg, name, arg_opcodes, negated, state) do
{{result_reg, arg_start_reg, arg_regs}, state} =
explain_function_registers(arg_opcodes, state)
arg_count = length(arg_opcodes)
function_addr = addr + arg_count + 1
output_addr = function_addr + 1
next_addr = output_addr + 1
arg_rows =
arg_opcodes
|> Enum.zip(arg_regs)
|> Enum.with_index()
|> Enum.map(fn {{opcode, reg}, index} ->
explain_literal_row(addr + 1 + index, opcode, reg)
end)
{output_row, state} =
explain_function_output_row(output_addr, result_reg, output_reg, negated, state)
rows =
[
[addr, "Once", 0, output_addr, 0, nil, 0, nil]
] ++
arg_rows ++
[
[
function_addr,
"Function",
explain_function_const_mask(arg_count),
explain_function_start_reg(arg_start_reg, arg_count),
result_reg,
explain_function_p4(name, arg_count),
0,
explain_function_comment(result_reg, arg_start_reg, arg_count)
],
output_row
]
{rows, state, next_addr}
end
defp explain_literal_coalesce_rows(addr, output_reg, arg_opcodes, state) do
result_reg = state.next_reg
state = %{state | next_reg: result_reg + 1}
arg_count = length(arg_opcodes)
output_addr = addr + 2 * arg_count
arg_rows =
arg_opcodes
|> Enum.with_index()
|> Enum.flat_map(fn {opcode, index} ->
literal_addr = addr + 1 + 2 * index
if index == arg_count - 1 do
[explain_literal_row(literal_addr, opcode, result_reg)]
else
[
explain_literal_row(literal_addr, opcode, result_reg),
[
literal_addr + 1,
"NotNull",
result_reg,
output_addr,
0,
nil,
0,
explain_null_test_comment("NotNull", result_reg, output_addr)
]
]
end
end)
{output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state)
rows = [[addr, "Once", 0, output_addr, 0, nil, 0, nil]] ++ arg_rows ++ [output_row]
{rows, state, output_addr + 1}
end
defp explain_literal_nullif_rows(
addr,
output_reg,
left_opcode,
right_opcode,
collation_p4,
state
) do
arg_opcodes = [left_opcode, right_opcode]
{{result_reg, arg_start_reg, arg_regs}, state} =
explain_function_registers(arg_opcodes, state)
function_addr = addr + 4
output_addr = function_addr + 1
arg_rows =
arg_opcodes
|> Enum.zip(arg_regs)
|> Enum.with_index()
|> Enum.map(fn {{opcode, reg}, index} ->
explain_literal_row(addr + 1 + index, opcode, reg)
end)
{output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state)
rows =
[
[addr, "Once", 0, output_addr, 0, nil, 0, nil]
] ++
arg_rows ++
[
[addr + 3, "CollSeq", 0, 0, 0, collation_p4, 0, nil],
[
function_addr,
"Function",
explain_function_const_mask(2),
arg_start_reg,
result_reg,
"nullif(2)",
0,
explain_function_comment(result_reg, arg_start_reg, 2)
],
output_row
]
{rows, state, output_addr + 1}
end
defp explain_literal_collated_function_rows(
addr,
output_reg,
name,
arg_opcodes,
collation_p4,
state
) do
{{result_reg, arg_start_reg, arg_regs}, state} =
explain_function_registers(arg_opcodes, state)
arg_count = length(arg_opcodes)
collseq_addr = addr + arg_count + 1
function_addr = collseq_addr + 1
output_addr = function_addr + 1
arg_rows =
arg_opcodes
|> Enum.zip(arg_regs)
|> Enum.with_index()
|> Enum.map(fn {{opcode, reg}, index} ->
explain_literal_row(addr + 1 + index, opcode, reg)
end)
{output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state)
rows =
[
[addr, "Once", 0, output_addr, 0, nil, 0, nil]
] ++
arg_rows ++
[
[collseq_addr, "CollSeq", 0, 0, 0, collation_p4, 0, nil],
[
function_addr,
"Function",
explain_function_const_mask(arg_count),
arg_start_reg,
result_reg,
explain_function_p4(name, arg_count),
0,
explain_function_comment(result_reg, arg_start_reg, arg_count)
],
output_row
]
{rows, state, output_addr + 1}
end
defp explain_literal_iif_rows(
addr,
output_reg,
condition_opcode,
true_opcode,
false_opcode,
state
) do
{{result_reg, condition_reg}, state} = explain_iif_registers(condition_opcode, state)
{rows, output_addr} =
case {explain_case_truth_opcode(condition_opcode), condition_reg} do
{true, nil} ->
output_addr = addr + 4
{[
[addr, "Once", 0, output_addr, 0, nil, 0, nil],
explain_literal_row(addr + 1, true_opcode, result_reg),
[addr + 2, "Goto", 0, output_addr, 0, nil, 0, nil],
explain_literal_row(addr + 3, false_opcode, result_reg)
], output_addr}
{false, nil} ->
false_addr = addr + 4
output_addr = addr + 5
{[
[addr, "Once", 0, output_addr, 0, nil, 0, nil],
[addr + 1, "Goto", 0, false_addr, 0, nil, 0, nil],
explain_literal_row(addr + 2, true_opcode, result_reg),
[addr + 3, "Goto", 0, output_addr, 0, nil, 0, nil],
explain_literal_row(false_addr, false_opcode, result_reg)
], output_addr}
{:dynamic, condition_reg} ->
false_addr = addr + 5
output_addr = addr + 6
{[
[addr, "Once", 0, output_addr, 0, nil, 0, nil],
explain_literal_row(addr + 1, condition_opcode, condition_reg),
[addr + 2, "IfNot", condition_reg, false_addr, 1, nil, 0, nil],
explain_literal_row(addr + 3, true_opcode, result_reg),
[addr + 4, "Goto", 0, output_addr, 0, nil, 0, nil],
explain_literal_row(false_addr, false_opcode, result_reg)
], output_addr}
end
{output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state)
{rows ++ [output_row], state, output_addr + 1}
end
defp explain_function_output_row(addr, result_reg, output_reg, true, state) do
row = [
addr,
"Not",
result_reg,
output_reg,
0,
nil,
0,
"r[#{output_reg}]= !r[#{result_reg}]"
]
{row, state}
end
defp explain_function_output_row(addr, result_reg, output_reg, false, state),
do: explain_computed_output_row(addr, result_reg, output_reg, state)
defp explain_computed_output_row(addr, result_reg, output_reg, state) do
opcode = if MapSet.member?(state.function_output_regs, output_reg), do: "SCopy", else: "Copy"
row = [
addr,
opcode,
result_reg,
output_reg,
0,
nil,
0,
"r[#{output_reg}]=r[#{result_reg}]"
]
state = %{state | function_output_regs: MapSet.put(state.function_output_regs, output_reg)}
{row, state}
end
defp explain_function_const_mask(arg_count), do: Integer.pow(2, arg_count) - 1
defp explain_function_start_reg(_arg_start_reg, 0), do: 0
defp explain_function_start_reg(arg_start_reg, _arg_count), do: arg_start_reg
defp explain_function_p4(name, _arg_count) when name in ["char", "format", "printf"],
do: "#{name}(-1)"
defp explain_function_p4("concat", _arg_count), do: "concat(-3)"
defp explain_function_p4("concat_ws", _arg_count), do: "concat_ws(-4)"
defp explain_function_p4(name, _arg_count) when name in ["min", "max"], do: "#{name}(-3)"
defp explain_function_p4(name, arg_count), do: "#{name}(#{arg_count})"
defp explain_function_comment(result_reg, _arg_start_reg, 0), do: "r[#{result_reg}]=func()"
defp explain_function_comment(result_reg, arg_start_reg, 1),
do: "r[#{result_reg}]=func(r[#{arg_start_reg}])"
defp explain_function_comment(result_reg, arg_start_reg, arg_count),
do: "r[#{result_reg}]=func(r[#{arg_start_reg}..#{arg_start_reg + arg_count - 1}])"
defp explain_iif_registers(condition_opcode, state) do
result_reg = state.next_reg
state = %{state | next_reg: result_reg + 1}
case explain_case_truth_opcode(condition_opcode) do
:dynamic ->
explain_iif_condition_register(result_reg, state)
_constant ->
{{result_reg, nil}, state}
end
end
defp explain_iif_condition_register(result_reg, %{iif_condition_reg: reg} = state)
when not is_nil(reg),
do: {{result_reg, reg}, state}
defp explain_iif_condition_register(result_reg, state) do
condition_reg = state.next_reg
state = %{state | next_reg: condition_reg + 1, iif_condition_reg: condition_reg}
{{result_reg, condition_reg}, state}
end
defp explain_literal_row(addr, {opcode, p1, p4}, reg),
do: [addr, opcode, p1, reg, 0, p4, 0, explain_literal_comment(opcode, p1, p4, reg)]
defp explain_values_program(rows, limit) do
cond do
Enum.any?(rows, &Enum.any?(&1, fn item -> item == :error end)) ->
:error
rows |> Enum.map(&length/1) |> Enum.uniq() |> length() != 1 ->
:error
true ->
width = rows |> hd() |> length()
row_count = length(rows)
limit_register_width = explain_limit_register_width(limit)
limit_setup_width = explain_limit_setup_width(limit)
coroutine_reg = limit_register_width + 1
producer_reg_start = coroutine_reg + 3
operand_count = explain_expression_operand_count(rows)
base_output_start = producer_reg_start + width * 2
output_start =
if operand_count == 0 do
base_output_start
else
base_output_start + operand_count
end
{producer_rows, state, producer_end_addr} =
rows
|> Enum.with_index()
|> Enum.reduce(
{[], explain_operand_state(base_output_start), limit_setup_width + 2},
fn {row, _row_index}, {acc, state, row_start_addr} ->
{literal_rows, state, yield_row_addr} =
explain_literal_expression_rows(row, row_start_addr, producer_reg_start, state)
producer_rows =
acc ++
literal_rows ++ [[yield_row_addr, "Yield", coroutine_reg, 0, 0, nil, 0, nil]]
{producer_rows, state, yield_row_addr + 1}
end
)
consumer_start_addr = producer_end_addr + 1
yield_addr = consumer_start_addr + 1
copy_start_addr = yield_addr + 1 + explain_limit_offset_width(limit)
result_addr = copy_start_addr + width
decr_addr = result_addr + 1
loop_addr = result_addr + 1 + explain_limit_decr_width(limit)
halt_addr = loop_addr + 1
start_addr = halt_addr + 1
post_rows = explain_literal_post_rows(state, start_addr, output_start + width)
goto_addr = start_addr + length(post_rows)
copy_rows =
0..(width - 1)
|> Enum.map(fn column_index ->
src_reg = producer_reg_start + column_index
dest_reg = output_start + column_index
[
copy_start_addr + column_index,
"Copy",
src_reg,
dest_reg,
0,
nil,
2,
"r[#{dest_reg}]=r[#{src_reg}]"
]
end)
{:ok,
[[0, "Init", 0, start_addr, 0, nil, 0, "Start at #{start_addr}"]] ++
explain_limit_rows(limit, halt_addr) ++
[
[
limit_setup_width + 1,
"InitCoroutine",
coroutine_reg,
consumer_start_addr,
2,
nil,
0,
nil
]
] ++
producer_rows ++
[
[producer_end_addr, "EndCoroutine", coroutine_reg, 0, 0, nil, 0, nil],
[consumer_start_addr, "InitCoroutine", coroutine_reg, 0, 2, nil, 0, nil],
[
yield_addr,
"Yield",
coroutine_reg,
halt_addr,
0,
nil,
0,
"next row of #{row_count}-ROW VALUES CLAUSE"
]
] ++
explain_limit_offset_rows(limit, yield_addr + 1, loop_addr) ++
copy_rows ++
[
[
result_addr,
"ResultRow",
output_start,
width,
0,
nil,
0,
explain_result_comment(output_start, width)
]
] ++
explain_limit_decr_rows(limit, decr_addr, halt_addr) ++
[
[loop_addr, "Goto", 0, yield_addr, 0, nil, 0, nil],
[halt_addr, "Halt", 0, 0, 0, nil, 0, nil]
] ++
post_rows ++
[
[goto_addr, "Goto", 0, 1, 0, nil, 0, nil]
]}
end
end
defp explain_expression_operand_count(rows) do
rows
|> List.flatten()
|> Enum.reduce(explain_operand_state(0), fn item, state ->
{_regs, state} = explain_expression_registers(item, state)
state
end)
|> Map.fetch!(:next_reg)
end
defp explain_expression_registers({:negate, opcode}, state) do
{zero_reg, state} = explain_operand_register(explain_zero_opcode(), state)
{operand_reg, state} = explain_operand_register(opcode, state)
{[zero_reg, operand_reg], state}
end
defp explain_expression_registers({:bitnot, opcode}, state) do
{operand_reg, state} = explain_operand_register(opcode, state)
{[operand_reg], state}
end
defp explain_expression_registers({:not, opcode}, state) do
{operand_reg, state} = explain_operand_register(opcode, state)
{[operand_reg], state}
end
defp explain_expression_registers({:in_list, value_opcode, list_opcodes}, state) do
{{value_reg, bitand_reg, list_regs}, state} =
explain_in_registers(value_opcode, list_opcodes, state)
{[value_reg, bitand_reg | list_regs], state}
end
defp explain_expression_registers({:not_in_list, value_opcode, list_opcodes}, state) do
{in_reg, state} =
explain_post_in_register(value_opcode, list_opcodes, state)
{[in_reg], state}
end
defp explain_expression_registers({:case, nil, branches, _else_opcode}, state) do
Enum.map_reduce(branches, state, fn {when_opcode, _then_opcode}, state ->
case explain_case_truth_opcode(when_opcode) do
:dynamic -> explain_operand_register(when_opcode, state)
_constant -> {nil, state}
end
end)
end
defp explain_expression_registers({:case, operand_opcode, branches, _else_opcode}, state) do
when_opcodes = Enum.map(branches, &elem(&1, 0))
{{operand_reg, when_regs}, state} =
explain_case_operand_registers(operand_opcode, when_opcodes, state)
{[operand_reg | when_regs], state}
end
defp explain_expression_registers({:literal_function, _name, arg_opcodes, _negated}, state) do
{{result_reg, _arg_start_reg, arg_regs}, state} =
explain_function_registers(arg_opcodes, state)
{[result_reg | arg_regs], state}
end
defp explain_expression_registers({:coalesce_function, _arg_opcodes}, state) do
result_reg = state.next_reg
{[result_reg], %{state | next_reg: result_reg + 1}}
end
defp explain_expression_registers(
{:nullif_function, left_opcode, right_opcode, _collation_p4},
state
) do
{{result_reg, _arg_start_reg, arg_regs}, state} =
explain_function_registers([left_opcode, right_opcode], state)
{[result_reg | arg_regs], state}
end
defp explain_expression_registers(
{:collated_function, _name, arg_opcodes, _collation_p4},
state
) do
{{result_reg, _arg_start_reg, arg_regs}, state} =
explain_function_registers(arg_opcodes, state)
{[result_reg | arg_regs], state}
end
defp explain_expression_registers(
{:iif_function, condition_opcode, _true_opcode, _false_opcode},
state
) do
{{result_reg, condition_reg}, state} = explain_iif_registers(condition_opcode, state)
regs = if is_nil(condition_reg), do: [result_reg], else: [result_reg, condition_reg]
{regs, state}
end
defp explain_expression_registers({:binary, _opcode, left_opcode, right_opcode}, state) do
{left_reg, state} = explain_operand_register(left_opcode, state)
{right_reg, state} = explain_operand_register(right_opcode, state)
{[left_reg, right_reg], state}
end
defp explain_expression_registers(
{tag, _opcode, left_opcode, right_opcode, _collation_p4},
state
)
when tag in [:compare, :is_compare] do
{left_reg, state} = explain_operand_register(left_opcode, state)
{right_reg, state} = explain_operand_register(right_opcode, state)
{[left_reg, right_reg], state}
end
defp explain_expression_registers({:null_test, _opcode, operand_opcode}, state) do
{operand_reg, state} = explain_operand_register(operand_opcode, state)
{[operand_reg], state}
end
defp explain_expression_registers({:between, value_opcode, low_opcode, high_opcode}, state) do
{{value_reg, lower_temp, upper_temp, low_reg, high_reg}, state} =
explain_between_registers(value_opcode, low_opcode, high_opcode, state)
{[value_reg, lower_temp, upper_temp, low_reg, high_reg], state}
end
defp explain_expression_registers({:not_between, value_opcode, low_opcode, high_opcode}, state) do
{between_reg, state} =
explain_post_between_register(value_opcode, low_opcode, high_opcode, state)
{[between_reg], state}
end
defp explain_expression_registers(_item, state), do: {[], state}
defp explain_zero_opcode, do: {"Integer", 0, nil}
defp explain_binary_comment("Add", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{right_reg}]+r[#{left_reg}]"
defp explain_binary_comment("Subtract", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{left_reg}]-r[#{right_reg}]"
defp explain_binary_comment("Multiply", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{right_reg}]*r[#{left_reg}]"
defp explain_binary_comment("Divide", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{left_reg}]/r[#{right_reg}]"
defp explain_binary_comment("Remainder", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{left_reg}]%r[#{right_reg}]"
defp explain_binary_comment("Concat", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{left_reg}]+r[#{right_reg}]"
defp explain_binary_comment("BitAnd", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{right_reg}]&r[#{left_reg}]"
defp explain_binary_comment("BitOr", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{right_reg}]|r[#{left_reg}]"
defp explain_binary_comment("ShiftLeft", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{left_reg}]<<r[#{right_reg}]"
defp explain_binary_comment("ShiftRight", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=r[#{left_reg}]>>r[#{right_reg}]"
defp explain_binary_comment("And", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=(r[#{right_reg}] && r[#{left_reg}])"
defp explain_binary_comment("Or", output_reg, left_reg, right_reg),
do: "r[#{output_reg}]=(r[#{right_reg}] || r[#{left_reg}])"
defp explain_compare_comment("Eq", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]==r[#{right_reg}] goto #{next_addr}"
defp explain_compare_comment("Ne", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]!=r[#{right_reg}] goto #{next_addr}"
defp explain_compare_comment("Lt", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]<r[#{right_reg}] goto #{next_addr}"
defp explain_compare_comment("Le", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]<=r[#{right_reg}] goto #{next_addr}"
defp explain_compare_comment("Gt", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]>r[#{right_reg}] goto #{next_addr}"
defp explain_compare_comment("Ge", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]>=r[#{right_reg}] goto #{next_addr}"
defp explain_null_test_comment("IsNull", operand_reg, next_addr),
do: "if r[#{operand_reg}]==NULL goto #{next_addr}"
defp explain_null_test_comment("NotNull", operand_reg, next_addr),
do: "if r[#{operand_reg}]!=NULL goto #{next_addr}"
defp explain_literal_expr({:collate, expr, _name}), do: explain_literal_expr(expr)
defp explain_literal_expr({:cast, expr, affinity}) do
case explain_literal_opcode(expr) do
:error -> :error
opcode -> {:cast, opcode, affinity}
end
end
defp explain_literal_expr({:negate, {:literal, value}} = expr)
when is_integer(value) or is_float(value) do
{:literal, explain_literal_opcode(expr)}
end
defp explain_literal_expr({:negate, {:literal, value}}) do
case explain_literal_opcode({:literal, value}) do
:error -> :error
opcode -> {:negate, opcode}
end
end
defp explain_literal_expr({:bitnot, expr}) do
case explain_literal_opcode(expr) do
:error -> :error
opcode -> {:bitnot, opcode}
end
end
defp explain_literal_expr({:not, expr}) do
case explain_literal_opcode(expr) do
:error -> :error
opcode -> {:not, opcode}
end
end
defp explain_literal_expr({:binary, :and, left, right}) do
with left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
if explain_zero_integer_opcode?(left_opcode) or explain_zero_integer_opcode?(right_opcode) do
{:literal, explain_zero_opcode()}
else
{:binary, "And", left_opcode, right_opcode}
end
else
_other -> :error
end
end
defp explain_literal_expr({:binary, :or, left, right}) do
with left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
{:binary, "Or", left_opcode, right_opcode}
else
_other -> :error
end
end
defp explain_literal_expr({:in, _expr, [], false}), do: {:literal, explain_zero_opcode()}
defp explain_literal_expr({:in, _expr, [], true}), do: {:literal, {"Integer", 1, nil}}
defp explain_literal_expr({:in, expr, list, false}) when is_list(list) do
with value_opcode when value_opcode != :error <- explain_literal_opcode(expr),
list_opcodes <- Enum.map(list, &explain_literal_opcode/1),
false <- Enum.any?(list_opcodes, &(&1 == :error)) do
{:in_list, value_opcode, list_opcodes}
else
_other -> :error
end
end
defp explain_literal_expr({:in, expr, list, true}) when is_list(list) do
with value_opcode when value_opcode != :error <- explain_literal_opcode(expr),
list_opcodes <- Enum.map(list, &explain_literal_opcode/1),
false <- Enum.any?(list_opcodes, &(&1 == :error)) do
{:not_in_list, value_opcode, list_opcodes}
else
_other -> :error
end
end
defp explain_literal_expr({:case, operand, branches, else_expr}) do
with operand_opcode <- explain_case_operand_opcode(operand),
false <- operand_opcode == :error,
branch_opcodes <- explain_case_branch_opcodes(branches),
false <- branch_opcodes == :error,
else_opcode <- explain_case_else_opcode(else_expr),
false <- else_opcode == :error do
{:case, operand_opcode, branch_opcodes, else_opcode}
else
_other -> :error
end
end
defp explain_literal_expr({:like, expr, pattern, nil, negated}) do
with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern),
expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr) do
{:literal_function, "like", [pattern_opcode, expr_opcode], negated}
else
_other -> :error
end
end
defp explain_literal_expr({:like, expr, pattern, escape, negated}) do
with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern),
expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr),
escape_opcode when escape_opcode != :error <- explain_literal_opcode(escape) do
{:literal_function, "like", [pattern_opcode, expr_opcode, escape_opcode], negated}
else
_other -> :error
end
end
defp explain_literal_expr({:glob, expr, pattern, negated}) do
with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern),
expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr) do
{:literal_function, "glob", [pattern_opcode, expr_opcode], negated}
else
_other -> :error
end
end
defp explain_literal_expr({:regexp, expr, pattern, negated}) do
with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern),
expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr) do
{:literal_function, "regexp", [pattern_opcode, expr_opcode], negated}
else
_other -> :error
end
end
defp explain_literal_expr({:function, name, args})
when name in ["coalesce", "ifnull"] and is_list(args) do
with {:ok, arity_range} <- Map.fetch(@scalar_arity, name),
true <- length(args) in arity_range,
arg_opcodes <- Enum.map(args, &explain_literal_opcode/1),
false <- Enum.any?(arg_opcodes, &(&1 == :error)) do
{:coalesce_function, arg_opcodes}
else
_other -> :error
end
end
defp explain_literal_expr({:function, "nullif", [left, right]}) do
with left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
{:nullif_function, left_opcode, right_opcode, explain_nullif_collation_p4(left, right)}
else
_other -> :error
end
end
defp explain_literal_expr({:function, name, [_, _ | _] = args}) when name in ["min", "max"] do
with true <- length(args) in 2..127,
arg_opcodes <- Enum.map(args, &explain_literal_opcode/1),
false <- Enum.any?(arg_opcodes, &(&1 == :error)) do
{:collated_function, name, arg_opcodes, explain_collated_function_p4(args)}
else
_other -> :error
end
end
defp explain_literal_expr({:function, "iif", [condition, true_expr, false_expr]}) do
with condition_opcode when condition_opcode != :error <- explain_literal_opcode(condition),
true_opcode when true_opcode != :error <- explain_literal_opcode(true_expr),
false_opcode when false_opcode != :error <- explain_literal_opcode(false_expr) do
{:iif_function, condition_opcode, true_opcode, false_opcode}
else
_other -> :error
end
end
defp explain_literal_expr({:function, name, args}) when is_list(args) do
with true <- name in @explain_literal_scalar_functions,
{:ok, arity_range} <- Map.fetch(@scalar_arity, name),
true <- length(args) in arity_range,
arg_opcodes <- Enum.map(args, &explain_literal_opcode/1),
false <- Enum.any?(arg_opcodes, &(&1 == :error)) do
{:literal_function, name, arg_opcodes, false}
else
_other -> :error
end
end
defp explain_literal_expr({:binary, op, left, right})
when op in [:add, :sub, :mul, :div, :mod, :concat, :bitand, :bitor, :shl, :shr] do
with {:ok, opcode} <- explain_binary_opcode(op),
left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
{:binary, opcode, left_opcode, right_opcode}
else
_other -> :error
end
end
defp explain_literal_expr({:binary, op, left, right})
when op in [:eq, :ne, :lt, :le, :gt, :ge] do
with {:ok, opcode} <- explain_compare_opcode(op),
left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
{:compare, opcode, left_opcode, right_opcode, explain_compare_collation_p4(left, right)}
else
_other -> :error
end
end
defp explain_literal_expr({:is, left, {:literal, nil}}) do
case explain_literal_opcode(left) do
:error -> :error
opcode -> {:null_test, "IsNull", opcode}
end
end
defp explain_literal_expr({:is_not, left, {:literal, nil}}) do
case explain_literal_opcode(left) do
:error -> :error
opcode -> {:null_test, "NotNull", opcode}
end
end
defp explain_literal_expr({:is, left, right}) do
with left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
{:is_compare, "Eq", left_opcode, right_opcode, explain_compare_collation_p4(left, right)}
else
_other -> :error
end
end
defp explain_literal_expr({:is_not, left, right}) do
with left_opcode when left_opcode != :error <- explain_literal_opcode(left),
right_opcode when right_opcode != :error <- explain_literal_opcode(right) do
{:is_compare, "Ne", left_opcode, right_opcode, explain_compare_collation_p4(left, right)}
else
_other -> :error
end
end
defp explain_literal_expr({:between, expr, low, high, false}) do
with value_opcode when value_opcode != :error <- explain_literal_opcode(expr),
low_opcode when low_opcode != :error <- explain_literal_opcode(low),
high_opcode when high_opcode != :error <- explain_literal_opcode(high) do
{:between, value_opcode, low_opcode, high_opcode}
else
_other -> :error
end
end
defp explain_literal_expr({:between, expr, low, high, true}) do
with value_opcode when value_opcode != :error <- explain_literal_opcode(expr),
low_opcode when low_opcode != :error <- explain_literal_opcode(low),
high_opcode when high_opcode != :error <- explain_literal_opcode(high) do
{:not_between, value_opcode, low_opcode, high_opcode}
else
_other -> :error
end
end
defp explain_literal_expr(expr) do
case explain_literal_opcode(expr) do
:error -> :error
opcode -> {:literal, opcode}
end
end
defp explain_case_operand_opcode(nil), do: nil
defp explain_case_operand_opcode(expr), do: explain_literal_opcode(expr)
defp explain_case_branch_opcodes(branches) do
branch_opcodes =
Enum.map(branches, fn {when_expr, then_expr} ->
{explain_literal_opcode(when_expr), explain_literal_opcode(then_expr)}
end)
if Enum.any?(branch_opcodes, fn {when_opcode, then_opcode} ->
when_opcode == :error or then_opcode == :error
end) do
:error
else
branch_opcodes
end
end
defp explain_case_else_opcode(nil), do: nil
defp explain_case_else_opcode(expr), do: explain_literal_opcode(expr)
defp explain_compare_collation_p4(left, right) do
case explain_explicit_collation_name(left) || explain_explicit_collation_name(right) do
nil -> nil
name -> "#{String.upcase(to_string(name))}-8"
end
end
defp explain_nullif_collation_p4(left, right),
do: explain_compare_collation_p4(left, right) || "BINARY-8"
defp explain_collated_function_p4(args) do
args
|> Enum.find_value(&explain_explicit_collation_name/1)
|> case do
nil -> "BINARY-8"
name -> "#{String.upcase(to_string(name))}-8"
end
end
defp explain_explicit_collation_name({:collate, _expr, name}), do: name
defp explain_explicit_collation_name(_expr), do: nil
defp explain_binary_opcode(:add), do: {:ok, "Add"}
defp explain_binary_opcode(:sub), do: {:ok, "Subtract"}
defp explain_binary_opcode(:mul), do: {:ok, "Multiply"}
defp explain_binary_opcode(:div), do: {:ok, "Divide"}
defp explain_binary_opcode(:mod), do: {:ok, "Remainder"}
defp explain_binary_opcode(:concat), do: {:ok, "Concat"}
defp explain_binary_opcode(:bitand), do: {:ok, "BitAnd"}
defp explain_binary_opcode(:bitor), do: {:ok, "BitOr"}
defp explain_binary_opcode(:shl), do: {:ok, "ShiftLeft"}
defp explain_binary_opcode(:shr), do: {:ok, "ShiftRight"}
defp explain_compare_opcode(:eq), do: {:ok, "Eq"}
defp explain_compare_opcode(:ne), do: {:ok, "Ne"}
defp explain_compare_opcode(:lt), do: {:ok, "Lt"}
defp explain_compare_opcode(:le), do: {:ok, "Le"}
defp explain_compare_opcode(:gt), do: {:ok, "Gt"}
defp explain_compare_opcode(:ge), do: {:ok, "Ge"}
defp explain_literal_opcode({:literal, nil}), do: {"Null", 0, nil}
defp explain_literal_opcode({:literal, value}) when is_integer(value),
do: explain_integer_literal_opcode(value)
defp explain_literal_opcode({:literal, value}) when is_boolean(value),
do: {"Integer", bool_int(value), nil}
# EXPLAIN renders a Real literal at full (round-trippable) precision — unlike
# value display / `CAST(x AS TEXT)`, which uses SQLite's `%.15g`.
defp explain_literal_opcode({:literal, value}) when is_float(value),
do: {"Real", 0, Float.to_string(value)}
defp explain_literal_opcode({:literal, {:blob, value}}) when is_binary(value),
do: {"Blob", byte_size(value), value}
defp explain_literal_opcode({:literal, value}) when is_binary(value),
do: {"String8", 0, value}
defp explain_literal_opcode({:collate, expr, _name}), do: explain_literal_opcode(expr)
defp explain_literal_opcode({:negate, {:literal, value}}) when is_integer(value),
do: explain_integer_literal_opcode(-value)
defp explain_literal_opcode({:negate, {:literal, value}}) when is_float(value),
do: {"Real", 0, Float.to_string(-value)}
defp explain_literal_opcode(_expr), do: :error
defp explain_zero_integer_opcode?({"Integer", 0, nil}), do: true
defp explain_zero_integer_opcode?(_opcode), do: false
defp explain_cast_affinity_code(:blob), do: 65
defp explain_cast_affinity_code(:text), do: 66
defp explain_cast_affinity_code(:numeric), do: 67
defp explain_cast_affinity_code(:integer), do: 68
defp explain_cast_affinity_code(:real), do: 69
defp explain_integer_literal_opcode(value)
when value >= -2_147_483_647 and value <= 2_147_483_647,
do: {"Integer", value, nil}
defp explain_integer_literal_opcode(value), do: {"Int64", 0, Value.to_text(value)}
defp explain_literal_comment("Null", _p1, _p4, reg), do: "r[#{reg}]=NULL"
defp explain_literal_comment("Integer", value, _p4, reg), do: "r[#{reg}]=#{value}"
defp explain_literal_comment("Int64", _p1, value, reg), do: "r[#{reg}]=#{value}"
defp explain_literal_comment("Real", _p1, value, reg), do: "r[#{reg}]=#{value}"
defp explain_literal_comment("String8", _p1, value, reg), do: "r[#{reg}]='#{value}'"
defp explain_literal_comment("Blob", size, value, reg), do: "r[#{reg}]=#{value} (len=#{size})"
defp explain_limit(nil, nil), do: {:ok, nil}
defp explain_limit(limit, nil) do
case explain_limit_integer(limit) do
{:ok, limit} -> {:ok, {:limit, limit}}
:error -> :error
end
end
defp explain_limit(limit_expr, offset_expr) do
with {:ok, limit} <- explain_limit_integer(limit_expr),
{:ok, offset} <- explain_limit_integer(offset_expr) do
{:ok, {:limit_offset, limit, offset}}
else
_other -> :error
end
end
defp explain_limit_integer({:literal, value}) when is_integer(value), do: {:ok, value}
defp explain_limit_integer({:negate, {:literal, value}}) when is_integer(value),
do: {:ok, -value}
defp explain_limit_integer(_expr), do: :error
defp explain_table_projection(table, name, alias_name, columns) do
columns
|> Enum.reduce_while({:ok, []}, fn
:star, {:ok, acc} ->
{:cont, {:ok, acc ++ explain_all_table_columns(table)}}
{:qualified_star, qualifier}, {:ok, acc} ->
if explain_table_qualifier?(qualifier, table, name, alias_name) do
{:cont, {:ok, acc ++ explain_all_table_columns(table)}}
else
{:halt, :error}
end
{{:column, qualifier, column_name}, _alias}, {:ok, acc} ->
if qualifier == nil or explain_table_qualifier?(qualifier, table, name, alias_name) do
case explain_table_column(table, column_name) do
nil -> {:halt, :error}
column -> {:cont, {:ok, acc ++ [column]}}
end
else
{:halt, :error}
end
_other, _acc ->
{:halt, :error}
end)
end
defp explain_all_table_columns(table) do
table.columns
|> Enum.with_index()
|> Enum.map(fn {column, index} ->
if Table.key(column.name) == table.rowid_alias do
{:rowid, column.name}
else
{:column, index}
end
end)
end
defp explain_table_column(table, column_name) do
key = Table.key(column_name)
case Enum.find_index(table.columns, &(Table.key(&1.name) == key)) do
nil when key in @rowid_names and not table.without_rowid -> {:rowid, table.name}
nil -> nil
_index when key == table.rowid_alias -> {:rowid, table.name}
index -> {:column, index}
end
end
defp explain_table_qualifier?(qualifier, table, name, alias_name) do
key = Table.key(qualifier)
key in [Table.key(alias_name || name), Table.key(table.name)]
end
defp explain_table_filter(_table, _name, _alias_name, nil), do: {:ok, nil}
defp explain_table_filter(table, name, alias_name, {:binary, :and, _left, _right} = where) do
where
|> where_conjuncts()
|> Enum.reduce_while({:ok, []}, fn term, {:ok, acc} ->
case explain_table_filter_term(table, name, alias_name, term) do
{:ok, filter} -> {:cont, {:ok, [filter | acc]}}
:error -> {:halt, :error}
end
end)
|> case do
{:ok, filters} -> explain_combine_filters(Enum.reverse(filters))
:error -> :error
end
end
defp explain_table_filter(table, name, alias_name, where),
do: explain_table_filter_term(table, name, alias_name, where)
defp explain_table_filter_term(table, name, alias_name, {:binary, op, left, right})
when op in [:eq, :ne, :lt, :le, :gt, :ge] do
cond do
op == :eq and explain_filter_rowid?(table, name, alias_name, left) ->
explain_rowid_filter(right)
op == :eq and explain_filter_rowid?(table, name, alias_name, right) ->
explain_rowid_filter(left)
op in [:lt, :le, :gt, :ge] and explain_filter_rowid?(table, name, alias_name, left) ->
explain_rowid_range_filter(op, right)
op in [:lt, :le, :gt, :ge] and explain_filter_rowid?(table, name, alias_name, right) ->
explain_rowid_range_filter(explain_flip_comparison_op(op), left)
column = explain_filter_column(table, name, alias_name, left) ->
explain_comparison_filter(column, op, right)
column = explain_filter_column(table, name, alias_name, right) ->
explain_comparison_filter(column, explain_flip_comparison_op(op), left)
true ->
:error
end
end
defp explain_table_filter_term(table, name, alias_name, {:binary, :or, _left, _right} = where) do
where
|> where_disjuncts()
|> explain_rowid_or_filter(table, name, alias_name)
end
defp explain_table_filter_term(
table,
name,
alias_name,
{:between, expr, low, high, false}
) do
if explain_filter_rowid?(table, name, alias_name, expr) do
with {:ok, lower} <- explain_rowid_range_filter(:ge, low),
{:ok, upper} <- explain_rowid_range_filter(:le, high) do
case explain_merge_rowid_ranges([lower, upper]) do
:error -> :error
range -> {:ok, range}
end
end
else
:error
end
end
defp explain_table_filter_term(table, name, alias_name, {:in, expr, list, false})
when is_list(list) do
if explain_filter_rowid?(table, name, alias_name, expr) do
explain_rowid_in_filter(list)
else
:error
end
end
defp explain_table_filter_term(table, name, alias_name, {:is, expr, {:literal, nil}}) do
case explain_filter_column(table, name, alias_name, expr) do
nil -> :error
column -> {:ok, {:null, column, "NotNull"}}
end
end
defp explain_table_filter_term(table, name, alias_name, {:is_not, expr, {:literal, nil}}) do
case explain_filter_column(table, name, alias_name, expr) do
nil -> :error
column -> {:ok, {:null, column, "IsNull"}}
end
end
defp explain_table_filter_term(_table, _name, _alias_name, _where), do: :error
defp explain_combine_filters(filters) do
{rowid_filters, residual_filters} =
Enum.split_with(filters, fn
{:rowid_eq, _literal_opcode} -> true
{:rowid_range, _lower, _upper} -> true
{:rowid_in, _literal_opcodes} -> true
_filter -> false
end)
case explain_combine_rowid_filters(rowid_filters) do
{:ok, nil} ->
case residual_filters do
[] -> {:ok, nil}
[filter] -> {:ok, filter}
filters -> {:ok, {:filters, filters}}
end
{:ok, {:rowid_eq, literal_opcode}} ->
case residual_filters do
[] -> {:ok, {:rowid_eq, literal_opcode}}
filters -> {:ok, {:rowid_eq, literal_opcode, filters}}
end
{:ok, {:rowid_range, lower, upper}} ->
case residual_filters do
[] -> {:ok, {:rowid_range, lower, upper}}
filters -> {:ok, {:rowid_range, lower, upper, filters}}
end
{:ok, {:rowid_in, literal_opcodes}} ->
case residual_filters do
[] -> {:ok, {:rowid_in, literal_opcodes}}
filters -> {:ok, {:rowid_in, literal_opcodes, filters}}
end
:error ->
:error
end
end
defp explain_combine_rowid_filters([]), do: {:ok, nil}
defp explain_combine_rowid_filters([{:rowid_eq, literal_opcode}]),
do: {:ok, {:rowid_eq, literal_opcode}}
defp explain_combine_rowid_filters([{:rowid_in, literal_opcodes}]),
do: {:ok, {:rowid_in, literal_opcodes}}
defp explain_combine_rowid_filters(filters) do
if Enum.any?(
filters,
&(match?({:rowid_eq, _literal_opcode}, &1) or match?({:rowid_in, _literal_opcodes}, &1))
) do
:error
else
case explain_merge_rowid_ranges(filters) do
:error -> :error
{:rowid_range, nil, nil} -> :error
range -> {:ok, range}
end
end
end
defp explain_merge_rowid_ranges(filters) do
Enum.reduce_while(filters, {:rowid_range, nil, nil}, fn
{:rowid_range, next_lower, next_upper}, {:rowid_range, lower, upper} ->
cond do
next_lower != nil and lower != nil ->
{:halt, :error}
next_upper != nil and upper != nil ->
{:halt, :error}
true ->
{:cont, {:rowid_range, next_lower || lower, next_upper || upper}}
end
end)
end
defp explain_rowid_filter(literal) do
case explain_literal_opcode(literal) do
:error -> :error
opcode -> {:ok, {:rowid_eq, opcode}}
end
end
defp explain_rowid_in_filter([]), do: :error
defp explain_rowid_in_filter(list) do
list
|> Enum.reduce_while([], fn expr, acc ->
case explain_literal_opcode(expr) do
:error -> {:halt, :error}
opcode -> {:cont, [opcode | acc]}
end
end)
|> case do
:error -> :error
literal_opcodes -> {:ok, {:rowid_in, Enum.reverse(literal_opcodes)}}
end
end
defp explain_rowid_or_filter(disjuncts, table, name, alias_name) do
if multiple_terms?(disjuncts) do
disjuncts
|> Enum.reduce_while([], fn disjunct, acc ->
case explain_rowid_or_disjunct(table, name, alias_name, disjunct) do
{:ok, literal_opcodes} -> {:cont, acc ++ literal_opcodes}
:error -> {:halt, :error}
end
end)
|> case do
:error -> :error
[] -> :error
literal_opcodes -> {:ok, {:rowid_in, literal_opcodes}}
end
else
:error
end
end
defp explain_rowid_or_disjunct(table, name, alias_name, {:binary, :eq, left, right}) do
cond do
explain_filter_rowid?(table, name, alias_name, left) ->
explain_rowid_or_literal(right)
explain_filter_rowid?(table, name, alias_name, right) ->
explain_rowid_or_literal(left)
true ->
:error
end
end
defp explain_rowid_or_disjunct(table, name, alias_name, {:in, expr, list, false})
when is_list(list) do
if explain_filter_rowid?(table, name, alias_name, expr) do
list
|> Enum.reduce_while([], fn item, acc ->
case explain_rowid_or_literal(item) do
{:ok, [literal_opcode]} -> {:cont, acc ++ [literal_opcode]}
:error -> {:halt, :error}
end
end)
|> case do
:error -> :error
[] -> :error
literal_opcodes -> {:ok, literal_opcodes}
end
else
:error
end
end
defp explain_rowid_or_disjunct(_table, _name, _alias_name, _disjunct), do: :error
defp explain_rowid_or_literal(expr) do
case explain_literal_opcode(expr) do
:error -> :error
literal_opcode -> {:ok, [literal_opcode]}
end
end
defp explain_rowid_range_filter(op, literal) do
case explain_literal_opcode(literal) do
:error -> :error
opcode -> {:ok, explain_rowid_range_bound(op, opcode)}
end
end
defp explain_rowid_range_bound(:gt, literal_opcode),
do: {:rowid_range, {"SeekGT", literal_opcode}, nil}
defp explain_rowid_range_bound(:ge, literal_opcode),
do: {:rowid_range, {"SeekGE", literal_opcode}, nil}
defp explain_rowid_range_bound(:lt, literal_opcode),
do: {:rowid_range, nil, {"Ge", literal_opcode}}
defp explain_rowid_range_bound(:le, literal_opcode),
do: {:rowid_range, nil, {"Gt", literal_opcode}}
defp explain_comparison_filter(column, op, literal) do
case explain_literal_opcode(literal) do
:error -> :error
opcode -> {:ok, {:compare, column, explain_jump_opcode(op), opcode}}
end
end
defp explain_filter_column(table, name, alias_name, {:collate, expr, _collation}),
do: explain_filter_column(table, name, alias_name, expr)
defp explain_filter_column(table, name, alias_name, {:column, qualifier, column_name}) do
if qualifier == nil or explain_table_qualifier?(qualifier, table, name, alias_name) do
case explain_table_column(table, column_name) do
{:column, _index} = column -> column
_other -> nil
end
end
end
defp explain_filter_column(_table, _name, _alias_name, _expr), do: nil
defp explain_filter_rowid?(table, name, alias_name, {:collate, expr, _collation}),
do: explain_filter_rowid?(table, name, alias_name, expr)
defp explain_filter_rowid?(table, name, alias_name, {:column, qualifier, column_name}) do
(qualifier == nil or explain_table_qualifier?(qualifier, table, name, alias_name)) and
match?({:rowid, _display}, explain_table_column(table, column_name))
end
defp explain_filter_rowid?(_table, _name, _alias_name, _expr), do: false
defp explain_flip_comparison_op(:lt), do: :gt
defp explain_flip_comparison_op(:le), do: :ge
defp explain_flip_comparison_op(:gt), do: :lt
defp explain_flip_comparison_op(:ge), do: :le
defp explain_flip_comparison_op(op), do: op
defp explain_jump_opcode(:eq), do: "Ne"
defp explain_jump_opcode(:ne), do: "Eq"
defp explain_jump_opcode(:lt), do: "Ge"
defp explain_jump_opcode(:le), do: "Gt"
defp explain_jump_opcode(:gt), do: "Le"
defp explain_jump_opcode(:ge), do: "Lt"
defp explain_table_scan_program(db, table, projection, filter, limit) do
case filter do
{:rowid_eq, literal_opcode} ->
explain_table_rowid_seek_program(db, table, projection, literal_opcode, [], limit)
{:rowid_eq, literal_opcode, residual_filters} ->
explain_table_rowid_seek_program(
db,
table,
projection,
literal_opcode,
residual_filters,
limit
)
{:rowid_range, lower, upper} ->
explain_table_rowid_range_program(db, table, projection, lower, upper, [], limit)
{:rowid_range, lower, upper, residual_filters} ->
explain_table_rowid_range_program(
db,
table,
projection,
lower,
upper,
residual_filters,
limit
)
{:rowid_in, literal_opcodes} ->
explain_table_rowid_in_program(db, table, projection, literal_opcodes, [], limit)
{:rowid_in, literal_opcodes, residual_filters} ->
explain_table_rowid_in_program(
db,
table,
projection,
literal_opcodes,
residual_filters,
limit
)
_other ->
explain_table_scan_loop_program(db, table, projection, filter, limit)
end
end
defp explain_table_scan_loop_program(db, table, projection, filter, limit) do
width = length(projection)
filter_conditions = explain_filter_conditions(filter)
filter_width = explain_filter_width(filter_conditions)
const_width = explain_filter_const_width(filter_conditions)
setup_width = explain_limit_setup_width(limit)
register_width = explain_limit_register_width(limit)
offset_width = explain_limit_offset_width(limit)
open_addr = 1 + setup_width
rewind_addr = open_addr + 1
filter_start_addr = rewind_addr + 1
filter_column_reg = 1 + register_width
filter_const_start_reg =
filter_column_reg + explain_filter_column_register_width(filter_conditions)
output_start = filter_const_start_reg + const_width
offset_addr = filter_start_addr + filter_width
projection_addr = offset_addr + offset_width
result_addr = projection_addr + width
decr_addr = result_addr + 1
next_addr = result_addr + 1 + explain_limit_decr_width(limit)
halt_addr = result_addr + 2 + explain_limit_decr_width(limit)
transaction_addr = halt_addr + 1
start_addr = transaction_addr
rootpage = explain_table_rootpage(db, table)
open_p4 = explain_table_open_p4(projection ++ explain_filter_projection(filter))
filter_rows =
explain_filter_rows(
table,
filter_conditions,
filter_start_addr,
filter_column_reg,
filter_const_start_reg,
next_addr
)
projection_rows =
projection
|> Enum.with_index()
|> Enum.map(fn {column, index} ->
explain_table_projection_row(
table,
column,
output_start + index,
projection_addr + index,
0
)
end)
const_rows =
explain_filter_const_rows(filter_conditions, transaction_addr + 1, filter_const_start_reg)
[[0, "Init", 0, start_addr, 0, nil, 0, "Start at #{start_addr}"]] ++
explain_limit_rows(limit, halt_addr) ++
[
[
open_addr,
"OpenRead",
0,
rootpage,
0,
open_p4,
0,
"root=#{rootpage} iDb=0; #{table.name}"
]
] ++
[[rewind_addr, "Rewind", 0, halt_addr, 0, nil, 0, nil]] ++
filter_rows ++
explain_limit_offset_rows(limit, offset_addr, next_addr) ++
projection_rows ++
[
[
result_addr,
"ResultRow",
output_start,
width,
0,
nil,
0,
explain_result_comment(output_start, width)
]
] ++
explain_limit_decr_rows(limit, decr_addr, halt_addr) ++
[
[next_addr, "Next", 0, filter_start_addr, 0, nil, 1, nil],
[halt_addr, "Halt", 0, 0, 0, nil, 0, nil],
[transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"]
] ++
const_rows ++
[[transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil]]
end
defp explain_filter_width(conditions), do: length(conditions) * 2
defp explain_filter_column_register_width([]), do: 0
defp explain_filter_column_register_width(_conditions), do: 1
defp explain_filter_register_width([]), do: 0
defp explain_filter_register_width(conditions),
do: 1 + explain_filter_const_width(conditions)
defp explain_filter_const_width(conditions),
do: Enum.count(conditions, &match?({:compare, _column, _jump_opcode, _literal_opcode}, &1))
defp explain_filter_projection({:rowid_eq, _literal_opcode}), do: []
defp explain_filter_projection({:rowid_eq, _literal_opcode, filters}),
do: explain_filters_projection(filters)
defp explain_filter_projection({:rowid_in, _literal_opcodes}), do: []
defp explain_filter_projection({:rowid_in, _literal_opcodes, filters}),
do: explain_filters_projection(filters)
defp explain_filter_projection({:compare, column, _jump_opcode, _literal_opcode}), do: [column]
defp explain_filter_projection({:null, column, _jump_opcode}), do: [column]
defp explain_filter_projection({:filters, filters}), do: explain_filters_projection(filters)
defp explain_filter_projection(nil), do: []
defp explain_filters_projection(filters),
do: Enum.flat_map(filters, &explain_filter_projection/1)
defp explain_filter_conditions(nil), do: []
defp explain_filter_conditions({:rowid_eq, _literal_opcode}), do: []
defp explain_filter_conditions({:rowid_eq, _literal_opcode, filters}), do: filters
defp explain_filter_conditions({:rowid_in, _literal_opcodes}), do: []
defp explain_filter_conditions({:rowid_in, _literal_opcodes, filters}), do: filters
defp explain_filter_conditions({:filters, filters}), do: filters
defp explain_filter_conditions({:compare, _column, _jump_opcode, _literal_opcode} = filter),
do: [filter]
defp explain_filter_conditions({:null, _column, _jump_opcode} = filter), do: [filter]
defp explain_filter_rows(table, conditions, start_addr, column_reg, const_start_reg, next_addr) do
{rows, _next_const_reg} =
conditions
|> Enum.with_index()
|> Enum.reduce({[], const_start_reg}, fn {condition, index}, {acc, const_reg} ->
addr = start_addr + index * 2
{condition_rows, next_const_reg} =
explain_filter_condition_rows(table, condition, addr, column_reg, const_reg, next_addr)
{acc ++ condition_rows, next_const_reg}
end)
rows
end
defp explain_filter_condition_rows(
table,
{:compare, column, jump_opcode, _literal_opcode},
addr,
column_reg,
const_reg,
next_addr
) do
rows = [
explain_table_projection_row(table, column, column_reg, addr, 0),
[
addr + 1,
jump_opcode,
const_reg,
next_addr,
column_reg,
"BINARY-8",
81,
explain_jump_comment(jump_opcode, column_reg, const_reg, next_addr)
]
]
{rows, const_reg + 1}
end
defp explain_filter_condition_rows(
table,
{:null, column, jump_opcode},
addr,
column_reg,
const_reg,
next_addr
) do
rows = [
explain_table_projection_row(table, column, column_reg, addr, 128),
[
addr + 1,
jump_opcode,
column_reg,
next_addr,
0,
nil,
0,
explain_null_jump_comment(jump_opcode, column_reg, next_addr)
]
]
{rows, const_reg}
end
defp explain_filter_const_rows(conditions, addr, const_start_reg) do
{rows, _next_addr, _next_const_reg} =
Enum.reduce(conditions, {[], addr, const_start_reg}, fn
{:compare, _column, _jump_opcode, literal_opcode}, {acc, next_addr, const_reg} ->
{opcode, p1, p4} = literal_opcode
row = [
next_addr,
opcode,
p1,
const_reg,
0,
p4,
0,
explain_literal_comment(opcode, p1, p4, const_reg)
]
{acc ++ [row], next_addr + 1, const_reg + 1}
{:null, _column, _jump_opcode}, acc ->
acc
end)
rows
end
defp explain_table_rowid_seek_program(
db,
table,
projection,
literal_opcode,
residual_filters,
limit
) do
width = length(projection)
setup_width = explain_limit_setup_width(limit)
register_width = explain_limit_register_width(limit)
offset_width = explain_limit_offset_width(limit)
filter_width = explain_filter_width(residual_filters)
filter_register_width = explain_filter_register_width(residual_filters)
const_width = explain_filter_const_width(residual_filters)
key_reg = 1 + register_width
filter_column_reg = key_reg + 1
filter_const_start_reg =
filter_column_reg + explain_filter_column_register_width(residual_filters)
output_start = key_reg + 1 + filter_register_width
open_addr = 1 + setup_width
literal_addr = open_addr + 1
seek_addr = literal_addr + 1
offset_addr = seek_addr + 1 + filter_width
projection_addr = offset_addr + offset_width
result_addr = projection_addr + width
decr_addr = result_addr + 1
halt_addr = result_addr + 1 + explain_limit_decr_width(limit)
transaction_addr = halt_addr + 1
rootpage = explain_table_rootpage(db, table)
open_p4 = explain_table_open_p4(projection ++ explain_filters_projection(residual_filters))
{literal_op, literal_p1, literal_p4} = literal_opcode
filter_rows =
explain_filter_rows(
table,
residual_filters,
seek_addr + 1,
filter_column_reg,
filter_const_start_reg,
halt_addr
)
projection_rows =
projection
|> Enum.with_index()
|> Enum.map(fn {column, index} ->
explain_table_projection_row(
table,
column,
output_start + index,
projection_addr + index,
0
)
end)
[
[0, "Init", 0, transaction_addr, 0, nil, 0, "Start at #{transaction_addr}"]
] ++
explain_limit_rows(limit, halt_addr) ++
[
[
open_addr,
"OpenRead",
0,
rootpage,
0,
open_p4,
0,
"root=#{rootpage} iDb=0; #{table.name}"
],
[
literal_addr,
literal_op,
literal_p1,
key_reg,
0,
literal_p4,
0,
explain_literal_comment(literal_op, literal_p1, literal_p4, key_reg)
],
[seek_addr, "SeekRowid", 0, halt_addr, key_reg, nil, 0, "intkey=r[#{key_reg}]"]
] ++
filter_rows ++
explain_limit_offset_rows(limit, offset_addr, halt_addr) ++
projection_rows ++
[
[
result_addr,
"ResultRow",
output_start,
width,
0,
nil,
0,
explain_result_comment(output_start, width)
]
] ++
explain_limit_decr_rows(limit, decr_addr, halt_addr) ++
[
[halt_addr, "Halt", 0, 0, 0, nil, 0, nil],
[transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"]
] ++
explain_filter_const_rows(residual_filters, transaction_addr + 1, filter_const_start_reg) ++
[
[transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil]
]
end
defp explain_table_rowid_in_program(
db,
table,
projection,
literal_opcodes,
residual_filters,
limit
) do
width = length(projection)
key_count = length(literal_opcodes)
setup_width = explain_limit_setup_width(limit)
register_width = explain_limit_register_width(limit)
offset_width = explain_limit_offset_width(limit)
filter_width = explain_filter_width(residual_filters)
residual_const_width = explain_filter_const_width(residual_filters)
key_start_reg = 1 + register_width
filter_column_reg = key_start_reg + key_count
filter_const_start_reg =
filter_column_reg + explain_filter_column_register_width(residual_filters)
output_start = filter_const_start_reg + residual_const_width
open_addr = 1 + setup_width
seek_start_addr = open_addr + 1
candidate_width =
1 + filter_width + offset_width + width + 1 + explain_limit_decr_width(limit) + 1
halt_addr = seek_start_addr + key_count * candidate_width
transaction_addr = halt_addr + 1
rootpage = explain_table_rootpage(db, table)
open_p4 = explain_table_open_p4(projection ++ explain_filters_projection(residual_filters))
candidate_rows =
literal_opcodes
|> Enum.with_index()
|> Enum.flat_map(fn {_literal_opcode, index} ->
candidate_addr = seek_start_addr + index * candidate_width
next_candidate_addr = candidate_addr + candidate_width
key_reg = key_start_reg + index
offset_addr = candidate_addr + 1 + filter_width
projection_addr = offset_addr + offset_width
result_addr = projection_addr + width
decr_addr = result_addr + 1
loop_addr = result_addr + 1 + explain_limit_decr_width(limit)
filter_rows =
explain_filter_rows(
table,
residual_filters,
candidate_addr + 1,
filter_column_reg,
filter_const_start_reg,
next_candidate_addr
)
projection_rows =
projection
|> Enum.with_index()
|> Enum.map(fn {column, projection_index} ->
explain_table_projection_row(
table,
column,
output_start + projection_index,
projection_addr + projection_index,
0
)
end)
[
[
candidate_addr,
"SeekRowid",
0,
next_candidate_addr,
key_reg,
nil,
0,
"intkey=r[#{key_reg}]"
]
] ++
filter_rows ++
explain_limit_offset_rows(limit, offset_addr, next_candidate_addr) ++
projection_rows ++
[
[
result_addr,
"ResultRow",
output_start,
width,
0,
nil,
0,
explain_result_comment(output_start, width)
]
] ++
explain_limit_decr_rows(limit, decr_addr, halt_addr) ++
[[loop_addr, "Goto", 0, next_candidate_addr, 0, nil, 0, nil]]
end)
literal_rows =
literal_opcodes
|> Enum.with_index()
|> Enum.map(fn {literal_opcode, index} ->
{literal_op, literal_p1, literal_p4} = literal_opcode
reg = key_start_reg + index
[
transaction_addr + 1 + index,
literal_op,
literal_p1,
reg,
0,
literal_p4,
0,
explain_literal_comment(literal_op, literal_p1, literal_p4, reg)
]
end)
const_start_addr = transaction_addr + 1 + key_count
const_width = key_count + residual_const_width
[
[0, "Init", 0, transaction_addr, 0, nil, 0, "Start at #{transaction_addr}"]
] ++
explain_limit_rows(limit, halt_addr) ++
[
[
open_addr,
"OpenRead",
0,
rootpage,
0,
open_p4,
0,
"root=#{rootpage} iDb=0; #{table.name}"
]
] ++
candidate_rows ++
[
[halt_addr, "Halt", 0, 0, 0, nil, 0, nil],
[transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"]
] ++
literal_rows ++
explain_filter_const_rows(residual_filters, const_start_addr, filter_const_start_reg) ++
[
[transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil]
]
end
defp explain_table_rowid_range_program(
db,
table,
projection,
lower,
upper,
residual_filters,
limit
) do
width = length(projection)
residual_conditions = explain_filter_conditions({:filters, residual_filters})
filter_width = explain_filter_width(residual_conditions)
residual_const_width = explain_filter_const_width(residual_conditions)
setup_width = explain_limit_setup_width(limit)
register_width = explain_limit_register_width(limit)
offset_width = explain_limit_offset_width(limit)
{lower_reg, next_reg} = explain_optional_register(lower, 1 + register_width)
{upper_reg, next_reg} = explain_optional_register(upper, next_reg)
{upper_rowid_reg, next_reg} = explain_optional_register(upper, next_reg)
{filter_column_reg, next_reg} = explain_optional_register(residual_conditions, next_reg)
filter_const_start_reg = next_reg
output_start = filter_const_start_reg + residual_const_width
open_addr = 1 + setup_width
start_addr = open_addr + 1
upper_start_addr = start_addr + 1
upper_width = if upper, do: 3, else: 0
loop_body_addr = upper_start_addr + upper_width
offset_addr = loop_body_addr + filter_width
projection_addr = offset_addr + offset_width
result_addr = projection_addr + width
decr_addr = result_addr + 1
next_addr = result_addr + 1 + explain_limit_decr_width(limit)
halt_addr = result_addr + 2 + explain_limit_decr_width(limit)
transaction_addr = halt_addr + 1
rootpage = explain_table_rootpage(db, table)
open_p4 = explain_table_open_p4(projection ++ explain_filters_projection(residual_filters))
start_row =
case lower do
{seek_opcode, _literal_opcode} ->
[start_addr, seek_opcode, 0, halt_addr, lower_reg, nil, 0, "key=r[#{lower_reg}]; pk"]
nil ->
[start_addr, "Rewind", 0, halt_addr, 0, nil, 0, nil]
end
{upper_rows, next_target_addr} =
explain_rowid_upper_bound_rows(
upper,
upper_start_addr,
upper_reg,
upper_rowid_reg,
halt_addr
)
filter_rows =
explain_filter_rows(
table,
residual_conditions,
loop_body_addr,
filter_column_reg,
filter_const_start_reg,
next_addr
)
projection_rows =
projection
|> Enum.with_index()
|> Enum.map(fn {column, index} ->
explain_table_projection_row(
table,
column,
output_start + index,
projection_addr + index,
0
)
end)
post_transaction_rows =
[]
|> explain_rowid_lower_const_row(lower, transaction_addr + 1, lower_reg)
|> then(fn {rows, addr} ->
rows ++ explain_filter_const_rows(residual_conditions, addr, filter_const_start_reg)
end)
const_width = if(lower, do: 1, else: 0) + residual_const_width
[
[0, "Init", 0, transaction_addr, 0, nil, 0, "Start at #{transaction_addr}"]
] ++
explain_limit_rows(limit, halt_addr) ++
[
[
open_addr,
"OpenRead",
0,
rootpage,
0,
open_p4,
0,
"root=#{rootpage} iDb=0; #{table.name}"
],
start_row
] ++
upper_rows ++
filter_rows ++
explain_limit_offset_rows(limit, offset_addr, next_addr) ++
projection_rows ++
[
[
result_addr,
"ResultRow",
output_start,
width,
0,
nil,
0,
explain_result_comment(output_start, width)
]
] ++
explain_limit_decr_rows(limit, decr_addr, halt_addr) ++
[
[next_addr, "Next", 0, next_target_addr, 0, nil, 0, nil],
[halt_addr, "Halt", 0, 0, 0, nil, 0, nil],
[transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"]
] ++
post_transaction_rows ++
[[transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil]]
end
defp explain_optional_register(nil, next_reg), do: {nil, next_reg}
defp explain_optional_register([], next_reg), do: {nil, next_reg}
defp explain_optional_register(_value, next_reg), do: {next_reg, next_reg + 1}
defp explain_limit_setup_width(nil), do: 0
defp explain_limit_setup_width({:limit, 0}), do: 2
defp explain_limit_setup_width({:limit, _limit}), do: 1
defp explain_limit_setup_width({:limit_offset, 0, _offset}), do: 5
defp explain_limit_setup_width({:limit_offset, _limit, _offset}), do: 4
defp explain_limit_register_width(nil), do: 0
defp explain_limit_register_width({:limit, _limit}), do: 1
defp explain_limit_register_width({:limit_offset, _limit, _offset}), do: 3
defp explain_limit_decr_width(nil), do: 0
defp explain_limit_decr_width(_limit), do: 1
defp explain_limit_offset_width({:limit_offset, _limit, _offset}), do: 1
defp explain_limit_offset_width(_limit), do: 0
defp explain_limit_rows(nil, _halt_addr), do: []
defp explain_limit_rows({:limit, 0}, halt_addr),
do: [
[1, "Integer", 0, 1, 0, nil, 0, "r[1]=0; LIMIT counter"],
[2, "Goto", 0, halt_addr, 0, nil, 0, nil]
]
defp explain_limit_rows({:limit, value}, _halt_addr),
do: [[1, "Integer", value, 1, 0, nil, 0, "r[1]=#{value}; LIMIT counter"]]
defp explain_limit_rows({:limit_offset, 0, offset}, halt_addr) do
[
[1, "Integer", 0, 1, 0, nil, 0, "r[1]=0; LIMIT counter"],
[2, "Goto", 0, halt_addr, 0, nil, 0, nil],
[3, "Integer", offset, 2, 0, nil, 0, "r[2]=#{offset}"],
[4, "MustBeInt", 2, 0, 0, nil, 0, "OFFSET counter"],
[
5,
"OffsetLimit",
1,
3,
2,
nil,
0,
"if r[1]>0 then r[3]=r[1]+max(0,r[2]) else r[3]=(-1); LIMIT+OFFSET"
]
]
end
defp explain_limit_rows({:limit_offset, limit, offset}, _halt_addr) do
[
[1, "Integer", limit, 1, 0, nil, 0, "r[1]=#{limit}; LIMIT counter"],
[2, "Integer", offset, 2, 0, nil, 0, "r[2]=#{offset}"],
[3, "MustBeInt", 2, 0, 0, nil, 0, "OFFSET counter"],
[
4,
"OffsetLimit",
1,
3,
2,
nil,
0,
"if r[1]>0 then r[3]=r[1]+max(0,r[2]) else r[3]=(-1); LIMIT+OFFSET"
]
]
end
defp explain_limit_offset_rows(nil, _addr, _jump_addr), do: []
defp explain_limit_offset_rows({:limit, _limit}, _addr, _jump_addr), do: []
defp explain_limit_offset_rows({:limit_offset, _limit, _offset}, addr, jump_addr),
do: [
[
addr,
"IfPos",
2,
jump_addr,
1,
nil,
0,
"if r[2]>0 then r[2]-=1, goto #{jump_addr}; OFFSET"
]
]
defp explain_limit_decr_rows(nil, _addr, _halt_addr), do: []
defp explain_limit_decr_rows(_limit, addr, halt_addr),
do: [[addr, "DecrJumpZero", 1, halt_addr, 0, nil, 0, "if (--r[1])==0 goto #{halt_addr}"]]
defp explain_rowid_upper_bound_rows(nil, addr, _upper_reg, _rowid_reg, _halt_addr),
do: {[], addr}
defp explain_rowid_upper_bound_rows(
{jump_opcode, literal_opcode},
addr,
upper_reg,
rowid_reg,
halt_addr
) do
{literal_op, literal_p1, literal_p4} = literal_opcode
rows = [
[
addr,
literal_op,
literal_p1,
upper_reg,
0,
literal_p4,
0,
explain_literal_comment(literal_op, literal_p1, literal_p4, upper_reg)
],
[addr + 1, "Rowid", 0, rowid_reg, 0, nil, 0, "r[#{rowid_reg}]= rowid of 0"],
[
addr + 2,
jump_opcode,
upper_reg,
halt_addr,
rowid_reg,
nil,
83,
explain_jump_comment(jump_opcode, rowid_reg, upper_reg, halt_addr)
]
]
{rows, addr + 1}
end
defp explain_rowid_lower_const_row(rows, nil, addr, _lower_reg), do: {rows, addr}
defp explain_rowid_lower_const_row(rows, {_seek_opcode, literal_opcode}, addr, lower_reg) do
{literal_op, literal_p1, literal_p4} = literal_opcode
row = [
addr,
literal_op,
literal_p1,
lower_reg,
0,
literal_p4,
0,
explain_literal_comment(literal_op, literal_p1, literal_p4, lower_reg)
]
{rows ++ [row], addr + 1}
end
defp explain_jump_comment("Eq", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]==r[#{right_reg}] goto #{next_addr}"
defp explain_jump_comment("Ne", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]!=r[#{right_reg}] goto #{next_addr}"
defp explain_jump_comment("Lt", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]<r[#{right_reg}] goto #{next_addr}"
defp explain_jump_comment("Le", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]<=r[#{right_reg}] goto #{next_addr}"
defp explain_jump_comment("Gt", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]>r[#{right_reg}] goto #{next_addr}"
defp explain_jump_comment("Ge", left_reg, right_reg, next_addr),
do: "if r[#{left_reg}]>=r[#{right_reg}] goto #{next_addr}"
defp explain_null_jump_comment("IsNull", reg, next_addr),
do: "if r[#{reg}]==NULL goto #{next_addr}"
defp explain_null_jump_comment("NotNull", reg, next_addr),
do: "if r[#{reg}]!=NULL goto #{next_addr}"
defp explain_table_projection_row(table, {:rowid, _display}, reg, addr, p5),
do: [addr, "Rowid", 0, reg, 0, nil, p5, "r[#{reg}]=#{table.name}.rowid"]
defp explain_table_projection_row(_table, {:column, index}, reg, addr, p5),
do: [
addr,
"Column",
0,
index,
reg,
nil,
p5,
"r[#{reg}]= cursor 0 column #{index}"
]
defp explain_result_comment(start, 1), do: "output=r[#{start}]"
defp explain_result_comment(start, width), do: "output=r[#{start}..#{start + width - 1}]"
defp explain_table_open_p4(projection) do
projection
|> Enum.flat_map(fn
{:column, index} -> [index]
{:rowid, _display} -> []
end)
|> case do
[] -> 0
indices -> Enum.max(indices) + 1
end
end
defp explain_table_rootpage(db, table) do
table_key = Database.table_storage_key(table.schema, table.name)
db.tables
|> Map.values()
|> Enum.filter(
&(Database.table_storage_key(&1.schema, "") == Database.table_storage_key(table.schema, ""))
)
|> Enum.sort_by(&Table.key(&1.name))
|> Enum.find_index(&(Database.table_storage_key(&1.schema, &1.name) == table_key))
|> case do
nil -> 0
index -> index + 1
end
end
defp internal_sqlite_object_name?(name),
do: name |> Table.key() |> String.starts_with?("sqlite_")
defp drop_index_owner(db, :any, name), do: ordered_index_owner(db, name)
defp drop_index_owner(db, schema, name),
do: Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name)
defp exec_insert(db, %Insert{} = stmt) do
# Materialize index entries up front so the reduce can keep them current
# row-by-row (incremental add + O(1) unique-conflict lookup) instead of a
# full rebuild per row.
table = db |> fetch_table!(stmt.schema, stmt.table) |> then(&ensure_index_entries(db, &1))
targets = insert_targets(table, stmt)
rows = insert_rows(db, stmt, table, targets)
on_conflict = stmt.or_conflict || :abort
old_rows = table.rows
before_triggers = triggers_for(db, stmt, :before, :insert)
after_triggers = triggers_for(db, stmt, :after, :insert)
{db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} =
Enum.reduce(
rows,
{db, table, 0, nil, [], [], MapSet.new()},
fn row, acc ->
insert_row_step(row, acc, stmt, targets, on_conflict, before_triggers, after_triggers)
end
)
db =
db
|> insert_put_table(
table,
count,
last_insert_rowid,
upserted_rowids,
fk_pairs,
before_triggers,
after_triggers
)
|> apply_replace_deleted_actions(table, old_rows, upserted_rowids)
|> apply_fk_update_actions(table, Enum.reverse(fk_pairs))
|> Database.record_changes(count, last_insert_rowid)
{dml_result(db, table, stmt.returning, Enum.reverse(returning_rows), :insert, count), db}
end
# The insert reduce keeps index entries current row-by-row (insert adds an
# entry; an upsert UPDATE rebuilds), so a plain insert needs no full rebuild
# here — making a bulk load of an indexed table O(n) rather than O(n²). When
# post-insert work mutates rows the per-row maintenance did not see (REPLACE/FK
# cascade deletions), or entries were never materialized, fall back to the full
# rebuild so the stored entries are correct.
defp insert_put_table(db, table, _count, _rowid, upserted_rowids, fk_pairs, _before, _after) do
if MapSet.size(upserted_rowids) == 0 and fk_pairs == [] and indexes_have_entries?(table) do
Database.put_table(db, table)
else
put_table(db, table)
end
end
defp insert_row_step(row, acc, stmt, targets, on_conflict, before_triggers, after_triggers) do
{db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} = acc
{values, explicit_rowid} = insert_values(targets, row)
# Build a candidate row map for CHECK validation
candidate =
table
|> build_candidate_row(values, db)
|> with_explicit_rowid(table, explicit_rowid)
|> apply_generated_columns(db, table, explicit_rowid)
check_strict_types!(table, candidate)
{trigger_status, db, table} =
if before_triggers == [] do
{:ok, db, table}
else
db = put_table(db, table)
{status, db} =
fire_triggers(
db,
before_triggers,
table,
nil,
trigger_row(before_insert_trigger_rowid(table, explicit_rowid), candidate)
)
{status, db, refetch_table(db, table)}
end
if trigger_status == :ignored do
{db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids}
else
values = candidate
check_env = table_env(db, table, nil, candidate)
case check_violations(db, table, candidate, check_env, on_conflict) do
:ok ->
before_update_triggers = triggers_for(db, stmt, :before, :update)
after_update_triggers = triggers_for(db, stmt, :after, :update)
case insert_or_upsert(
db,
table,
values,
explicit_rowid,
stmt,
on_conflict,
before_update_triggers,
after_update_triggers
) do
{:inserted, db, table, rowid} ->
stored_row = Table.fetch_row!(table, rowid)
returning_rows = [{rowid, stored_row} | returning_rows]
{db, table} =
fire_after_row_triggers(
db,
table,
after_triggers,
nil,
trigger_row(rowid, stored_row)
)
last_insert_rowid = last_insert_rowid_for_table(table, rowid, last_insert_rowid)
{db, table, count + 1, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids}
{:updated, db, table, rowid, {old_rowid, old_row, returning_row}} ->
returning_rows = [{rowid, returning_row} | returning_rows]
{db, table, count + 1, last_insert_rowid, returning_rows,
[{old_row, returning_row} | fk_pairs],
upserted_rowids |> MapSet.put(rowid) |> MapSet.put(old_rowid)}
:ignore ->
{db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids}
{:error, message} ->
conflict_fail!(db, table, on_conflict, message)
end
:ignore ->
{db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids}
{:error, message} ->
conflict_fail!(db, table, on_conflict, message)
end
end
end
defp fire_after_row_triggers(db, table, [], _old_row, _new_row), do: {db, table}
defp fire_after_row_triggers(db, table, triggers, old_row, new_row) do
db = put_table(db, table)
{_status, db} = fire_triggers(db, triggers, table, old_row, new_row)
{db, refetch_table(db, table)}
end
defp trigger_row(rowid, row), do: {:trigger_row, rowid, row}
defp before_insert_trigger_rowid(%{without_rowid: true}, _explicit_rowid), do: nil
defp before_insert_trigger_rowid(_table, rowid) when is_integer(rowid), do: rowid
defp before_insert_trigger_rowid(_table, _rowid), do: -1
defp last_insert_rowid_for_table(%{without_rowid: true}, _rowid, previous), do: previous
defp last_insert_rowid_for_table(_table, rowid, _previous), do: rowid
# DML against a view runs its INSTEAD OF triggers, once per affected row;
# without a matching trigger the view is read-only.
defp exec_view_dml(db, view, stmt, event) do
triggers = triggers_for(db, view.name, :instead_of, event)
if triggers == [] do
fail("cannot modify #{view.name} because it is a view")
end
pseudo = view_pseudo_table(db, view)
{db, count, returning_rows} =
case event do
:insert ->
targets = insert_targets(pseudo, stmt)
rows = insert_rows(db, stmt, pseudo, targets)
Enum.reduce(rows, {db, 0, []}, fn row, {db, count, returning_rows} ->
{values, _explicit_rowid} = insert_values(targets, row)
candidate = build_candidate_row(pseudo, values, db)
{_status, db} = fire_triggers(db, triggers, pseudo, nil, candidate)
{db, count + 1, [{count + 1, candidate} | returning_rows]}
end)
:update ->
changed_keys = Enum.map(stmt.assignments, fn {name, _expr} -> Table.key(name) end)
Enum.each(stmt.assignments, fn {name, _expr} ->
Table.column(pseudo, name) || fail("no such column: #{name}")
end)
db
|> update_target_rows(pseudo, stmt)
|> Enum.reduce({db, 0, []}, fn {rowid, row, env}, {db, count, returning_rows} ->
new_row =
Enum.reduce(stmt.assignments, row, fn {name, expr}, acc ->
Map.put(acc, Table.key(name), eval(expr, env))
end)
{_status, db} = fire_triggers(db, triggers, pseudo, row, new_row, changed_keys)
{db, count + 1, [{rowid, new_row} | returning_rows]}
end)
:delete ->
db
|> dml_target_rows(pseudo, stmt)
|> Enum.reduce({db, 0, []}, fn {rowid, row}, {db, count, returning_rows} ->
{_status, db} = fire_triggers(db, triggers, pseudo, row, nil)
{db, count + 1, [{rowid, row} | returning_rows]}
end)
end
db = Database.record_changes(db, count)
{dml_result(db, pseudo, stmt.returning, Enum.reverse(returning_rows), event, count), db}
end
# A view materialized as a throwaway table value, for INSTEAD OF trigger
# row iteration and OLD./NEW. construction.
defp view_pseudo_table(db, view) do
result = query_result(db, view.query, nil)
names = view.columns || result.columns
columns = Enum.map(names, &%ColumnDef{name: &1, affinity: :blob})
# Rows are stored positionally; `row` is already in `names`/`columns` order.
rows =
result.rows
|> Enum.with_index(1)
|> Map.new(fn {row, index} -> {index, List.to_tuple(row)} end)
%Table{name: view.name, columns: columns, rows: rows, next_rowid: map_size(rows) + 1}
end
defp exec_update(db, %Update{} = stmt) do
check_window_placement!(stmt.where)
table = fetch_table!(db, stmt.schema, stmt.table)
assignments = Enum.map(stmt.assignments, &update_assignment(table, &1))
on_conflict = stmt.or_conflict || :abort
old_rows = table.rows
target_rows = update_target_rows(db, table, stmt)
target_rowids = MapSet.new(target_rows, fn {rowid, _row, _env} -> rowid end)
changed_keys = Enum.map(assignments, &update_assignment_key/1)
before_triggers = triggers_for(db, stmt, :before, :update)
after_triggers = triggers_for(db, stmt, :after, :update)
# Maintain index entries incrementally (remove the old row's entries, add the
# new row's) on the common path instead of rebuilding every index from a full
# scan per statement — that `put_table`/`refresh_index_entries` was O(n) per
# UPDATE, i.e. O(n²) over a table. REPLACE (deletes conflicting rows mid-loop)
# and trigger-bearing updates (re-fetch/refresh mid-loop) fall back to the
# full rebuild, where the incremental bookkeeping is too fragile to be worth it.
incremental? = on_conflict != :replace and before_triggers == [] and after_triggers == []
table = if incremental?, do: ensure_index_entries(db, table), else: table
{db, table, count, returning_rows, fk_pairs} =
target_rows
|> Enum.reduce({db, table, 0, [], []}, fn {rowid, row, env},
{db, table, count, returning_rows, fk_pairs} ->
{new_row, explicit_rowid} = updated_row_and_rowid(table, assignments, row, env)
new_row =
apply_generated_columns(new_row, db, table, update_rowid_value(explicit_rowid, rowid))
check_strict_types!(table, new_row)
{trigger_status, db, table} =
if before_triggers == [] do
{:ok, db, table}
else
db = put_table(db, table)
{status, db} =
fire_triggers(
db,
before_triggers,
table,
trigger_row(rowid, row),
trigger_row(rowid, new_row),
changed_keys
)
{status, db, refetch_table(db, table)}
end
skip_row = {db, table, count, returning_rows, fk_pairs}
cond do
trigger_status == :ignored ->
skip_row
# A BEFORE trigger may have deleted the row out from under us.
not Map.has_key?(table.rows, rowid) ->
skip_row
true ->
new_row =
if before_triggers == [] do
new_row
else
table
|> Table.fetch_row!(rowid)
|> rebase_updated_row(table, assignments, new_row)
|> apply_generated_columns(
db,
table,
update_conflict_rowid(table, rowid, new_row, explicit_rowid)
)
end
check_strict_types!(table, new_row)
check_env = table_env(db, table, rowid, new_row)
case check_violations(db, table, new_row, check_env, on_conflict) do
:ok ->
conflict_rowid = update_conflict_rowid(table, rowid, new_row, explicit_rowid)
case resolve_unique_indexes_for_dml(
db,
table,
conflict_rowid,
new_row,
on_conflict,
excluding_rowids: [rowid]
) do
{:ok, db, table} ->
opts = [on_conflict: stmt.or_conflict]
opts = update_rowid_opts(opts, explicit_rowid)
case Table.update_row(table, rowid, new_row, opts) do
{:ok, table} ->
new_rowid = updated_rowid(table, rowid, new_row, explicit_rowid)
stored_row = Table.fetch_row!(table, new_rowid)
# Incrementally retarget this row's index entries: drop the
# old row's, add the new row's. Keeps entries current so the
# end-of-statement commit skips the full rebuild.
table =
if incremental? do
table = remove_index_entries(db, table, [{rowid, row}])
add_index_entries(db, table, [new_rowid])
else
table
end
{db, table} =
if after_triggers == [] do
{db, table}
else
db = put_table(db, table)
{_status, db} =
fire_triggers(
db,
after_triggers,
table,
trigger_row(rowid, row),
trigger_row(new_rowid, stored_row),
changed_keys
)
{db, refetch_table(db, table)}
end
returning_rows = [{new_rowid, stored_row} | returning_rows]
{db, table, count + 1, returning_rows, [{row, stored_row} | fk_pairs]}
:ignore ->
skip_row
{:error, message} ->
conflict_fail!(db, table, on_conflict, message)
end
:ignore ->
skip_row
{:error, message} ->
conflict_fail!(db, table, on_conflict, message)
end
:ignore ->
skip_row
{:error, message} ->
conflict_fail!(db, table, on_conflict, message)
end
end
end)
# Incremental path kept entries current → store without the full refresh;
# otherwise `put_table` rebuilds them.
db = if incremental?, do: Database.put_table(db, table), else: put_table(db, table)
db =
db
|> apply_replace_deleted_actions(table, old_rows, target_rowids)
|> apply_fk_update_actions(table, Enum.reverse(fk_pairs))
|> Database.record_changes(count)
{dml_result(db, table, stmt.returning, Enum.reverse(returning_rows), :update, count), db}
end
defp exec_delete(db, %Delete{} = stmt) do
check_window_placement!(stmt.where)
table = fetch_table!(db, stmt.schema, stmt.table)
before_triggers = triggers_for(db, stmt, :before, :delete)
after_triggers = triggers_for(db, stmt, :after, :delete)
returning_rows = dml_target_rows(db, table, stmt)
{db, table, deleted_rows, deleted_pairs} =
if before_triggers == [] and after_triggers == [] do
rowids = Enum.map(returning_rows, &elem(&1, 0))
# `deleted_pairs` (the deleted `{rowid, row}`s) lets the index entries be
# maintained incrementally below, instead of rebuilt by a full scan.
{db, Table.delete_rows(table, rowids), Enum.map(returning_rows, &elem(&1, 1)),
returning_rows}
else
{db, table, deleted} =
Enum.reduce(returning_rows, {db, table, []}, fn {rowid, row}, {db, table, deleted} ->
# A trigger fired for an earlier row may have deleted this one.
if Map.has_key?(table.rows, rowid) do
db = put_table(db, table)
{status, db} =
fire_triggers(db, before_triggers, table, trigger_row(rowid, row), nil)
table = refetch_table(db, table)
if status == :ignored or not Map.has_key?(table.rows, rowid) do
{db, table, deleted}
else
table = Table.delete_rows(table, [rowid])
{db, table} =
fire_after_row_triggers(db, table, after_triggers, trigger_row(rowid, row), nil)
{db, table, [row | deleted]}
end
else
{db, table, deleted}
end
end)
# Triggers can delete arbitrary rows mid-loop, so fall back to a rebuild.
{db, table, Enum.reverse(deleted), nil}
end
count = length(deleted_rows)
db_after_put =
if deleted_pairs do
Database.put_table(db, remove_index_entries(db, table, deleted_pairs))
else
put_table(db, table)
end
db =
db_after_put
|> apply_fk_delete_actions(table, deleted_rows)
|> Database.record_changes(count)
{dml_result(db, table, stmt.returning, returning_rows, :delete, count), db}
end
defp exec_sqlite_sequence_update(db, %Update{} = stmt) do
ensure_sqlite_sequence_exists!(db)
sequence_table = sqlite_sequence_table(db)
assignments =
Enum.map(stmt.assignments, fn {name, expr} ->
column = Table.column(sequence_table, name) || fail("no such column: #{name}")
if Table.key(column.name) == "name" do
fail("cannot UPDATE sqlite_sequence.name")
end
{column, expr}
end)
{db, count, returning_rows} =
db
|> update_target_rows(sequence_table, stmt)
|> Enum.reduce({db, 0, []}, fn {rowid, row, env}, {db, count, returning_rows} ->
new_row =
Enum.reduce(assignments, row, fn {column, expr}, new_row ->
value = expr |> eval(env) |> Value.apply_affinity(column.affinity)
unless is_integer(value) do
fail("datatype mismatch")
end
Map.put(new_row, Table.key(column.name), value)
end)
db = put_sqlite_sequence(db, Map.fetch!(row, "name"), Map.fetch!(new_row, "seq"), true)
{db, count + 1, [{rowid, new_row} | returning_rows]}
end)
result_table = sqlite_sequence_table(db)
db = Database.record_changes(db, count)
{dml_result(db, result_table, stmt.returning, Enum.reverse(returning_rows), :update, count),
db}
end
defp exec_sqlite_sequence_delete(db, %Delete{} = stmt) do
ensure_sqlite_sequence_exists!(db)
sequence_table = sqlite_sequence_table(db)
returning_rows = dml_target_rows(db, sequence_table, stmt)
db =
Enum.reduce(returning_rows, db, fn {_rowid, row}, db ->
put_sqlite_sequence(db, Map.fetch!(row, "name"), 0, false)
end)
count = length(returning_rows)
db = Database.record_changes(db, count)
{dml_result(db, sequence_table, stmt.returning, returning_rows, :delete, count), db}
end
defp exec_sqlite_sequence_insert(db, %Insert{} = stmt) do
ensure_sqlite_sequence_exists!(db)
sequence_table = sqlite_sequence_table(db)
targets = insert_targets(sequence_table, stmt)
rows = insert_rows(db, stmt, sequence_table, targets)
{db, count, returning_rows} =
rows
|> Enum.reduce({db, 0, []}, fn row, {db, count, returning_rows} ->
{values, _explicit_rowid} = insert_values(targets, row)
new_row =
sequence_table
|> build_candidate_row(values, db)
|> Map.update!("name", &Value.apply_affinity(&1, :text))
|> Map.update!("seq", &Value.apply_affinity(&1, :integer))
unless is_integer(Map.fetch!(new_row, "seq")) do
fail("datatype mismatch")
end
db =
put_sqlite_sequence(db, Map.fetch!(new_row, "name"), Map.fetch!(new_row, "seq"), true)
{db, count + 1, [{count + 1, new_row} | returning_rows]}
end)
result_table = sqlite_sequence_table(db)
db = Database.record_changes(db, count)
{dml_result(db, result_table, stmt.returning, Enum.reverse(returning_rows), :insert, count),
db}
end
defp table_info_rows(table, include_hidden) do
pk_positions = primary_key_positions(table)
table.columns
|> Enum.reject(&(not include_hidden and &1.generated))
|> Enum.with_index()
|> Enum.map(fn {column, index} ->
key = Table.key(column.name)
row = [
index,
column.name,
column.declared_type || "",
if(column.not_null, do: 1, else: 0),
pragma_default(column.default),
Map.get(pk_positions, key, 0)
]
if include_hidden, do: row ++ [generated_hidden(column)], else: row
end)
end
defp generated_hidden(%{generated: {:virtual, _}}), do: 2
defp generated_hidden(%{generated: {:stored, _}}), do: 3
defp generated_hidden(_column), do: 0
defp foreign_key_list_rows(table) do
table
|> foreign_key_specs()
|> Enum.with_index()
|> Enum.flat_map(fn {spec, id} ->
Enum.with_index(spec.child_keys)
|> Enum.map(fn {child_key, seq} ->
[
id,
seq,
spec.parent_table,
display_column_name(table, child_key),
Enum.at(spec.parent_keys, seq),
fk_action_name(spec.on_update),
fk_action_name(spec.on_delete),
"NONE"
]
end)
end)
end
defp foreign_key_check_rows(db, nil) do
db.tables
|> Map.values()
|> Enum.filter(&main_schema?(&1.schema))
|> Enum.sort_by(&Table.key(&1.name))
|> Enum.flat_map(&foreign_key_check_table_rows(db, &1))
end
defp foreign_key_check_rows(db, {:schema, schema, nil}) do
ensure_schema_exists!(db, schema)
db.tables
|> Map.values()
|> Enum.filter(&(Table.key(&1.schema || "main") == Table.key(schema)))
|> Enum.sort_by(&Table.key(&1.name))
|> Enum.flat_map(&foreign_key_check_table_rows(db, &1))
end
defp foreign_key_check_rows(db, {:schema, schema, table_name}) do
ensure_schema_exists!(db, schema)
case Map.fetch(db.tables, Database.table_storage_key(schema, table_name)) do
{:ok, table} -> foreign_key_check_table_rows(db, table)
:error -> fail("no such table: #{table_name}")
end
end
defp foreign_key_check_rows(db, table_name) do
case pragma_fetch_table(db, table_name) do
{:ok, table} -> foreign_key_check_table_rows(db, table)
:error -> fail("no such table: #{table_name}")
end
end
defp foreign_key_check_table_rows(db, child_table) do
child_table
|> foreign_key_specs()
|> Enum.with_index()
|> Enum.flat_map(fn {spec, id} ->
{parent_table, parent_columns} = foreign_key_check_parent(db, child_table, spec)
child_table
|> Table.scan()
|> Enum.flat_map(fn {rowid, row} ->
child_values = Enum.map(spec.child_keys, &Map.get(row, &1))
cond do
Enum.any?(child_values, &is_nil/1) ->
[]
parent_table == nil ->
[[child_table.name, rowid, spec.parent_table, id]]
parent_row_exists?(parent_table, parent_columns, child_values) ->
[]
true ->
[[child_table.name, rowid, parent_table.name, id]]
end
end)
end)
end
defp foreign_key_check_parent(db, child_table, spec) do
case fetch_fk_parent_table(db, child_table, spec.parent_table) do
{:ok, parent_table} -> {parent_table, referenced_columns!(child_table, spec, parent_table)}
{:error, _message} -> {nil, []}
end
end
defp primary_key_positions(%{composite_keys: [{_name, keys} | _]}) do
keys
|> Enum.with_index(1)
|> Map.new()
end
defp primary_key_positions(table) do
table.columns
|> Enum.filter(& &1.primary_key)
|> Enum.map(&Table.key(&1.name))
|> Enum.with_index(1)
|> Map.new()
end
defp pragma_default(nil), do: nil
defp pragma_default({:literal, nil}), do: "NULL"
defp pragma_default({:literal, value}) when is_binary(value),
do: "'#{String.replace(value, "'", "''")}'"
defp pragma_default({:literal, value}), do: Value.to_text(value)
defp pragma_default({:negate, {:literal, value}}), do: "-" <> Value.to_text(value)
defp pragma_default({:column, nil, word}), do: word
# An expression default (function call, etc.): render it parenthesized, the
# form SQLite stores in `sqlite_master`/`PRAGMA table_info` and re-parses.
defp pragma_default(expr), do: "(#{expr_name(expr)})"
defp index_list_rows(table) do
(table.indexes ++ Enum.reverse(table.autoindexes))
|> Enum.with_index()
|> Enum.map(fn {index, seq} ->
[
seq,
index.name,
if(index.unique, do: 1, else: 0),
Map.get(index, :origin, "c"),
if(index.where, do: 1, else: 0)
]
end)
end
defp put_autoindexes(table, constraints) do
%{table | autoindexes: autoindexes_for_table(table, constraints)}
end
defp autoindexes_for_table(table, constraints) do
inline_specs =
table.columns
|> Enum.flat_map(fn column ->
key = Table.key(column.name)
cond do
column.primary_key and key != table.rowid_alias -> [{:pk, [key]}]
column.unique -> [{:u, [key]}]
true -> []
end
end)
table_specs =
Enum.flat_map(constraints, fn
{:primary_key, _name, keys} ->
if keys == [table.rowid_alias], do: [], else: [{:pk, keys}]
{:unique, _name, keys} ->
[{:u, keys}]
_constraint ->
[]
end)
table.name
|> autoindex_names(inline_specs ++ table_specs)
|> Enum.map(fn {name, origin, keys} -> autoindex(table, name, origin, keys) end)
end
defp autoindex_names(table_name, specs) do
specs
|> Enum.with_index(1)
|> Enum.map(fn {{origin, keys}, seq} ->
{"sqlite_autoindex_#{table_name}_#{seq}", origin, keys}
end)
end
defp autoindex(table, name, origin, keys) do
%{
name: name,
columns: keys,
members: Enum.map(keys, &{:column, &1}),
collations: Enum.map(keys, &column_collation_name(table, &1)),
directions: List.duplicate(:asc, length(keys)),
unique: true,
where: nil,
origin: Atom.to_string(origin),
autoindex: true
}
end
defp rename_autoindexes(table) do
%{table | autoindexes: autoindexes_with_table_name(table.autoindexes, table.name)}
end
defp autoindexes_with_table_name(autoindexes, table_name) do
autoindexes
|> Enum.with_index(1)
|> Enum.map(fn {index, seq} -> %{index | name: "sqlite_autoindex_#{table_name}_#{seq}"} end)
end
defp rename_index_column(index, old_key, new_key) do
columns = Enum.map(index.columns, &if(&1 == old_key, do: new_key, else: &1))
members =
Enum.map(index_members(index), fn
{:column, ^old_key} -> {:column, new_key}
member -> member
end)
%{index | columns: columns, members: members}
end
defp drop_from_autoindexes(autoindexes, col_key) do
autoindexes
|> Enum.map(fn index ->
columns = Enum.reject(index.columns, &(&1 == col_key))
members = Enum.reject(index_members(index), &(&1 == {:column, col_key}))
%{index | columns: columns, members: members}
end)
|> Enum.reject(&(&1.columns == []))
end
defp index_info_rows(table, index) do
index
|> index_members()
|> Enum.with_index()
|> Enum.map(fn {member, seqno} -> index_info_row(table, member, seqno) end)
end
defp index_xinfo_rows(table, index) do
member_rows =
index
|> index_members()
|> Enum.with_index()
|> Enum.map(fn {member, seqno} ->
[seqno, cid, name] = index_info_row(table, member, seqno)
direction = Enum.at(Map.get(index, :directions, []), seqno, :asc)
collation = Enum.at(Map.get(index, :collations, []), seqno) || :binary
[seqno, cid, name, if(direction == :desc, do: 1, else: 0), pragma_collation(collation), 1]
end)
member_rows ++ [[length(member_rows), -1, nil, 0, "BINARY", 0]]
end
defp index_info_row(table, {:column, column_key}, seqno) do
[seqno, column_position(table, column_key), display_column_name(table, column_key)]
end
defp index_info_row(_table, {:expr, _expr}, seqno) do
[seqno, -2, nil]
end
defp column_position(table, column_key) do
Enum.find_index(table.columns, &(Table.key(&1.name) == column_key)) || -1
end
defp pragma_collation(:binary), do: "BINARY"
defp pragma_collation(collation), do: collation |> to_string() |> String.upcase()
defp display_column_name(table, column_key) do
case Enum.find(table.columns, &(Table.key(&1.name) == column_key)) do
nil -> column_key
column -> column.name
end
end
defp collation_list_rows(db) do
custom =
db.collations
|> Map.values()
|> Enum.map(&pragma_collation(&1.name))
|> Enum.reject(&(&1 in ["BINARY", "NOCASE", "RTRIM"]))
|> Enum.sort()
["BINARY", "NOCASE", "RTRIM"] ++ custom
end
defp function_list_rows(db) do
scalar_rows =
@scalar_arity
|> Enum.map(fn {name, arity} -> function_list_row(name, 1, "s", function_narg(arity)) end)
aggregate_rows =
@aggregate_functions
|> Enum.flat_map(fn name ->
Enum.map(aggregate_nargs(name), &function_list_row(name, 1, "w", &1))
end)
window_rows =
@window_functions
|> Enum.flat_map(fn name ->
Enum.map(window_nargs(name), &function_list_row(name, 1, "w", &1))
end)
custom_scalar_rows =
db.scalar_functions
|> Map.values()
|> Enum.map(&function_list_row(&1.name, 0, "s", &1.arity))
custom_aggregate_rows =
db.aggregate_functions
|> Map.values()
|> Enum.map(&function_list_row(&1.name, 0, "w", &1.arity))
(scalar_rows ++ aggregate_rows ++ window_rows ++ custom_scalar_rows ++ custom_aggregate_rows)
|> Enum.sort_by(fn [name, builtin, type, _enc, narg, _flags] ->
{name, builtin, type, narg}
end)
end
defp function_list_row(name, builtin, type, narg), do: [name, builtin, type, "utf8", narg, 0]
defp function_narg(%Range{first: first, last: last}) when first == last, do: first
defp function_narg(%Range{}), do: -1
defp aggregate_nargs("count"), do: [0, 1]
defp aggregate_nargs(name) when name in ["group_concat", "string_agg"], do: [1, 2]
defp aggregate_nargs(name) when name in ["json_group_object", "jsonb_group_object"], do: [2]
defp aggregate_nargs(_name), do: [1]
defp window_nargs(name)
when name in ["row_number", "rank", "dense_rank", "percent_rank", "cume_dist"],
do: [0]
defp window_nargs("ntile"), do: [1]
defp window_nargs(name) when name in ["lag", "lead"], do: [1, 2, 3]
defp window_nargs(name) when name in ["first_value", "last_value"], do: [1]
defp window_nargs("nth_value"), do: [2]
defp pragma_page_count(db, schema) do
tables =
db.tables
|> Map.values()
|> Enum.filter(&(Table.key(&1.schema || "main") == Table.key(schema || "main")))
if tables == [] do
0
else
table_pages = length(tables)
index_pages =
tables
|> Enum.map(&length(&1.indexes))
|> Enum.sum()
max(2, 1 + table_pages + index_pages)
end
end
defp database_list_rows(db) do
[[0, "main", ""]] ++
Enum.map(db.attached_databases, fn attached ->
[attached.seq, attached.name, attached.file]
end)
end
defp attached_database_key(name), do: String.downcase(name)
defp attached_database?(db, key) do
Enum.any?(db.attached_databases, fn attached ->
attached_database_key(attached.name) == key
end)
end
defp next_attached_database_seq(db) do
used = MapSet.new(Enum.map(db.attached_databases, & &1.seq))
Enum.find(2..125, &(not MapSet.member?(used, &1)))
end
defp ensure_schema_exists!(_db, schema) when schema in [nil, "main", "temp"], do: :ok
defp ensure_schema_exists!(db, schema) do
unless attached_database?(db, attached_database_key(schema)) do
fail("unknown database #{schema}")
end
end
# When resolving a `schema.table` reference in FROM, SQLite reports an unknown
# schema as a missing table (`no such table: schema.table`), not "unknown
# database" — that wording is reserved for DDL/ATTACH-level operations.
defp ensure_table_schema!(_db, schema, _name) when schema in [nil, "main", "temp"], do: :ok
defp ensure_table_schema!(db, schema, name) do
unless attached_database?(db, attached_database_key(schema)) do
fail("no such table: #{schema}.#{name}")
end
end
defp main_schema?(schema), do: schema in [nil, "main"]
defp temp_schema?(schema), do: schema == "temp"
defp trigger_schema(schema) when schema in [nil, "main"], do: nil
defp trigger_schema(schema), do: schema
defp ensure_trigger_schema_exists!(_db, "temp"), do: :ok
defp ensure_trigger_schema_exists!(db, schema), do: ensure_schema_exists!(db, schema)
defp trigger_target_schema!(db, %CreateTrigger{} = stmt) do
schema = trigger_schema(stmt.schema)
cond do
temp_schema?(stmt.schema) and stmt.table_schema == nil ->
case trigger_target_lookup_schema(db, stmt.table) do
{:ok, schema} -> schema
:error -> nil
end
temp_schema?(stmt.schema) ->
ensure_schema_exists!(db, stmt.table_schema)
stmt.table_schema
stmt.table_schema != nil and trigger_schema(stmt.table_schema) != schema ->
fail("trigger #{stmt.name} cannot reference objects in database #{stmt.table_schema}")
true ->
ensure_schema_exists!(db, schema)
schema
end
end
defp trigger_target_lookup_schema(db, table_name) do
Enum.find_value(table_lookup_order(db), :error, fn schema ->
key = Database.table_storage_key(schema, table_name)
if Map.has_key?(db.tables, key) or Map.has_key?(db.views, key) do
{:ok, schema}
else
nil
end
end)
end
defp trigger_target_label(%CreateTrigger{schema: "temp", table_schema: nil, table: table}, nil),
do: table
defp trigger_target_label(%CreateTrigger{table: table}, schema),
do: "#{schema || "main"}.#{table}"
defp attach_filename(":memory:"), do: ""
defp attach_filename(nil), do: ""
defp attach_filename({:blob, blob}), do: blob
defp attach_filename(value), do: Value.to_text(value)
defp qualify_view_query(query, nil), do: qualify_view_query(query, "main")
defp qualify_view_query(%Select{} = query, schema) do
%{query | from: qualify_view_source(query.from, schema)}
end
defp qualify_view_query(%Compound{} = query, schema) do
%{
query
| left: qualify_view_query(query.left, schema),
right: qualify_view_query(query.right, schema)
}
end
defp qualify_view_query(query, _schema), do: query
defp qualify_view_source(nil, _schema), do: nil
defp qualify_view_source({:table, {:schema, _source_schema, _name}, _alias} = source, _schema),
do: source
defp qualify_view_source({:table, name, alias_name}, schema),
do: {:table, {:schema, schema, name}, alias_name}
defp qualify_view_source({:subquery, query, alias_name}, schema),
do: {:subquery, qualify_view_query(query, schema), alias_name}
defp qualify_view_source({:join, type, left, right, constraint}, schema),
do:
{:join, type, qualify_view_source(left, schema), qualify_view_source(right, schema),
constraint}
defp qualify_trigger_statement(stmt, "temp"), do: stmt
defp qualify_trigger_statement(stmt, nil), do: qualify_trigger_statement(stmt, "main")
defp qualify_trigger_statement(%Insert{schema: nil, source: {:select, query}} = stmt, schema),
do: %{stmt | schema: schema, source: {:select, qualify_view_query(query, schema)}}
defp qualify_trigger_statement(%Insert{source: {:select, query}} = stmt, schema),
do: %{stmt | source: {:select, qualify_view_query(query, schema)}}
defp qualify_trigger_statement(%Insert{schema: nil} = stmt, schema),
do: %{stmt | schema: schema}
defp qualify_trigger_statement(%Update{schema: nil, from: from} = stmt, schema),
do: %{stmt | schema: schema, from: qualify_view_source(from, schema)}
defp qualify_trigger_statement(%Update{from: from} = stmt, schema),
do: %{stmt | from: qualify_view_source(from, schema)}
defp qualify_trigger_statement(%Delete{schema: nil} = stmt, schema),
do: %{stmt | schema: schema}
defp qualify_trigger_statement(%Select{} = stmt, schema), do: qualify_view_query(stmt, schema)
defp qualify_trigger_statement(%Compound{} = stmt, schema), do: qualify_view_query(stmt, schema)
defp qualify_trigger_statement(stmt, _schema), do: stmt
defp integrity_check_rows(%{ignore_check_constraints: true}, _arg), do: []
defp integrity_check_rows(db, {:schema, schema, arg}) do
ensure_schema_exists!(db, schema)
integrity_check_rows(db, arg, schema)
end
defp integrity_check_rows(db, arg), do: integrity_check_rows(db, arg, :all)
defp integrity_check_rows(db, nil, scope), do: integrity_check_scope_rows(db, scope, nil)
defp integrity_check_rows(db, arg, scope) when is_integer(arg),
do: integrity_check_scope_rows(db, scope, integrity_check_limit(arg))
defp integrity_check_rows(db, table_name, scope) when is_binary(table_name) do
case integrity_check_target(db, scope, table_name) do
{:table, table} -> integrity_check_table_rows(db, table)
:view -> []
:error -> fail("no such table: #{table_name}")
end
end
defp integrity_check_scope_rows(db, scope, limit) do
db.tables
|> Map.values()
|> Enum.filter(&integrity_check_scope_match?(&1.schema, scope))
|> Enum.sort_by(fn table -> {Table.key(table.schema || "main"), Table.key(table.name)} end)
|> Enum.flat_map(&integrity_check_table_rows(db, &1))
|> limit_integrity_check_rows(limit)
end
defp integrity_check_table_rows(db, table) do
table
|> Table.scan()
|> Enum.flat_map(fn {rowid, row} ->
env = table_env(db, table, rowid, row)
if Enum.any?(table.checks, fn {_name, expr} -> truth(expr, env) == false end) do
[["CHECK constraint failed in #{table.name}"]]
else
[]
end
end)
end
defp integrity_check_target(db, :all, table_name) do
Enum.find_value(table_lookup_order(db), :error, fn schema ->
integrity_check_target(db, schema, table_name, :soft)
end)
end
defp integrity_check_target(db, schema, table_name),
do: integrity_check_target(db, schema, table_name, :strict)
defp integrity_check_target(db, schema, table_name, mode) do
key = Database.table_storage_key(schema, table_name)
cond do
Map.has_key?(db.tables, key) -> {:table, Map.fetch!(db.tables, key)}
Map.has_key?(db.views, key) -> :view
mode == :soft -> nil
true -> :error
end
end
defp integrity_check_scope_match?(_schema, :all), do: true
defp integrity_check_scope_match?(schema, wanted), do: schema_matches?(schema, wanted)
defp integrity_check_limit(0), do: nil
defp integrity_check_limit(limit) when limit < 0, do: 1
defp integrity_check_limit(limit), do: limit
defp limit_integrity_check_rows(rows, nil), do: rows
defp limit_integrity_check_rows(rows, limit), do: Enum.take(rows, limit)
defp schema_matches?(schema, wanted), do: Table.key(schema || "main") == Table.key(wanted)
defp table_list_rows(db) do
table_rows =
db.tables
|> Map.values()
|> Enum.sort_by(&table_list_sort_key/1)
|> Enum.map(fn table ->
[
table.schema || "main",
table.name,
"table",
length(table.columns),
bool_int(table.without_rowid),
bool_int(table.strict)
]
end)
sequence_rows =
if sqlite_sequence_exists?(db) do
[["main", "sqlite_sequence", "table", 2, 0, 0]]
else
[]
end
view_rows =
db.views
|> Map.values()
|> Enum.sort_by(&table_list_sort_key/1)
|> Enum.map(fn view ->
[view.schema || "main", view.name, "view", view_column_count(db, view), 0, 0]
end)
table_rows ++ sequence_rows ++ view_rows ++ schema_table_list_rows(db)
end
defp schema_table_list_rows(db) do
attached_rows =
Enum.map(db.attached_databases, fn attached ->
[attached.name, "sqlite_schema", "table", 5, 0, 0]
end)
[
["main", "sqlite_schema", "table", 5, 0, 0],
["temp", "sqlite_temp_schema", "table", 5, 0, 0]
] ++ attached_rows
end
defp table_list_sort_key(table), do: {schema_sort_rank(table.schema), Table.key(table.name)}
defp schema_sort_rank(schema) when schema in [nil, "main"], do: {0, "main"}
defp schema_sort_rank("temp"), do: {1, "temp"}
defp schema_sort_rank(schema), do: {2, Table.key(schema)}
defp pragma_fetch_table(db, {:schema, schema, name}) do
ensure_schema_exists!(db, schema)
case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do
{:ok, table} -> {:ok, table}
:error -> :error
end
end
defp pragma_fetch_table(db, name) do
Enum.find_value(table_lookup_order(db), :error, fn schema ->
case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do
{:ok, table} -> {:ok, table}
:error -> nil
end
end)
end
defp pragma_find_index_owner(db, {:schema, schema, name}) do
ensure_schema_exists!(db, schema)
Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name)
end
defp pragma_find_index_owner(db, name), do: ordered_index_owner(db, name)
defp find_autoindex_owner(db, schema, index_name) do
key = Table.key(index_name)
Enum.find_value(db.tables, fn {_table_key, table} ->
if index_schema_matches?(table.schema, schema) do
case Enum.find(table.autoindexes, &(Table.key(&1.name) == key)) do
nil -> nil
index -> {table, index}
end
else
nil
end
end)
end
defp index_schema_matches?(_table_schema, :any), do: true
defp index_schema_matches?(nil, nil), do: true
defp index_schema_matches?(nil, "main"), do: true
defp index_schema_matches?("main", nil), do: true
defp index_schema_matches?(table_schema, schema),
do: Table.key(table_schema || "main") == Table.key(schema || "main")
defp validate_analyze_target!(_db, nil), do: :ok
defp validate_analyze_target!(db, name) do
{schema, target} = operational_target(name)
ensure_schema_exists!(db, schema)
with :error <- operational_table_lookup(db, schema, target),
nil <- operational_index_lookup(db, schema, target) do
fail("no such table: #{target}")
else
_ -> :ok
end
end
defp validate_reindex_target!(_db, nil), do: :ok
defp validate_reindex_target!(db, name) do
{schema, target} = operational_target(name)
ensure_schema_exists!(db, schema)
with :error <- operational_table_lookup(db, schema, target),
nil <- operational_index_lookup(db, schema, target) do
fail("unable to identify the object to be reindexed")
else
_ -> :ok
end
end
defp operational_target(name) do
case String.split(name, ".", parts: 2) do
[target] -> {nil, target}
[schema, target] -> {schema, target}
end
end
defp operational_table_lookup(db, nil, name), do: pragma_fetch_table(db, name)
defp operational_table_lookup(db, schema, name) do
case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do
{:ok, table} -> {:ok, table}
:error -> :error
end
end
defp operational_index_lookup(db, nil, name), do: ordered_index_owner(db, name)
defp operational_index_lookup(db, schema, name),
do: Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name)
defp table_lookup_order(db), do: ["temp", nil] ++ Enum.map(db.attached_databases, & &1.name)
defp ordered_index_owner(db, name) do
Enum.find_value(table_lookup_order(db), fn schema ->
Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name)
end)
end
defp drop_object_schema(_db, schema, _name) when schema != nil, do: schema
defp drop_object_schema(db, nil, name) do
case Enum.find_value(table_lookup_order(db), fn schema ->
key = Database.table_storage_key(schema, name)
if Map.has_key?(db.tables, key) or Map.has_key?(db.views, key) do
{:ok, schema}
end
end) do
{:ok, schema} -> schema
nil -> nil
end
end
defp filter_pragma_table_list(rows, _db, nil), do: rows
defp filter_pragma_table_list(rows, db, {:schema, schema, name}) do
ensure_schema_exists!(db, schema)
rows
|> Enum.filter(fn [row_schema, _row_name, _type, _ncol, _wr, _strict] ->
row_schema == schema
end)
|> filter_pragma_table_list(db, name)
end
defp filter_pragma_table_list(rows, _db, name) do
key = Table.key(name)
Enum.filter(rows, fn [_schema, row_name, _type, _ncol, _wr, _strict] ->
Table.key(row_name) == key
end)
end
defp view_column_count(_db, %{columns: columns}) when is_list(columns), do: length(columns)
defp view_column_count(db, view) do
db
|> query_result(view.query, nil)
|> then(&length(&1.columns))
end
defp bool_int(true), do: 1
defp bool_int(false), do: 0
defp pragma_enabled?(value) when is_integer(value), do: value != 0
defp pragma_enabled?(value) when is_binary(value) do
value
|> String.downcase()
|> then(&(&1 in ["1", "on", "true", "yes"]))
end
defp pragma_enabled?({:literal, value}), do: pragma_enabled?(value)
defp pragma_enabled?(_value), do: false
defp pragma_auto_vacuum(value, current) when is_integer(value) do
case value do
1 -> 1
2 -> 2
0 -> if(current == 0, do: 0, else: current)
_other -> current
end
end
defp pragma_auto_vacuum(value, current) when is_binary(value) do
case String.downcase(value) do
"full" -> 1
"incremental" -> 2
"none" -> if(current == 0, do: 0, else: current)
"off" -> if(current == 0, do: 0, else: current)
"false" -> if(current == 0, do: 0, else: current)
"0" -> if(current == 0, do: 0, else: current)
"1" -> 1
"2" -> 2
_other -> current
end
end
defp pragma_auto_vacuum({:literal, value}, current), do: pragma_auto_vacuum(value, current)
defp pragma_auto_vacuum(_value, current), do: current
defp pragma_header_field("schema_version"), do: :schema_version
defp pragma_header_field("user_version"), do: :user_version
defp pragma_header_field("application_id"), do: :application_id
defp pragma_header_value(value)
when is_integer(value) and value in -2_147_483_648..2_147_483_647,
do: value
defp pragma_header_value(_value), do: 0
@valid_page_sizes MapSet.new([512, 1024, 2048, 4096, 8192, 16_384, 32_768, 65_536])
defp pragma_page_size(value) when is_integer(value) do
if MapSet.member?(@valid_page_sizes, value), do: value
end
defp pragma_page_size(_value), do: nil
defp pragma_cache_size(value) when is_integer(value), do: value
defp pragma_cache_size(_value), do: 0
defp pragma_default_cache_size(value) when is_integer(value), do: abs(value)
defp pragma_default_cache_size(_value), do: 2_000
defp pragma_cache_spill(value, _current) when is_integer(value) and value <= 0, do: 0
defp pragma_cache_spill(value, current) when is_integer(value),
do: max(value, min(current, 2_000))
defp pragma_cache_spill(value, _current) when is_binary(value) do
case String.downcase(value) do
"off" -> 0
"false" -> 0
"no" -> 0
"0" -> 0
"on" -> 2_000
"true" -> 2_000
"yes" -> 2_000
"1" -> 2_000
_other -> 2_000
end
end
defp pragma_cache_spill({:literal, value}, current), do: pragma_cache_spill(value, current)
defp pragma_cache_spill(_value, current), do: current
defp pragma_max_page_count(value, _current) when is_integer(value) and value > 0, do: value
defp pragma_max_page_count(_value, current), do: current
defp pragma_journal_mode(value, current) when is_binary(value) do
case String.downcase(value) do
"memory" -> "memory"
"off" -> "off"
"wal" -> current
mode when mode in ["delete", "truncate", "persist"] -> current
_other -> current
end
end
defp pragma_journal_mode(_value, current), do: current
defp pragma_journal_size_limit(value) when is_integer(value) and value >= -1, do: value
defp pragma_journal_size_limit(_value), do: 0
defp journal_mode_result(mode, db) do
{%Result{
command: :select,
columns: ["journal_mode"],
rows: [[mode]],
rows_affected: 0,
affinities: [:text]
}, db}
end
defp pragma_locking_mode(value, current) when is_binary(value) do
case String.downcase(value) do
mode when mode in ["normal", "exclusive"] -> mode
_other -> current
end
end
defp pragma_locking_mode(_value, current), do: current
defp locking_mode_result(mode, db) do
{%Result{
command: :select,
columns: ["locking_mode"],
rows: [[mode]],
rows_affected: 0,
affinities: [:text]
}, %{db | locking_mode: mode}}
end
defp pragma_synchronous(value) when is_integer(value) and value in 0..3, do: value
defp pragma_synchronous(value) when is_binary(value) do
case String.downcase(value) do
"off" -> 0
"normal" -> 1
"full" -> 2
"extra" -> 3
_other -> 0
end
end
defp pragma_synchronous(_value), do: 0
defp pragma_temp_store(value) when is_integer(value) and value in 0..2, do: value
defp pragma_temp_store(value) when is_binary(value) do
case String.downcase(value) do
"default" -> 0
"file" -> 1
"memory" -> 2
_other -> 0
end
end
defp pragma_temp_store(_value), do: 0
defp non_negative_pragma_integer(value) when is_integer(value) and value >= 0, do: value
defp non_negative_pragma_integer(_value), do: 0
defp pragma_analysis_limit(value, _current) when is_integer(value) and value >= 0, do: value
defp pragma_analysis_limit(_value, current), do: current
defp pragma_threads(value, _current) when is_integer(value) and value >= 0, do: min(value, 8)
defp pragma_threads(_value, current), do: current
defp pragma_secure_delete(value) when is_integer(value) do
cond do
value == 0 -> 0
value > 0 -> 1
true -> 0
end
end
defp pragma_secure_delete(value) when is_binary(value) do
case String.downcase(value) do
"fast" -> 2
"on" -> 1
"true" -> 1
"yes" -> 1
"1" -> 1
_other -> 0
end
end
defp pragma_secure_delete({:literal, value}), do: pragma_secure_delete(value)
defp pragma_secure_delete(_value), do: 0
defp pragma_bool_result(name, value, db) do
{%Result{
command: :select,
columns: [name],
rows: [[bool_int(value)]],
rows_affected: 0,
affinities: [:integer]
}, db}
end
defp pragma_integer_result(name, value, db) do
{%Result{
command: :select,
columns: [name],
rows: [[value]],
rows_affected: 0,
affinities: [:integer]
}, db}
end
defp explain_query_plan_rows(db, %With{query: query}), do: explain_query_plan_rows(db, query)
defp explain_query_plan_rows(_db, %Values{}), do: [[1, 0, 0, "SCAN CONSTANT ROW"]]
defp explain_query_plan_rows(db, %Compound{} = stmt) do
{left_rows, next_id} =
db
|> explain_query_plan_rows(stmt.left)
|> renumber_query_plan_rows(3, 2)
op_id = next_id
{right_rows, _next_id} =
db
|> explain_query_plan_rows(stmt.right)
|> renumber_query_plan_rows(op_id + 1, op_id)
[[1, 0, 0, "COMPOUND QUERY"], [2, 1, 0, "LEFT-MOST SUBQUERY"]] ++
left_rows ++ [[op_id, 1, 0, compound_query_plan_detail(stmt.op)]] ++ right_rows
end
defp explain_query_plan_rows(_db, %Select{from: nil}), do: [[1, 0, 0, "SCAN CONSTANT ROW"]]
defp explain_query_plan_rows(db, %Select{from: from, where: where}) do
{rows, _next_id} = explain_from_rows(db, from, where, 2)
rows
end
defp explain_from_rows(db, {:table, name, alias_name}, where, id) do
{[[id, 0, 0, explain_table_detail(db, name, alias_name, where)]], id + 1}
end
defp explain_from_rows(_db, {:table_function, name, _args, alias_name}, _where, id) do
{[[id, 0, 0, "SCAN #{alias_name || name} VIRTUAL TABLE INDEX 1:"]], id + 1}
end
defp explain_from_rows(db, {:subquery, query, alias_name}, _where, id) do
{rows, next_id} =
db
|> explain_query_plan_rows(query)
|> renumber_query_plan_rows(id, 0)
rows =
Enum.map(rows, fn [row_id, parent, notused, detail] ->
[row_id, parent, notused, subquery_detail(detail, alias_name)]
end)
{rows, next_id}
end
defp explain_from_rows(
db,
{:join, type, left, {:table, right_name, right_alias}, constraint},
where,
id
) do
case explain_join_right_table_detail(
db,
type,
left,
right_name,
right_alias,
constraint,
where
) do
nil ->
case explain_join_left_table_detail(
db,
type,
left,
right_name,
right_alias,
constraint,
where
) do
{_left_name, _left_alias, left_detail} ->
right_detail = explain_table_detail(db, right_name, right_alias, nil)
{[[id, 0, 0, right_detail], [id + 1, 0, 0, left_detail]], id + 2}
nil ->
{left_rows, next_id} = explain_from_rows(db, left, nil, id)
right_detail = explain_table_detail(db, right_name, right_alias, nil)
{left_rows ++ [[next_id, 0, 0, right_detail]], next_id + 1}
end
right_detail ->
{left_rows, next_id} = explain_from_rows(db, left, nil, id)
{left_rows ++ [[next_id, 0, 0, right_detail]], next_id + 1}
end
end
defp explain_from_rows(db, {:join, _type, left, right, _constraint}, _where, id) do
{left_rows, next_id} = explain_from_rows(db, left, nil, id)
{right_rows, next_id} = explain_from_rows(db, right, nil, next_id)
{left_rows ++ right_rows, next_id}
end
defp explain_table_detail(db, name, alias_name, where) do
display = table_source_display(name, alias_name)
case plain_table(db, table_source_key(name)) do
nil -> "SCAN #{display}"
table -> access_path_detail(table, display, table_access_path(db, table, where))
end
end
defp explain_join_right_table_detail(db, type, left, right_name, right_alias, constraint, where) do
with join_kind when join_kind in [:inner, :left, :right, :full] <- indexed_join_kind(type),
%Table{} = table <- plain_table(db, table_source_key(right_name)),
ltmpls when ltmpls != [] <- explain_templates(db, left) do
table = ensure_index_entries(db, table)
rtmpl = table_frame(table, right_alias)
using = using_columns(type, constraint, ltmpls, rtmpl)
right_qualifier = table_source_qualifier(right_name, right_alias)
right_display = table_source_display(right_name, right_alias)
lookup_terms =
join_lookup_terms(join_kind, constraint, where) ++
using_lookup_terms(using, right_qualifier)
case join_rowid_lookup_plan(table, right_qualifier, ltmpls, lookup_terms) do
{:ok, {:eq, expr}} ->
access_path_detail(table, right_display, {:rowid_eq, expr})
{:ok, {:in, exprs}} ->
access_path_detail(table, right_display, {:rowid_in, exprs})
{:ok, {:range, bounds}} ->
access_path_detail(table, right_display, {:rowid_range, bounds})
:error ->
join_index_right_table_detail(table, right_name, right_alias, ltmpls, lookup_terms)
end
else
_ -> nil
end
end
defp join_index_right_table_detail(table, right_name, right_alias, ltmpls, lookup_terms) do
right_display = table_source_display(right_name, right_alias)
case join_index_lookup_plan(table, right_name, right_alias, ltmpls, lookup_terms) do
{:ok, index, {:eq, prefix}} ->
access_path_detail(table, right_display, {:index_member_eq, index, prefix})
{:ok, index, {:in, prefix, member, exprs}} ->
access_path_detail(
table,
right_display,
{:index_member_in, index, prefix, member, exprs}
)
{:ok, index, {:range, prefix, range_member, bounds}} ->
access_path_detail(
table,
right_display,
{:index_member_range, index, prefix, range_member, bounds}
)
:error ->
nil
end
end
defp explain_join_left_table_detail(
db,
type,
{:table, left_name, left_alias},
right_name,
right_alias,
constraint,
where
) do
with :inner <- indexed_join_kind(type),
%Table{} = table <- plain_table(db, table_source_key(left_name)),
%Table{} = right_table <- plain_table(db, table_source_key(right_name)) do
table = ensure_index_entries(db, table)
ltmpl = table_frame(table, left_alias)
rtmpl = table_frame(right_table, right_alias)
using = using_columns(type, constraint, [ltmpl], rtmpl)
left_qualifier = table_source_qualifier(left_name, left_alias)
left_display = table_source_display(left_name, left_alias)
rtmpls = [%{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}]
lookup_terms =
join_lookup_terms(:inner, constraint, where) ++
using_lookup_terms(using, left_qualifier)
case join_rowid_lookup_plan(table, left_qualifier, rtmpls, lookup_terms) do
{:ok, {:eq, expr}} ->
{left_name, left_alias, access_path_detail(table, left_display, {:rowid_eq, expr})}
{:ok, {:in, exprs}} ->
{left_name, left_alias, access_path_detail(table, left_display, {:rowid_in, exprs})}
{:ok, {:range, bounds}} ->
{left_name, left_alias, access_path_detail(table, left_display, {:rowid_range, bounds})}
:error ->
case join_index_lookup_plan(table, left_name, left_alias, rtmpls, lookup_terms) do
{:ok, index, {:eq, prefix}} ->
{left_name, left_alias,
access_path_detail(
table,
left_display,
{:index_member_eq, index, prefix}
)}
{:ok, index, {:in, prefix, member, exprs}} ->
{left_name, left_alias,
access_path_detail(
table,
left_display,
{:index_member_in, index, prefix, member, exprs}
)}
{:ok, index, {:range, prefix, range_member, bounds}} ->
{left_name, left_alias,
access_path_detail(
table,
left_display,
{:index_member_range, index, prefix, range_member, bounds}
)}
:error ->
nil
end
end
else
_ -> nil
end
end
defp explain_join_left_table_detail(
_db,
_type,
_left,
_right_name,
_right_alias,
_constraint,
_where
),
do: nil
defp explain_templates(db, {:table, name, alias_name}) do
case plain_table(db, Table.key(name)) do
%Table{} = table -> [table_frame(table, alias_name)]
_ -> []
end
end
defp explain_templates(db, {:join, type, left, {:table, name, alias_name}, constraint}) do
ltmpls = explain_templates(db, left)
case plain_table(db, Table.key(name)) do
%Table{} = table ->
rtmpl = table_frame(table, alias_name)
using = using_columns(type, constraint, ltmpls, rtmpl)
ltmpls ++ [%{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}]
_ ->
ltmpls
end
end
defp explain_templates(_db, _from), do: []
defp access_path_detail(_table, display, {:rowid_eq, _value}),
do: "SEARCH #{display} USING INTEGER PRIMARY KEY (rowid=?)"
defp access_path_detail(_table, display, {:rowid_in, _exprs}),
do: "SEARCH #{display} USING INTEGER PRIMARY KEY (rowid=?)"
defp access_path_detail(_table, display, {:rowid_range, [{op, _expr} | _bounds]}) do
op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op]
"SEARCH #{display} USING INTEGER PRIMARY KEY (rowid#{op_text}?)"
end
defp access_path_detail(table, display, {:index_eq, index, n_columns}) do
terms =
index.columns
|> Enum.take(n_columns)
|> Enum.map_join(" AND ", &"#{display_column_name(table, &1)}=?")
"SEARCH #{display} USING INDEX #{index.name} (#{terms})"
end
defp access_path_detail(table, display, {:index_range, index, [{op, _expr} | _bounds]}) do
op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op]
column = display_column_name(table, List.first(index.columns))
"SEARCH #{display} USING INDEX #{index.name} (#{column}#{op_text}?)"
end
defp access_path_detail(table, display, {:index_in, index, _exprs}) do
column = display_column_name(table, List.first(index.columns))
"SEARCH #{display} USING INDEX #{index.name} (#{column}=?)"
end
defp access_path_detail(_table, display, {:expr_index_eq, index, _expr, _value}),
do: "SEARCH #{display} USING INDEX #{index.name} (<expr>=?)"
defp access_path_detail(_table, display, {:expr_index_in, index, _expr, _values}),
do: "SEARCH #{display} USING INDEX #{index.name} (<expr>=?)"
defp access_path_detail(_table, display, {:expr_index_or, index, _expr, _values}),
do: "SEARCH #{display} USING INDEX #{index.name} (<expr>=?)"
defp access_path_detail(
_table,
display,
{:expr_index_range, index, _expr, [{op, _bound} | _bounds]}
) do
op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op]
"SEARCH #{display} USING INDEX #{index.name} (<expr>#{op_text}?)"
end
defp access_path_detail(table, display, {:index_member_eq, index, prefix}) do
terms =
prefix
|> Enum.map_join(" AND ", fn
{{:column, key}, _expr} -> "#{display_column_name(table, key)}=?"
{{:expr, _indexed_expr}, _expr} -> "<expr>=?"
end)
"SEARCH #{display} USING INDEX #{index.name} (#{terms})"
end
defp access_path_detail(table, display, {:index_member_in, index, prefix, in_member, _exprs}) do
prefix_terms =
prefix
|> Enum.map(fn
{{:column, key}, _expr} -> "#{display_column_name(table, key)}=?"
{{:expr, _indexed_expr}, _expr} -> "<expr>=?"
end)
in_term =
case in_member do
{:column, key} -> "#{display_column_name(table, key)}=?"
{:expr, _indexed_expr} -> "<expr>=?"
end
terms = Enum.join(prefix_terms ++ [in_term], " AND ")
"SEARCH #{display} USING INDEX #{index.name} (#{terms})"
end
defp access_path_detail(
table,
display,
{:index_member_range, index, prefix, range_member, [{op, _bound} | _bounds]}
) do
prefix_terms =
prefix
|> Enum.map(fn
{{:column, key}, _expr} -> "#{display_column_name(table, key)}=?"
{{:expr, _indexed_expr}, _expr} -> "<expr>=?"
end)
op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op]
range_term =
case range_member do
{:column, key} -> "#{display_column_name(table, key)}#{op_text}?"
{:expr, _indexed_expr} -> "<expr>#{op_text}?"
end
terms = Enum.join(prefix_terms ++ [range_term], " AND ")
"SEARCH #{display} USING INDEX #{index.name} (#{terms})"
end
defp access_path_detail(_table, display, :scan), do: "SCAN #{display}"
defp subquery_detail("SCAN CONSTANT ROW", nil), do: "SCAN CONSTANT ROW"
defp subquery_detail("SCAN CONSTANT ROW", alias_name), do: "SCAN #{alias_name}"
defp subquery_detail(detail, _alias_name), do: detail
defp compound_query_plan_detail(:union_all), do: "UNION ALL"
defp compound_query_plan_detail(:union), do: "UNION USING TEMP B-TREE"
defp compound_query_plan_detail(:intersect), do: "INTERSECT USING TEMP B-TREE"
defp compound_query_plan_detail(:except), do: "EXCEPT USING TEMP B-TREE"
defp renumber_query_plan_rows(rows, start_id, root_parent) do
id_map =
rows
|> Enum.map(&hd/1)
|> Enum.with_index(start_id)
|> Map.new()
rows =
Enum.map(rows, fn [old_id, old_parent, notused, detail] ->
parent = if old_parent == 0, do: root_parent, else: Map.fetch!(id_map, old_parent)
[Map.fetch!(id_map, old_id), parent, notused, detail]
end)
{rows, start_id + length(rows)}
end
# -- CREATE TABLE helpers -------------------------------------------------------
# Splits table-level constraints into composite_keys and composite_uniques.
defp partition_table_constraints(constraints) do
Enum.reduce(constraints, {[], []}, fn
{:primary_key, name, cols}, {pks, uqs} -> {pks ++ [{name, cols}], uqs}
{:unique, name, cols}, {pks, uqs} -> {pks, uqs ++ [{name, cols}]}
{:check, _name, _expr}, acc -> acc
{:foreign_key, _name, _cols, _ref_table, _ref_cols, _actions}, acc -> acc
end)
end
# Returns only the CHECK constraints from table-level constraints.
defp table_check_constraints(constraints) do
for {:check, name, expr} <- constraints, do: {name, expr}
end
defp table_foreign_keys(constraints) do
for {:foreign_key, _name, cols, ref_table, ref_cols, actions} <- constraints do
{cols, ref_table, ref_cols, actions}
end
end
defp ensure_valid_check_constraints!(table_name, columns, constraints) do
Enum.each(columns, fn column ->
ensure_valid_check_expr!(table_name, columns, column.check)
end)
Enum.each(constraints, fn
{:check, _name, expr} -> ensure_valid_check_expr!(table_name, columns, expr)
_constraint -> :ok
end)
end
defp ensure_valid_check_expr!(_table_name, _columns, nil), do: :ok
defp ensure_valid_check_expr!(table_name, columns, expr) do
if contains_bind_parameter?(expr) do
fail("parameters prohibited in CHECK constraints")
end
validate_check_expr_columns!(
expr,
Table.key(table_name),
MapSet.new(columns, &Table.key(&1.name))
)
end
defp validate_check_expr_columns!({:column, nil, name}, _table_key, column_keys) do
unless MapSet.member?(column_keys, Table.key(name)) do
fail("no such column: #{name}")
end
end
defp validate_check_expr_columns!({:column, qualifier, name}, table_key, column_keys) do
qualified_name = "#{qualifier}.#{name}"
unless Table.key(qualifier) == table_key and MapSet.member?(column_keys, Table.key(name)) do
fail("no such column: #{qualified_name}")
end
end
defp validate_check_expr_columns!({:subquery, _query}, _table_key, _column_keys),
do: fail("subqueries prohibited in CHECK constraints")
defp validate_check_expr_columns!({:exists, _query}, _table_key, _column_keys),
do: fail("subqueries prohibited in CHECK constraints")
defp validate_check_expr_columns!(
{:in, _expr, {:select, _query}, _negated},
_table_key,
_column_keys
),
do: fail("subqueries prohibited in CHECK constraints")
defp validate_check_expr_columns!(%module{}, _table_key, _column_keys)
when module in [Select, Compound, Values, With],
do: fail("subqueries prohibited in CHECK constraints")
defp validate_check_expr_columns!(tuple, table_key, column_keys) when is_tuple(tuple) do
tuple
|> Tuple.to_list()
|> Enum.each(&validate_check_expr_columns!(&1, table_key, column_keys))
end
defp validate_check_expr_columns!(list, table_key, column_keys) when is_list(list) do
Enum.each(list, &validate_check_expr_columns!(&1, table_key, column_keys))
end
defp validate_check_expr_columns!(_other, _table_key, _column_keys), do: :ok
# -- ALTER TABLE helpers --------------------------------------------------------
defp rename_in_composites(composites, old_key, new_key) do
Enum.map(composites, fn {cname, cols} ->
{cname, Enum.map(cols, fn k -> if k == old_key, do: new_key, else: k end)}
end)
end
defp alter_add_column(db, table, col_def) do
ensure_valid_check_expr!(table.name, table.columns ++ [col_def], col_def.check)
# Restrictions
if col_def.primary_key do
fail("Cannot add a PRIMARY KEY column")
end
if col_def.unique do
fail("Cannot add a UNIQUE column")
end
# NOT NULL without a usable default is only an error if rows exist
has_rows = table.rows != %{}
if has_rows and col_def.not_null and not has_constant_default?(col_def) do
fail("Cannot add a NOT NULL column with default value NULL")
end
if has_rows and not constant_default?(col_def) do
fail("Cannot add a column with non-constant default")
end
# With foreign keys on, existing rows would all take the default value,
# which cannot be checked against the parent table.
if has_rows and db.foreign_keys and col_def.references != nil and
has_constant_default?(col_def) do
fail("Cannot add a REFERENCES column with non-NULL default value")
end
new_col_key = Table.key(col_def.name)
default = column_default_value(db, col_def)
new_rows =
Map.new(Table.scan(table), fn {rowid, row} ->
{rowid, Map.put(row, new_col_key, default)}
end)
new_checks =
if col_def.check != nil do
table.checks ++ [{col_def.check_name, col_def.check}]
else
table.checks
end
new_table =
Table.narrow_all_rows(%{
table
| columns: table.columns ++ [col_def],
rows: new_rows,
checks: new_checks,
frame_columns: nil,
column_index: nil
})
{%Result{command: :alter_table}, db |> put_table(new_table) |> Database.schema_changed()}
end
defp alter_drop_column(db, table, col_name) do
col = Table.column(table, col_name)
unless col do
fail(~s(no such column: "#{col_name}"))
end
if length(table.columns) == 1 do
fail(~s(cannot drop column "#{col_name}": no other columns exist))
end
if col.primary_key do
fail(~s(cannot drop PRIMARY KEY column: "#{col_name}"))
end
if col.unique do
fail(~s(cannot drop UNIQUE column: "#{col_name}"))
end
col_key = Table.key(col_name)
# An index over the column would be left dangling; SQLite errors too.
index_uses_column? = fn index ->
Enum.any?(index_members(index), fn
{:column, key} -> key == col_key
{:expr, expr} -> expr_references_column?(expr, col_key)
end)
end
case Enum.find(table.indexes, index_uses_column?) do
nil -> :ok
index -> fail("error in index #{index.name} after drop column: no such column: #{col_name}")
end
new_columns = Enum.reject(table.columns, &(Table.key(&1.name) == col_key))
new_rowid_alias = if table.rowid_alias == col_key, do: nil, else: table.rowid_alias
# Remove column from all rows
new_rows =
Map.new(Table.scan(table), fn {rowid, row} -> {rowid, Map.delete(row, col_key)} end)
# Remove column from composite constraints
new_composite_keys = drop_from_composites(table.composite_keys, col_key)
new_composite_uniques = drop_from_composites(table.composite_uniques, col_key)
# Remove checks that reference only the dropped column (heuristic: drop column-level checks
# on that specific column; table-level checks are kept as SQLite would error on them)
new_checks = Enum.reject(table.checks, fn {_name, _expr} -> false end)
new_table =
Table.narrow_all_rows(%{
table
| columns: new_columns,
rowid_alias: new_rowid_alias,
rows: new_rows,
composite_keys: new_composite_keys,
composite_uniques: new_composite_uniques,
autoindexes: drop_from_autoindexes(table.autoindexes, col_key),
checks: new_checks,
frame_columns: nil,
column_index: nil
})
{%Result{command: :alter_table}, db |> put_table(new_table) |> Database.schema_changed()}
end
defp drop_from_composites(composites, col_key) do
composites
|> Enum.map(fn {cname, cols} -> {cname, Enum.reject(cols, &(&1 == col_key))} end)
|> Enum.reject(fn {_cname, cols} -> cols == [] end)
end
# Returns true if the column has a constant (literal) default
defp constant_default?(%{default: nil}), do: true
defp constant_default?(%{default: {:literal, _}}), do: true
defp constant_default?(%{default: {:negate, {:literal, _}}}), do: true
defp constant_default?(%{default: {:column, nil, _}}), do: true
defp constant_default?(_), do: false
# Returns true if the column has a non-NULL constant default (satisfies NOT NULL)
defp has_constant_default?(%{default: nil}), do: false
defp has_constant_default?(%{default: {:literal, nil}}), do: false
defp has_constant_default?(%{default: {:literal, _}}), do: true
defp has_constant_default?(%{default: {:negate, {:literal, _}}}), do: true
defp has_constant_default?(%{default: {:column, nil, _}}), do: true
defp has_constant_default?(_), do: false
# The value a DEFAULT clause produces for an omitted column. Literal and
# bare-word forms are resolved directly; any other form is an expression
# default (e.g. `DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now'))`) and is
# evaluated like SQLite does at insert time, in an empty row environment
# (DEFAULT expressions may not reference columns). Evaluation failures fall
# back to NULL rather than aborting the insert.
defp column_default_value(_db, %{default: nil}), do: nil
defp column_default_value(_db, %{default: {:literal, value}, affinity: aff}),
do: Value.apply_affinity(value, aff)
defp column_default_value(_db, %{default: {:negate, {:literal, value}}, affinity: aff})
when is_number(value),
do: Value.apply_affinity(-value, aff)
defp column_default_value(_db, %{default: {:column, nil, word}, affinity: aff}),
do: Value.apply_affinity(word, aff)
defp column_default_value(db, %{default: expr, affinity: aff}) when expr != nil do
env = %{db: db, frames: [], group: nil, outer: nil}
expr |> eval(env) |> Value.apply_affinity(aff)
rescue
_ -> nil
catch
_ -> nil
end
defp column_default_value(_db, _), do: nil
# Extract an integer LIMIT from the outermost SELECT/Compound/With for
# use as a recursive CTE row cap. Returns nil if not present or not an integer literal.
defp extract_query_limit(%Select{limit: {:literal, n}}, _db) when is_integer(n) and n > 0, do: n
defp extract_query_limit(%Compound{limit: {:literal, n}}, _db) when is_integer(n) and n > 0,
do: n
defp extract_query_limit(%With{query: inner}, db), do: extract_query_limit(inner, db)
defp extract_query_limit(_, _), do: nil
# Any place a query can appear (subquery, EXISTS, IN, FROM) accepts a
# simple select, a compound, bare VALUES, or a WITH.
defp query_result(db, %Select{} = stmt, outer), do: select_result(db, stmt, outer)
defp query_result(db, %Compound{} = stmt, outer), do: compound_result(db, stmt, outer)
defp query_result(db, %Values{} = stmt, outer), do: values_result(db, stmt, outer)
defp query_result(db, %With{} = stmt, outer) do
outer_limit = extract_query_limit(stmt.query, db)
# Inner WITH CTEs can shadow outer CTEs. We need to allow the inner names
# to be evaluated even if they already exist in db.ctes. Save the outer ctes,
# remove any keys that this WITH will shadow, then evaluate.
inner_keys = MapSet.new(stmt.ctes, &Table.key(&1.name))
db_for_inner = %{db | ctes: Map.drop(db.ctes, MapSet.to_list(inner_keys))}
db_with_ctes = resolve_ctes(db_for_inner, stmt.ctes, stmt.recursive, outer_limit)
# Run the body with inner scope; discard inner CTEs from result (restore outer).
result = query_result(db_with_ctes, stmt.query, outer)
result
end
# A bare VALUES select: output columns are named column1..columnN.
defp values_result(db, %Values{} = stmt, outer) do
env = %{db: db, frames: [], group: nil, outer: outer}
case stmt.rows |> Enum.map(&length/1) |> Enum.uniq() do
[width] ->
rows = Enum.map(stmt.rows, fn exprs -> Enum.map(exprs, &eval(&1, env)) end)
names = Enum.map(1..width, &"column#{&1}")
rows =
rows
|> compound_order(stmt.order_by, names, [names])
|> clamp(db, stmt.limit, stmt.offset)
%Result{command: :select, columns: names, rows: rows, rows_affected: 0}
_mixed ->
fail("all VALUES must have the same number of terms")
end
end
defp savepoint_index(stack, name) do
target = {:savepoint, Table.key(name)}
Enum.find_index(stack, fn {kind, _snapshot} -> kind == target end)
end
# -- INSERT helpers -------------------------------------------------------------
# The rows to insert, as evaluated value lists (or `:default` for
# DEFAULT VALUES), with SQLite's count-mismatch errors.
defp insert_rows(_db, %Insert{source: :default_values}, _table, _targets), do: [:default]
defp insert_rows(db, %Insert{source: {:values, expr_rows}} = stmt, table, targets) do
case expr_rows |> Enum.map(&length/1) |> Enum.uniq() do
[width] ->
check_insert_width(stmt, table, targets, width)
env = constant_env(db)
Enum.map(expr_rows, fn exprs -> Enum.map(exprs, &eval(&1, env)) end)
_mixed ->
fail("all VALUES must have the same number of terms")
end
end
defp insert_rows(db, %Insert{source: {:select, query}} = stmt, table, targets) do
result = query_result(db, query, nil)
check_insert_width(stmt, table, targets, length(result.columns))
result.rows
end
defp check_insert_width(stmt, table, targets, width) do
cond do
stmt.columns == nil and width != length(targets) ->
fail(
"table #{table.name} has #{length(targets)} columns " <>
"but #{width} values were supplied"
)
stmt.columns != nil and width != length(targets) ->
fail("#{width} values for #{length(targets)} columns")
true ->
:ok
end
end
# Builds a full candidate row map (all columns, with defaults for missing values)
# for CHECK constraint evaluation, without going through Table.insert.
defp build_candidate_row(table, values, db) do
# Column keys are static per table; reuse the cached folded keys
# (`frame_columns`) rather than re-folding `Table.key(column.name)` for every
# column on every inserted row — a hot spot on bulk/repeated inserts.
table
|> Table.frame_columns()
|> Enum.zip(table.columns)
|> Map.new(fn {{col_key, _name, _aff, _coll}, column} ->
value =
case Map.fetch(values, col_key) do
{:ok, v} -> v
:error -> column_default_value(db, column)
end
{col_key, value}
end)
end
defp with_explicit_rowid(row, %{rowid_alias: alias_key}, rowid)
when alias_key != nil and is_integer(rowid),
do: Map.put(row, alias_key, rowid)
defp with_explicit_rowid(row, _table, _rowid), do: row
defp apply_generated_columns(row, db, table, rowid) do
Enum.reduce(table.columns, row, fn
%{generated: {_, expr}} = column, row ->
env = table_env(db, table, rowid, row)
value =
expr
|> eval(env)
|> Value.apply_affinity(column.affinity)
Map.put(row, Table.key(column.name), value)
_column, row ->
row
end)
end
defp check_strict_types!(%{strict: false}, _row), do: :ok
defp check_strict_types!(table, row) do
Enum.each(table.columns, fn column ->
value = Map.get(row, Table.key(column.name))
if column.generated == nil and value != nil and not strict_value_allowed?(column, value) do
storage_type = value |> Value.type_of() |> Atom.to_string() |> String.upcase()
declared_type = String.upcase(column.declared_type)
fail(
"cannot store #{storage_type} value in #{declared_type} column #{table.name}.#{column.name}"
)
end
end)
end
defp strict_value_allowed?(%{declared_type: type}, value) do
case {String.upcase(type), Value.type_of(value)} do
{"ANY", _storage_class} -> true
{"INT", :integer} -> true
{"INTEGER", :integer} -> true
{"REAL", :real} -> true
{"TEXT", :text} -> true
{"BLOB", :blob} -> true
_ -> false
end
end
# -- foreign keys -------------------------------------------------------------
#
# SQLite checks immediate foreign keys at the end of each statement: the
# statement fails if it leaves more violations than it found (fkey.c keeps
# a per-statement counter; the delta scan below is the tree-walking
# equivalent). Deferred constraints wait until the outermost transaction
# commits. RESTRICT actions fire as parent rows are deleted or updated,
# regardless of deferral; CASCADE / SET NULL / SET DEFAULT rewrite child
# rows before the statement-end check runs.
defp with_fk_statement_check(db, table_name, fun) when is_binary(table_name) do
entries =
if db.foreign_keys do
db
|> relevant_fk_entries(Database.table_storage_key(nil, table_name))
|> immediate_fk_entries(db)
else
[]
end
if entries == [] do
fun.()
else
# Unresolvable references (missing parent table, key mismatch) only
# error on DML against the child table itself, as in SQLite.
target_key = Table.key(table_name)
{strict, lax} =
Enum.split_with(entries, fn {child_key, _parent_key, _spec} -> child_key == target_key end)
count = fn db ->
fk_violation_count(db, strict, :raise) + fk_violation_count(db, lax, :skip)
end
violations_before = count.(db)
{result, new_db} = fun.()
if count.(new_db) > violations_before do
fail("FOREIGN KEY constraint failed")
end
{result, new_db}
end
end
defp with_fk_statement_check(db, %{schema: schema, table: table_name}, fun)
when is_binary(table_name) do
with_fk_statement_check(db, Database.table_storage_key(schema, table_name), fun)
end
defp with_fk_statement_check(_db, _stmt, fun), do: fun.()
defp check_deferred_foreign_keys!(%{foreign_keys: false}), do: :ok
defp check_deferred_foreign_keys!(db) do
{_kind, snapshot} = List.last(db.txn_stack)
db_before = Database.restore_schema(db, snapshot)
violations_now = fk_violation_count(db, all_fk_entries(db), :skip)
violations_before = fk_violation_count(db_before, all_fk_entries(db_before), :skip)
if violations_now > violations_before do
fail("FOREIGN KEY constraint failed")
end
:ok
end
defp all_fk_entries(db) do
Enum.flat_map(db.tables, fn {child_key, child_table} ->
Enum.map(foreign_key_specs(child_table), fn spec ->
{child_key, fk_parent_key(child_table, spec), spec}
end)
end)
end
# Foreign keys a DML statement on `table_name` can affect: those whose
# child or parent is the table itself or any table reachable from it
# through delete/update actions.
defp relevant_fk_entries(db, table_key) do
entries = all_fk_entries(db)
closure = fk_action_closure(entries, MapSet.new([table_key]))
Enum.filter(entries, fn {child_key, parent_key, _spec} ->
MapSet.member?(closure, child_key) or MapSet.member?(closure, parent_key)
end)
end
defp fk_action_closure(entries, set) do
additions =
for {child_key, parent_key, _spec} <- entries,
MapSet.member?(set, parent_key),
not MapSet.member?(set, child_key),
do: child_key
case additions do
[] -> set
keys -> fk_action_closure(entries, MapSet.union(set, MapSet.new(keys)))
end
end
# Inside a transaction, constraints declared DEFERRABLE INITIALLY DEFERRED
# (or all of them under PRAGMA defer_foreign_keys) wait until COMMIT.
# Outside a transaction every constraint is effectively immediate.
defp immediate_fk_entries(entries, db) do
if db.txn_stack == [] do
entries
else
Enum.reject(entries, fn {_child_key, _parent_key, spec} ->
spec.deferred or db.defer_foreign_keys
end)
end
end
defp fk_violation_count(db, entries, on_missing) do
Enum.reduce(entries, 0, fn {child_key, _parent_key, spec}, acc ->
case Map.fetch(db.tables, child_key) do
{:ok, child_table} -> acc + fk_spec_violation_count(db, child_table, spec, on_missing)
:error -> acc
end
end)
end
defp fk_parent_key(child_table, spec) do
Database.table_storage_key(child_table.schema, spec.parent_table)
end
# `:skip` tolerates unresolvable references (missing parent table, key
# mismatch) by not counting them — used at COMMIT, where SQLite only sees
# the counter accumulated by statements that resolved successfully.
defp fk_spec_violation_count(db, child_table, spec, :skip) do
fk_spec_violation_count(db, child_table, spec, :raise)
rescue
_e in Error -> 0
end
defp fk_spec_violation_count(db, child_table, spec, :raise) do
{parent_table, parent_columns} = referenced_parent!(db, child_table, spec, child_table)
Enum.count(Table.scan(child_table), fn {_rowid, row} ->
child_values = Enum.map(spec.child_keys, &Map.get(row, &1))
not Enum.any?(child_values, &is_nil/1) and
not parent_row_exists?(parent_table, parent_columns, child_values)
end)
end
# With foreign keys on, DROP TABLE performs an implicit DELETE FROM the
# table first: delete actions fire and remaining references fail the drop.
defp drop_table_fk_cleanup(%{foreign_keys: false} = db, _schema, _name), do: db
defp drop_table_fk_cleanup(db, schema, name) do
table_key = Database.table_storage_key(schema, name)
case Map.fetch(db.tables, table_key) do
:error ->
db
{:ok, table} when map_size(table.rows) == 0 ->
db
{:ok, table} ->
{_result, db} =
with_fk_statement_check(db, table_key, fn ->
deleted_rows = Enum.map(Table.scan(table), fn {_rowid, row} -> row end)
emptied = Table.delete_rows(table, Map.keys(table.rows))
db =
db
|> put_table(emptied)
|> apply_fk_delete_actions(emptied, deleted_rows)
{nil, db}
end)
db
end
end
defp apply_fk_delete_actions(%{foreign_keys: false} = db, _parent_table, _deleted_rows), do: db
defp apply_fk_delete_actions(db, _parent_table, []), do: db
defp apply_fk_delete_actions(db, parent_table, deleted_rows) do
Enum.reduce(parent_reference_checks(db, parent_table), db, fn
{child_table, spec, parent_columns}, db ->
child_table = refetch_table(db, child_table)
deleted_keys =
deleted_rows
|> Enum.map(fn row -> Enum.map(parent_columns, &Map.get(row, Table.key(&1.name))) end)
|> Enum.reject(fn values -> Enum.any?(values, &is_nil/1) end)
if deleted_keys == [] do
db
else
matches = matching_child_rows(child_table, spec, parent_columns, deleted_keys)
case spec.on_delete do
:restrict ->
if matches != [] or
restrict_among_deleted?(
child_table,
parent_table,
spec,
parent_columns,
deleted_rows
) do
fail("FOREIGN KEY constraint failed")
end
db
:cascade ->
rowids = Enum.map(matches, &elem(&1, 0))
child_table = Table.delete_rows(child_table, rowids)
db = put_table(db, child_table)
apply_fk_delete_actions(db, child_table, Enum.map(matches, &elem(&1, 1)))
:set_null ->
fk_set_child_columns(db, child_table, spec, matches, fn _column -> nil end)
:set_default ->
fk_set_child_columns(db, child_table, spec, matches, &column_default_value(db, &1))
:no_action ->
db
end
end
end)
end
# A self-referential RESTRICT fires as each parent row is deleted, so a row
# deleted later in the same statement still counts as a referencing child
# of one deleted earlier.
defp restrict_among_deleted?(child_table, parent_table, spec, parent_columns, deleted_rows) do
Database.table_storage_key(child_table.schema, child_table.name) ==
Database.table_storage_key(parent_table.schema, parent_table.name) and
deleted_rows
|> Enum.with_index()
|> Enum.any?(fn {row, index} ->
parent_values = Enum.map(parent_columns, &Map.get(row, Table.key(&1.name)))
not Enum.any?(parent_values, &is_nil/1) and
deleted_rows
|> Enum.drop(index + 1)
|> Enum.any?(fn later ->
child_values = Enum.map(spec.child_keys, &Map.get(later, &1))
not Enum.any?(child_values, &is_nil/1) and
foreign_key_values_match?(child_values, parent_values, parent_columns)
end)
end)
end
defp apply_fk_update_actions(%{foreign_keys: false} = db, _parent_table, _changed_pairs),
do: db
defp apply_fk_update_actions(db, _parent_table, []), do: db
defp apply_fk_update_actions(db, parent_table, changed_pairs) do
Enum.reduce(parent_reference_checks(db, parent_table), db, fn
{child_table, spec, parent_columns}, db ->
changes =
changed_pairs
|> Enum.map(fn {old_row, new_row} ->
{Enum.map(parent_columns, &Map.get(old_row, Table.key(&1.name))),
Enum.map(parent_columns, &Map.get(new_row, Table.key(&1.name)))}
end)
|> Enum.reject(fn {old_values, new_values} ->
Enum.any?(old_values, &is_nil/1) or
foreign_key_parent_values_equal?(old_values, new_values)
end)
Enum.reduce(changes, db, fn {old_values, new_values}, db ->
child_table = refetch_table(db, child_table)
matches = matching_child_rows(child_table, spec, parent_columns, [old_values])
case spec.on_update do
:restrict ->
if matches != [], do: fail("FOREIGN KEY constraint failed")
db
:cascade ->
new_by_key = spec.child_keys |> Enum.zip(new_values) |> Map.new()
fk_set_child_columns(db, child_table, spec, matches, fn column ->
new_by_key
|> Map.fetch!(Table.key(column.name))
|> Value.apply_affinity(column.affinity)
end)
:set_null ->
fk_set_child_columns(db, child_table, spec, matches, fn _column -> nil end)
:set_default ->
fk_set_child_columns(db, child_table, spec, matches, &column_default_value(db, &1))
:no_action ->
db
end
end)
end)
end
# REPLACE conflict resolution is a DELETE followed by an INSERT, so the
# rows it removes fire ON DELETE foreign key actions, as in SQLite. The
# removed rows are found by diffing the table against its pre-statement
# rows, excluding rowids the statement updated in place.
defp apply_replace_deleted_actions(%{foreign_keys: false} = db, _table, _old_rows, _excluded),
do: db
defp apply_replace_deleted_actions(db, table, old_rows, excluded_rowids) do
deleted =
for {rowid, tuple} <- old_rows,
not MapSet.member?(excluded_rowids, rowid),
Map.get(table.rows, rowid) != tuple,
do: Table.row_to_map(table, tuple)
apply_fk_delete_actions(db, table, deleted)
end
defp matching_child_rows(child_table, spec, parent_columns, parent_keys) do
Enum.filter(Table.scan(child_table), fn {_rowid, row} ->
child_values = Enum.map(spec.child_keys, &Map.get(row, &1))
not Enum.any?(child_values, &is_nil/1) and
Enum.any?(parent_keys, &foreign_key_values_match?(child_values, &1, parent_columns))
end)
end
# Applies a SET NULL / SET DEFAULT / cascading-update rewrite to child rows,
# enforcing the child table's own constraints, then propagates the change
# to foreign keys that reference the child table in turn.
defp fk_set_child_columns(db, child_table, spec, matches, value_fun) do
{child_table, pairs} =
Enum.reduce(matches, {child_table, []}, fn {rowid, row}, {table, pairs} ->
new_row =
spec.child_keys
|> Enum.reduce(row, fn key, row ->
Map.put(row, key, value_fun.(Table.column(table, key)))
end)
|> apply_generated_columns(db, table, rowid)
check_strict_types!(table, new_row)
check_env = table_env(db, table, rowid, new_row)
case check_violations(db, table, new_row, check_env, :abort) do
:ok -> :ok
{:error, message} -> fail(message)
end
case resolve_unique_indexes(db, table, rowid, new_row, :abort) do
{:ok, table} ->
case Table.update_row(table, rowid, new_row) do
{:ok, table} ->
new_rowid = updated_rowid(table, rowid, new_row)
{table, [{row, Table.fetch_row!(table, new_rowid)} | pairs]}
{:error, message} ->
fail(message)
end
{:error, message} ->
fail(message)
end
end)
db
|> put_table(child_table)
|> apply_fk_update_actions(child_table, Enum.reverse(pairs))
end
defp refetch_table(db, table) do
case Map.fetch(db.tables, Database.table_storage_key(table.schema, table.name)) do
{:ok, table} -> table
:error -> table
end
end
# -- triggers -----------------------------------------------------------------
#
# Triggers fire per affected row. OLD./NEW. references in the WHEN clause
# and body are substituted with the row's values before execution, so body
# statements run through the ordinary exec path. RAISE(IGNORE) abandons
# the row operation and any later triggers without rolling back changes
# already made (thrown as :raise_ignore, caught per body statement).
defp triggers_for(db, table_name, timing, event) when is_binary(table_name) do
triggers_for(db, nil, table_name, timing, event)
end
defp triggers_for(db, %{schema: schema, table: table_name}, timing, event),
do: triggers_for(db, schema, table_name, timing, event)
defp triggers_for(db, schema, table_name, timing, event) do
key = Database.table_storage_key(schema, table_name)
db.triggers
|> Map.values()
|> Enum.filter(&(&1.table_key == key and &1.timing == timing and &1.event == event))
|> Enum.sort_by(& &1.seq)
end
defp validate_trigger_statement!(stmt, trigger_schema, trigger_name) do
case stmt do
%Insert{source: :default_values} ->
fail(~s(near "DEFAULT": syntax error))
%Insert{returning: [_ | _]} ->
fail("cannot use RETURNING in a trigger")
%Insert{target_qualified: true} ->
fail(qualified_trigger_dml_message())
%Insert{} = insert ->
validate_trigger_insert_statement!(insert, trigger_schema, trigger_name)
%Update{target_qualified: true} ->
fail(qualified_trigger_dml_message())
%Delete{target_qualified: true} ->
fail(qualified_trigger_dml_message())
%Update{index_hint: :not_indexed} ->
fail(not_indexed_trigger_dml_message())
%Delete{index_hint: :not_indexed} ->
fail(not_indexed_trigger_dml_message())
%Update{index_hint: {:indexed_by, _}} ->
fail(indexed_by_trigger_dml_message())
%Update{} = update ->
validate_trigger_update_statement!(update)
validate_trigger_update_sources!(update, trigger_schema, trigger_name)
%Delete{index_hint: {:indexed_by, _}} ->
fail(indexed_by_trigger_dml_message())
%Delete{returning: [_ | _]} ->
fail(~s(near "RETURNING": syntax error))
%Delete{order_by: [_ | _]} ->
fail(~s(near "ORDER": syntax error))
%Delete{limit: limit} when not is_nil(limit) ->
fail(~s(near "LIMIT": syntax error))
%Delete{} = delete ->
validate_trigger_delete_sources!(delete, trigger_schema, trigger_name)
%Select{} = query ->
validate_trigger_query_sources!(query, trigger_schema, trigger_name)
%Compound{} = query ->
validate_trigger_query_sources!(query, trigger_schema, trigger_name)
%Values{} ->
:ok
%With{query: %Insert{}} ->
fail(~s(near "INSERT": syntax error))
%With{query: %Update{}} ->
fail(~s(near "UPDATE": syntax error))
%With{query: %Delete{}} ->
fail(~s(near "DELETE": syntax error))
%With{query: query} = with_query ->
validate_trigger_query_sources!(with_query, trigger_schema, trigger_name)
validate_trigger_statement!(query, trigger_schema, trigger_name)
_other ->
fail("unsupported statement in trigger body")
end
end
defp validate_trigger_update_statement!(%Update{returning: [_ | _]}),
do: fail(~s(near "RETURNING": syntax error))
defp validate_trigger_update_statement!(%Update{order_by: [_ | _]}),
do: fail(~s(near "ORDER": syntax error))
defp validate_trigger_update_statement!(%Update{limit: limit}) when not is_nil(limit),
do: fail(~s(near "LIMIT": syntax error))
defp validate_trigger_update_statement!(%Update{}), do: :ok
defp validate_trigger_insert_statement!(%Insert{} = stmt, trigger_schema, trigger_name) do
source_schemas =
case stmt.source do
{:select, query} -> trigger_query_source_schemas(query)
{:values, rows} -> rows |> List.flatten() |> trigger_exprs_source_schemas()
_other -> []
end
(source_schemas ++ trigger_upsert_source_schemas(stmt.upsert))
|> validate_trigger_source_schemas!(trigger_schema, trigger_name)
end
defp validate_trigger_update_sources!(%Update{} = stmt, trigger_schema, trigger_name) do
assignment_exprs = Enum.map(stmt.assignments, fn {_name, expr} -> expr end)
(trigger_from_source_schemas(stmt.from) ++
trigger_exprs_source_schemas(assignment_exprs) ++
trigger_expr_source_schemas(stmt.where))
|> validate_trigger_source_schemas!(trigger_schema, trigger_name)
end
defp validate_trigger_delete_sources!(%Delete{} = stmt, trigger_schema, trigger_name) do
stmt.where
|> trigger_expr_source_schemas()
|> validate_trigger_source_schemas!(trigger_schema, trigger_name)
end
defp trigger_upsert_source_schemas(upserts) do
Enum.flat_map(upserts, fn
{:nothing, target} ->
trigger_upsert_target_source_schemas(target)
{:update, target, assignments, where} ->
assignment_exprs = Enum.map(assignments, fn {_name, expr} -> expr end)
trigger_upsert_target_source_schemas(target) ++
trigger_exprs_source_schemas(assignment_exprs) ++ trigger_expr_source_schemas(where)
end)
end
defp trigger_upsert_target_source_schemas({_columns, where}),
do: trigger_expr_source_schemas(where)
defp trigger_upsert_target_source_schemas(_target), do: []
defp validate_trigger_query_sources!(_query, "temp", _trigger_name), do: :ok
defp validate_trigger_query_sources!(query, trigger_schema, trigger_name) do
query
|> trigger_query_source_schemas()
|> validate_trigger_source_schemas!(trigger_schema, trigger_name)
end
defp validate_trigger_source_schemas!(_source_schemas, "temp", _trigger_name), do: :ok
defp validate_trigger_source_schemas!(source_schemas, trigger_schema, trigger_name) do
trigger_schema = trigger_schema(trigger_schema)
Enum.each(source_schemas, fn source_schema ->
normalized_source_schema = trigger_schema(source_schema)
if source_schema != nil and normalized_source_schema != trigger_schema do
fail("trigger #{trigger_name} cannot reference objects in database #{source_schema}")
end
end)
end
defp trigger_from_source_schemas(nil), do: []
defp trigger_from_source_schemas({:table, {:schema, schema, _name}, _alias}), do: [schema]
defp trigger_from_source_schemas({:table, _name, _alias}), do: []
defp trigger_from_source_schemas({:subquery, query, _alias}),
do: trigger_query_source_schemas(query)
defp trigger_from_source_schemas({:join, _type, left, right, constraint}),
do:
trigger_from_source_schemas(left) ++
trigger_from_source_schemas(right) ++ trigger_join_constraint_source_schemas(constraint)
defp trigger_from_source_schemas(_other), do: []
defp trigger_join_constraint_source_schemas({:on, expr}), do: trigger_expr_source_schemas(expr)
defp trigger_join_constraint_source_schemas(_constraint), do: []
defp trigger_query_source_schemas(%Select{} = query) do
trigger_from_source_schemas(query.from) ++
trigger_select_column_source_schemas(query.columns) ++
trigger_expr_source_schemas(query.where) ++
trigger_exprs_source_schemas(query.group_by) ++
trigger_expr_source_schemas(query.having) ++
trigger_order_source_schemas(query.order_by) ++
trigger_expr_source_schemas(query.limit) ++
trigger_expr_source_schemas(query.offset)
end
defp trigger_query_source_schemas(%Compound{left: left, right: right}),
do: trigger_query_source_schemas(left) ++ trigger_query_source_schemas(right)
defp trigger_query_source_schemas(%With{ctes: ctes, query: query}) do
Enum.flat_map(ctes, &trigger_query_source_schemas(&1.query)) ++
trigger_query_source_schemas(query)
end
defp trigger_query_source_schemas(_query), do: []
defp trigger_select_column_source_schemas(columns) do
Enum.flat_map(columns, fn
{expr, _alias} -> trigger_expr_source_schemas(expr)
_other -> []
end)
end
defp trigger_order_source_schemas(order_by) do
Enum.flat_map(order_by, fn {expr, _direction} -> trigger_expr_source_schemas(expr) end)
end
defp trigger_exprs_source_schemas(exprs),
do: Enum.flat_map(exprs, &trigger_expr_source_schemas/1)
defp trigger_expr_source_schemas({:select, query}), do: trigger_query_source_schemas(query)
defp trigger_expr_source_schemas({:subquery, query}), do: trigger_query_source_schemas(query)
defp trigger_expr_source_schemas({:exists, query}), do: trigger_query_source_schemas(query)
defp trigger_expr_source_schemas(%Select{} = query), do: trigger_query_source_schemas(query)
defp trigger_expr_source_schemas(%Compound{} = query), do: trigger_query_source_schemas(query)
defp trigger_expr_source_schemas(%With{} = query), do: trigger_query_source_schemas(query)
defp trigger_expr_source_schemas(term) when is_tuple(term) do
term
|> Tuple.to_list()
|> Enum.flat_map(&trigger_expr_source_schemas/1)
end
defp trigger_expr_source_schemas(term) when is_list(term),
do: Enum.flat_map(term, &trigger_expr_source_schemas/1)
defp trigger_expr_source_schemas(term) when is_map(term) do
term
|> Map.values()
|> Enum.flat_map(&trigger_expr_source_schemas/1)
end
defp trigger_expr_source_schemas(_term), do: []
defp validate_trigger_definition!(%CreateTrigger{} = stmt) do
if contains_bind_parameter?(stmt.when) or Enum.any?(stmt.body, &contains_bind_parameter?/1) do
fail("trigger cannot use variables")
end
Enum.each(stmt.body, &validate_trigger_statement!(&1, stmt.schema, stmt.name))
end
defp qualified_trigger_dml_message do
"qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers"
end
defp not_indexed_trigger_dml_message do
"the NOT INDEXED clause is not allowed on UPDATE or DELETE statements within triggers"
end
defp indexed_by_trigger_dml_message do
"the INDEXED BY clause is not allowed on UPDATE or DELETE statements within triggers"
end
defp contains_bind_parameter?({:param, _index, _raw}), do: true
defp contains_bind_parameter?(term) when is_tuple(term) do
term
|> Tuple.to_list()
|> Enum.any?(&contains_bind_parameter?/1)
end
defp contains_bind_parameter?(term) when is_list(term),
do: Enum.any?(term, &contains_bind_parameter?/1)
defp contains_bind_parameter?(term) when is_map(term) do
map =
if Map.has_key?(term, :__struct__) do
Map.from_struct(term)
else
term
end
map
|> Map.values()
|> Enum.any?(&contains_bind_parameter?/1)
end
defp contains_bind_parameter?(_term), do: false
defp drop_trigger_key(db, nil, name) do
Enum.find_value(trigger_lookup_order(db), fn schema ->
key = Database.table_storage_key(schema, name)
if Map.has_key?(db.triggers, key), do: key
end)
end
defp drop_trigger_key(db, schema, name) do
key = Database.table_storage_key(schema, name)
if Map.has_key?(db.triggers, key), do: key
end
defp trigger_lookup_order(db), do: ["temp", nil] ++ Enum.map(db.attached_databases, & &1.name)
defp drop_triggers_on(db, schema, table_name) do
key = Database.table_storage_key(schema, table_name)
%{
db
| triggers:
db.triggers
|> Enum.reject(fn {_k, trigger} -> trigger.table_key == key end)
|> Map.new()
}
end
defp fire_triggers(db, triggers, table, old_row, new_row, changed_keys \\ nil) do
Enum.reduce_while(triggers, {:ok, db}, fn trigger, {:ok, db} ->
skip? =
(trigger.key in db.active_triggers and not db.recursive_triggers) or
(trigger.update_columns != nil and changed_keys != nil and
not Enum.any?(trigger.update_columns, &(&1 in changed_keys)))
if skip? do
{:cont, {:ok, db}}
else
case fire_trigger(db, trigger, table, old_row, new_row) do
{:ok, db} -> {:cont, {:ok, db}}
{:ignored, db} -> {:halt, {:ignored, db}}
end
end
end)
end
defp fire_trigger(db, trigger, table, old_row, new_row) do
if length(db.active_triggers) >= @max_trigger_depth do
fail("too many levels of trigger recursion")
end
saved_active = db.active_triggers
db = %{db | active_triggers: [trigger.key | saved_active]}
fires? =
trigger.when == nil or
trigger.when
|> trigger_substitute(table, old_row, new_row)
|> truth(%{db: db, frames: [], group: nil, outer: nil})
|> Kernel.==(true)
result =
if fires? do
Enum.reduce_while(trigger.body, {:ok, db}, fn stmt, {:ok, db} ->
stmt =
stmt
|> trigger_substitute(table, old_row, new_row)
|> qualify_trigger_statement(trigger.schema)
try do
{_result, db} = exec(db, stmt)
{:cont, {:ok, db}}
catch
:raise_ignore -> {:halt, {:ignored, db}}
end
end)
else
{:ok, db}
end
{status, db} = result
{status, %{db | active_triggers: saved_active}}
end
defp trigger_substitute(ast, table, old_row, new_row) do
trig_walk(ast, table, old_row, new_row)
end
defp trig_walk({:column, qualifier, name} = node, table, old_row, new_row)
when is_binary(qualifier) do
case Table.key(qualifier) do
"old" when old_row != nil -> {:literal, trigger_row_value(table, old_row, name, qualifier)}
"new" when new_row != nil -> {:literal, trigger_row_value(table, new_row, name, qualifier)}
_ -> node
end
end
defp trig_walk(tuple, table, old_row, new_row) when is_tuple(tuple) do
tuple
|> Tuple.to_list()
|> Enum.map(&trig_walk(&1, table, old_row, new_row))
|> List.to_tuple()
end
defp trig_walk(list, table, old_row, new_row) when is_list(list) do
Enum.map(list, &trig_walk(&1, table, old_row, new_row))
end
defp trig_walk(%module{} = node, table, old_row, new_row) do
struct!(
module,
node
|> Map.from_struct()
|> Enum.map(fn {key, value} -> {key, trig_walk(value, table, old_row, new_row)} end)
)
end
defp trig_walk(%{} = map, table, old_row, new_row) do
Map.new(map, fn {key, value} -> {key, trig_walk(value, table, old_row, new_row)} end)
end
defp trig_walk(other, _table, _old_row, _new_row), do: other
defp trigger_row_value(table, {:trigger_row, rowid, row}, name, qualifier) do
key = Table.key(name)
cond do
table.rowid_alias != nil and key == table.rowid_alias and is_nil(Map.get(row, key)) ->
rowid
Map.has_key?(row, key) ->
Map.get(row, key)
key in @rowid_names and table.rowid_alias != nil ->
Map.get(row, table.rowid_alias) || rowid
key in @rowid_names and not table.without_rowid ->
rowid
true ->
fail("no such column: #{qualifier}.#{name}")
end
end
defp trigger_row_value(table, row, name, qualifier) do
key = Table.key(name)
cond do
Map.has_key?(row, key) ->
Map.get(row, key)
key in @rowid_names and table.rowid_alias != nil ->
Map.get(row, table.rowid_alias)
true ->
fail("no such column: #{qualifier}.#{name}")
end
end
defp parent_reference_checks(db, parent_table) do
Enum.flat_map(db.tables, fn {_key, child_table} ->
child_table
|> foreign_key_specs()
|> Enum.flat_map(fn spec ->
if fk_parent_key(child_table, spec) ==
Database.table_storage_key(parent_table.schema, parent_table.name) do
[{child_table, spec, referenced_columns!(child_table, spec, parent_table)}]
else
[]
end
end)
end)
end
defp referenced_parent!(db, child_table, spec, current_table) do
referenced_table_name = spec.parent_table
parent_table =
if fk_parent_key(child_table, spec) ==
Database.table_storage_key(current_table.schema, current_table.name) do
current_table
else
case fetch_fk_parent_table(db, child_table, referenced_table_name) do
{:ok, table} ->
table
{:error, _message} ->
fail("no such table: #{child_table.schema || "main"}.#{referenced_table_name}")
end
end
{parent_table, referenced_columns!(child_table, spec, parent_table)}
end
defp fetch_fk_parent_table(db, child_table, parent_name) do
key = Database.table_storage_key(child_table.schema, parent_name)
case Map.fetch(db.tables, key) do
{:ok, table} -> {:ok, table}
:error -> {:error, "no such table: #{child_table.schema || "main"}.#{parent_name}"}
end
end
defp referenced_columns!(child_table, spec, parent_table) do
parent_column_keys =
case spec.parent_keys do
[] ->
primary_key_column_keys(parent_table) ||
foreign_key_mismatch!(child_table, parent_table)
keys ->
keys
end
if length(parent_column_keys) != length(spec.child_keys) do
foreign_key_mismatch!(child_table, parent_table)
end
parent_columns =
Enum.map(parent_column_keys, fn key ->
Table.column(parent_table, key) ||
foreign_key_mismatch!(child_table, parent_table)
end)
unless referenced_key_unique?(parent_table, parent_column_keys) do
foreign_key_mismatch!(child_table, parent_table)
end
parent_columns
end
defp referenced_key_unique?(%{rowid_alias: rowid_alias}, [rowid_alias]) when rowid_alias != nil,
do: true
defp referenced_key_unique?(parent_table, keys) do
inline_unique? =
case keys do
[key] ->
case Table.column(parent_table, key) do
%{primary_key: true} -> true
%{unique: true} -> true
_ -> false
end
_ ->
false
end
composite_unique? =
Enum.any?(parent_table.composite_keys ++ parent_table.composite_uniques, fn {_name, cols} ->
cols == keys
end)
index_unique? =
Enum.any?(parent_table.indexes, fn index ->
index.unique and index.columns == keys
end)
inline_unique? or composite_unique? or index_unique?
end
defp primary_key_column_keys(%{rowid_alias: rowid_alias}) when rowid_alias != nil,
do: [rowid_alias]
defp primary_key_column_keys(table) do
inline_keys =
table.columns
|> Enum.filter(& &1.primary_key)
|> Enum.map(&Table.key(&1.name))
composite_keys =
case table.composite_keys do
[{_name, keys}] -> keys
_ -> []
end
case inline_keys ++ composite_keys do
[] -> nil
keys -> keys
end
end
defp parent_row_exists?(parent_table, parent_columns, child_values) do
Enum.any?(Table.scan(parent_table), fn {_rowid, parent_row} ->
parent_values = Enum.map(parent_columns, &Map.get(parent_row, Table.key(&1.name)))
foreign_key_values_match?(child_values, parent_values, parent_columns)
end)
end
defp foreign_key_values_match?(child_values, parent_values, parent_columns) do
Enum.zip([child_values, parent_values, parent_columns])
|> Enum.all?(fn {child_value, parent_value, parent_column} ->
child_value
|> Value.apply_affinity(parent_column.affinity)
|> Value.compare(parent_value)
|> Kernel.==(:eq)
end)
end
defp foreign_key_parent_values_equal?(old_values, new_values) do
Enum.zip(old_values, new_values)
|> Enum.all?(fn {old_value, new_value} -> Value.compare(old_value, new_value) == :eq end)
end
defp foreign_key_specs(table) do
column_specs =
table.columns
|> Enum.filter(& &1.references)
|> Enum.map(fn column ->
{parent_table, parent_keys, actions} = column.references
Map.merge(actions, %{
child_keys: [Table.key(column.name)],
parent_table: parent_table,
parent_keys: Enum.map(parent_keys, &Table.key/1)
})
end)
table_specs =
Enum.map(table.foreign_keys, fn {child_keys, parent_table, parent_keys, actions} ->
Map.merge(actions, %{
child_keys: child_keys,
parent_table: parent_table,
parent_keys: parent_keys
})
end)
column_specs ++ table_specs
end
defp foreign_key_mismatch!(child_table, parent_table) do
fail(~s(foreign key mismatch - "#{child_table.name}" referencing "#{parent_table.name}"))
end
defp resolve_unique_indexes(db, table, rowid, row, on_conflict) do
conflicts = unique_index_conflicts(db, table, rowid, row)
case {conflicts, on_conflict} do
{[], _} ->
{:ok, table}
{_conflicts, :ignore} ->
:ignore
{conflicts, :replace} ->
rowids = Enum.flat_map(conflicts, &elem(&1, 0)) |> Enum.uniq()
{:ok, Table.delete_rows(table, rowids)}
{[{_rowids, message} | _], _} ->
{:error, message}
end
end
defp resolve_unique_indexes_for_dml(db, table, rowid, row, on_conflict, opts \\ []) do
case {replacement_conflict_rowids(db, table, rowid, row, opts), on_conflict} do
{[], _} ->
case resolve_unique_indexes(db, table, rowid, row, on_conflict) do
{:ok, table} -> {:ok, db, table}
other -> other
end
{_rowids, :ignore} ->
:ignore
{rowids, :replace} ->
fire_replace_delete_triggers(db, table, rowids)
{_rowids, _} ->
case resolve_unique_indexes(db, table, rowid, row, on_conflict) do
{:ok, table} -> {:ok, db, table}
other -> other
end
end
end
defp replacement_conflict_rowids(db, table, rowid, row, opts) do
excluded_rowids =
opts
|> Keyword.get(:excluding_rowids, [])
|> Enum.reject(&is_nil/1)
|> MapSet.new()
(table_constraint_conflict_rowids(table, rowid, row, excluded_rowids) ++
explicit_index_conflict_rowids(db, table, rowid, row))
|> Enum.reject(&MapSet.member?(excluded_rowids, &1))
|> Enum.uniq()
end
defp fire_replace_delete_triggers(db, table, rowids) do
if db.recursive_triggers do
before_triggers = triggers_for(db, table.schema, table.name, :before, :delete)
after_triggers = triggers_for(db, table.schema, table.name, :after, :delete)
Enum.reduce(rowids, {:ok, db, table}, fn rowid, {:ok, db, table} ->
case Table.fetch_row(table, rowid) do
:error ->
{:ok, db, table}
{:ok, row} ->
db = put_table(db, table)
{status, db} =
fire_triggers(db, before_triggers, table, trigger_row(rowid, row), nil)
table = refetch_table(db, table)
if status == :ignored or not Map.has_key?(table.rows, rowid) do
{:ok, db, table}
else
table = Table.delete_rows(table, [rowid])
{db, table} =
fire_after_row_triggers(db, table, after_triggers, trigger_row(rowid, row), nil)
{:ok, db, table}
end
end
end)
else
{:ok, db, Table.delete_rows(table, rowids)}
end
end
defp table_constraint_conflict_rowids(table, rowid, row, excluded_rowids) do
rowid_conflicts =
if is_integer(rowid) and Map.has_key?(table.rows, rowid) do
[rowid]
else
[]
end
single_unique_conflicts =
for column <- table.columns,
unique_constraint_column?(table, column),
column_key = Table.key(column.name),
value = Map.fetch!(row, column_key),
not is_nil(value),
{conflicting_rowid, existing} <- Table.scan(table),
not MapSet.member?(excluded_rowids, conflicting_rowid),
Value.compare(Map.fetch!(existing, column_key), value) == :eq do
conflicting_rowid
end
composite_key_conflicts =
table.composite_keys
|> Enum.reject(fn {_name, column_keys} ->
length(column_keys) == 1 and hd(column_keys) == table.rowid_alias
end)
|> composite_constraint_conflict_rowids(table, row, excluded_rowids)
composite_unique_conflicts =
composite_constraint_conflict_rowids(table.composite_uniques, table, row, excluded_rowids)
rowid_conflicts ++
single_unique_conflicts ++ composite_key_conflicts ++ composite_unique_conflicts
end
defp unique_constraint_column?(table, column),
do: (column.primary_key or column.unique) and Table.key(column.name) != table.rowid_alias
defp composite_constraint_conflict_rowids(constraints, table, row, excluded_rowids) do
Enum.flat_map(constraints, fn {_name, column_keys} ->
values = Enum.map(column_keys, &Map.fetch!(row, &1))
if Enum.any?(values, &is_nil/1) do
[]
else
for {rowid, existing} <- Table.scan(table),
not MapSet.member?(excluded_rowids, rowid),
Enum.zip(column_keys, values)
|> Enum.all?(fn {key, value} ->
Value.compare(Map.fetch!(existing, key), value) == :eq
end) do
rowid
end
end
end)
end
defp explicit_index_conflict_rowids(db, table, rowid, row) do
db
|> unique_index_conflicts(table, rowid, row)
|> Enum.flat_map(&elem(&1, 0))
end
defp insert_conflict_rowid(_table, rowid, _row) when is_integer(rowid), do: rowid
defp insert_conflict_rowid(table, nil, row) do
case table.rowid_alias do
nil ->
table.next_rowid
alias_key ->
case Map.fetch!(row, alias_key) do
nil -> next_insert_rowid(table)
rowid when is_integer(rowid) -> rowid
_other -> nil
end
end
end
defp insert_conflict_rowid(_table, _rowid, _row), do: nil
defp next_insert_rowid(table) do
cond do
table.autoincrement and table.sequence_row -> max(table.sequence + 1, table.next_rowid)
table.autoincrement -> table_next_available_rowid(table)
true -> table.next_rowid
end
end
defp table_next_available_rowid(table) do
table.rows
|> Map.keys()
|> Enum.max(fn -> 0 end)
|> Kernel.+(1)
end
# Explicit UNIQUE indexes are enforced here because index members can carry
# collations, including connection-local callbacks stored on the database.
defp unique_index_conflicts(db, table, rowid, row) do
for index <- table.indexes,
index.unique,
index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where),
index_values = index_member_values(db, table, rowid, row, index),
not Enum.any?(index_values, &is_nil/1),
conflicts = unique_index_duplicates(db, table, index, index_values, rowid),
conflicts != [] do
{conflicts, index_conflict_message(table, index)}
end
end
# Fast path: for a binary-collation index with materialized entries, the
# conflicting rowids are exactly `entries[values]` (a hash lookup), turning a
# bulk insert into a unique-indexed table from O(n²) into O(n). The entry key
# uses affinity-canonical member values, which match binary `Value.compare`
# equality, so this is exact for binary collation; collated/custom indexes
# (where two distinct keys may be equal) and un-materialized entries take the
# row-scan fallback below. Entries are kept current by the per-row maintenance
# in the insert/update paths, so a duplicate inserted earlier in the same
# statement is already visible here.
defp unique_index_duplicates(db, table, index, values, excluding_rowid) do
entries = Map.get(index, :entries)
if entries != nil and binary_collation_index?(index) do
entries
|> Map.get(List.to_tuple(values), [])
|> Enum.reject(&(&1 == excluding_rowid))
else
for {existing_rowid, existing_row} <- Table.scan(table),
existing_rowid != excluding_rowid,
index.where == nil or
row_matches_partial_index?(db, table, existing_rowid, existing_row, index.where),
existing_values = index_member_values(db, table, existing_rowid, existing_row, index),
not Enum.any?(existing_values, &is_nil/1),
index_values_equal?(db, index, existing_values, values) do
existing_rowid
end
end
end
defp binary_collation_index?(index) do
case Map.get(index, :collations) do
nil ->
true
collations ->
Enum.all?(collations, fn
nil -> true
name when is_binary(name) -> String.downcase(name) == "binary"
_ -> false
end)
end
end
defp row_matches_partial_index?(db, table, rowid, row, where) do
env = table_env(db, table, rowid, row)
truth(where, env) == true
end
defp index_members(index),
do: Map.get(index, :members) || Enum.map(index.columns, &{:column, &1})
defp lookup_indexes(table), do: table.indexes ++ table.autoindexes
defp index_member_values(db, table, rowid, row, index) do
Enum.map(index_members(index), fn
# `row` may be the raw stored tuple (the hot per-insert maintenance path) or
# a `key => value` map (full rebuilds via `Table.scan`).
{:column, key} when is_tuple(row) ->
Table.cell(table, row, key)
{:column, key} ->
case row do
%{^key => value} -> value
_ -> nil
end
{:expr, expr} ->
eval(expr, table_env(db, table, rowid, row))
end)
end
defp index_values_equal?(db, index, left_values, right_values) do
collations = Map.get(index, :collations) || List.duplicate(nil, length(left_values))
values_equal_with_collations?(db, left_values, right_values, collations)
end
defp values_equal_with_collations?(db, left_values, right_values, collations) do
Enum.zip([left_values, right_values, collations])
|> Enum.all?(fn {left, right, collation} ->
Value.compare(left, right, normalize_collation!(collation, %{db: db})) == :eq
end)
end
defp index_conflict_message(table, index) do
if index.columns != [] do
"UNIQUE constraint failed: #{column_list(table, index.columns)}"
else
"UNIQUE constraint failed: index '#{index.name}'"
end
end
# Deep scan for a column reference, for DROP COLUMN's dangling-index check.
defp expr_references_column?({:column, _qualifier, name}, col_key),
do: Table.key(name) == col_key
defp expr_references_column?(tuple, col_key) when is_tuple(tuple) do
tuple |> Tuple.to_list() |> Enum.any?(&expr_references_column?(&1, col_key))
end
defp expr_references_column?(list, col_key) when is_list(list) do
Enum.any?(list, &expr_references_column?(&1, col_key))
end
defp expr_references_column?(_other, _col_key), do: false
defp validate_index_expression!(table, expr) do
case expr do
{:column, nil, name} ->
unless Table.column(table, name), do: fail("no such column: #{name}")
{:column, _qualifier, name} ->
unless Table.column(table, name), do: fail("no such column: #{name}")
tuple when is_tuple(tuple) ->
tuple |> Tuple.to_list() |> Enum.each(&validate_index_expression!(table, &1))
list when is_list(list) ->
Enum.each(list, &validate_index_expression!(table, &1))
_other ->
:ok
end
end
defp index_member_collation(_table, %{collate: collation}, _member)
when is_binary(collation),
do: collation
defp index_member_collation(table, _column_spec, {:column, key}) do
case Table.column(table, key) do
nil -> nil
column -> column.collate
end
end
defp index_member_collation(_table, %{collate: collation}, _member), do: collation
defp column_list(table, col_keys) do
Enum.map_join(col_keys, ", ", fn col_key ->
"#{table.name}.#{display_column_name(table, col_key)}"
end)
end
defp insert_or_upsert(
db,
table,
values,
explicit_rowid,
%{upsert: []},
on_conflict,
_before_update_triggers,
_after_update_triggers
) do
rowid = insert_conflict_rowid(table, explicit_rowid, values)
case resolve_unique_indexes_for_dml(db, table, rowid, values, on_conflict) do
{:ok, db, table} ->
case Table.insert(table, values, rowid: explicit_rowid, on_conflict: on_conflict) do
{:ok, table, rowid} ->
{:inserted, db, maybe_add_index_entries(db, table, rowid), rowid}
other ->
other
end
other ->
other
end
end
defp insert_or_upsert(
db,
table,
values,
explicit_rowid,
%{upsert: clauses},
on_conflict,
before_update_triggers,
after_update_triggers
) do
candidate = upsert_candidate(db, table, values, explicit_rowid)
Enum.each(clauses, &validate_upsert_target!(table, elem(&1, 1)))
# The clauses chain: the first whose target the candidate conflicts with
# handles the row; conflicts on untargeted constraints fail normally.
match =
Enum.find_value(clauses, fn clause ->
case upsert_conflict_rowid(db, table, elem(clause, 1), candidate, explicit_rowid) do
nil -> nil
rowid -> {clause, rowid}
end
end)
case match do
nil ->
# A trailing catch-all DO NOTHING also swallows partial-index
# conflicts the targeted probe above cannot see.
insert_algorithm =
if Enum.any?(clauses, &(elem(&1, 1) == nil)), do: :ignore, else: on_conflict
rowid = insert_conflict_rowid(table, explicit_rowid, values)
case resolve_unique_indexes_for_dml(db, table, rowid, values, insert_algorithm) do
{:ok, db, table} ->
case Table.insert(table, values, rowid: explicit_rowid, on_conflict: on_conflict) do
{:ok, table, rowid} ->
# Keep index entries current as we go so the next row's unique
# check (and the final store) need no full rebuild.
{:inserted, db, maybe_add_index_entries(db, table, rowid), rowid}
other ->
other
end
other ->
other
end
{{:nothing, _target}, _rowid} ->
:ignore
{{:update, _target, assignments, where}, rowid} ->
existing = Table.fetch_row!(table, rowid)
env = upsert_env(db, table, rowid, existing, candidate)
if where != nil and truth(where, env) != true do
:ignore
else
assignments = Enum.map(assignments, &update_assignment(table, &1))
{new_row, explicit_rowid} = updated_row_and_rowid(table, assignments, existing, env)
new_row =
apply_generated_columns(new_row, db, table, update_rowid_value(explicit_rowid, rowid))
check_strict_types!(table, new_row)
changed_keys = Enum.map(assignments, &update_assignment_key/1)
{trigger_status, db, table} =
if before_update_triggers == [] do
{:ok, db, table}
else
db = put_table(db, table)
{status, db} =
fire_triggers(db, before_update_triggers, table, existing, new_row, changed_keys)
{status, db, refetch_table(db, table)}
end
cond do
trigger_status == :ignored ->
:ignore
not Map.has_key?(table.rows, rowid) ->
:ignore
true ->
new_row =
if before_update_triggers == [] do
new_row
else
table
|> Table.fetch_row!(rowid)
|> rebase_updated_row(table, assignments, new_row)
|> apply_generated_columns(
db,
table,
update_conflict_rowid(table, rowid, new_row, explicit_rowid)
)
end
check_strict_types!(table, new_row)
conflict_rowid = update_conflict_rowid(table, rowid, new_row, explicit_rowid)
case resolve_unique_indexes_for_dml(
db,
table,
conflict_rowid,
new_row,
on_conflict,
excluding_rowids: [rowid]
) do
{:ok, db, table} ->
opts = update_rowid_opts([on_conflict: on_conflict], explicit_rowid)
case Table.update_row(table, rowid, new_row, opts) do
{:ok, table} ->
# An upsert UPDATE moves a row; rebuild this table's index
# entries so later rows in the batch see a consistent set
# (the incremental add path only knows about plain inserts).
table = refresh_index_entries(db, table)
new_rowid = updated_rowid(table, rowid, new_row, explicit_rowid)
returning_row = Table.fetch_row!(table, new_rowid)
{db, table} =
fire_after_row_triggers(
db,
table,
after_update_triggers,
existing,
returning_row
)
{:updated, db, table, new_rowid, {rowid, existing, returning_row}}
other ->
other
end
other ->
other
end
end
end
end
end
defp upsert_candidate(db, table, values, explicit_rowid) do
candidate = build_candidate_row(table, values, db)
case {table.rowid_alias, explicit_rowid} do
{alias_key, rowid} when alias_key != nil and is_integer(rowid) ->
Map.put(candidate, alias_key, rowid)
_ ->
candidate
end
end
defp upsert_env(db, table, rowid, existing, candidate) do
target_frame = %{table_frame(table, nil) | row: existing, rowid: rowid}
excluded_frame = %{
table_frame(table, "excluded")
| row: candidate,
rowid: nil,
hidden: MapSet.new(Enum.map(table.columns, &Table.key(&1.name)))
}
%{db: db, frames: [target_frame, excluded_frame], group: nil, outer: nil}
end
defp validate_upsert_target!(_table, nil), do: :ok
defp validate_upsert_target!(table, {columns, target_where}) do
Enum.each(columns, fn column_name ->
unless Table.column(table, column_name) do
fail("no such column: #{column_name}")
end
end)
target_keys = Enum.map(columns, &Table.key/1)
# A bare target matches full uniqueness constraints only; targeting a
# partial unique index requires the WHERE clause, as in SQLite.
matched? =
if target_where == nil do
Enum.any?(upsert_unique_targets(table), &(&1 == target_keys))
else
Enum.any?(table.indexes, fn index ->
index.unique and Map.get(index, :where) != nil and index.columns == target_keys
end)
end
unless matched? do
fail("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint")
end
end
defp upsert_conflict_rowid(db, table, nil, candidate, explicit_rowid) do
table
|> upsert_unique_targets()
|> Enum.find_value(&upsert_conflict_rowid(db, table, {&1, nil}, candidate, explicit_rowid))
end
defp upsert_conflict_rowid(db, table, {target_columns, target_where}, candidate, explicit_rowid) do
target_keys = Enum.map(target_columns, &Table.key/1)
cond do
target_keys == [table.rowid_alias] and target_where == nil ->
rowid_value = explicit_rowid || Map.get(candidate, table.rowid_alias)
if is_integer(rowid_value) and Map.has_key?(table.rows, rowid_value), do: rowid_value
true ->
values = Enum.map(target_keys, &Map.get(candidate, &1))
cond do
Enum.any?(values, &is_nil/1) ->
nil
# A partial index only conflicts when both the candidate and the
# existing row satisfy the index predicate.
target_where != nil and
truth(target_where, table_env(db, table, nil, candidate)) != true ->
nil
true ->
collations = upsert_target_collations(table, target_keys, target_where)
Enum.find_value(Table.scan(table), fn {rowid, row} ->
existing_values = Enum.map(target_keys, &Map.get(row, &1))
if values_equal_with_collations?(db, existing_values, values, collations) and
(target_where == nil or
truth(target_where, table_env(db, table, rowid, row)) == true) do
rowid
end
end)
end
end
end
defp upsert_unique_targets(table) do
rowid_targets =
if table.rowid_alias != nil do
[[table.rowid_alias]]
else
[]
end
column_targets =
for column <- table.columns,
(column.primary_key or column.unique) and Table.key(column.name) != table.rowid_alias,
do: [Table.key(column.name)]
index_targets =
for index <- table.indexes, index.unique, is_nil(Map.get(index, :where)), do: index.columns
composite_pk_targets =
for {_name, keys} <- table.composite_keys,
not (length(keys) == 1 and hd(keys) == table.rowid_alias),
do: keys
composite_unique_targets = for {_name, keys} <- table.composite_uniques, do: keys
rowid_targets ++
column_targets ++ index_targets ++ composite_pk_targets ++ composite_unique_targets
end
defp upsert_target_collations(table, target_keys, target_where) do
case matching_unique_index(table, target_keys, target_where) do
nil ->
Enum.map(target_keys, &column_collation_name(table, &1))
index ->
Map.get(index, :collations) ||
Enum.map(target_keys, &column_collation_name(table, &1))
end
end
defp matching_unique_index(table, target_keys, target_where) do
Enum.find(table.indexes, fn index ->
index.unique and index.columns == target_keys and
((target_where == nil and is_nil(Map.get(index, :where))) or
(target_where != nil and Map.get(index, :where) != nil))
end)
end
defp column_collation_name(table, key) do
case Table.column(table, key) do
nil -> nil
column -> column.collate
end
end
# Validates CHECK constraints for the candidate row.
# Returns :ok, :ignore (on_conflict == :ignore), or {:error, message}.
# SQLite: OR IGNORE suppresses CHECK failures; OR REPLACE does NOT bypass CHECK.
defp check_violations(
%{ignore_check_constraints: true},
_table,
_candidate_row,
_check_env,
_on_conflict
),
do: :ok
defp check_violations(_db, table, _candidate_row, check_env, on_conflict) do
Enum.find_value(table.checks, :ok, fn {cname, expr} ->
result = truth(expr, check_env)
# NULL result passes (only false fails)
if result == false do
if on_conflict == :ignore do
:ignore
else
message =
if cname do
"CHECK constraint failed: #{cname}"
else
"CHECK constraint failed: #{check_text(expr)}"
end
{:error, message}
end
end
end)
end
defp insert_values(_targets, :default), do: {%{}, nil}
defp insert_values(targets, row) do
targets
|> Enum.zip(row)
|> Enum.reduce({%{}, nil}, fn
{:rowid, value}, {values, _rowid} ->
if value == nil or is_integer(value), do: {values, value}, else: fail("datatype mismatch")
{{column, key}, value}, {values, rowid} ->
coerced = Value.apply_affinity(value, column.affinity)
{Map.put(values, key, coerced), rowid}
end)
end
defp select_result(db, stmt, outer) do
# Try the JIT codegen path first (uncorrelated SELECTs only); it self-gates
# and returns :fallback for unsupported shapes / one-shot queries.
with nil <- outer,
{:ok, result} <- ExSQL.Codegen.run_select(db, stmt) do
result
else
_ ->
case compile_vdbe(db, stmt, outer) do
{:ok, plan} -> run_vdbe(plan)
:unsupported -> select_result_treewalk(db, stmt, outer)
end
end
end
defp select_result_treewalk(db, stmt, outer) do
check_window_placement!(stmt.where)
Enum.each(stmt.group_by, &check_window_placement!/1)
check_window_placement!(stmt.having)
{templates, frame_rows} = planned_relation(db, stmt.from, stmt.where, outer)
# Resolve the WHERE's column references once (single-table scans only) and
# compile the predicate to a closure, rather than re-resolving names +
# affinity/collation and re-dispatching the AST for every row.
precompiled_where = precompile_scan_where(stmt.where, db, templates, outer)
filter = compile_scan_filter(precompiled_where)
# A column-free WHERE (e.g. `WHERE NULL IS NOT NULL`) is constant for the
# whole scan — evaluate it once instead of per row. Constant false/NULL → no
# rows; constant true → drop the filter (every row passes).
{filter, frame_rows} =
case constant_filter_value(precompiled_where, db) do
:dynamic -> {filter, frame_rows}
true -> {nil, frame_rows}
_false_or_null -> {nil, []}
end
# Fuse the per-row env wrap and the WHERE filter into one pass, so a
# filtered-out row never lands in an intermediate list. When the predicate
# is row-local (only this frame's columns — no subquery, outer reference,
# or db-dependent call), it reads nothing but `frames`, so filter against a
# minimal map and build the full env only for surviving rows — skipping the
# 4-key allocation for every row a selective filter rejects.
envs =
cond do
filter == nil ->
Enum.map(frame_rows, &%{db: db, frames: &1, group: nil, outer: outer})
match?([_], templates) and row_local?(precompiled_where) ->
for frames <- frame_rows,
filter.(%{group: nil, frames: frames}),
do: %{db: db, frames: frames, group: nil, outer: outer}
true ->
for frames <- frame_rows,
env = %{db: db, frames: frames, group: nil, outer: outer},
filter.(env),
do: env
end
columns =
stmt.columns
|> expand_columns(templates)
|> resolve_window_refs(stmt.windows)
names = Enum.map(columns, &result_column_name(db, templates, &1))
envs = maybe_reverse_unordered_envs(db, stmt, columns, envs)
aggregate? = aggregate_query?(db, stmt, columns)
# ORDER BY expressions may reference output aliases (`ORDER BY 10-(x+y)`).
# For non-aggregate queries the keys are evaluated per row against the scan
# frames, so fastify their column refs (as projection/WHERE already are) to
# skip per-row name resolution during the sort. Aggregate ORDER BY runs
# against grouped envs, so leave it on the normal path.
order_by =
if aggregate? do
Enum.map(stmt.order_by, fn {expr, direction} ->
{substitute_aliases(expr, columns, templates), direction}
end)
else
lookup = frame_column_lookup(db, templates)
Enum.map(stmt.order_by, fn {expr, direction} ->
{expr |> substitute_aliases(columns, templates) |> rewrite_frame_columns(lookup),
direction}
end)
end
window_exprs = collect_windows(columns)
envs = apply_window_output_order(envs, window_exprs)
window_values = compute_windows(window_exprs, envs)
projected =
if aggregate? do
grouped =
db
|> grouped_envs(stmt, columns, templates, envs, outer)
grouped
|> Enum.map(fn genv ->
{genv, Enum.map(columns, fn {expr, _} -> expr |> eval(genv) |> sql_value() end)}
end)
else
eval_columns = precompile_scan_columns(columns, db, templates)
# The vast majority of queries have no window functions; skip the per-row
# `:windows` injection (and the `with_index`/`Map.get`/`Map.put` it needs)
# entirely in that case — it's pure overhead on every projected row. Also
# compile each output column to a closure once so the per-row projection
# is a direct call rather than re-dispatching `eval/2` on the AST per
# column per row (`compile_pred/1` falls back to `eval` for anything it
# doesn't specialize, so the result is identical).
if window_exprs == [] do
compiled = Enum.map(eval_columns, fn {expr, _} -> compile_pred(expr) end)
Enum.map(envs, fn env ->
{env, Enum.map(compiled, fn c -> c.(env) |> sql_value() end)}
end)
else
envs
|> Enum.with_index()
|> Enum.map(fn {env, index} ->
env = Map.put(env, :windows, Map.get(window_values, index, %{}))
{env, Enum.map(eval_columns, fn {expr, _} -> expr |> eval(env) |> sql_value() end)}
end)
end
end
template_env = %{db: db, frames: templates, group: nil, outer: outer}
rows =
projected
|> distinct(stmt.distinct, columns, template_env)
|> order(order_by, columns, names)
|> Enum.map(&elem(&1, 1))
|> clamp(db, stmt.limit, stmt.offset)
affinities = Enum.map(columns, fn {expr, _} -> expr_affinity(expr, template_env) end)
%Result{
command: :select,
columns: names,
rows: rows,
rows_affected: 0,
affinities: affinities
}
end
# -- VDBE compilation -----------------------------------------------------------
#
# Compiles the common single-table scan/filter/project shape to an
# `ExSQL.Vdbe` opcode program. Anything outside the supported subset returns
# `:unsupported` so `select_result/3` falls back to the tree walker. The
# compiler reuses the tree walker's own affinity/collation/name helpers, so a
# compiled query produces byte-for-byte the same result as the interpreter.
#
# Supported: no correlation/DISTINCT/GROUP BY/HAVING/ORDER BY/window; FROM a
# single plain table (not a view, CTE, or subquery); projection of `*` or
# plain columns; WHERE nil or a conjunction of `column <cmp> literal` terms;
# integer LIMIT/OFFSET. The `reverse_unordered_selects` shuffle is honored by
# declining (the tree walker owns that behavior).
defp compile_vdbe(_db, _stmt, outer) when outer != nil, do: :unsupported
defp compile_vdbe(db, %Select{} = stmt, _outer) do
with false <- vdbe_disabled?(),
true <- vdbe_simple_shape?(db, stmt),
{:table, name, alias_name} when is_binary(name) <- stmt.from,
%Table{} = table <- plain_table(db, relation_unqualified_table_key(db, name)),
false <- vdbe_rowid_table?(table),
false <- vdbe_seek_available?(db, table, stmt.where),
template = table_frame(table, alias_name),
columns = expand_columns(stmt.columns, [template]),
false <- aggregate_query?(db, stmt, columns),
[] <- collect_windows(columns),
{:ok, limit} <- vdbe_int_literal(stmt.limit),
{:ok, offset} <- vdbe_int_literal(stmt.offset),
{:ok, projection} <- vdbe_projection(db, columns, [template], table),
{:ok, filter} <- vdbe_filter(stmt.where, db, table, template),
{:ok, order} <-
vdbe_order(stmt.order_by, db, table, template, columns, projection.names) do
program = vdbe_program(projection.loads, filter, order.key_loads)
{:ok,
%{
table: table,
program: program,
names: projection.names,
affinities: projection.affinities,
limit: limit,
offset: offset,
order: order.directions
}}
else
_ -> :unsupported
end
end
defp compile_vdbe(_db, _stmt, _outer), do: :unsupported
# The register VM is disabled by default: the tree walker now compiles a
# single-table scan's WHERE/projection columns to direct lookups, compiles the
# predicate to a closure, and sorts via decorate-sort-undecorate — and that
# measures faster than the VM at every scale tested (the VM rebuilds its state
# map per opcode, more per-row allocation than one env + baked-in closures).
# Set `EXSQL_USE_VDBE=1` to re-enable for A/B comparison.
defp vdbe_disabled?, do: System.get_env("EXSQL_USE_VDBE") == nil
defp vdbe_simple_shape?(db, %Select{} = stmt) do
not db.reverse_unordered_selects and not stmt.distinct and stmt.group_by == [] and
stmt.having == nil and map_size(stmt.windows) == 0
end
# The per-row environment a compiled closure evaluates against: one frame for
# the scanned table, no group/outer (single-table, uncorrelated).
defp vdbe_env(db, template, row, rowid) do
%{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil}
end
# Selecting an INTEGER PRIMARY KEY (rowid alias) reads the rowid, not the
# stored column; leave those to the tree walker for now.
defp vdbe_rowid_table?(%Table{rowid_alias: alias}), do: alias != nil
# Decline whenever the tree walker would satisfy this WHERE with a rowid or
# index seek. The VDBE only knows how to full-scan, so compiling these would
# silently drop the seek — changing not just performance but observable
# behavior (e.g. how many rows a side-effecting WHERE term like a UDF visits).
# `table_access_path/3` inspects index *definitions* (not materialized
# entries), so this is a cheap check that mirrors the planner's own decision.
defp vdbe_seek_available?(_db, _table, nil), do: false
defp vdbe_seek_available?(db, table, where), do: table_access_path(db, table, where) != :scan
defp vdbe_int_literal(nil), do: {:ok, nil}
defp vdbe_int_literal({:literal, n}) when is_integer(n) and n >= 0, do: {:ok, n}
defp vdbe_int_literal(_other), do: :unsupported
# Resolves the projection to register loads plus result names and affinities.
# A plain column of this table loads with the fast `:column` opcode; any other
# expression compiles to an `:eval` closure over the tree walker's evaluator.
defp vdbe_projection(db, columns, [template] = templates, table) do
template_env = %{db: db, frames: templates, group: nil, outer: nil}
{names, affs, loads, _reg} =
Enum.reduce(columns, {[], [], [], 0}, fn {expr, _alias} = item, {names, affs, loads, reg} ->
load = vdbe_proj_load(expr, table, db, template, reg)
name = result_column_name(db, templates, item)
affinity = expr_affinity(expr, template_env)
{[name | names], [affinity | affs], [load | loads], reg + 1}
end)
{:ok,
%{names: Enum.reverse(names), affinities: Enum.reverse(affs), loads: Enum.reverse(loads)}}
end
defp vdbe_proj_load({:column, _qualifier, name} = expr, table, db, template, reg) do
key = Table.key(name)
if Table.column(table, key),
do: {:column, key, reg},
else: vdbe_eval_load(expr, db, template, reg)
end
defp vdbe_proj_load(expr, _table, db, template, reg),
do: vdbe_eval_load(expr, db, template, reg)
defp vdbe_eval_load(expr, db, template, reg) do
{:eval, fn row, rowid -> eval(expr, vdbe_env(db, template, row, rowid)) end, reg}
end
# WHERE compiles to the fast `:cmp` chain when it is a conjunction of
# `column <cmp> literal` (binary collation); otherwise the whole predicate
# becomes one `:filter` closure so any WHERE shape is still handled.
defp vdbe_filter(nil, _db, _table, _template), do: {:ok, :none}
defp vdbe_filter(where, db, table, template) do
case vdbe_cmp_terms(where, db, table, template) do
{:ok, terms} ->
{:ok, {:cmp, terms}}
:unsupported ->
{:ok,
{:filter, fn row, rowid -> matches_where?(where, vdbe_env(db, template, row, rowid)) end}}
end
end
defp vdbe_cmp_terms({:binary, :and, left, right}, db, table, template) do
with {:ok, l} <- vdbe_cmp_terms(left, db, table, template),
{:ok, r} <- vdbe_cmp_terms(right, db, table, template) do
{:ok, l ++ r}
end
end
defp vdbe_cmp_terms({:binary, op, left, right}, db, table, template)
when op in [:eq, :ne, :lt, :le, :gt, :ge] do
env = %{db: db, frames: [template], group: nil, outer: nil}
# Only the binary-collation hot path; user/column collations need the
# collation callback resolved per comparison, which the tree walker owns.
case {vdbe_operand(left, table, env), vdbe_operand(right, table, env),
comparison_collation(left, right, env)} do
{{:col, key, aff_a}, {:lit, value, aff_b}, :binary} ->
{:ok, [{op, {:col, key, aff_a}, {:lit, value, aff_b}, :binary}]}
{{:lit, value, aff_a}, {:col, key, aff_b}, :binary} ->
{:ok, [{op, {:lit, value, aff_a}, {:col, key, aff_b}, :binary}]}
_other ->
:unsupported
end
end
defp vdbe_cmp_terms(_other, _db, _table, _template), do: :unsupported
defp vdbe_operand({:column, _qualifier, name} = expr, table, env) do
key = Table.key(name)
if Table.column(table, key), do: {:col, key, expr_affinity(expr, env)}, else: :error
end
defp vdbe_operand({:literal, value} = expr, _table, env)
when not is_tuple(value) or value == nil,
do: {:lit, value, expr_affinity(expr, env)}
defp vdbe_operand(_expr, _table, _env), do: :error
# ORDER BY plan: for each term, either reuse a projection register (integer
# position, `ORDER BY 2`) or compile the (alias-substituted) expression to a
# key `:eval` load. Only binary collation is handled; anything else declines
# so the tree walker keeps ownership of collated ordering.
defp vdbe_order([], _db, _table, _template, _columns, _names),
do: {:ok, %{key_loads: [], directions: nil}}
defp vdbe_order(order_by, db, table, template, columns, names) do
env = %{db: db, frames: [template], group: nil, outer: nil}
nproj = length(columns)
# Key `:eval` registers sit above the projection regs and the two cmp scratch regs.
key_base = nproj + 2
Enum.reduce_while(order_by, {[], [], key_base}, fn {expr, direction}, {loads, dirs, reg} ->
if order_collation(expr, env, columns, names) == :binary do
{load, next_reg} = vdbe_order_key(expr, db, table, template, columns, nproj, reg)
{:cont, {[load | loads], [direction | dirs], next_reg}}
else
{:halt, :unsupported}
end
end)
|> case do
:unsupported ->
:unsupported
{loads, dirs, _reg} ->
{:ok, %{key_loads: Enum.reverse(loads), directions: Enum.reverse(dirs)}}
end
end
# `ORDER BY <n>` references the n-th projected column's register directly.
defp vdbe_order_key({:literal, n}, _db, _table, _template, _columns, nproj, reg)
when is_integer(n) and n >= 1 and n <= nproj,
do: {{:reuse, n - 1}, reg}
defp vdbe_order_key(expr, db, _table, template, columns, _nproj, reg) do
subbed = substitute_aliases(expr, columns, [template])
{{:eval, fn row, rowid -> eval(subbed, vdbe_env(db, template, row, rowid)) end, reg}, reg + 1}
end
# Lays out the program: rewind; the filter (a `:cmp` chain or one `:filter`
# closure, each failure jumping to Next); the projection loads; any ORDER BY
# key `:eval` loads; ResultRow (projection + key registers); Next; Halt.
defp vdbe_program(proj_loads, filter, key_loads) do
nproj = length(proj_loads)
proj_regs = Enum.to_list(0..(nproj - 1))
filter_ops = vdbe_filter_ops(filter, nproj)
{key_eval_ops, key_regs} = vdbe_key_ops(key_loads, proj_regs)
body = filter_ops ++ proj_loads ++ key_eval_ops ++ [{:result_row, proj_regs, key_regs}]
next_addr = 1 + length(body)
halt_addr = next_addr + 1
ops =
[{:rewind, halt_addr}] ++
patch_filter_fail(body, next_addr) ++
[{:next, 1}, {:halt}]
List.to_tuple(ops)
end
defp vdbe_filter_ops(:none, _scratch), do: []
defp vdbe_filter_ops({:filter, fun}, _scratch), do: [{:filter, fun, :fail}]
defp vdbe_filter_ops({:cmp, terms}, scratch),
do: Enum.flat_map(terms, &vdbe_cmp_ops(&1, scratch))
defp vdbe_cmp_ops({op, a, b, collation}, scratch) do
{load_a, aff_a} = vdbe_operand_op(a, scratch)
{load_b, aff_b} = vdbe_operand_op(b, scratch + 1)
[load_a, load_b, {:cmp, op, scratch, aff_a, scratch + 1, aff_b, collation, :fail}]
end
# Returns the ORDER BY key-eval opcodes and the register list `result_row`
# reads keys from (reused projection regs need no opcode).
defp vdbe_key_ops(key_loads, proj_regs) do
{ops, regs} =
Enum.reduce(key_loads, {[], []}, fn
{:reuse, proj_index}, {ops, regs} ->
{ops, [Enum.at(proj_regs, proj_index) | regs]}
{:eval, _fun, reg} = op, {ops, regs} ->
{[op | ops], [reg | regs]}
end)
{Enum.reverse(ops), Enum.reverse(regs)}
end
defp vdbe_operand_op({:col, key, aff}, reg), do: {{:column, key, reg}, aff}
defp vdbe_operand_op({:lit, value, aff}, reg), do: {{:value, value, reg}, aff}
defp patch_filter_fail(ops, next_addr) do
Enum.map(ops, fn
{:cmp, op, ra, aa, rb, ab, coll, :fail} -> {:cmp, op, ra, aa, rb, ab, coll, next_addr}
{:filter, fun, :fail} -> {:filter, fun, next_addr}
other -> other
end)
end
defp run_vdbe(%{table: table, program: program} = plan) do
rows = ExSQL.Vdbe.run(Table.scan(table), program, plan.limit, plan.offset, plan.order)
%Result{
command: :select,
columns: plan.names,
rows: rows,
rows_affected: 0,
affinities: plan.affinities
}
end
defp maybe_reverse_unordered_envs(%{reverse_unordered_selects: false}, _stmt, _columns, envs),
do: envs
defp maybe_reverse_unordered_envs(db, stmt, columns, envs) do
if stmt.order_by == [] or aggregate_query?(db, stmt, columns) do
Enum.reverse(envs)
else
envs
end
end
defp sql_value({:json, text}), do: text
defp sql_value(value), do: value
defp dml_result(%{count_changes: false}, _table, [], _returning_rows, command, count),
do: %Result{command: command, rows_affected: count}
defp dml_result(%{count_changes: true}, _table, [], _returning_rows, command, count) do
%Result{
command: :select,
columns: ["rows #{dml_count_changes_verb(command)}"],
rows: [[count]],
rows_affected: count,
affinities: [:integer]
}
end
defp dml_result(db, table, returning, returning_rows, _command, count) do
template = table_frame(table, nil)
columns = expand_columns(returning, [template])
names = Enum.map(columns, &result_column_name(db, [template], &1))
rows =
Enum.map(returning_rows, fn {rowid, row} ->
env = %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil}
Enum.map(columns, fn {expr, _alias_name} -> expr |> eval(env) |> sql_value() end)
end)
template_env = %{db: db, frames: [template], group: nil, outer: nil}
affinities = Enum.map(columns, fn {expr, _} -> expr_affinity(expr, template_env) end)
%Result{
command: :select,
columns: names,
rows: rows,
rows_affected: count,
affinities: affinities
}
end
defp dml_count_changes_verb(:insert), do: "inserted"
defp dml_count_changes_verb(:update), do: "updated"
defp dml_count_changes_verb(:delete), do: "deleted"
defp update_assignment(table, {name, expr}) do
case Table.column(table, name) do
%{} = column ->
if column.generated, do: fail("cannot UPDATE generated column \"#{column.name}\"")
{:column, column, expr}
nil ->
key = Table.key(name)
if key in @rowid_names and not table.without_rowid do
{:rowid, key, expr}
else
fail("no such column: #{name}")
end
end
end
defp update_assignment_key({:column, column, _expr}), do: Table.key(column.name)
defp update_assignment_key({:rowid, key, _expr}), do: key
defp updated_row_and_rowid(table, assignments, row, env) do
Enum.reduce(assignments, {row, :not_set}, fn
{:column, column, expr}, {new_row, explicit_rowid} ->
value = expr |> eval(env) |> Value.apply_affinity(column.affinity)
{Map.put(new_row, Table.key(column.name), value), explicit_rowid}
{:rowid, _key, expr}, {new_row, _explicit_rowid} ->
value = expr |> eval(env) |> Value.apply_affinity(:integer)
new_row =
case table.rowid_alias do
nil -> new_row
alias_key -> Map.put(new_row, alias_key, value)
end
{new_row, value}
end)
end
defp rebase_updated_row(current_row, table, assignments, new_row) do
Enum.reduce(assignments, current_row, fn
{:column, column, _expr}, row ->
key = Table.key(column.name)
Map.put(row, key, Map.fetch!(new_row, key))
{:rowid, _key, _expr}, row ->
case table.rowid_alias do
nil -> row
alias_key -> Map.put(row, alias_key, Map.fetch!(new_row, alias_key))
end
end)
end
defp update_rowid_value(:not_set, rowid), do: rowid
defp update_rowid_value(explicit_rowid, _rowid), do: explicit_rowid
defp update_conflict_rowid(_table, rowid, _row, :not_set) when is_nil(rowid), do: nil
defp update_conflict_rowid(table, rowid, row, :not_set) do
case table.rowid_alias do
nil -> rowid
alias_key -> Map.fetch!(row, alias_key)
end
end
defp update_conflict_rowid(_table, rowid, _row, explicit_rowid),
do: update_rowid_value(explicit_rowid, rowid)
defp update_rowid_opts(opts, :not_set), do: opts
defp update_rowid_opts(opts, explicit_rowid), do: Keyword.put(opts, :rowid, explicit_rowid)
defp updated_rowid(table, old_rowid, row), do: updated_rowid(table, old_rowid, row, :not_set)
defp updated_rowid(_table, _old_rowid, _row, explicit_rowid) when explicit_rowid != :not_set,
do: explicit_rowid
defp updated_rowid(table, old_rowid, row, :not_set) do
case table.rowid_alias do
nil -> old_rowid
alias_key -> Map.fetch!(row, alias_key)
end
end
# No LIMIT/OFFSET: the entire matched set is affected regardless of order, so
# take the index/rowid access path (`planned_relation`) instead of a full
# scan — the difference between O(matches·log n) and O(n) per statement.
# Residual conjuncts the index doesn't cover are still applied by `filter`.
defp dml_target_rows(db, table, %{limit: nil, offset: nil, schema: schema} = stmt)
when schema in [nil, "main"] do
{templates, frame_rows} = planned_relation(db, {:table, table.name, nil}, stmt.where, nil)
filter = compile_scan_filter(precompile_scan_where(stmt.where, db, templates, nil))
matched =
for [frame] <- frame_rows,
keep_row?(filter, %{db: db, frames: [frame], group: nil, outer: nil}),
do: {frame.rowid, frame_row_map(frame)}
Enum.sort_by(matched, &elem(&1, 0))
end
defp dml_target_rows(db, table, stmt) do
# Build the frame template once and fastify/compile the predicate once,
# rather than rebuilding the frame and re-resolving column names (with their
# `downcase`) for every row — the same treatment SELECT scans get.
template = table_frame(table, nil)
filter = compile_scan_filter(precompile_scan_where(stmt.where, db, [template], nil))
matched =
for {rowid, row} <- Table.scan(table),
env = %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil},
keep_row?(filter, env),
do: {rowid, row}
limited =
matched
|> order_dml_targets(db, table, stmt.order_by)
|> clamp(db, stmt.limit, stmt.offset)
selected = MapSet.new(limited, &elem(&1, 0))
Enum.filter(matched, fn {rowid, _row} -> MapSet.member?(selected, rowid) end)
end
defp keep_row?(nil, _env), do: true
defp keep_row?(filter, env), do: filter.(env)
# Take the planner's rowid/index access path (not a full scan) when there's no
# LIMIT/OFFSET to order by — so `UPDATE ... WHERE pk = ?` is an O(1) seek
# instead of O(n), which (with incremental index maintenance) makes a table of
# single-row updates O(n) rather than O(n²).
defp update_target_rows(
db,
%Table{without_rowid: false} = table,
%{from: nil, limit: nil, offset: nil, schema: schema} = stmt
)
when schema in [nil, "main"] do
{templates, frame_rows} = planned_relation(db, {:table, table.name, nil}, stmt.where, nil)
filter = compile_scan_filter(precompile_scan_where(stmt.where, db, templates, nil))
matched =
for [frame] <- frame_rows,
env = %{db: db, frames: [frame], group: nil, outer: nil},
keep_row?(filter, env),
do: {frame.rowid, frame_row_map(frame), env}
finalize_update_targets(matched, db, stmt)
end
defp update_target_rows(db, table, %{from: nil} = stmt) do
template = table_frame(table, nil)
filter = compile_scan_filter(precompile_scan_where(stmt.where, db, [template], nil))
matched =
table
|> Table.scan()
|> Enum.flat_map(fn {rowid, row} ->
env = %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil}
if keep_row?(filter, env), do: [{rowid, row, env}], else: []
end)
finalize_update_targets(matched, db, stmt)
end
defp update_target_rows(db, table, stmt) do
matched =
table
|> Table.scan()
|> Enum.flat_map(fn {rowid, row} ->
case update_match_env(db, table, rowid, row, stmt) do
nil -> []
env -> [{rowid, row, env}]
end
end)
finalize_update_targets(matched, db, stmt)
end
defp finalize_update_targets(matched, db, stmt) do
limited =
matched
|> order_update_targets(stmt.order_by)
|> clamp(db, stmt.limit, stmt.offset)
selected = MapSet.new(limited, fn {rowid, _row, _env} -> rowid end)
Enum.filter(matched, fn {rowid, _row, _env} -> MapSet.member?(selected, rowid) end)
end
defp update_match_env(db, table, rowid, row, %{from: from, where: where}) do
target_frame = %{table_frame(table, nil) | row: row, rowid: rowid}
{_templates, source_rows} = relation(db, from, nil)
source_rows
|> Enum.map(fn source_frames ->
%{db: db, frames: [target_frame | source_frames], group: nil, outer: nil}
end)
|> Enum.filter(&matches_where?(where, &1))
|> List.last()
end
defp order_update_targets(rows, []), do: rows
defp order_update_targets(rows, order_by) do
Enum.sort(rows, fn {_rowid_a, _row_a, env_a}, {_rowid_b, _row_b, env_b} ->
compare_term_values(order_by, env_a, env_b)
end)
end
defp order_dml_targets(rows, _db, _table, []), do: rows
defp order_dml_targets(rows, db, table, order_by) do
Enum.sort(rows, fn {rowid_a, row_a}, {rowid_b, row_b} ->
env_a = table_env(db, table, rowid_a, row_a)
env_b = table_env(db, table, rowid_b, row_b)
compare_term_values(order_by, env_a, env_b)
end)
end
defp aggregate_query?(db, stmt, columns) do
stmt.group_by != [] or stmt.having != nil or
Enum.any?(columns, fn {expr, _} -> contains_aggregate?(expr, db) end)
end
defp collect_windows(columns) do
columns
|> Enum.flat_map(fn {expr, _} -> windows_in(expr) end)
|> Enum.uniq()
end
defp windows_in({:window, _name, _args, _spec, _filter} = expr), do: [expr]
defp windows_in(expr) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.flat_map(fn
element when is_tuple(element) -> windows_in(element)
elements when is_list(elements) -> Enum.flat_map(elements, &windows_in/1)
_ -> []
end)
end
defp windows_in(_), do: []
defp contains_window?(%_struct{} = term), do: term |> Map.from_struct() |> contains_window?()
defp contains_window?({:window, _name, _args, _spec, _filter}), do: true
defp contains_window?(tuple) when is_tuple(tuple),
do: tuple |> Tuple.to_list() |> contains_window?()
defp contains_window?(list) when is_list(list), do: Enum.any?(list, &contains_window?/1)
defp contains_window?(map) when is_map(map), do: map |> Map.values() |> contains_window?()
defp contains_window?(_term), do: false
defp apply_window_output_order(envs, []), do: envs
defp apply_window_output_order(envs, [{:window, _name, _args, spec, _filter} | _]) do
terms = Enum.map(spec.partition_by, &{&1, :asc}) ++ spec.order_by
sort_envs_by_terms(envs, terms)
end
# Window functions may only appear in the SELECT list and ORDER BY;
# WHERE, GROUP BY, and HAVING reject them. Subqueries get their own check
# when they execute, so the walk does not descend into them.
defp check_window_placement!(nil), do: :ok
defp check_window_placement!({:window, name, _args, _spec, _filter}) do
fail("misuse of window function #{name}()")
end
defp check_window_placement!(%Select{}), do: :ok
defp check_window_placement!(%Compound{}), do: :ok
defp check_window_placement!(%Values{}), do: :ok
defp check_window_placement!(%With{}), do: :ok
defp check_window_placement!(tuple) when is_tuple(tuple) do
tuple |> Tuple.to_list() |> Enum.each(&check_window_placement!/1)
end
defp check_window_placement!(list) when is_list(list) do
Enum.each(list, &check_window_placement!/1)
end
defp check_window_placement!(_other), do: :ok
defp compute_windows([], _envs), do: %{}
defp compute_windows(window_exprs, envs) do
indexed_envs = Enum.with_index(envs)
db = envs |> List.first(%{db: Database.new()}) |> Map.fetch!(:db)
Enum.reduce(window_exprs, %{}, fn {:window, name, args, spec, filter} = expr, acc ->
values =
cond do
aggregate_call?(db, name, args) ->
aggregate_window_values(name, args, spec, filter, indexed_envs)
name in @window_functions and filter == nil ->
built_in_window_values(name, args, spec, indexed_envs)
name in @window_functions ->
fail("FILTER clause may only be used with aggregate window functions")
true ->
fail("#{name}() may not be used as a window function")
end
Enum.reduce(values, acc, fn {index, value}, acc ->
Map.update(acc, index, %{expr => value}, &Map.put(&1, expr, value))
end)
end)
end
defp aggregate_window_values(name, args, spec, filter, indexed_envs) do
db = indexed_envs |> List.first({%{db: Database.new()}, 0}) |> elem(0) |> Map.fetch!(:db)
case fetch_window_aggregate_function(db, name, args) do
{:ok, %{kind: :incremental_window} = function} ->
incremental_window_values(function, args, spec, filter, indexed_envs)
_other ->
spec
|> window_partitions(indexed_envs)
|> Enum.flat_map(fn {_key, ordered} ->
Enum.with_index(ordered)
|> Enum.map(fn {{env, index}, position} ->
frame =
spec
|> window_frame_indexed(ordered, position)
|> Enum.map(&elem(&1, 0))
|> filter_window_frame(filter)
{index, aggregate(name, args, frame, env)}
end)
end)
end
end
defp fetch_window_aggregate_function(db, name, args) when is_list(args),
do: Database.fetch_aggregate_function(db, name, length(args))
defp fetch_window_aggregate_function(_db, _name, _args), do: :error
defp filter_window_frame(frame, nil), do: frame
defp filter_window_frame(frame, filter), do: Enum.filter(frame, &(truth(filter, &1) == true))
defp incremental_window_values(function, args, spec, filter, indexed_envs) do
spec
|> window_partitions(indexed_envs)
|> Enum.flat_map(fn {_key, ordered} ->
{_state, _previous_frame, values} =
ordered
|> Enum.with_index()
|> Enum.reduce({call_incremental_window_init(function), [], []}, fn {{_env, index},
position},
{state,
previous_frame,
values} ->
frame =
spec
|> window_frame_indexed(ordered, position)
|> filter_window_indexed_frame(filter)
state =
previous_frame
|> frame_difference(frame)
|> Enum.reduce(state, fn {env, _index}, state ->
call_incremental_window_update(
function,
:inverse,
state,
incremental_args(args, env)
)
end)
state =
frame
|> frame_difference(previous_frame)
|> Enum.reduce(state, fn {env, _index}, state ->
call_incremental_window_update(function, :step, state, incremental_args(args, env))
end)
value = call_incremental_window_value(function, state)
{state, frame, [{index, value} | values]}
end)
Enum.reverse(values)
end)
end
defp filter_window_indexed_frame(frame, nil), do: frame
defp filter_window_indexed_frame(frame, filter) do
Enum.filter(frame, fn {env, _index} -> truth(filter, env) == true end)
end
defp frame_difference(left, right) do
right_indexes = MapSet.new(right, &elem(&1, 1))
Enum.reject(left, fn {_env, index} -> MapSet.member?(right_indexes, index) end)
end
defp incremental_args(args, env), do: Enum.map(args, &eval(&1, env))
defp built_in_window_values(name, args, spec, indexed_envs) do
validate_window_args!(name, args)
spec
|> window_partitions(indexed_envs)
|> Enum.flat_map(fn {_key, ordered} ->
peers = rank_peers(ordered, spec.order_by)
ordered
|> Enum.with_index()
|> Enum.map(fn {{env, index}, position} ->
frame = window_frame_indexed(spec, ordered, position)
value = built_in_window_value(name, args, ordered, frame, peers, env, position)
{index, value}
end)
end)
end
defp window_partitions(spec, indexed_envs) do
indexed_envs
|> Enum.group_by(fn {env, _index} -> Enum.map(spec.partition_by, &eval(&1, env)) end)
|> Enum.map(fn {key, partition} ->
{key, sort_indexed_envs_by_terms(partition, spec.order_by)}
end)
end
defp rank_peers(ordered, []), do: Enum.map(ordered, fn _ -> 1 end)
defp rank_peers(ordered, order_by) do
ordered
|> Enum.reduce({[], nil, 0, 0}, fn {env, _index}, {ranks, previous_key, row_number, rank} ->
key = Enum.map(order_by, fn {expr, _direction} -> eval(expr, env) end)
row_number = row_number + 1
rank = if key == previous_key, do: rank, else: row_number
{[rank | ranks], key, row_number, rank}
end)
|> elem(0)
|> Enum.reverse()
end
defp validate_window_args!("row_number", []), do: :ok
defp validate_window_args!("row_number", _args),
do: fail("wrong number of arguments to function row_number()")
defp validate_window_args!("rank", []), do: :ok
defp validate_window_args!("dense_rank", []), do: :ok
defp validate_window_args!("percent_rank", []), do: :ok
defp validate_window_args!("cume_dist", []), do: :ok
defp validate_window_args!("lead", args) when length(args) in 1..3, do: :ok
defp validate_window_args!("lag", args) when length(args) in 1..3, do: :ok
defp validate_window_args!("first_value", [_arg]), do: :ok
defp validate_window_args!("last_value", [_arg]), do: :ok
defp validate_window_args!("nth_value", [_arg, _n]), do: :ok
defp validate_window_args!("ntile", [_arg]), do: :ok
defp validate_window_args!(name, _args),
do: fail("wrong number of arguments to function #{name}()")
defp built_in_window_value("row_number", [], _ordered, _frame, _peers, _env, position),
do: position + 1
defp built_in_window_value("rank", [], _ordered, _frame, peers, _env, position),
do: Enum.at(peers, position)
defp built_in_window_value("dense_rank", [], _ordered, _frame, peers, _env, position),
do: peers |> Enum.take(position + 1) |> Enum.uniq() |> length()
defp built_in_window_value("lead", args, ordered, _frame, _peers, env, position) do
offset = window_offset(args, env, 1)
default = window_default(args, env)
target = Enum.at(ordered, position + offset)
window_arg_value(target, List.first(args), default)
end
defp built_in_window_value("lag", args, ordered, _frame, _peers, env, position) do
offset = window_offset(args, env, 1)
default = window_default(args, env)
index = position - offset
# Guard the negative index: `Enum.at/2` treats a negative index as counting
# from the end, but a lag before the partition start must yield the default.
target = if index >= 0, do: Enum.at(ordered, index)
window_arg_value(target, List.first(args), default)
end
defp built_in_window_value("ntile", [arg], ordered, _frame, _peers, env, position) do
buckets = eval(arg, env)
unless is_integer(buckets) and buckets > 0 do
fail("argument of ntile must be a positive integer")
end
ntile_bucket(position, length(ordered), buckets)
end
defp built_in_window_value("first_value", [arg], _ordered, frame, _peers, _env, _position) do
window_arg_value(List.first(frame), arg, nil)
end
defp built_in_window_value("last_value", [arg], _ordered, frame, _peers, _env, _position) do
frame |> List.last() |> window_arg_value(arg, nil)
end
defp built_in_window_value(
"nth_value",
[_arg, n_expr] = args,
_ordered,
frame,
_peers,
env,
_position
) do
n = eval(n_expr, env)
unless is_integer(n) and n > 0 do
fail("second argument to nth_value must be a positive integer")
end
frame |> Enum.at(n - 1) |> window_arg_value(List.first(args), nil)
end
defp built_in_window_value("percent_rank", [], ordered, _frame, peers, _env, position) do
total = length(ordered)
if total <= 1, do: 0.0, else: (Enum.at(peers, position) - 1) / (total - 1)
end
defp built_in_window_value("cume_dist", [], ordered, _frame, peers, _env, position) do
rank = Enum.at(peers, position)
last_peer =
peers |> Enum.with_index() |> Enum.filter(fn {peer, _} -> peer == rank end) |> List.last()
{_peer, last_index} = last_peer
(last_index + 1) / length(ordered)
end
defp window_frame_indexed(%{frame: nil, order_by: []}, ordered, _position), do: ordered
# The default frame with ORDER BY is RANGE BETWEEN UNBOUNDED PRECEDING AND
# CURRENT ROW: peer rows of the current row are included.
defp window_frame_indexed(%{frame: nil, order_by: order_by}, ordered, position) do
{_first, last} = peer_bounds(ordered, order_by, position)
Enum.take(ordered, last + 1)
end
defp window_frame_indexed(%{frame: frame, order_by: order_by}, ordered, position) do
total = length(ordered)
current_env = ordered |> Enum.at(position) |> elem(0)
{first, last} =
case frame.unit do
:rows ->
{rows_frame_index(frame.start, :start, current_env, position, total),
rows_frame_index(frame.finish, :finish, current_env, position, total)}
:range ->
range_frame_bounds(frame, order_by, ordered, position, total)
:groups ->
groups_frame_bounds(frame, order_by, ordered, position, total)
end
positions = if last < first, do: [], else: Enum.to_list(first..last)
positions =
case Map.get(frame, :exclude, :no_others) do
:no_others ->
positions
:current_row ->
List.delete(positions, position)
:group ->
{peer_first, peer_last} = peer_bounds(ordered, order_by, position)
Enum.reject(positions, &(&1 >= peer_first and &1 <= peer_last))
:ties ->
{peer_first, peer_last} = peer_bounds(ordered, order_by, position)
Enum.reject(positions, &(&1 != position and &1 >= peer_first and &1 <= peer_last))
end
Enum.map(positions, &Enum.at(ordered, &1))
end
defp rows_frame_index(:unbounded_preceding, _side, _env, _position, _total), do: 0
defp rows_frame_index(:unbounded_following, _side, _env, _position, total), do: total - 1
defp rows_frame_index(:current_row, _side, _env, position, _total), do: position
defp rows_frame_index({:preceding, expr}, side, env, position, _total) do
max(position - frame_offset!(expr, env, side), 0)
end
defp rows_frame_index({:following, expr}, side, env, position, total) do
min(position + frame_offset!(expr, env, side), total - 1)
end
defp frame_offset!(expr, env, side) do
case eval(expr, env) do
n when is_integer(n) and n >= 0 ->
n
_ ->
fail(
"frame #{if side == :start, do: "starting", else: "ending"} offset must be a non-negative integer"
)
end
end
# First and last position of the current row's peer group (rows with equal
# ORDER BY keys). With no ORDER BY every row is a peer of every other.
defp peer_bounds(ordered, [], _position), do: {0, length(ordered) - 1}
defp peer_bounds(ordered, order_by, position) do
keys = order_keys(ordered, order_by)
current = Enum.at(keys, position)
indexed = Enum.with_index(keys)
first = Enum.find_value(indexed, fn {key, i} -> if key == current, do: i end)
last =
indexed
|> Enum.reverse()
|> Enum.find_value(fn {key, i} -> if key == current, do: i end)
{first, last}
end
defp order_keys(ordered, order_by) do
Enum.map(ordered, fn {env, _index} ->
Enum.map(order_by, fn {expr, _direction} -> eval(expr, env) end)
end)
end
# RANGE frames: CURRENT ROW means the current peer group; numeric offsets
# require exactly one ORDER BY term and select rows whose key is within
# the offset of the current row's key.
defp range_frame_bounds(frame, order_by, ordered, position, total) do
offset_frame? =
match?({:preceding, _}, frame.start) or match?({:following, _}, frame.start) or
match?({:preceding, _}, frame.finish) or match?({:following, _}, frame.finish)
if offset_frame? do
unless match?([_], order_by) do
fail("RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression")
end
range_offset_bounds(frame, order_by, ordered, position, total)
else
{peer_first, peer_last} = peer_bounds(ordered, order_by, position)
first =
case frame.start do
:unbounded_preceding -> 0
:current_row -> peer_first
:unbounded_following -> total - 1
end
last =
case frame.finish do
:unbounded_following -> total - 1
:current_row -> peer_last
:unbounded_preceding -> 0
end
{first, last}
end
end
defp range_offset_bounds(frame, [{order_expr, direction}], ordered, position, total) do
current_env = ordered |> Enum.at(position) |> elem(0)
current_key = eval(order_expr, current_env)
if current_key == nil do
# NULLs are peers of one another; an offset frame on a NULL key covers
# exactly the NULL peer group.
peer_bounds(ordered, [{order_expr, direction}], position)
else
# Signed distance from the current key, oriented along the sort
# direction; NULL keys sort before everything and never match offsets.
deltas =
Enum.map(ordered, fn {env, _index} ->
case eval(order_expr, env) do
nil ->
nil
key ->
if direction == :desc,
do: numeric(current_key) - numeric(key),
else: numeric(key) - numeric(current_key)
end
end)
start_value =
case frame.start do
:unbounded_preceding -> nil
{:preceding, expr} -> -range_offset!(expr, current_env, :start)
:current_row -> 0
{:following, expr} -> range_offset!(expr, current_env, :start)
end
finish_value =
case frame.finish do
:unbounded_following -> nil
{:preceding, expr} -> -range_offset!(expr, current_env, :finish)
:current_row -> 0
{:following, expr} -> range_offset!(expr, current_env, :finish)
end
first =
if start_value == nil do
0
else
Enum.find_index(deltas, fn delta -> delta != nil and delta >= start_value end) || total
end
last =
if finish_value == nil do
total - 1
else
case Enum.with_index(deltas)
|> Enum.filter(fn {delta, _i} -> delta != nil and delta <= finish_value end)
|> List.last() do
{_delta, i} -> i
nil -> -1
end
end
{first, last}
end
end
defp numeric(value) when is_number(value), do: value
defp numeric(_value), do: 0
defp range_offset!(expr, env, side) do
case eval(expr, env) do
n when is_number(n) and n >= 0 ->
n
_ ->
fail(
"frame #{if side == :start, do: "starting", else: "ending"} offset must be a non-negative number"
)
end
end
# GROUPS frames count whole peer groups instead of rows.
defp groups_frame_bounds(frame, order_by, ordered, position, total) do
keys = order_keys(ordered, order_by)
{group_numbers, _last_key, _n} =
Enum.reduce(keys, {[], :none, -1}, fn key, {numbers, last_key, n} ->
n = if key == last_key, do: n, else: n + 1
{[n | numbers], key, n}
end)
group_numbers = Enum.reverse(group_numbers)
current_group = Enum.at(group_numbers, position)
max_group = List.last(group_numbers)
current_env = ordered |> Enum.at(position) |> elem(0)
start_group =
case frame.start do
:unbounded_preceding -> 0
{:preceding, expr} -> current_group - frame_offset!(expr, current_env, :start)
:current_row -> current_group
{:following, expr} -> current_group + frame_offset!(expr, current_env, :start)
end
finish_group =
case frame.finish do
:unbounded_following -> max_group
{:preceding, expr} -> current_group - frame_offset!(expr, current_env, :finish)
:current_row -> current_group
{:following, expr} -> current_group + frame_offset!(expr, current_env, :finish)
end
indexed = Enum.with_index(group_numbers)
first =
Enum.find_value(indexed, total, fn {group, i} ->
if group >= start_group, do: i
end)
last =
indexed
|> Enum.reverse()
|> Enum.find_value(-1, fn {group, i} -> if group <= finish_group, do: i end)
{first, last}
end
defp ntile_bucket(position, total, buckets) when buckets >= total, do: position + 1
defp ntile_bucket(position, total, buckets) do
base_size = div(total, buckets)
larger_buckets = rem(total, buckets)
larger_rows = larger_buckets * (base_size + 1)
if position < larger_rows do
div(position, base_size + 1) + 1
else
larger_buckets + div(position - larger_rows, base_size) + 1
end
end
defp window_offset([_arg, offset_expr | _rest], env, _default) do
case eval(offset_expr, env) do
n when is_integer(n) and n >= 0 -> n
_ -> 1
end
end
defp window_offset(_args, _env, default), do: default
defp window_default([_arg, _offset, default_expr], env), do: eval(default_expr, env)
defp window_default(_args, _env), do: nil
defp window_arg_value(nil, _arg, default), do: default
defp window_arg_value({target_env, _index}, arg, _default), do: eval(arg, target_env)
# -- compound selects ----------------------------------------------------------
#
# UNION/INTERSECT/EXCEPT run their distinct rows through a sorted temp
# B-tree in SQLite, so their unordered output comes back sorted; UNION ALL
# is plain concatenation. ORDER BY terms must name output columns (by
# position, output name, or any component select's column name).
defp compound_result(db, %Compound{} = stmt, outer) do
{rows, leaf_names} = compound_rows(db, stmt, outer)
names = hd(leaf_names)
rows =
rows
|> compound_order(stmt.order_by, names, leaf_names)
|> clamp(db, stmt.limit, stmt.offset)
%Result{command: :select, columns: names, rows: rows, rows_affected: 0}
end
defp compound_rows(db, %Compound{} = stmt, outer) do
{left_rows, left_names} = compound_rows(db, stmt.left, outer)
{right_rows, right_names} = compound_rows(db, stmt.right, outer)
if length(hd(left_names)) != length(hd(right_names)) do
fail(
"SELECTs to the left and right of #{compound_name(stmt.op)} " <>
"do not have the same number of result columns"
)
end
rows =
case stmt.op do
:union_all ->
left_rows ++ right_rows
:union ->
distinct_rows(left_rows ++ right_rows)
:intersect ->
right_set = row_key_set(right_rows)
left_rows |> distinct_rows() |> Enum.filter(&MapSet.member?(right_set, row_key(&1)))
:except ->
right_set = row_key_set(right_rows)
left_rows |> distinct_rows() |> Enum.reject(&MapSet.member?(right_set, row_key(&1)))
end
{rows, left_names ++ right_names}
end
defp compound_rows(db, stmt, outer) do
result = query_result(db, stmt, outer)
{result.rows, [result.columns]}
end
defp compound_name(:union_all), do: "UNION ALL"
defp compound_name(:union), do: "UNION"
defp compound_name(:intersect), do: "INTERSECT"
defp compound_name(:except), do: "EXCEPT"
defp distinct_rows(rows) do
rows
|> Enum.sort(&(compare_keys(&1, &2) != :gt))
|> Enum.reduce([], fn row, acc ->
case acc do
[previous | _] ->
if compare_keys(row, previous) == :eq, do: acc, else: [row | acc]
[] ->
[row]
end
end)
|> Enum.reverse()
end
# O(n+m) set membership for INTERSECT/EXCEPT: a canonical hash key per row
# whose equality matches `compare_keys/2` (i.e. element-wise `Value.compare`
# under the default binary collation, which is what compound set ops use).
# NULLs are equal to each other; an integer and a numerically-equal float
# collapse to the same key (Value.compare treats `1` and `1.0` as `:eq`);
# text/JSON compare as text; blobs by bytes — each in its own rank bucket so
# cross-type rows never collide.
defp row_key_set(rows), do: MapSet.new(rows, &row_key/1)
defp row_key(row), do: Enum.map(row, &value_key/1)
defp value_key(nil), do: :null
defp value_key(value) when is_integer(value), do: {:num, value}
defp value_key(value) when is_float(value) do
truncated = trunc(value)
if truncated == value, do: {:num, truncated}, else: {:num, value}
end
defp value_key({:json, text}) when is_binary(text), do: {:text, text}
defp value_key(value) when is_binary(value), do: {:text, value}
defp value_key({:blob, bytes}), do: {:blob, bytes}
defp value_key(other), do: {:other, other}
defp compound_order(rows, [], _names, _leaf_names), do: rows
defp compound_order(rows, order_by, names, leaf_names) do
keys =
order_by
|> Enum.with_index(1)
|> Enum.map(fn {{expr, direction}, term} ->
{compound_order_position(expr, term, names, leaf_names), direction}
end)
Enum.sort(rows, fn a, b ->
Enum.reduce_while(keys, true, fn {index, direction}, _ ->
case Value.compare(Enum.at(a, index), Enum.at(b, index)) do
:eq -> {:cont, true}
:lt -> {:halt, direction == :asc}
:gt -> {:halt, direction == :desc}
end
end)
end)
end
defp compound_order_position({:literal, n}, term, names, _leaf_names) when is_integer(n) do
if n < 1 or n > length(names) do
fail(
"#{ordinal(term)} ORDER BY term out of range - should be between 1 and #{length(names)}"
)
end
n - 1
end
# The qualifier is ignored: `ORDER BY t1.log` matches output column `log`.
defp compound_order_position({:column, _qualifier, name}, term, names, leaf_names) do
key = Table.key(name)
Enum.find_value([names | leaf_names], fn columns ->
Enum.find_index(columns, &(Table.key(&1) == key))
end) || fail_unmatched_order_term(term)
end
defp compound_order_position(_expr, term, _names, _leaf_names),
do: fail_unmatched_order_term(term)
defp fail_unmatched_order_term(term) do
fail("#{ordinal(term)} ORDER BY term does not match any column in the result set")
end
# -- FROM: tables, subqueries, joins -----------------------------------------
#
# A relation is {templates, rows}: one frame template per source (carrying
# name/columns/hidden), and each row as a list of frame instances. Joins are
# nested loops; NATURAL/USING mark the right side's join columns hidden so
# they appear once in `*` expansion and resolve unambiguously.
# -- access planning -------------------------------------------------------------
#
# The first sliver of a query planner: a single-table FROM whose WHERE
# constrains the rowid with `=` becomes a point lookup into the row map
# instead of a full scan. The WHERE clause is still applied afterwards, so
# the lookup only needs to return a superset-safe restriction.
# A provably constant-false WHERE (`WHERE NULL IS NOT NULL`, `WHERE NOT 35 IS
# NOT NULL`, …) yields no rows regardless of the FROM, so skip materializing
# the relation's rows entirely — critical for an unconstrained comma join,
# whose cartesian product (e.g. `FROM t, t` over 1000 rows = 1e6 frames) would
# otherwise be built only to be discarded. Templates still come from the table
# schema (cheap) so the result columns resolve. Falls back to the normal
# builder for FROM shapes whose templates aren't trivially derivable
# (subqueries, views, table functions, NATURAL/USING joins).
defp planned_relation(db, from, where, outer) do
with true <- constant_false_where?(where, db),
templates when is_list(templates) <- relation_templates(db, from) do
{templates, []}
else
_ -> planned_relation_dispatch(db, from, where, outer)
end
end
defp constant_false_where?(where, db) when not is_nil(where) do
const_predicate?(where) and Value.truthy(eval(where, constant_env(db))) != true
end
defp constant_false_where?(_where, _db), do: false
defp relation_templates(db, {:table, {:schema, schema, name}, alias_name}) do
ensure_table_schema!(db, schema, name)
table_template_only(db, Database.table_storage_key(schema, name), alias_name, name)
end
defp relation_templates(db, {:table, name, alias_name}) do
relation_templates_unqualified(db, name, alias_name)
end
defp relation_templates(
db,
{:join, %{natural: false, left: false, right: false}, left, right, nil}
) do
with l when is_list(l) <- relation_templates(db, left),
r when is_list(r) <- relation_templates(db, right) do
l ++ r
else
_ -> :unsupported
end
end
defp relation_templates(_db, _other), do: :unsupported
defp relation_templates_unqualified(db, name, alias_name) do
key = relation_unqualified_table_key(db, name)
table_template_only(db, key, alias_name, name)
end
defp table_template_only(db, table_key, alias_name, _name) do
case Map.fetch(db.tables, table_key) do
{:ok, table} -> [table_frame(table, alias_name)]
:error -> :unsupported
end
end
defp planned_relation_dispatch(
db,
{:table, {:schema, schema, name}, alias_name} = from,
where,
outer
) do
ensure_table_schema!(db, schema, name)
planned_named_relation(
db,
Database.table_storage_key(schema, name),
alias_name,
from,
where,
outer
)
end
defp planned_relation_dispatch(db, {:table, name, alias_name} = from, where, outer) do
planned_named_relation(db, Table.key(name), alias_name, from, where, outer)
end
defp planned_relation_dispatch(
db,
{:join, type, left, {:table, name, alias_name}, constraint} = from,
where,
outer
)
when constraint != nil or not (type.left or type.right) do
# The rowid/index join planners drive a per-left-row probe of the right
# table. That only pays off when a term actually correlates the right table
# with the left (`right.col = left.col`, or an explicit ON/USING). Without a
# correlation a single-table filter like `right.col IN (consts)` would be
# re-probed once per left row, which is far slower than filtering the right
# table once — so fall straight to the reordering/pushdown fallback.
# A per-left-row probe must first materialize the whole left relation. When
# the left is itself a multi-table comma join, that means building its cross
# product up front — disastrous when those tables have no join predicates
# tying them together (e.g. select4's 7-way joins blow up to ~10^7 rows).
# For a plain inner comma join we therefore only peel when the left is a
# single base table; a multi-table left goes to the reorder/hash-join
# fallback, which orders and hashes the whole join globally. Explicit-ON and
# OUTER joins keep the probe path (the inner-only fallback can't express
# their semantics).
peel? =
join_correlated?(db, type, name, alias_name, constraint, where) and
(constraint != nil or type.left or type.right or match?({:table, _, _}, left))
if peel? do
# When the left is itself a join, build it once and share it across both
# planners, otherwise an n-way join rebuilds the left ~2^n times.
left_rel =
case left do
{:join, _t, _l, _r, _c} -> planned_relation(db, left, where, outer)
_other -> nil
end
planned_rowid_join_relation(
db,
type,
left,
left_rel,
name,
alias_name,
constraint,
where,
outer
) ||
planned_index_join_relation(
db,
type,
left,
left_rel,
name,
alias_name,
constraint,
where,
outer
) ||
planned_left_rowid_inner_join_relation(
db,
type,
left,
{:table, name, alias_name},
constraint,
where,
outer
) ||
planned_left_index_inner_join_relation(
db,
type,
left,
{:table, name, alias_name},
constraint,
where,
outer
) ||
join_fallback_relation(db, from, where, outer)
else
join_fallback_relation(db, from, where, outer)
end
end
defp planned_relation_dispatch(
db,
{:join, _type, _left, {:table, _name, _alias_name}, _constraint} = from,
where,
outer
),
do: join_fallback_relation(db, from, where, outer)
defp planned_relation_dispatch(db, from, where, outer),
do: join_fallback_relation(db, from, where, outer)
# True when a term ties the right table to another table — the precondition
# for a useful per-left-row index/rowid probe. An explicit ON/USING join is
# always treated as correlated; otherwise we look for an equality between a
# right-table column and a column of some other table.
defp join_correlated?(_db, %{natural: true}, _name, _alias_name, _constraint, _where), do: true
defp join_correlated?(_db, _type, _name, _alias_name, constraint, _where)
when constraint != nil,
do: true
defp join_correlated?(db, _type, name, alias_name, _constraint, where) do
case plain_table(db, table_source_key(name)) do
%Table{} = table ->
right_qual = table_source_qualifier(name, alias_name)
right_cols = MapSet.new(table.columns, &Table.key(&1.name))
Enum.any?(where_conjuncts(where), fn
{:binary, :eq, {:column, _, _} = a, {:column, _, _} = b} ->
ar = right_column?(a, right_qual, right_cols)
br = right_column?(b, right_qual, right_cols)
# A correlation only justifies peeling the right table to a
# per-left-row probe when the probe can actually *seek* one side:
# the right column drives a probe of this table, or the other
# (left) column is the rowid/index prefix of its own table (the
# left-index probe planner). A plain-column equi-join where neither
# side is rowid/indexed is far better handled by the fallback's hash
# join — peeling it strands the equi-join in a separate scope and
# forces a cross product of the remaining tables.
cond do
ar and not br ->
right_probe_column?(table, a) or column_probe_eligible?(db, b)
br and not ar ->
right_probe_column?(table, b) or column_probe_eligible?(db, a)
true ->
false
end
_other ->
false
end)
_not_plain_table ->
false
end
end
defp right_column?({:column, nil, name}, _right_qual, right_cols),
do: MapSet.member?(right_cols, Table.key(name))
defp right_column?({:column, qualifier, _name}, right_qual, _right_cols),
do: Table.key(qualifier) == right_qual
# The given column can drive a rowid/index seek probe on `table`.
defp right_probe_column?(table, {:column, _, name}) do
key = Table.key(name)
cond do
table.rowid_alias == key -> true
key in @rowid_names and not table.without_rowid -> true
Enum.any?(lookup_indexes(table), &(List.first(&1.columns) == key)) -> true
true -> false
end
end
# The other (left) side of a correlation can drive a left-table seek probe:
# resolve the column's table(s) and ask whether it is rowid/index-eligible
# there. Qualified names pick the named table; unqualified names check every
# base table (the column name is unique to its table in practice).
defp column_probe_eligible?(db, {:column, qualifier, _name} = column) do
tables =
case qualifier do
nil ->
for %Table{} = t <- Map.values(db.tables), do: t
qual ->
case plain_table(db, table_source_key(qual)) do
%Table{} = t -> [t]
_ -> []
end
end
Enum.any?(tables, &right_probe_column?(&1, column))
end
# The generic nested-loop fallback (used once index/rowid join planning has
# declined). Reorders comma joins to apply selective filters and equi-joins
# early, pushes single-table WHERE conjuncts into each base table, and marks
# equi-join nodes for hashing. When the join order is changed, the resulting
# frames are permuted back to the original FROM order so projection (`*`,
# positional refs) is unaffected.
defp join_fallback_relation(db, from, where, outer) do
{tree, perm} = optimize_inner_join(db, from, where)
relation = relation(db, tree, outer)
if perm, do: permute_relation(relation, perm), else: relation
end
# Reorders {tmpls, rows} so the per-source frames match `target_quals`
# (original FROM order) instead of the optimized execution order.
defp permute_relation({tmpls, rows}, target_quals) do
order = Enum.map(target_quals, fn q -> Enum.find_index(tmpls, &(&1.name == q)) end)
if Enum.any?(order, &is_nil/1) do
{tmpls, rows}
else
new_tmpls = Enum.map(order, &Enum.at(tmpls, &1))
new_rows = Enum.map(rows, fn frames -> Enum.map(order, &Enum.at(frames, &1)) end)
{new_tmpls, new_rows}
end
end
# Composite hash-join key for one row; `:null` if any key column is NULL,
# since SQL equality never matches on NULL.
defp hash_join_key(exprs, env) do
Enum.reduce_while(exprs, [], fn expr, acc ->
case eval(expr, env) do
nil -> {:halt, :null}
value -> {:cont, [value | acc]}
end
end)
end
defp optimize_inner_join(db, from, where) do
# An all-inner join's `ON` equi-conditions can drive hash joins just like a
# comma join's WHERE keys, so explicit `a JOIN b ON a.k=b.k` hashes too. The
# ON conjuncts feed *only* the hash-join annotation — not pushdown — so the
# join node keeps evaluating its full ON exactly once (a side-effecting ON
# term isn't duplicated into a base-table prefilter). WHERE conjuncts still
# drive reordering and pushdown as before.
where_conjuncts = where_conjuncts(where)
hash_conjuncts = where_conjuncts ++ inner_join_on_conjuncts(from)
with [_ | _] <- hash_conjuncts,
{:ok, [_ | _] = sources, has_opaque?} <- inner_join_sources(db, from) do
source_lookup = source_lookup(sources, has_opaque?)
{reordered, perm} = reorder_comma_join(from, where_conjuncts, source_lookup)
tree =
reordered
|> pushdown_predicates(where_conjuncts, source_lookup)
|> annotate_hashjoins(hash_conjuncts, source_lookup)
{tree, perm}
else
_ -> {from, nil}
end
end
defp inner_join_on_conjuncts({:join, _type, left, right, {:on, expr}}) do
inner_join_on_conjuncts(left) ++ inner_join_on_conjuncts(right) ++ where_conjuncts(expr)
end
defp inner_join_on_conjuncts({:join, _type, left, right, _constraint}) do
inner_join_on_conjuncts(left) ++ inner_join_on_conjuncts(right)
end
defp inner_join_on_conjuncts(_other), do: []
# Greedy join ordering for pure comma joins over base tables: start from the
# most-filtered table, then always extend along a join predicate (avoiding
# cartesian blowups), preferring the most-filtered candidate. Inner joins are
# freely reorderable and WHERE is re-applied downstream, so this only changes
# execution order; `permute_relation/2` restores the original column order.
defp reorder_comma_join(from, conjuncts, source_lookup) do
with false <- source_lookup.has_opaque?,
{:ok, [_, _ | _] = leaves} <- comma_join_leaves(from) do
original = Enum.map(leaves, &leaf_qualifier/1)
leaf_by_qual = Map.new(Enum.zip(original, leaves))
qsets =
conjuncts
|> Enum.map(&conjunct_qualifiers(&1, source_lookup))
|> Enum.reject(&(MapSet.size(&1) == 0))
filter_counts = filter_counts_by_qualifier(original, qsets)
order = greedy_join_order(original, qsets, filter_counts)
if order == original do
{from, nil}
else
tree = order |> Enum.map(&Map.fetch!(leaf_by_qual, &1)) |> rebuild_left_deep()
{tree, original}
end
else
_ -> {from, nil}
end
end
# Only pure comma joins (no NATURAL/USING/ON, no outer side) are freely
# reorderable; a NATURAL join's shared-column semantics must not be rebuilt
# into a plain cross join.
defp comma_join_leaves({:join, %{natural: false, left: false, right: false}, left, right, nil}) do
with {:ok, l} <- comma_join_leaves(left),
{:ok, r} <- comma_join_leaves(right) do
{:ok, l ++ r}
end
end
defp comma_join_leaves({:table, _name, _alias} = table), do: {:ok, [table]}
defp comma_join_leaves(_other), do: :no
defp leaf_qualifier({:table, name, alias_name}), do: table_source_qualifier(name, alias_name)
defp conjunct_qualifiers(expr, source_lookup) do
expr
|> expr_column_refs([])
|> Enum.reduce(MapSet.new(), fn ref, acc ->
case column_owner(ref, source_lookup) do
{:ok, qualifier} -> MapSet.put(acc, qualifier)
_unknown -> acc
end
end)
end
defp greedy_join_order(quals, qsets, filter_counts) do
start = Enum.max_by(quals, &Map.get(filter_counts, &1, 0))
extend_join_order(
[start],
List.delete(quals, start),
MapSet.new([start]),
join_adjacency(quals, qsets),
filter_counts
)
end
defp extend_join_order(order, [], _chosen, _adjacency, _filter_counts), do: Enum.reverse(order)
defp extend_join_order(order, remaining, chosen, adjacency, filter_counts) do
connected_set =
chosen
|> Enum.reduce(MapSet.new(), fn qualifier, acc ->
MapSet.union(acc, Map.get(adjacency, qualifier, MapSet.new()))
end)
|> MapSet.difference(chosen)
connected = Enum.filter(remaining, &MapSet.member?(connected_set, &1))
pool = if connected == [], do: remaining, else: connected
next = Enum.max_by(pool, &Map.get(filter_counts, &1, 0))
extend_join_order(
[next | order],
List.delete(remaining, next),
MapSet.put(chosen, next),
adjacency,
filter_counts
)
end
defp filter_counts_by_qualifier(qualifiers, qsets) do
empty_counts = Map.new(qualifiers, &{&1, 0})
Enum.reduce(qsets, empty_counts, fn qset, counts ->
if MapSet.size(qset) == 1 do
[qualifier] = MapSet.to_list(qset)
Map.update!(counts, qualifier, &(&1 + 1))
else
counts
end
end)
end
defp join_adjacency(qualifiers, qsets) do
empty = Map.new(qualifiers, &{&1, MapSet.new()})
Enum.reduce(qsets, empty, fn qset, adjacency ->
if MapSet.size(qset) > 1 do
Enum.reduce(qset, adjacency, fn qualifier, adjacency ->
Map.update!(adjacency, qualifier, &MapSet.union(&1, MapSet.delete(qset, qualifier)))
end)
else
adjacency
end
end)
end
@comma_join_type %{natural: false, left: false, right: false}
defp rebuild_left_deep([first | rest]) do
Enum.reduce(rest, first, fn leaf, acc -> {:join, @comma_join_type, acc, leaf, nil} end)
end
defp pushdown_predicates(from, conjuncts, source_lookup) do
assignments =
conjuncts
|> Enum.filter(&pushable_predicate?/1)
|> Enum.reduce(%{}, fn pred, acc ->
case predicate_owner(pred, source_lookup) do
{:ok, qualifier} -> Map.update(acc, qualifier, [pred], &[pred | &1])
:none -> acc
end
end)
if assignments == %{}, do: from, else: apply_pushdown(from, assignments)
end
# Annotates each inner-join node with the cross-table equi-join keys it can
# use for a hash join. Only same-affinity numeric plain-column equalities
# qualify (canonical storage, no collation, NULLs excluded), so the hash key
# is exactly SQLite `=`. The full WHERE is re-applied later as a backstop.
defp annotate_hashjoins(from, conjuncts, source_lookup) do
{from, _qualifiers} = annotate_hashjoins_with_qualifiers(from, conjuncts, source_lookup)
from
end
defp annotate_hashjoins_with_qualifiers(
{:join, type, left, right, constraint},
conjuncts,
source_lookup
) do
{left, la} = annotate_hashjoins_with_qualifiers(left, conjuncts, source_lookup)
{right, ra} = annotate_hashjoins_with_qualifiers(right, conjuncts, source_lookup)
qualifiers = la ++ ra
if type.left or type.right do
{{:join, type, left, right, constraint}, qualifiers}
else
equi =
Enum.flat_map(conjuncts, fn conj ->
case equi_join_key(conj, la, ra, source_lookup) do
{:ok, lexpr, rexpr} -> [{lexpr, rexpr}]
:no -> []
end
end)
node =
if equi == [],
do: {:join, type, left, right, constraint},
else: {:hashjoin, type, left, right, constraint, equi}
{node, qualifiers}
end
end
defp annotate_hashjoins_with_qualifiers({:prefiltered, src, preds}, _conjuncts, _source_lookup),
do: {{:prefiltered, src, preds}, subtree_qualifiers(src)}
defp annotate_hashjoins_with_qualifiers(
{:table, name, alias_name} = table,
_conjuncts,
_source_lookup
),
do: {table, [table_source_qualifier(name, alias_name)]}
defp annotate_hashjoins_with_qualifiers(from, _conjuncts, _source_lookup), do: {from, []}
defp subtree_qualifiers({:join, _type, left, right, _constraint}),
do: subtree_qualifiers(left) ++ subtree_qualifiers(right)
# annotate_hashjoins/4 rewrites inner nodes bottom-up, so a node's left child
# may already be a `:hashjoin`. Without this clause it falls through to the
# `_opaque -> []` catch-all, the parent sees no left qualifiers, and every join
# above the first hash join silently degrades to a cross product — the
# dominant blowup in wide comma joins (select4's 8-way joins).
defp subtree_qualifiers({:hashjoin, _type, left, right, _constraint, _keys}),
do: subtree_qualifiers(left) ++ subtree_qualifiers(right)
defp subtree_qualifiers({:prefiltered, src, _preds}), do: subtree_qualifiers(src)
defp subtree_qualifiers({:table, name, alias_name}),
do: [table_source_qualifier(name, alias_name)]
defp subtree_qualifiers(_opaque), do: []
defp equi_join_key(
{:binary, :eq, {:column, _, _} = e1, {:column, _, _} = e2},
la,
ra,
source_lookup
) do
with {:ok, q1} <- column_owner(e1, source_lookup),
{:ok, q2} <- column_owner(e2, source_lookup),
a1 when a1 != nil <- column_affinity_in_sources(e1, source_lookup),
^a1 <- column_affinity_in_sources(e2, source_lookup),
true <- hashable_equi_join?(a1, e1, e2, source_lookup) do
cond do
q1 in la and q2 in ra -> {:ok, e1, e2}
q1 in ra and q2 in la -> {:ok, e2, e1}
true -> :no
end
else
_ -> :no
end
end
defp equi_join_key(_conj, _la, _ra, _source_lookup), do: :no
# The hash key is the raw evaluated value, so a value equal under `=` must
# produce the same key. Numeric same-affinity columns are stored canonically
# (the affinity equality above prevents int/float mixing). TEXT/BLOB are only
# hashable under BINARY collation, where `=` is byte equality; NOCASE/RTRIM
# would group equal values under different keys, so they keep the nested loop.
defp hashable_equi_join?(affinity, _e1, _e2, _sources)
when affinity in [:integer, :real, :numeric],
do: true
defp hashable_equi_join?(affinity, e1, e2, source_lookup) when affinity in [:text, :blob],
do:
binary_collation_in_sources?(e1, source_lookup) and
binary_collation_in_sources?(e2, source_lookup)
defp hashable_equi_join?(_affinity, _e1, _e2, _sources), do: false
defp binary_collation_in_sources?({:column, nil, name}, source_lookup) do
key = Table.key(name)
case Map.get(source_lookup.by_column, key) do
%{collations: collations} -> binary_collation_name?(Map.get(collations, key))
_ -> false
end
end
defp binary_collation_in_sources?({:column, qualifier, name}, source_lookup) do
qkey = Table.key(qualifier)
key = Table.key(name)
case Map.get(source_lookup.by_qualifier, qkey) do
%{collations: collations} -> binary_collation_name?(Map.get(collations, key))
nil -> false
end
end
defp column_affinity_in_sources({:column, nil, name}, source_lookup) do
key = Table.key(name)
case Map.get(source_lookup.by_column, key) do
%{affinities: affinities} -> Map.get(affinities, key)
_ -> nil
end
end
defp column_affinity_in_sources({:column, qualifier, name}, source_lookup) do
qkey = Table.key(qualifier)
key = Table.key(name)
case Map.get(source_lookup.by_qualifier, qkey) do
%{affinities: affinities} -> Map.get(affinities, key)
nil -> nil
end
end
# Walks an all-inner-join tree, returning the base tables (qualifier key +
# column-key set) plus whether any opaque source (subquery/view/CTE/table
# function) is present. Bails (`:not_inner`) on any outer/right join so we
# never reshape null-extended semantics.
defp inner_join_sources(db, from) do
case inner_join_walk(db, from, [], false) do
{:ok, sources, has_opaque?} -> {:ok, sources, has_opaque?}
:not_inner -> :not_inner
end
end
defp inner_join_walk(db, {:join, type, left, right, _constraint}, acc, opaque?) do
if type.left or type.right do
:not_inner
else
case inner_join_walk(db, left, acc, opaque?) do
{:ok, acc, opaque?} -> inner_join_walk(db, right, acc, opaque?)
:not_inner -> :not_inner
end
end
end
defp inner_join_walk(db, {:table, {:schema, schema, name}, alias_name}, acc, opaque?) do
inner_join_source(
db,
Database.table_storage_key(schema, name),
name,
alias_name,
acc,
opaque?
)
end
defp inner_join_walk(db, {:table, name, alias_name}, acc, opaque?) do
inner_join_source(
db,
relation_unqualified_table_key(db, name),
name,
alias_name,
acc,
opaque?
)
end
defp inner_join_walk(_db, _other, acc, _opaque?), do: {:ok, acc, true}
defp inner_join_source(db, table_key, name, alias_name, acc, opaque?) do
case plain_table(db, table_key) do
%Table{} = table ->
source = %{
qualifier: table_source_qualifier(name, alias_name),
columns: MapSet.new(table.columns, &Table.key(&1.name)),
affinities: Map.new(table.columns, &{Table.key(&1.name), &1.affinity}),
collations: Map.new(table.columns, &{Table.key(&1.name), &1.collate || "BINARY"})
}
{:ok, [source | acc], opaque?}
_not_plain_table ->
# A view/CTE materialized as a table-name source: treat as opaque.
{:ok, acc, true}
end
end
defp source_lookup(sources, has_opaque?) do
by_qualifier = Map.new(sources, &{&1.qualifier, &1})
by_column =
Enum.reduce(sources, %{}, fn source, columns ->
Enum.reduce(source.columns, columns, fn key, columns ->
case Map.fetch(columns, key) do
:error -> Map.put(columns, key, source)
{:ok, _existing} -> Map.put(columns, key, :ambiguous)
end
end)
end)
%{has_opaque?: has_opaque?, by_qualifier: by_qualifier, by_column: by_column}
end
# A predicate is safe to evaluate early when it is row-local: no subquery,
# aggregate, or window references.
defp pushable_predicate?(expr), do: not expr_unpushable?(expr)
defp expr_unpushable?({:subquery, _, _}), do: true
defp expr_unpushable?({:subquery, _}), do: true
defp expr_unpushable?({:exists, _}), do: true
defp expr_unpushable?({:scalar_subquery, _}), do: true
defp expr_unpushable?({:in, _expr, {:select, _}, _negated}), do: true
defp expr_unpushable?({:window, _, _, _, _}), do: true
defp expr_unpushable?({:function, name, _args}) when is_binary(name),
do: name in @aggregate_functions
defp expr_unpushable?(tuple) when is_tuple(tuple),
do: tuple |> Tuple.to_list() |> Enum.any?(&expr_unpushable?/1)
defp expr_unpushable?(list) when is_list(list), do: Enum.any?(list, &expr_unpushable?/1)
defp expr_unpushable?(_other), do: false
# Returns `{:ok, qualifier}` when every column the predicate references
# belongs to a single base table; `:none` otherwise.
defp predicate_owner(expr, source_lookup) do
case expr_column_refs(expr, []) do
[] ->
:none
refs ->
owners = Enum.map(refs, &column_owner(&1, source_lookup))
case Enum.uniq(owners) do
[{:ok, qualifier}] -> {:ok, qualifier}
_ambiguous_or_unknown -> :none
end
end
end
defp column_owner({:column, nil, name}, source_lookup) do
# Unqualified: only safe when ownership is unambiguous and there is no
# opaque source that might also expose the column.
key = Table.key(name)
case Map.get(source_lookup.by_column, key) do
%{qualifier: qualifier} when not source_lookup.has_opaque? -> {:ok, qualifier}
_ -> :unknown
end
end
defp column_owner({:column, qualifier, _name}, source_lookup) do
key = Table.key(qualifier)
case Map.get(source_lookup.by_qualifier, key) do
%{qualifier: qualifier} -> {:ok, qualifier}
nil -> :unknown
end
end
defp expr_column_refs({:column, _qualifier, _name} = ref, acc), do: [ref | acc]
defp expr_column_refs(tuple, acc) when is_tuple(tuple),
do: tuple |> Tuple.to_list() |> Enum.reduce(acc, &expr_column_refs/2)
defp expr_column_refs(list, acc) when is_list(list),
do: Enum.reduce(list, acc, &expr_column_refs/2)
defp expr_column_refs(_other, acc), do: acc
defp apply_pushdown({:join, type, left, right, constraint}, assignments) do
{:join, type, apply_pushdown(left, assignments), apply_pushdown(right, assignments),
constraint}
end
defp apply_pushdown({:table, name, alias_name} = table, assignments) do
case Map.get(assignments, table_source_qualifier(name, alias_name)) do
nil -> table
preds -> {:prefiltered, table, preds}
end
end
defp apply_pushdown(other, _assignments), do: other
defp table_source_key({:schema, schema, name}), do: Database.table_storage_key(schema, name)
defp table_source_key(name), do: Table.key(name)
defp table_source_name({:schema, _schema, name}), do: name
defp table_source_name(name), do: name
defp table_source_display(name, alias_name), do: alias_name || table_source_name(name)
defp table_source_qualifier(name, alias_name),
do: Table.key(table_source_display(name, alias_name))
defp planned_named_relation(db, key, alias_name, from, where, outer) do
case plain_table(db, key) do
%Table{} = table ->
case table_access_path(db, table, where) do
{:rowid_eq, {:literal, value}} ->
planned_rowid_eq_relation(table, alias_name, value)
{:rowid_in, exprs} ->
planned_rowid_in_relation(table, alias_name, exprs)
{:rowid_range, bounds} ->
planned_rowid_range_relation(table, alias_name, bounds)
_other ->
planned_index_relation(db, table, alias_name, where) || relation(db, from, outer)
end
_not_plain_table ->
relation(db, from, outer)
end
end
defp planned_rowid_eq_relation(table, alias_name, value) do
tmpl = table_frame(table, alias_name)
rows =
case Value.apply_affinity(value, :integer) do
rowid when is_integer(rowid) ->
case Table.fetch_row(table, rowid) do
{:ok, row} -> [[%{tmpl | row: row, rowid: rowid}]]
:error -> []
end
_not_integer ->
[]
end
{[tmpl], rows}
end
defp planned_rowid_in_relation(table, alias_name, exprs) do
tmpl = table_frame(table, alias_name)
rows =
exprs
|> rowid_in_lookup_values()
|> Enum.flat_map(fn rowid ->
case Table.fetch_row(table, rowid) do
{:ok, row} -> [[%{tmpl | row: row, rowid: rowid}]]
:error -> []
end
end)
{[tmpl], rows}
end
defp planned_rowid_range_relation(table, alias_name, bounds) do
tmpl = table_frame(table, alias_name)
rows =
case rowid_range_lookup_bounds(bounds) do
{:ok, bounds} ->
table
|> Table.scan()
|> Enum.filter(fn {rowid, _row} -> rowid_range_match?(rowid, bounds) end)
|> Enum.map(fn {rowid, row} -> [%{tmpl | row: row, rowid: rowid}] end)
:error ->
[]
end
{[tmpl], rows}
end
defp rowid_in_lookup_values(exprs) do
exprs
|> Enum.flat_map(fn
{:literal, value} ->
case Value.apply_affinity(value, :integer) do
rowid when is_integer(rowid) -> [rowid]
_not_integer -> []
end
_expr ->
[]
end)
|> Enum.uniq()
|> Enum.sort()
end
defp rowid_range_lookup_bounds(bounds) do
bounds
|> Enum.reduce_while({:ok, []}, fn
{op, {:literal, value}}, {:ok, acc} ->
{:cont, {:ok, [{op, Value.apply_affinity(value, :integer)} | acc]}}
_bound, _acc ->
{:halt, :error}
end)
|> case do
{:ok, bounds} -> {:ok, Enum.reverse(bounds)}
:error -> :error
end
end
defp rowid_range_match?(rowid, bounds) do
Enum.all?(bounds, fn {op, value} ->
Value.compare_op(op, rowid, value) == true
end)
end
defp rowid_in_constraint(table, conjuncts) do
Enum.find_value(conjuncts, fn
{:in, expr, list, false} when is_list(list) ->
expr = strip_collation(expr)
if rowid_column_ref?(table, expr) and Enum.all?(list, &constant_expr?/1) do
list
end
_other ->
nil
end)
end
defp rowid_or_literal_constraint(table, where) do
where
|> where_conjuncts()
|> Enum.find_value(fn term ->
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
values <- Enum.map(disjuncts, &rowid_or_literal_disjunct(table, &1)),
true <- Enum.all?(values, &match?([_ | _], &1)) do
Enum.flat_map(values, & &1)
else
_other -> nil
end
end)
end
defp rowid_or_literal_disjunct(table, {:binary, :eq, left, right}) do
cond do
rowid_column_ref?(table, strip_collation(left)) and constant_expr?(strip_collation(right)) ->
[strip_collation(right)]
rowid_column_ref?(table, strip_collation(right)) and constant_expr?(strip_collation(left)) ->
[strip_collation(left)]
true ->
[]
end
end
defp rowid_or_literal_disjunct(table, {:in, expr, list, false}) when is_list(list) do
expr = strip_collation(expr)
if rowid_column_ref?(table, expr) and Enum.all?(list, &constant_expr?/1) do
list
else
[]
end
end
defp rowid_or_literal_disjunct(_table, _term), do: []
defp rowid_range_constraints(table, conjuncts) do
conjuncts
|> Enum.flat_map(fn
{:binary, op, left, right} when op in [:lt, :le, :gt, :ge] ->
case rowid_range_constraint(table, left, right, op) do
nil -> []
bound -> [bound]
end
{:between, expr, low, high, false} ->
case rowid_between_constraint(table, expr, low, high) do
nil -> []
bounds -> bounds
end
_other ->
[]
end)
|> case do
[] -> nil
bounds -> bounds
end
end
defp rowid_range_constraint(table, left, right, op) do
left_base = strip_collation(left)
right_base = strip_collation(right)
cond do
rowid_column_ref?(table, left_base) and constant_expr?(right_base) ->
{op, right_base}
rowid_column_ref?(table, right_base) and constant_expr?(left_base) ->
{flip_range_op(op), left_base}
true ->
nil
end
end
defp rowid_between_constraint(table, expr, low, high) do
expr_base = strip_collation(expr)
low_base = strip_collation(low)
high_base = strip_collation(high)
if rowid_column_ref?(table, expr_base) and constant_expr?(low_base) and
constant_expr?(high_base) do
[{:ge, low_base}, {:le, high_base}]
end
end
defp planned_index_relation(db, table, alias_name, where) do
table = ensure_index_entries(db, table)
with {:ok, index, values} <- planned_index_lookup(table, where) do
tmpl = table_frame(table, alias_name)
rows =
db
|> index_lookup_rowids(index, values)
|> Enum.flat_map(fn rowid ->
case Table.fetch_row(table, rowid) do
{:ok, row} -> [[%{tmpl | row: row, rowid: rowid}]]
:error -> []
end
end)
{[tmpl], rows}
else
_ -> nil
end
end
defp planned_rowid_join_relation(
db,
type,
left,
left_rel,
right_name,
right_alias,
constraint,
where,
outer
) do
with join_kind when join_kind in [:inner, :left, :right, :full] <- indexed_join_kind(type),
%Table{without_rowid: false} = table <- plain_table(db, table_source_key(right_name)) do
rtmpl = table_frame(table, right_alias)
{ltmpls, lrows} = left_rel || planned_relation(db, left, where, outer)
using = using_columns(type, constraint, ltmpls, rtmpl)
right_qualifier = table_source_qualifier(right_name, right_alias)
rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}
lookup_terms =
join_lookup_terms(join_kind, constraint, where) ++
using_lookup_terms(using, right_qualifier)
case join_rowid_lookup_plan(table, right_qualifier, ltmpls, lookup_terms) do
{:ok, lookup_plan} ->
{rows, matched_right_rowids} =
Enum.map_reduce(lrows, MapSet.new(), fn lframes, matched_right_rowids ->
env = %{db: db, frames: lframes, group: nil, outer: outer}
raw_matches =
join_rowid_matches(db, table, rtmpl, lookup_plan, env, constraint, using, outer)
matches =
case raw_matches do
[] when join_kind in [:left, :full] -> [lframes ++ [null_frame(rtmpl)]]
matches -> matches
end
matched_right_rowids = track_matched_right_rowids(raw_matches, matched_right_rowids)
{matches, matched_right_rowids}
end)
|> then(fn {rows, matched_right_rowids} ->
{Enum.flat_map(rows, & &1), matched_right_rowids}
end)
right_rows =
if join_kind in [:right, :full] do
for {rowid, row} <- Table.scan(table),
not MapSet.member?(matched_right_rowids, rowid) do
right_unmatched_row(ltmpls, %{rtmpl | row: row, rowid: rowid}, using)
end
else
[]
end
{ltmpls ++ [rtmpl], rows ++ right_rows}
:error ->
nil
end
else
_ -> nil
end
end
defp join_rowid_matches(db, table, rtmpl, lookup_plan, env, constraint, using, outer) do
lookup_plan
|> join_rowid_lookup_rowids(table, env)
|> Enum.flat_map(fn rowid ->
case Table.fetch_row(table, rowid) do
{:ok, row} ->
rframe = %{rtmpl | row: row, rowid: rowid}
if join_match?(db, constraint, using, env.frames, rframe, outer) do
[env.frames ++ [rframe]]
else
[]
end
:error ->
[]
end
end)
end
defp join_rowid_lookup_rowids({:eq, expr}, table, env) do
case join_rowid_lookup_value(table, expr, env) do
rowid when is_integer(rowid) -> [rowid]
_not_integer -> []
end
end
defp join_rowid_lookup_rowids({:in, exprs}, table, env) do
exprs
|> Enum.flat_map(fn expr ->
case join_rowid_lookup_value(table, expr, env) do
rowid when is_integer(rowid) -> [rowid]
_not_integer -> []
end
end)
|> Enum.uniq()
|> Enum.sort()
end
defp join_rowid_lookup_rowids({:range, bounds}, table, env) do
case join_rowid_range_bounds(table, bounds, env) do
{:ok, bounds} ->
table
|> Table.scan()
|> Enum.filter(fn {rowid, _row} -> rowid_range_match?(rowid, bounds) end)
|> Enum.map(fn {rowid, _row} -> rowid end)
:error ->
[]
end
end
defp join_rowid_range_bounds(table, bounds, env) do
bounds
|> Enum.reduce_while({:ok, []}, fn {op, expr}, {:ok, acc} ->
case join_rowid_lookup_value(table, expr, env) do
value when is_integer(value) -> {:cont, {:ok, [{op, value} | acc]}}
_not_integer -> {:halt, :error}
end
end)
|> case do
{:ok, bounds} -> {:ok, Enum.reverse(bounds)}
:error -> :error
end
end
defp join_rowid_lookup_value(table, expr, env) do
expr
|> eval(env)
|> Value.apply_affinity(:integer)
|> rowid_lookup_value(table)
end
defp rowid_lookup_value(value, %{without_rowid: false}) when is_integer(value), do: value
defp rowid_lookup_value(_value, _table), do: nil
defp track_matched_right_rowids(raw_matches, matched_right_rowids) do
Enum.reduce(raw_matches, matched_right_rowids, fn frames, matched_right_rowids ->
frames
|> List.last()
|> Map.fetch!(:rowid)
|> then(&MapSet.put(matched_right_rowids, &1))
end)
end
defp planned_index_join_relation(
db,
type,
left,
left_rel,
right_name,
right_alias,
constraint,
where,
outer
) do
with join_kind when join_kind in [:inner, :left, :right, :full] <- indexed_join_kind(type),
%Table{} = table <- plain_table(db, table_source_key(right_name)) do
table = ensure_index_entries(db, table)
rtmpl = table_frame(table, right_alias)
{ltmpls, lrows} = left_rel || planned_relation(db, left, where, outer)
using = using_columns(type, constraint, ltmpls, rtmpl)
right_qualifier = table_source_qualifier(right_name, right_alias)
rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}
lookup_terms =
join_lookup_terms(join_kind, constraint, where) ++
using_lookup_terms(using, right_qualifier)
with {:ok, index, lookup_plan} <-
join_index_lookup_plan(table, right_name, right_alias, ltmpls, lookup_terms) do
{rows, matched_right_rowids} =
Enum.map_reduce(lrows, MapSet.new(), fn lframes, matched_right_rowids ->
env = %{db: db, frames: lframes, group: nil, outer: outer}
raw_matches =
db
|> join_probe_rowids(index, table, lookup_plan, env)
|> Enum.flat_map(fn rowid ->
case Table.fetch_row(table, rowid) do
{:ok, row} ->
rframe = %{rtmpl | row: row, rowid: rowid}
if join_match?(db, constraint, using, lframes, rframe, outer) do
[lframes ++ [rframe]]
else
[]
end
:error ->
[]
end
end)
matches =
case raw_matches do
[] when join_kind in [:left, :full] -> [lframes ++ [null_frame(rtmpl)]]
matches -> matches
end
matched_right_rowids = track_matched_right_rowids(raw_matches, matched_right_rowids)
{matches, matched_right_rowids}
end)
|> then(fn {rows, matched_right_rowids} ->
{Enum.flat_map(rows, & &1), matched_right_rowids}
end)
right_rows =
if join_kind in [:right, :full] do
for {rowid, row} <- Table.scan(table),
not MapSet.member?(matched_right_rowids, rowid) do
right_unmatched_row(ltmpls, %{rtmpl | row: row, rowid: rowid}, using)
end
else
[]
end
{ltmpls ++ [rtmpl], rows ++ right_rows}
else
_ -> nil
end
else
_ -> nil
end
end
defp planned_left_index_inner_join_relation(
db,
type,
{:table, left_name, left_alias},
{:table, right_name, right_alias},
constraint,
where,
outer
) do
with :inner <- indexed_join_kind(type),
%Table{} = table <- plain_table(db, table_source_key(left_name)) do
table = ensure_index_entries(db, table)
ltmpl = table_frame(table, left_alias)
rfrom = {:table, right_name, right_alias}
{rtmpls, rrows} = planned_relation(db, rfrom, nil, outer)
[rtmpl] = rtmpls
using = using_columns(type, constraint, [ltmpl], rtmpl)
left_qualifier = table_source_qualifier(left_name, left_alias)
rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}
lookup_terms =
join_lookup_terms(:inner, constraint, where) ++
using_lookup_terms(using, left_qualifier)
with {:ok, index, lookup_plan} <-
join_index_lookup_plan(table, left_name, left_alias, rtmpls, lookup_terms) do
rows =
Enum.flat_map(rrows, fn [rframe] ->
env = %{db: db, frames: [rframe], group: nil, outer: outer}
db
|> join_probe_rowids(index, table, lookup_plan, env)
|> Enum.flat_map(fn rowid ->
case Table.fetch_row(table, rowid) do
{:ok, row} ->
lframe = %{ltmpl | row: row, rowid: rowid}
rframe = %{rframe | hidden: rtmpl.hidden}
if join_match?(db, constraint, using, [lframe], rframe, outer) do
[[lframe, rframe]]
else
[]
end
:error ->
[]
end
end)
end)
{[ltmpl, rtmpl], rows}
else
_ -> nil
end
else
_ -> nil
end
end
defp planned_left_index_inner_join_relation(
_db,
_type,
_left,
_right,
_constraint,
_where,
_outer
),
do: nil
defp planned_left_rowid_inner_join_relation(
db,
type,
{:table, left_name, left_alias},
{:table, right_name, right_alias},
constraint,
where,
outer
) do
with :inner <- indexed_join_kind(type),
%Table{without_rowid: false} = table <- plain_table(db, table_source_key(left_name)) do
ltmpl = table_frame(table, left_alias)
rfrom = {:table, right_name, right_alias}
{rtmpls, rrows} = planned_relation(db, rfrom, nil, outer)
[rtmpl] = rtmpls
using = using_columns(type, constraint, [ltmpl], rtmpl)
left_qualifier = table_source_qualifier(left_name, left_alias)
rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}
lookup_terms =
join_lookup_terms(:inner, constraint, where) ++
using_lookup_terms(using, left_qualifier)
case join_rowid_lookup_plan(table, left_qualifier, rtmpls, lookup_terms) do
{:ok, lookup_plan} ->
rows =
left_rowid_inner_join_rows(
db,
table,
ltmpl,
rtmpl,
rrows,
lookup_plan,
constraint,
using,
outer
)
{[ltmpl, rtmpl], rows}
:error ->
nil
end
else
_ -> nil
end
end
defp planned_left_rowid_inner_join_relation(
_db,
_type,
_left,
_right,
_constraint,
_where,
_outer
),
do: nil
defp left_rowid_inner_join_rows(
db,
table,
ltmpl,
rtmpl,
rrows,
lookup_plan,
constraint,
using,
outer
) do
Enum.flat_map(rrows, fn [rframe] ->
env = %{db: db, frames: [rframe], group: nil, outer: outer}
lookup_plan
|> join_rowid_lookup_rowids(table, env)
|> Enum.flat_map(
&left_rowid_inner_join_match(
db,
table,
ltmpl,
rtmpl,
&1,
rframe,
constraint,
using,
outer
)
)
end)
end
defp left_rowid_inner_join_match(
db,
table,
ltmpl,
rtmpl,
rowid,
rframe,
constraint,
using,
outer
) do
case Table.fetch_row(table, rowid) do
{:ok, row} ->
lframe = %{ltmpl | row: row, rowid: rowid}
rframe = %{rframe | hidden: rtmpl.hidden}
if join_match?(db, constraint, using, [lframe], rframe, outer) do
[[lframe, rframe]]
else
[]
end
:error ->
[]
end
end
defp indexed_join_kind(%{natural: false, left: false, right: false}), do: :inner
defp indexed_join_kind(%{natural: false, left: true, right: false}), do: :left
defp indexed_join_kind(%{natural: false, left: false, right: true}), do: :right
defp indexed_join_kind(%{natural: false, left: true, right: true}), do: :full
defp indexed_join_kind(%{natural: true, left: false, right: false}), do: :inner
defp indexed_join_kind(%{natural: true, left: true, right: false}), do: :left
defp indexed_join_kind(%{natural: true, left: false, right: true}), do: :right
defp indexed_join_kind(%{natural: true, left: true, right: true}), do: :full
defp indexed_join_kind(_type), do: :unsupported
defp join_lookup_terms(:inner, constraint, where),
do: join_constraint_terms(constraint) ++ where_conjuncts(where)
defp join_lookup_terms(join_kind, constraint, where) when join_kind in [:left, :right, :full],
do: join_constraint_terms(constraint) ++ where_conjuncts(where)
defp using_lookup_terms(using, right_qualifier) do
Enum.map(using, fn key ->
{:using_eq, key, right_qualifier}
end)
end
defp join_index_lookup_plan(table, right_name, right_alias, ltmpls, terms) do
case join_in_index_lookup(table, right_name, right_alias, ltmpls, terms) do
{:ok, index, prefix, member, exprs} ->
{:ok, index, {:in, prefix, member, exprs}}
:error ->
case join_index_lookup(table, right_name, right_alias, ltmpls, terms) do
{:ok, index, prefix} ->
{:ok, index, {:eq, prefix}}
:error ->
case join_range_index_lookup(table, right_name, right_alias, ltmpls, terms) do
{:ok, index, prefix, range_member, bounds} ->
{:ok, index, {:range, prefix, range_member, bounds}}
:error ->
:error
end
end
end
end
defp join_probe_rowids(db, index, table, {:eq, prefix}, env) do
lookup_values = Enum.map(prefix, &join_lookup_value(table, &1, env))
index_lookup_rowids(db, index, {:eq, lookup_values})
end
defp join_probe_rowids(db, index, table, {:range, prefix, range_member, bounds}, env) do
prefix_values = Enum.map(prefix, &join_lookup_value(table, &1, env))
bound_values =
Enum.map(bounds, fn {op, expr} ->
{op, join_lookup_value(table, {range_member, expr}, env)}
end)
lookup =
if prefix_values == [] do
{:range, bound_values}
else
{:member_range, prefix_values, bound_values}
end
index_lookup_rowids(db, index, lookup)
end
defp join_probe_rowids(db, index, table, {:in, prefix, member, exprs}, env) do
prefix_values = Enum.map(prefix, &join_lookup_value(table, &1, env))
values = Enum.map(exprs, &join_lookup_value(table, {member, &1}, env))
lookup =
if prefix_values == [] do
{:in, values}
else
{:member_in, prefix_values, values}
end
index_lookup_rowids(db, index, lookup)
end
defp join_index_lookup(table, right_name, right_alias, ltmpls, terms) do
right_qualifier = table_source_qualifier(right_name, right_alias)
local_terms = Enum.map(terms, &localize_index_term(&1, right_qualifier, table))
lookup_indexes(table)
|> Enum.filter(&join_index_usable?(table, &1, local_terms))
|> Enum.map(fn index ->
member_collations = index_member_collation_pairs(index)
prefix =
member_collations
|> Enum.reduce_while({:ok, []}, fn {member, collation}, {:ok, prefix} ->
case join_member_equality_constraint(
table,
right_qualifier,
ltmpls,
terms,
member,
collation
) do
nil -> {:halt, :error}
expr -> {:cont, {:ok, [{member, expr} | prefix]}}
end
end)
|> case do
{:ok, prefix} -> Enum.reverse(prefix)
:error -> []
end
{index, prefix}
end)
|> Enum.filter(fn {_index, prefix} -> prefix != [] end)
|> Enum.max_by(
fn {index, prefix} -> {length(prefix), if(index.unique, do: 1, else: 0)} end,
fn -> nil end
)
|> case do
nil -> :error
{index, prefix} -> {:ok, index, prefix}
end
end
defp join_range_index_lookup(table, right_name, right_alias, ltmpls, terms) do
right_qualifier = table_source_qualifier(right_name, right_alias)
local_terms = Enum.map(terms, &localize_index_term(&1, right_qualifier, table))
lookup_indexes(table)
|> Enum.filter(&join_index_usable?(table, &1, local_terms))
|> Enum.map(fn index ->
member_collations = index_member_collation_pairs(index)
{prefix, remaining} =
join_equality_prefix(table, right_qualifier, ltmpls, terms, member_collations)
range_member = List.first(remaining)
range_collation = index_member_collation(index, range_member)
bounds =
join_member_range_constraints(
table,
right_qualifier,
terms,
range_member,
range_collation
)
{index, prefix, range_member, bounds}
end)
|> Enum.filter(fn {_index, _prefix, range_member, bounds} ->
range_member != nil and bounds != []
end)
|> Enum.max_by(
fn {index, prefix, _range_member, bounds} ->
{length(prefix), length(bounds), if(index.unique, do: 1, else: 0)}
end,
fn -> nil end
)
|> case do
nil -> :error
{index, prefix, range_member, bounds} -> {:ok, index, prefix, range_member, bounds}
end
end
defp join_in_index_lookup(table, right_name, right_alias, ltmpls, terms) do
right_qualifier = table_source_qualifier(right_name, right_alias)
local_terms = Enum.map(terms, &localize_index_term(&1, right_qualifier, table))
lookup_indexes(table)
|> Enum.filter(&join_index_usable?(table, &1, local_terms))
|> Enum.map(fn index ->
member_collations = index_member_collation_pairs(index)
{prefix, remaining} =
join_equality_prefix(table, right_qualifier, ltmpls, terms, member_collations)
member = List.first(remaining)
member_collation = index_member_collation(index, member)
exprs =
join_member_in_constraint(
table,
right_qualifier,
ltmpls,
terms,
member,
member_collation
)
{index, prefix, member, exprs}
end)
|> Enum.filter(fn {_index, _prefix, member, exprs} ->
member != nil and exprs != []
end)
|> Enum.max_by(
fn {index, prefix, _member, exprs} ->
{length(prefix), length(exprs), if(index.unique, do: 1, else: 0)}
end,
fn -> nil end
)
|> case do
nil -> :error
{index, prefix, member, exprs} -> {:ok, index, prefix, member, exprs}
end
end
defp index_member_collation_pairs(index) do
members = index_members(index)
collations = Map.get(index, :collations) || []
members
|> Enum.with_index()
|> Enum.map(fn {member, index} -> {member, Enum.at(collations, index)} end)
end
defp index_member_collation(_index, nil), do: nil
defp index_member_collation(index, member) do
members = index_members(index)
collations = Map.get(index, :collations) || []
case Enum.find_index(members, &(&1 == member)) do
nil -> nil
member_index -> Enum.at(collations, member_index)
end
end
defp join_equality_prefix(table, right_qualifier, ltmpls, terms, member_collations) do
member_collations
|> Enum.reduce_while({[], member_collations}, fn {member, collation} = member_collation,
{prefix, [_member | rest]} ->
case join_member_equality_constraint(
table,
right_qualifier,
ltmpls,
terms,
member,
collation
) do
nil -> {:halt, {Enum.reverse(prefix), Enum.map([member_collation | rest], &elem(&1, 0))}}
expr -> {:cont, {[{member, expr} | prefix], rest}}
end
end)
|> case do
{prefix, []} -> {Enum.reverse(prefix), []}
other -> other
end
end
defp join_lookup_value(table, {{:column, key}, expr}, env) do
value = eval(expr, env)
column = Table.column(table, key)
Value.apply_affinity(value, column.affinity)
end
defp join_lookup_value(_table, {{:expr, _indexed_expr}, expr}, env), do: eval(expr, env)
defp join_constraint_terms({:on, expr}), do: where_conjuncts(expr)
defp join_constraint_terms(_constraint), do: []
defp join_member_equality_constraint(
table,
right_qualifier,
ltmpls,
terms,
{:column, key},
index_collation
) do
Enum.find_value(terms, fn term ->
case join_equality_constraint(table, right_qualifier, ltmpls, term, index_collation) do
{^key, expr} -> expr
_ -> nil
end
end)
end
defp join_member_equality_constraint(
table,
right_qualifier,
_ltmpls,
terms,
{:expr, indexed_expr},
_index_collation
) do
Enum.find_value(terms, fn
{:binary, :eq, left, right} ->
cond do
explicit_collation_node?(left) or explicit_collation_node?(right) ->
nil
expression_equivalent?(localize_index_term(left, right_qualifier, table), indexed_expr) and
not expr_references_right_table?(right, right_qualifier, table) ->
right
expression_equivalent?(localize_index_term(right, right_qualifier, table), indexed_expr) and
not expr_references_right_table?(left, right_qualifier, table) ->
left
true ->
nil
end
_term ->
nil
end)
end
defp join_member_range_constraints(_table, _right_qualifier, _terms, nil, _collation),
do: []
defp join_member_range_constraints(
table,
right_qualifier,
terms,
{:column, key},
index_collation
) do
terms
|> Enum.flat_map(fn
{:binary, op, left, right} when op in [:lt, :le, :gt, :ge] ->
cond do
right_join_column(table, right_qualifier, [], strip_collation(left)) == key and
collation_names_equal?(range_term_collation(table, key, left, right), index_collation) and
not expr_references_right_table?(right, right_qualifier, table) ->
[{op, right}]
right_join_column(table, right_qualifier, [], strip_collation(right)) == key and
collation_names_equal?(range_term_collation(table, key, left, right), index_collation) and
not expr_references_right_table?(left, right_qualifier, table) ->
[{flip_range_op(op), left}]
true ->
[]
end
{:between, expr, low, high, false} ->
if right_join_column(table, right_qualifier, [], strip_collation(expr)) == key and
collation_names_equal?(
range_term_collation(table, key, expr, low),
index_collation
) and
collation_names_equal?(
range_term_collation(table, key, expr, high),
index_collation
) and
not expr_references_right_table?(low, right_qualifier, table) and
not expr_references_right_table?(high, right_qualifier, table) do
[{:ge, low}, {:le, high}]
else
[]
end
_term ->
[]
end)
end
defp join_member_range_constraints(
table,
right_qualifier,
terms,
{:expr, indexed_expr},
_index_collation
) do
terms
|> Enum.flat_map(fn
{:binary, op, left, right} when op in [:lt, :le, :gt, :ge] ->
cond do
explicit_collation_node?(left) or explicit_collation_node?(right) ->
[]
expression_equivalent?(localize_index_term(left, right_qualifier, table), indexed_expr) and
not expr_references_right_table?(right, right_qualifier, table) ->
[{op, right}]
expression_equivalent?(localize_index_term(right, right_qualifier, table), indexed_expr) and
not expr_references_right_table?(left, right_qualifier, table) ->
[{flip_range_op(op), left}]
true ->
[]
end
{:between, expr, low, high, false} ->
if not (explicit_collation_node?(expr) or explicit_collation_node?(low) or
explicit_collation_node?(high)) and
expression_equivalent?(
localize_index_term(expr, right_qualifier, table),
indexed_expr
) and
not expr_references_right_table?(low, right_qualifier, table) and
not expr_references_right_table?(high, right_qualifier, table) do
[{:ge, low}, {:le, high}]
else
[]
end
_term ->
[]
end)
end
defp join_member_in_constraint(_table, _right_qualifier, _ltmpls, _terms, nil, _collation),
do: []
defp join_member_in_constraint(
table,
right_qualifier,
ltmpls,
terms,
{:column, key},
index_collation
) do
Enum.find_value(terms, [], fn
{:in, expr, list, false} when is_list(list) ->
if right_join_column(table, right_qualifier, ltmpls, strip_collation(expr)) == key and
not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation) and
Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do
list
end
term ->
join_member_or_lookup_constraint(
table,
right_qualifier,
ltmpls,
key,
index_collation,
term
)
end)
end
defp join_member_in_constraint(
table,
right_qualifier,
_ltmpls,
terms,
{:expr, indexed_expr},
_index_collation
) do
Enum.find_value(terms, [], fn
{:in, expr, list, false} when is_list(list) ->
if expression_equivalent?(localize_index_term(expr, right_qualifier, table), indexed_expr) and
not explicit_collation_node?(list) and
Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do
list
end
term ->
join_expression_or_lookup_constraint(table, right_qualifier, indexed_expr, term)
end)
end
defp join_expression_or_lookup_constraint(table, right_qualifier, indexed_expr, term) do
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
constraints <-
Enum.map(
disjuncts,
&join_expression_or_lookup_disjunct(table, right_qualifier, indexed_expr, &1)
),
true <- Enum.all?(constraints, &match?([_ | _], &1)) do
Enum.flat_map(constraints, & &1)
else
_other -> nil
end
end
defp join_expression_or_lookup_disjunct(
table,
right_qualifier,
indexed_expr,
{:binary, :eq, left, right}
) do
cond do
explicit_collation_node?(left) or explicit_collation_node?(right) ->
nil
expression_equivalent?(localize_index_term(left, right_qualifier, table), indexed_expr) and
join_lookup_expr_usable?(right, table, right_qualifier) ->
[right]
expression_equivalent?(localize_index_term(right, right_qualifier, table), indexed_expr) and
join_lookup_expr_usable?(left, table, right_qualifier) ->
[left]
true ->
nil
end
end
defp join_expression_or_lookup_disjunct(
table,
right_qualifier,
indexed_expr,
{:in, expr, list, false}
)
when is_list(list) do
if expression_equivalent?(localize_index_term(expr, right_qualifier, table), indexed_expr) and
not explicit_collation_node?(list) and
Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do
list
end
end
defp join_expression_or_lookup_disjunct(
_table,
_right_qualifier,
_indexed_expr,
_term
),
do: nil
defp join_member_or_lookup_constraint(
table,
right_qualifier,
ltmpls,
key,
index_collation,
term
) do
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
constraints <-
Enum.map(
disjuncts,
&join_member_or_lookup_disjunct(
table,
right_qualifier,
ltmpls,
key,
index_collation,
&1
)
),
true <- Enum.all?(constraints, &match?([_ | _], &1)) do
Enum.flat_map(constraints, & &1)
else
_other -> nil
end
end
defp join_member_or_lookup_disjunct(
table,
right_qualifier,
ltmpls,
key,
index_collation,
{:binary, :eq, left, right}
) do
left_base = strip_collation(left)
right_base = strip_collation(right)
cond do
right_join_column(table, right_qualifier, ltmpls, left_base) == key ->
join_member_or_lookup_candidate(
table,
right_qualifier,
index_collation,
join_term_collation(table, key, left, right),
right
)
right_join_column(table, right_qualifier, ltmpls, right_base) == key ->
join_member_or_lookup_candidate(
table,
right_qualifier,
index_collation,
join_term_collation(table, key, left, right),
left
)
true ->
nil
end
end
defp join_member_or_lookup_disjunct(
table,
right_qualifier,
ltmpls,
key,
index_collation,
{:in, expr, list, false}
)
when is_list(list) do
expr_base = strip_collation(expr)
if right_join_column(table, right_qualifier, ltmpls, expr_base) == key and
not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation) and
Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do
list
end
end
defp join_member_or_lookup_disjunct(
_table,
_right_qualifier,
_ltmpls,
_key,
_index_collation,
_term
),
do: nil
defp join_member_or_lookup_candidate(
table,
right_qualifier,
index_collation,
term_collation,
expr
) do
if collation_names_equal?(term_collation, index_collation) and
join_lookup_expr_usable?(expr, table, right_qualifier) do
[expr]
end
end
defp join_equality_constraint(
table,
right_qualifier,
ltmpls,
{:using_eq, key, right_qualifier},
_index_collation
) do
if Table.column(table, key) != nil and Enum.any?(ltmpls, &visible?(&1, key)),
do: {key, {:column, nil, key}},
else: nil
end
defp join_equality_constraint(
table,
right_qualifier,
ltmpls,
{:binary, :eq, left, right},
index_collation
) do
cond do
key = right_join_column(table, right_qualifier, ltmpls, strip_collation(left)) ->
join_equality_constraint_candidate(
table,
key,
right,
right_qualifier,
join_term_collation(table, key, left, right),
index_collation
)
key = right_join_column(table, right_qualifier, ltmpls, strip_collation(right)) ->
join_equality_constraint_candidate(
table,
key,
left,
right_qualifier,
join_term_collation(table, key, left, right),
index_collation
)
true ->
nil
end
end
defp join_equality_constraint(_table, _right_qualifier, _ltmpls, _term, _index_collation),
do: nil
defp join_equality_constraint_candidate(
table,
key,
left_expr,
right_qualifier,
term_collation,
index_collation
) do
if expr_references_right_table?(left_expr, right_qualifier, table) or
not collation_names_equal?(term_collation, index_collation) do
nil
else
{key, left_expr}
end
end
defp join_term_collation(table, key, left, right) do
explicit_collation_name(left) || explicit_collation_name(right) ||
column_collation_name(table, key) || :binary
end
defp join_index_usable?(table, index, local_terms) do
members = index_members(index)
collations = Map.get(index, :collations) || []
members != [] and index_predicate_usable?(index, terms_where(local_terms)) and
members
|> Enum.zip(collations)
|> Enum.all?(fn
{{:column, key}, collation} ->
if join_terms_have_explicit_collation_for_column?(table, key, local_terms) do
join_terms_match_index_collation?(table, key, collation, local_terms)
else
collation_names_equal?(column_collation_name(table, key), collation) or
join_terms_match_index_collation?(table, key, collation, local_terms)
end
{{:expr, _indexed_expr}, _collation} ->
true
end)
end
defp join_terms_have_explicit_collation_for_column?(table, key, terms) do
Enum.any?(terms, fn
{:binary, op, left, right} when op in [:eq, :lt, :le, :gt, :ge] ->
(localized_column_expr?(table, key, strip_collation(left)) and
explicit_collation_node?(left)) or
(localized_column_expr?(table, key, strip_collation(right)) and
explicit_collation_node?(right))
{:between, expr, _low, _high, false} ->
localized_column_expr?(table, key, strip_collation(expr)) and
explicit_collation_node?(expr)
{:in, expr, list, false} when is_list(list) ->
localized_column_expr?(table, key, strip_collation(expr)) and
explicit_collation_node?(expr)
term ->
disjuncts = where_disjuncts(term)
multiple_terms?(disjuncts) and
join_terms_have_explicit_collation_for_column?(table, key, disjuncts)
end)
end
defp join_terms_match_index_collation?(table, key, index_collation, terms) do
Enum.any?(terms, fn
{:binary, :eq, left, right} ->
(localized_column_expr?(table, key, strip_collation(left)) or
localized_column_expr?(table, key, strip_collation(right))) and
collation_names_equal?(
join_term_collation(table, key, left, right),
index_collation
)
{:binary, op, left, right} when op in [:lt, :le, :gt, :ge] ->
(localized_column_expr?(table, key, strip_collation(left)) or
localized_column_expr?(table, key, strip_collation(right))) and
collation_names_equal?(
range_term_collation(table, key, left, right),
index_collation
)
{:between, expr, low, high, false} ->
localized_column_expr?(table, key, strip_collation(expr)) and
collation_names_equal?(
range_term_collation(table, key, expr, low),
index_collation
) and
collation_names_equal?(
range_term_collation(table, key, expr, high),
index_collation
)
{:in, expr, list, false} when is_list(list) ->
localized_column_expr?(table, key, strip_collation(expr)) and
not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation)
term ->
join_or_terms_match_index_collation?(table, key, index_collation, term) or
member_or_literal_constraint(term, table, key, index_collation) != nil
end)
end
defp join_or_terms_match_index_collation?(table, key, index_collation, term) do
disjuncts = where_disjuncts(term)
multiple_terms?(disjuncts) and
Enum.all?(disjuncts, fn
{:binary, :eq, left, right} ->
(localized_column_expr?(table, key, strip_collation(left)) or
localized_column_expr?(table, key, strip_collation(right))) and
collation_names_equal?(join_term_collation(table, key, left, right), index_collation)
{:in, expr, list, false} when is_list(list) ->
localized_column_expr?(table, key, strip_collation(expr)) and
not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation)
_other ->
false
end)
end
defp localized_column_expr?(table, key, {:column, nil, name}),
do: Table.key(name) == key and Table.column(table, key) != nil
defp localized_column_expr?(_table, _key, _expr), do: false
defp terms_where([]), do: nil
defp terms_where([term]), do: term
defp terms_where([term | rest]), do: {:binary, :and, term, terms_where(rest)}
defp localize_index_term({:column, qualifier, name} = expr, target_qualifier, table) do
key = Table.key(name)
cond do
qualifier != nil and Table.key(qualifier) == target_qualifier ->
{:column, nil, name}
qualifier == nil and Table.column(table, key) != nil ->
{:column, nil, name}
true ->
expr
end
end
defp localize_index_term(expr, target_qualifier, table) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.map(&localize_index_term(&1, target_qualifier, table))
|> List.to_tuple()
end
defp localize_index_term(expr, target_qualifier, table) when is_list(expr),
do: Enum.map(expr, &localize_index_term(&1, target_qualifier, table))
defp localize_index_term(expr, _target_qualifier, _table), do: expr
defp join_rowid_lookup_plan(table, right_qualifier, ltmpls, terms) do
case join_rowid_point_lookup_plan(table, right_qualifier, ltmpls, terms) do
{:ok, lookup_plan} -> {:ok, lookup_plan}
:error -> join_rowid_range_lookup_plan(table, right_qualifier, ltmpls, terms)
end
end
defp join_rowid_point_lookup_plan(table, right_qualifier, ltmpls, terms) do
Enum.find_value(terms, :error, fn term ->
case join_rowid_lookup_term(table, right_qualifier, ltmpls, term) do
{:eq, _expr} = lookup_plan -> {:ok, lookup_plan}
{:in, _exprs} = lookup_plan -> {:ok, lookup_plan}
_other -> nil
end
end)
end
defp join_rowid_range_lookup_plan(table, right_qualifier, ltmpls, terms) do
terms
|> Enum.flat_map(fn term ->
case join_rowid_lookup_term(table, right_qualifier, ltmpls, term) do
{:range, bounds} -> bounds
_other -> []
end
end)
|> case do
[] -> :error
bounds -> {:ok, {:range, bounds}}
end
end
defp join_rowid_lookup_term(
table,
right_qualifier,
ltmpls,
{:using_eq, key, right_qualifier}
) do
if table.rowid_alias == key and Enum.any?(ltmpls, &has_column?(&1, key)) do
{:eq, {:column, nil, key}}
end
end
defp join_rowid_lookup_term(table, right_qualifier, ltmpls, {:binary, :eq, left, right}) do
left_base = strip_collation(left)
right_base = strip_collation(right)
cond do
right_join_rowid_column?(table, right_qualifier, ltmpls, left_base) ->
join_rowid_equality_candidate(table, right_qualifier, ltmpls, right)
right_join_rowid_column?(table, right_qualifier, ltmpls, right_base) ->
join_rowid_equality_candidate(table, right_qualifier, ltmpls, left)
true ->
nil
end
end
defp join_rowid_lookup_term(
table,
right_qualifier,
ltmpls,
{:in, expr, exprs, false}
)
when is_list(exprs) do
expr = strip_collation(expr)
if right_join_rowid_column?(table, right_qualifier, ltmpls, expr) and
Enum.all?(exprs, &join_rowid_lookup_expr_usable?(&1, table, right_qualifier, ltmpls)) do
{:in, exprs}
end
end
defp join_rowid_lookup_term(table, right_qualifier, ltmpls, {:binary, op, left, right})
when op in [:lt, :le, :gt, :ge] do
left_base = strip_collation(left)
right_base = strip_collation(right)
cond do
right_join_rowid_column?(table, right_qualifier, ltmpls, left_base) ->
join_rowid_range_candidate(table, right_qualifier, ltmpls, op, right)
right_join_rowid_column?(table, right_qualifier, ltmpls, right_base) ->
join_rowid_range_candidate(table, right_qualifier, ltmpls, flip_range_op(op), left)
true ->
nil
end
end
defp join_rowid_lookup_term(
table,
right_qualifier,
ltmpls,
{:between, expr, low, high, false}
) do
expr = strip_collation(expr)
if right_join_rowid_column?(table, right_qualifier, ltmpls, expr) and
join_rowid_lookup_expr_usable?(low, table, right_qualifier, ltmpls) and
join_rowid_lookup_expr_usable?(high, table, right_qualifier, ltmpls) do
{:range, [{:ge, low}, {:le, high}]}
end
end
defp join_rowid_lookup_term(
table,
right_qualifier,
ltmpls,
{:binary, :or, _left, _right} = term
) do
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
values <-
Enum.map(disjuncts, &join_rowid_or_lookup_disjunct(table, right_qualifier, ltmpls, &1)),
true <- Enum.all?(values, &match?([_ | _], &1)) do
{:in, Enum.flat_map(values, & &1)}
else
_other -> nil
end
end
defp join_rowid_lookup_term(_table, _right_qualifier, _ltmpls, _term), do: nil
defp join_rowid_equality_candidate(table, right_qualifier, ltmpls, expr) do
if expr_references_right_table?(expr, right_qualifier, table) or
not lookup_value_resolvable?(expr, ltmpls) do
nil
else
{:eq, expr}
end
end
defp join_rowid_range_candidate(table, right_qualifier, ltmpls, op, expr) do
if join_rowid_lookup_expr_usable?(expr, table, right_qualifier, ltmpls) do
{:range, [{op, expr}]}
end
end
defp join_rowid_or_lookup_disjunct(table, right_qualifier, ltmpls, disjunct) do
case join_rowid_lookup_term(table, right_qualifier, ltmpls, disjunct) do
{:eq, expr} -> [expr]
{:in, exprs} -> exprs
_other -> []
end
end
defp join_rowid_lookup_expr_usable?(expr, table, right_qualifier, ltmpls),
do:
join_lookup_expr_usable?(expr, table, right_qualifier) and
lookup_value_resolvable?(expr, ltmpls)
defp join_lookup_expr_usable?(expr, table, right_qualifier),
do: not expr_references_right_table?(expr, right_qualifier, table)
# A per-left-row seek value must be evaluable against the left relation alone
# (the probe runs before the right row is fetched). When a comma join is
# reordered/split, a join predicate can correlate the right table with a table
# that is NOT yet in this left relation (e.g. `t31.rowid = t55.b55` while only
# t51/t29 are built); using it as a seek key would evaluate `t55.b55` against
# the wrong frames and raise "no such column". Requiring every referenced
# column to resolve in `ltmpls` declines those — the full WHERE is still
# applied downstream once every table is joined, so results are unchanged.
defp lookup_value_resolvable?(expr, ltmpls) do
expr
|> expr_column_refs([])
|> Enum.all?(&column_resolvable_in_templates?(&1, ltmpls))
end
defp column_resolvable_in_templates?({:column, nil, name}, ltmpls) do
key = Table.key(name)
Enum.any?(ltmpls, &visible?(&1, key)) or
(key in @rowid_names and Enum.any?(ltmpls, & &1.has_rowid))
end
defp column_resolvable_in_templates?({:column, qualifier, name}, ltmpls) do
qkey = Table.key(qualifier)
key = Table.key(name)
Enum.any?(ltmpls, fn t ->
t.name == qkey and (has_column?(t, key) or (key in @rowid_names and t.has_rowid))
end)
end
defp right_join_rowid_column?(%{without_rowid: true}, _right_qualifier, _ltmpls, _expr),
do: false
defp right_join_rowid_column?(table, right_qualifier, ltmpls, {:column, qualifier, name}) do
key = Table.key(name)
cond do
qualifier != nil ->
Table.key(qualifier) == right_qualifier and
rowid_column_ref?(table, {:column, nil, name})
table.rowid_alias == key ->
not Enum.any?(ltmpls, &visible?(&1, key))
key in @rowid_names ->
not Enum.any?(ltmpls, & &1.has_rowid)
true ->
false
end
end
defp right_join_rowid_column?(_table, _right_qualifier, _ltmpls, _expr), do: false
defp right_join_column(table, right_qualifier, ltmpls, {:column, qualifier, name}) do
key = Table.key(name)
cond do
qualifier != nil ->
if Table.key(qualifier) == right_qualifier and Table.column(table, key) != nil, do: key
Table.column(table, key) != nil and not Enum.any?(ltmpls, &visible?(&1, key)) ->
key
true ->
nil
end
end
defp right_join_column(_table, _right_qualifier, _ltmpls, _expr), do: nil
defp expr_references_right_table?({:column, qualifier, name}, right_qualifier, table) do
cond do
qualifier != nil -> Table.key(qualifier) == right_qualifier
Table.column(table, name) -> true
true -> false
end
end
defp expr_references_right_table?(expr, right_qualifier, table) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.any?(&expr_references_right_table?(&1, right_qualifier, table))
end
defp expr_references_right_table?(expr, right_qualifier, table) when is_list(expr),
do: Enum.any?(expr, &expr_references_right_table?(&1, right_qualifier, table))
defp expr_references_right_table?(_expr, _right_qualifier, _table), do: false
defp planned_index_lookup(table, where) do
case table_access_path(nil, table, where) do
{:index_eq, index, n_columns} ->
with true <- index_lookup_usable?(table, index, n_columns, where),
{:ok, values} <- index_lookup_values(table, index, n_columns, where) do
{:ok, index, {:eq, values}}
end
{:index_range, index, bounds} ->
with true <- index_range_lookup_usable?(table, index, where),
{:ok, bounds} <- index_range_lookup_bounds(table, index, bounds) do
{:ok, index, {:range, bounds}}
end
{:index_in, index, exprs} ->
with true <- index_in_lookup_usable?(table, index, where),
{:ok, values} <- index_in_lookup_values(table, index, exprs) do
{:ok, index, {:in, values}}
end
{:expr_index_eq, index, _expr, {:literal, value}} ->
with true <- expression_index_lookup_usable?(index, where) do
{:ok, index, {:eq, [value]}}
end
{:expr_index_in, index, _expr, exprs} ->
with true <- expression_index_lookup_usable?(index, where),
{:ok, values} <- expression_index_in_lookup_values(exprs) do
{:ok, index, {:in, values}}
end
{:expr_index_or, index, _expr, values} ->
with true <- expression_index_lookup_usable?(index, where) do
{:ok, index, {:in, values}}
end
{:expr_index_range, index, _expr, bounds} ->
with true <- expression_index_lookup_usable?(index, where),
{:ok, bounds} <- expression_index_range_lookup_bounds(bounds) do
{:ok, index, {:range, bounds}}
end
{:index_member_eq, index, prefix} ->
with true <- member_index_lookup_usable?(table, index, prefix, where),
{:ok, values} <- index_member_lookup_values(table, prefix) do
{:ok, index, {:eq, values}}
end
{:index_member_range, index, prefix, range_member, bounds} ->
with true <- member_index_lookup_usable?(table, index, prefix, where),
{:ok, prefix_values} <- index_member_lookup_values(table, prefix),
{:ok, bounds} <- index_member_range_lookup_bounds(table, range_member, bounds) do
{:ok, index, {:member_range, prefix_values, bounds}}
end
{:index_member_in, index, prefix, in_member, exprs} ->
with true <- member_index_lookup_usable?(table, index, prefix, where),
{:ok, prefix_values} <- index_member_lookup_values(table, prefix),
{:ok, values} <- index_member_in_lookup_values(table, in_member, exprs) do
{:ok, index, {:member_in, prefix_values, values}}
end
_other ->
:error
end
end
# The table itself — not shadowed by a CTE or one of the virtual tables.
defp plain_table(db, key) do
if Map.has_key?(db.ctes, key) or Map.has_key?(db.pending_ctes, key) or
key in ["sqlite_schema", "sqlite_master", "sqlite_sequence"] do
nil
else
Map.get(db.tables, key)
end
end
# How a WHERE clause can drive access to `table`:
# `{:rowid_eq, expr}`, `{:index_eq, index, n_columns}`,
# `{:index_range, index, op_text}`, or `:scan`.
# Each access-path analysis is computed lazily inside its `cond` branch and
# short-circuits at the first match, rather than eagerly computing all ~15
# (several of which traverse the table's indexes) for every query. `eq_keys`
# stays eager — it's cheap (a conjunct scan) and the common `index_eq` branch
# needs it in both its guard and body.
defp table_access_path(_db, table, where) do
conjuncts = where_conjuncts(where)
eq_keys = equality_keys(table, conjuncts)
cond do
(rowid_eq = rowid_equality_constraint(table, conjuncts)) != nil ->
{:rowid_eq, rowid_eq}
(rowid_in = rowid_in_constraint(table, conjuncts)) != nil ->
{:rowid_in, rowid_in}
(rowid_or = rowid_or_literal_constraint(table, where)) != nil ->
{:rowid_in, rowid_or}
(rowid_range = rowid_range_constraints(table, conjuncts)) != nil ->
{:rowid_range, rowid_range}
(member_in = best_member_in_index(table, conjuncts)) != nil ->
{index, prefix, in_member, exprs} = member_in
{:index_member_in, index, prefix, in_member, exprs}
(member_range = best_member_range_index(table, conjuncts)) != nil ->
{index, prefix, range_member, bounds} = member_range
{:index_member_range, index, prefix, range_member, bounds}
index = best_equality_index(table, eq_keys, where) ->
prefix = Enum.take_while(index.columns, &MapSet.member?(eq_keys, &1))
{:index_eq, index, length(prefix)}
result = range_access_path(table, conjuncts, where) ->
result
result = in_access_path(table, conjuncts, where) ->
result
result = or_access_path(table, where) ->
result
(expression_eq = best_expression_equality_index(table, conjuncts)) != nil ->
{index, expr, value} = expression_eq
{:expr_index_eq, index, expr, value}
(expression_in = best_expression_in_index(table, conjuncts)) != nil ->
{index, expr, values} = expression_in
{:expr_index_in, index, expr, values}
(expression_or = best_expression_or_literal_index(table, conjuncts)) != nil ->
{index, expr, values} = expression_or
{:expr_index_or, index, expr, values}
(expression_range = best_expression_range_index(table, conjuncts)) != nil ->
{index, expr, bounds} = expression_range
{:expr_index_range, index, expr, bounds}
(member_eq = best_member_equality_index(table, conjuncts)) != nil ->
{index, prefix} = member_eq
{:index_member_eq, index, prefix}
true ->
:scan
end
end
defp equality_keys(table, conjuncts) do
Enum.reduce(conjuncts, MapSet.new(), fn
{:binary, :eq, left, right}, acc ->
case equality_constraint(table, left, right) do
{key, _expr} -> MapSet.put(acc, key)
nil -> acc
end
_other, acc ->
acc
end)
end
defp range_access_path(table, conjuncts, where) do
constraints = range_constraints(table, conjuncts)
index =
Enum.find(lookup_indexes(table), fn index ->
Map.has_key?(constraints, List.first(index.columns)) and
index_range_lookup_usable?(table, index, where)
end)
if index, do: {:index_range, index, Map.fetch!(constraints, List.first(index.columns))}
end
defp in_access_path(table, conjuncts, where) do
constraints = in_constraints(table, conjuncts)
index =
Enum.find(lookup_indexes(table), fn index ->
Map.has_key?(constraints, List.first(index.columns)) and
index_in_lookup_usable?(table, index, where)
end)
if index, do: {:index_in, index, Map.fetch!(constraints, List.first(index.columns))}
end
defp or_access_path(table, where) do
constraints = or_literal_constraints(table, where)
index =
Enum.find(
lookup_indexes(table),
&(Map.has_key?(constraints, List.first(&1.columns)) and
index_in_lookup_usable?(table, &1, where))
)
if index, do: {:index_in, index, Map.fetch!(constraints, List.first(index.columns))}
end
defp rowid_equality_constraint(table, conjuncts) do
Enum.find_value(conjuncts, fn
{:binary, :eq, left, right} ->
cond do
rowid_column_ref?(table, left) and constant_expr?(right) -> right
rowid_column_ref?(table, right) and constant_expr?(left) -> left
true -> nil
end
_other ->
nil
end)
end
defp where_conjuncts(nil), do: []
defp where_conjuncts({:binary, :and, left, right}),
do: where_conjuncts(left) ++ where_conjuncts(right)
defp where_conjuncts(expr), do: [expr]
defp where_disjuncts({:binary, :or, left, right}),
do: where_disjuncts(left) ++ where_disjuncts(right)
defp where_disjuncts(expr), do: [expr]
defp multiple_terms?([_, _ | _]), do: true
defp multiple_terms?(_terms), do: false
defp range_constraints(table, conjuncts) do
Enum.reduce(conjuncts, %{}, fn
{:binary, op, left, right}, acc when op in [:lt, :le, :gt, :ge] ->
case range_constraint(table, left, right, op) do
{key, constraint} -> Map.update(acc, key, [constraint], &[constraint | &1])
nil -> acc
end
{:between, expr, low, high, false}, acc ->
case between_constraint(table, expr, low, high) do
{key, constraints} ->
reversed_constraints = Enum.reverse(constraints)
Map.update(acc, key, reversed_constraints, &(reversed_constraints ++ &1))
nil ->
acc
end
_other, acc ->
acc
end)
|> Map.new(fn {key, constraints} -> {key, Enum.reverse(constraints)} end)
end
defp in_constraints(table, conjuncts) do
Enum.reduce(conjuncts, %{}, fn
{:in, expr, list, false}, acc when is_list(list) ->
case in_constraint(table, expr, list) do
{key, exprs} -> Map.put(acc, key, exprs)
nil -> acc
end
_other, acc ->
acc
end)
end
defp or_literal_constraints(table, where) do
where
|> where_conjuncts()
|> Enum.reduce(%{}, fn term, acc ->
Map.merge(acc, or_literal_constraint_group(table, term))
end)
end
defp or_literal_constraint_group(table, term) do
terms = where_disjuncts(term)
with true <- length(terms) > 1,
constraints <- Enum.map(terms, &or_literal_constraint(table, &1)),
true <- Enum.all?(constraints, &match?({_, [_ | _]}, &1)),
[{key, _exprs} | _rest] <- constraints,
true <- Enum.all?(constraints, &(elem(&1, 0) == key)) do
%{key => Enum.flat_map(constraints, &elem(&1, 1))}
else
_other -> %{}
end
end
defp constant_expr?({:literal, _value}), do: true
defp constant_expr?({:collate, expr, _name}), do: constant_expr?(expr)
defp constant_expr?(_expr), do: false
defp rowid_column_ref?(table, {:column, nil, name}) do
key = Table.key(name)
key in @rowid_names or (table.rowid_alias != nil and key == table.rowid_alias)
end
defp rowid_column_ref?(_table, _expr), do: false
defp constrained_column_key(table, left, right) do
case equality_constraint(table, left, right) do
{key, _expr} -> key
nil -> nil
end
end
defp equality_constraint(table, left, right) do
left_base = strip_collation(left)
right_base = strip_collation(right)
cond do
match?({:column, nil, _}, left_base) and constant_expr?(right_base) ->
{:column, nil, name} = left_base
if key = column_key(table, name), do: {key, right_base}
match?({:column, nil, _}, right_base) and constant_expr?(left_base) ->
{:column, nil, name} = right_base
if key = column_key(table, name), do: {key, left_base}
true ->
nil
end
end
defp strip_collation({:collate, expr, _name}), do: strip_collation(expr)
defp strip_collation(expr), do: expr
defp range_constraint(table, left, right, op) do
left_base = strip_collation(left)
right_base = strip_collation(right)
cond do
match?({:column, nil, _}, left_base) and constant_expr?(right_base) ->
{:column, nil, name} = left_base
if key = column_key(table, name), do: {key, {op, right_base}}
match?({:column, nil, _}, right_base) and constant_expr?(left_base) ->
{:column, nil, name} = right_base
if key = column_key(table, name), do: {key, {flip_range_op(op), left_base}}
true ->
nil
end
end
defp between_constraint(table, expr, low, high) do
expr_base = strip_collation(expr)
low_base = strip_collation(low)
high_base = strip_collation(high)
with {:column, nil, name} <- expr_base,
key when not is_nil(key) <- column_key(table, name),
true <- constant_expr?(low_base),
true <- constant_expr?(high_base) do
{key, [{:ge, low_base}, {:le, high_base}]}
else
_other -> nil
end
end
defp flip_range_op(:lt), do: :gt
defp flip_range_op(:le), do: :ge
defp flip_range_op(:gt), do: :lt
defp flip_range_op(:ge), do: :le
defp flip_range_op(:eq), do: :eq
defp in_constraint(table, expr, list) do
expr = strip_collation(expr)
with {:column, nil, name} <- expr,
key when not is_nil(key) <- column_key(table, name),
true <- Enum.all?(list, &constant_expr?/1) do
{key, list}
else
_other -> nil
end
end
defp or_literal_constraint(table, {:binary, :eq, left, right}) do
case equality_constraint(table, left, right) do
nil ->
nil
{key, expr} ->
{key, [expr]}
end
end
defp or_literal_constraint(table, {:in, expr, list, false}) when is_list(list) do
case in_constraint(table, expr, list) do
nil -> nil
{key, exprs} -> {key, exprs}
end
end
defp or_literal_constraint(_table, _expr), do: nil
defp column_key(table, name) do
if Table.column(table, name), do: Table.key(name)
end
defp index_lookup_usable?(table, index, n_columns, where) do
prefix_columns = Enum.take(index.columns, n_columns)
prefix_collations = Enum.take(Map.get(index, :collations) || [], n_columns)
index_predicate_usable?(index, where) and index.columns != [] and
equality_terms_match_index_collations?(table, prefix_columns, prefix_collations, where)
end
defp index_range_lookup_usable?(table, index, where) do
first_key = List.first(index.columns)
index_collation = List.first(Map.get(index, :collations) || [])
index_predicate_usable?(index, where) and index.columns != [] and
range_terms_match_index_collation?(table, first_key, index_collation, where)
end
defp index_in_lookup_usable?(table, index, where) do
first_key = List.first(index.columns)
index_collation = List.first(Map.get(index, :collations) || [])
index_predicate_usable?(index, where) and index.columns != [] and
(in_terms_match_index_collation?(table, first_key, index_collation, where) or
or_literal_terms_match_index_collation?(table, first_key, index_collation, where))
end
defp expression_index_lookup_usable?(index, where) do
index_predicate_usable?(index, where) and match?([{:expr, _expr}], index_members(index))
end
defp member_index_lookup_usable?(table, index, prefix, where),
do:
index_predicate_usable?(index, where) and
member_index_collations_compatible?(table, index, prefix)
defp member_index_collations_compatible?(table, index, prefix) do
collations = Map.get(index, :collations) || []
prefix
|> Enum.zip(collations)
|> Enum.all?(fn
{{{:column, key}, _expr}, collation} -> collation == column_collation_name(table, key)
{{{:expr, _indexed_expr}, _expr}, _collation} -> true
end)
end
defp index_predicate_usable?(%{where: nil}, _where), do: true
defp index_predicate_usable?(%{where: index_where}, query_where) do
query_terms = where_conjuncts(query_where)
index_disjuncts = where_disjuncts(index_where)
if length(index_disjuncts) > 1 do
query_terms_imply_index_or?(query_terms, index_disjuncts)
else
index_where
|> where_conjuncts()
|> Enum.all?(&query_terms_imply_index_conjunct?(query_terms, &1))
end
end
defp query_terms_imply_index_term?(query_terms, index_term) do
index_term
|> where_conjuncts()
|> Enum.all?(&query_terms_imply_index_conjunct?(query_terms, &1))
end
defp query_terms_imply_index_conjunct?(query_terms, index_conjunct) do
index_disjuncts = where_disjuncts(index_conjunct)
if length(index_disjuncts) > 1 do
query_terms_imply_index_or?(query_terms, index_disjuncts)
else
Enum.any?(query_terms, &query_term_implies_index_term?(&1, index_conjunct))
end
end
defp query_terms_imply_index_or?(query_terms, index_disjuncts) do
Enum.any?(query_terms, &query_term_implies_index_or?(&1, index_disjuncts)) or
Enum.any?(index_disjuncts, fn index_term ->
query_terms_imply_index_term?(query_terms, index_term)
end)
end
defp query_term_implies_index_term?(query_term, index_term) do
query_term_implies_index_term?(query_term, index_term, :default)
end
defp query_term_implies_index_term?(
{:binary, :and, left, right},
index_term,
:default
)
when not (is_tuple(index_term) and tuple_size(index_term) == 4 and
elem(index_term, 0) == :binary and
(elem(index_term, 1) == :and or elem(index_term, 1) == :or)),
do:
query_term_implies_index_term?(left, index_term, :default) or
query_term_implies_index_term?(right, index_term, :default)
defp query_term_implies_index_term?(
{:binary, :or, left, right},
index_term,
:default
)
when not (is_tuple(index_term) and tuple_size(index_term) == 4 and
elem(index_term, 0) == :binary and
(elem(index_term, 1) == :and or elem(index_term, 1) == :or)),
do:
query_term_implies_index_term?(left, index_term, :default) and
query_term_implies_index_term?(right, index_term, :default)
defp query_term_implies_index_term?(
{:binary, :and, _left, _right} = query_term,
{:binary, :and, index_left, index_right},
_mode
),
do:
query_term_implies_index_term?(query_term, index_left) and
query_term_implies_index_term?(query_term, index_right)
defp query_term_implies_index_term?(
{:binary, :or, _left, _right} = query_term,
{:binary, :or, index_left, index_right},
_mode
),
do:
query_term_implies_index_term?(query_term, index_left) or
query_term_implies_index_term?(query_term, index_right)
defp query_term_implies_index_term?(query_term, index_term, :default) do
expression_equivalent?(query_term, index_term) or
range_term_implies_index_term?(query_term, index_term) or
not_null_predicate_implied?(query_term, index_term) or
range_predicate_implied?(query_term, index_term) or
in_predicate_implied?(query_term, index_term)
end
defp range_term_implies_index_term?(
{:binary, query_op, query_left, query_right},
{:binary, index_op, index_left, index_right}
)
when query_op in [:lt, :le, :gt, :ge] and index_op in [:lt, :le, :gt, :ge] do
cond do
expression_equivalent?(query_left, index_left) and
expression_equivalent?(query_right, index_right) ->
range_operator_implies?(query_op, index_op)
expression_equivalent?(query_left, index_right) and
expression_equivalent?(query_right, index_left) ->
query_op
|> flip_range_op()
|> range_operator_implies?(index_op)
true ->
false
end
end
defp range_term_implies_index_term?(_query_term, _index_term), do: false
defp range_operator_implies?(:gt, :gt), do: true
defp range_operator_implies?(:gt, :ge), do: true
defp range_operator_implies?(:ge, :ge), do: true
defp range_operator_implies?(:lt, :lt), do: true
defp range_operator_implies?(:lt, :le), do: true
defp range_operator_implies?(:le, :le), do: true
defp range_operator_implies?(_query_op, _index_op), do: false
defp query_term_implies_index_or?(query_term, index_disjuncts) do
query_term_implies_index_or?(query_term, index_disjuncts, :default)
end
defp query_term_implies_index_or?({:binary, :and, _, _} = query_term, index_disjuncts, _mode),
do: Enum.any?(index_disjuncts, &query_term_implies_index_term?(query_term, &1))
defp query_term_implies_index_or?({:binary, :or, left, right}, index_disjuncts, _mode),
do:
query_term_implies_index_or?(left, index_disjuncts) and
query_term_implies_index_or?(right, index_disjuncts)
defp query_term_implies_index_or?(query_term, index_disjuncts, :default) do
with {:ok, {index_expr, index_values}} <- or_literal_values(index_disjuncts),
{:ok, {query_expr, query_values}} <- in_implication_values(query_term),
true <- expression_equivalent?(query_expr, index_expr) do
Enum.all?(query_values, fn query_value ->
Enum.any?(index_values, &literal_values_equal?(query_value, &1))
end)
else
_other -> Enum.any?(index_disjuncts, &query_term_implies_index_term?(query_term, &1))
end
end
defp or_literal_values(index_disjuncts) do
index_disjuncts
|> Enum.reduce_while({:ok, nil, []}, fn
{:binary, :eq, left, {:literal, value}}, {:ok, nil, values} ->
{:cont, {:ok, left, [value | values]}}
{:binary, :eq, left, {:literal, value}}, {:ok, expr, values} ->
if expression_equivalent?(left, expr) do
{:cont, {:ok, expr, [value | values]}}
else
{:halt, :error}
end
{:binary, :eq, {:literal, value}, right}, {:ok, nil, values} ->
{:cont, {:ok, right, [value | values]}}
{:binary, :eq, {:literal, value}, right}, {:ok, expr, values} ->
if expression_equivalent?(right, expr) do
{:cont, {:ok, expr, [value | values]}}
else
{:halt, :error}
end
{:in, expr, list, false}, {:ok, nil, values} when is_list(list) ->
case literal_values(list) do
{:ok, literal_values} -> {:cont, {:ok, expr, Enum.reverse(literal_values) ++ values}}
:error -> {:halt, :error}
end
{:in, in_expr, list, false}, {:ok, expr, values} when is_list(list) ->
with true <- expression_equivalent?(in_expr, expr),
{:ok, literal_values} <- literal_values(list) do
{:cont, {:ok, expr, Enum.reverse(literal_values) ++ values}}
else
_other -> {:halt, :error}
end
_other, _acc ->
{:halt, :error}
end)
|> case do
{:ok, nil, _values} -> :error
{:ok, expr, values} -> {:ok, {expr, Enum.reverse(values)}}
:error -> :error
end
end
defp not_null_predicate_implied?(query_term, {:is_not, expr, {:literal, nil}}),
do: comparison_excludes_null?(query_term, expr)
defp not_null_predicate_implied?(_query_term, _index_term), do: false
defp literal_boolean_constraint({:not, expr}), do: {:ok, {expr, false}}
defp literal_boolean_constraint(_query_term), do: :error
defp comparison_excludes_null?({:not, {:is, query_expr, {:literal, nil}}}, expr),
do: expression_equivalent?(query_expr, expr)
defp comparison_excludes_null?({:not, {:is_not, _query_expr, {:literal, nil}}}, _expr),
do: false
defp comparison_excludes_null?({:binary, op, left, right}, expr)
when op in [:eq, :ne, :lt, :le, :gt, :ge],
do: expression_equivalent?(left, expr) or expression_equivalent?(right, expr)
defp comparison_excludes_null?({:between, between_expr, _low, _high, _negated}, expr),
do: expression_equivalent?(between_expr, expr)
defp comparison_excludes_null?({:in, in_expr, _source, _negated}, expr),
do: expression_equivalent?(in_expr, expr)
defp comparison_excludes_null?({:is, is_expr, {:literal, value}}, expr) when not is_nil(value),
do: expression_equivalent?(is_expr, expr)
defp comparison_excludes_null?({:is, {:literal, value}, is_expr}, expr) when not is_nil(value),
do: expression_equivalent?(is_expr, expr)
defp comparison_excludes_null?({:like, like_expr, _pattern, _escape, _negated}, expr),
do: expression_equivalent?(like_expr, expr)
defp comparison_excludes_null?({:glob, glob_expr, _pattern, _negated}, expr),
do: expression_equivalent?(glob_expr, expr)
defp comparison_excludes_null?({:regexp, regexp_expr, _pattern, _negated}, expr),
do: expression_equivalent?(regexp_expr, expr)
defp comparison_excludes_null?(query_term, expr) do
with {:ok, {query_expr, _truth}} <- literal_boolean_constraint(query_term) do
expression_equivalent?(query_expr, expr)
else
_other -> false
end
end
defp range_predicate_implied?(query_term, index_term) do
with {:ok, query_constraints} <- comparison_constraints(query_term),
{:ok, index_constraints} <- comparison_constraints(index_term) do
Enum.all?(index_constraints, fn index_constraint ->
Enum.any?(query_constraints, &comparison_constraint_implies?(&1, index_constraint))
end)
else
_other -> false
end
end
defp comparison_constraints({:binary, op, left, {:literal, value}})
when op in [:eq, :lt, :le, :gt, :ge],
do: {:ok, [{left, op, value}]}
defp comparison_constraints({:binary, op, {:literal, value}, right})
when op in [:eq, :lt, :le, :gt, :ge],
do: {:ok, [{right, flip_range_op(op), value}]}
defp comparison_constraints({:between, expr, {:literal, low}, {:literal, high}, false}),
do: {:ok, [{expr, :ge, low}, {expr, :le, high}]}
defp comparison_constraints(_term), do: :error
defp comparison_constraint_implies?(
{query_expr, query_op, query_value},
{index_expr, index_op, index_value}
) do
expression_equivalent?(query_expr, index_expr) and
comparable_implication_values?(query_value, index_value) and
comparison_constraint_implies?(query_op, query_value, index_op, index_value)
end
defp comparable_implication_values?(left, right) when is_number(left) and is_number(right),
do: true
defp comparable_implication_values?(left, right) when is_binary(left) and is_binary(right),
do: true
defp comparable_implication_values?(_left, _right), do: false
defp comparison_constraint_implies?(:eq, query_value, index_op, index_value),
do: Value.compare_op(index_op, query_value, index_value) == true
defp comparison_constraint_implies?(query_op, query_value, index_op, index_value)
when query_op in [:gt, :ge] and index_op in [:gt, :ge],
do: lower_bound_implies?(query_op, Value.compare(query_value, index_value), index_op)
defp comparison_constraint_implies?(query_op, query_value, index_op, index_value)
when query_op in [:lt, :le] and index_op in [:lt, :le],
do: upper_bound_implies?(query_op, Value.compare(query_value, index_value), index_op)
defp comparison_constraint_implies?(_query_op, _query_value, _index_op, _index_value), do: false
defp lower_bound_implies?(:gt, cmp, :gt), do: cmp in [:gt, :eq]
defp lower_bound_implies?(:ge, :gt, :gt), do: true
defp lower_bound_implies?(:gt, cmp, :ge), do: cmp in [:gt, :eq]
defp lower_bound_implies?(:ge, cmp, :ge), do: cmp in [:gt, :eq]
defp lower_bound_implies?(_query_op, _cmp, _index_op), do: false
defp upper_bound_implies?(:lt, cmp, :lt), do: cmp in [:lt, :eq]
defp upper_bound_implies?(:le, :lt, :lt), do: true
defp upper_bound_implies?(:lt, cmp, :le), do: cmp in [:lt, :eq]
defp upper_bound_implies?(:le, cmp, :le), do: cmp in [:lt, :eq]
defp upper_bound_implies?(_query_op, _cmp, _index_op), do: false
defp in_predicate_implied?(query_term, {:in, index_expr, index_list, false})
when is_list(index_list) do
with {:ok, index_values} <- literal_values(index_list),
{:ok, {query_expr, query_values}} <- in_implication_values(query_term),
true <- expression_equivalent?(query_expr, index_expr) do
Enum.all?(query_values, fn query_value ->
Enum.any?(index_values, &literal_values_equal?(query_value, &1))
end)
else
_other -> false
end
end
defp in_predicate_implied?(_query_term, _index_term), do: false
defp in_implication_values({:binary, :eq, left, {:literal, value}}),
do: {:ok, {left, [value]}}
defp in_implication_values({:binary, :eq, {:literal, value}, right}),
do: {:ok, {right, [value]}}
defp in_implication_values({:in, expr, list, false}) when is_list(list) do
case literal_values(list) do
{:ok, values} -> {:ok, {expr, values}}
:error -> :error
end
end
defp in_implication_values(_term), do: :error
defp literal_values(list) do
list
|> Enum.reduce_while({:ok, []}, fn
{:literal, value}, {:ok, values} -> {:cont, {:ok, [value | values]}}
_expr, _acc -> {:halt, :error}
end)
|> case do
{:ok, values} -> {:ok, Enum.reverse(values)}
:error -> :error
end
end
defp literal_values_equal?(left, right),
do: Value.compare_op(:eq, left, right) == true
defp range_terms_match_index_collation?(table, key, index_collation, where) do
terms =
where
|> where_conjuncts()
|> Enum.flat_map(fn
{:binary, op, left, right} when op in [:lt, :le, :gt, :ge] ->
case range_constraint(table, left, right, op) do
{^key, _constraint} -> [{left, right}]
_other -> []
end
{:between, expr, low, high, false} ->
case between_constraint(table, expr, low, high) do
{^key, _constraints} -> [{expr, low}, {expr, high}]
_other -> []
end
_other ->
[]
end)
terms != [] and
Enum.all?(terms, fn {left, right} ->
collation_names_equal?(range_term_collation(table, key, left, right), index_collation)
end)
end
defp range_term_collation(table, key, left, right) do
explicit_collation_name(left) || explicit_collation_name(right) ||
column_collation_name(table, key) || :binary
end
defp in_terms_match_index_collation?(table, key, index_collation, where) do
terms =
where
|> where_conjuncts()
|> Enum.flat_map(fn
{:in, expr, list, false} when is_list(list) ->
case in_constraint(table, expr, list) do
{^key, _exprs} -> [{expr, list}]
_other -> []
end
_other ->
[]
end)
terms != [] and
Enum.all?(terms, fn {expr, list} ->
not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation)
end)
end
defp in_term_collation(table, key, expr),
do: explicit_collation_name(expr) || column_collation_name(table, key) || :binary
defp or_literal_terms_match_index_collation?(table, key, index_collation, where) do
terms =
where
|> where_conjuncts()
|> Enum.flat_map(fn term ->
disjuncts = where_disjuncts(term)
if multiple_terms?(disjuncts) do
Enum.flat_map(disjuncts, fn
{:binary, :eq, left, right} ->
case equality_constraint(table, left, right) do
{^key, _expr} -> [{:eq, left, right}]
_other -> []
end
{:in, expr, list, false} when is_list(list) ->
case in_constraint(table, expr, list) do
{^key, _exprs} -> [{:in, expr, list}]
_other -> []
end
_other ->
[]
end)
else
[]
end
end)
terms != [] and
Enum.all?(terms, fn
{:eq, left, right} ->
collation_names_equal?(
equality_term_collation(table, key, left, right),
index_collation
)
{:in, expr, list} ->
not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation)
end)
end
defp equality_terms_match_index_collations?(table, keys, collations, where) do
keys
|> Enum.zip(collations)
|> Enum.all?(fn {key, index_collation} ->
terms = equality_terms_for_key(table, key, where)
terms != [] and
Enum.all?(terms, fn {_expr, term_collation} ->
collation_names_equal?(term_collation, index_collation)
end)
end)
end
defp equality_terms_for_key(table, key, where) do
where
|> where_conjuncts()
|> Enum.flat_map(fn
{:binary, :eq, left, right} ->
case equality_constraint(table, left, right) do
{^key, expr} -> [{expr, equality_term_collation(table, key, left, right)}]
_other -> []
end
_other ->
[]
end)
end
defp equality_term_collation(table, key, left, right) do
explicit_collation_name(left) || explicit_collation_name(right) ||
column_collation_name(table, key) || :binary
end
defp explicit_collation_name({:collate, _expr, name}), do: name
defp explicit_collation_name(expr) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.find_value(&explicit_collation_name/1)
end
defp explicit_collation_name(expr) when is_list(expr),
do: Enum.find_value(expr, &explicit_collation_name/1)
defp explicit_collation_name(_expr), do: nil
defp collation_names_equal?(left, right),
do: normalize_collation_name(left) == normalize_collation_name(right)
defp normalize_collation_name(nil), do: "binary"
defp normalize_collation_name(:binary), do: "binary"
defp normalize_collation_name(name) when is_binary(name), do: String.downcase(name)
defp normalize_collation_name(name), do: name
defp explicit_collation_node?({:collate, _expr, _name}), do: true
defp explicit_collation_node?(expr) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.any?(&explicit_collation_node?/1)
end
defp explicit_collation_node?(expr) when is_list(expr),
do: Enum.any?(expr, &explicit_collation_node?/1)
defp explicit_collation_node?(_expr), do: false
defp best_expression_equality_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
case index_members(index) do
[{:expr, indexed_expr}] ->
expression_equality_constraint(index, indexed_expr, conjuncts)
_other ->
nil
end
end)
end
defp best_member_equality_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
members = index_members(index)
cond do
members == [] ->
nil
Enum.all?(members, &match?({:column, _key}, &1)) ->
nil
true ->
case member_equality_prefix(table, members, conjuncts) do
[] -> nil
prefix -> {index, prefix}
end
end
end)
end
defp member_equality_prefix(table, members, conjuncts) do
members
|> Enum.reduce_while([], fn member, acc ->
case member_equality_constraint(table, member, conjuncts) do
{:ok, expr} -> {:cont, [{member, expr} | acc]}
:error -> {:halt, acc}
end
end)
|> Enum.reverse()
end
defp best_member_range_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
members = index_members(index)
cond do
members == [] ->
nil
true ->
prefix = member_equality_prefix(table, members, conjuncts)
range_member = Enum.at(members, length(prefix))
cond do
prefix == [] or is_nil(range_member) ->
nil
bounds = member_range_constraints(table, index, range_member, conjuncts) ->
{index, prefix, range_member, bounds}
true ->
nil
end
end
end)
end
defp best_member_in_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
members = index_members(index)
cond do
members == [] ->
nil
true ->
prefix = member_equality_prefix(table, members, conjuncts)
in_member = Enum.at(members, length(prefix))
cond do
prefix == [] or is_nil(in_member) ->
nil
exprs = member_in_constraint(table, index, in_member, conjuncts) ->
{index, prefix, in_member, exprs}
true ->
nil
end
end
end)
end
defp member_equality_constraint(table, {:column, key}, conjuncts) do
Enum.find_value(conjuncts, :error, fn
{:binary, :eq, left, right} ->
constrained_key = constrained_column_key(table, left, right)
cond do
constrained_key != key ->
nil
explicit_collation_node?(left) or explicit_collation_node?(right) ->
nil
match?({:column, nil, _}, left) ->
{:ok, right}
true ->
{:ok, left}
end
_other ->
nil
end)
end
defp member_equality_constraint(_table, {:expr, indexed_expr}, conjuncts) do
Enum.find_value(conjuncts, :error, fn
{:binary, :eq, left, right} ->
cond do
expression_equivalent?(left, indexed_expr) and constant_expr?(right) and
not explicit_collation_node?(right) ->
{:ok, right}
expression_equivalent?(right, indexed_expr) and constant_expr?(left) and
not explicit_collation_node?(left) ->
{:ok, left}
true ->
nil
end
_other ->
nil
end)
end
defp member_range_constraints(table, index, {:column, key}, conjuncts) do
members = index_members(index)
member_index = Enum.find_index(members, &(&1 == {:column, key}))
index_collation = Enum.at(Map.get(index, :collations) || [], member_index || 0)
conjuncts
|> Enum.flat_map(fn
{:binary, op, left, right} when op in [:lt, :le, :gt, :ge] ->
case range_constraint(table, left, right, op) do
{^key, bound} ->
if collation_names_equal?(
range_term_collation(table, key, left, right),
index_collation
) do
[bound]
else
[]
end
_other ->
[]
end
{:between, expr, low, high, false} ->
case between_range_constraint(table, key, expr, low, high, index_collation) do
nil -> []
bounds -> bounds
end
_other ->
[]
end)
|> case do
[] -> nil
bounds -> bounds
end
end
defp member_range_constraints(_table, _index, {:expr, indexed_expr}, conjuncts) do
case expression_range_constraints(indexed_expr, conjuncts) do
[] -> nil
bounds -> bounds
end
end
defp member_range_constraints(_table, _index, _member, _conjuncts), do: nil
defp between_range_constraint(table, key, expr, low, high, index_collation) do
expr_base = strip_collation(expr)
low_base = strip_collation(low)
high_base = strip_collation(high)
with {:column, nil, name} <- expr_base,
^key <- column_key(table, name),
true <- constant_expr?(low_base),
true <- constant_expr?(high_base),
true <-
collation_names_equal?(range_term_collation(table, key, expr, low), index_collation),
true <-
collation_names_equal?(range_term_collation(table, key, expr, high), index_collation) do
[{:ge, low_base}, {:le, high_base}]
else
_other -> nil
end
end
defp member_in_constraint(table, index, {:column, key}, conjuncts) do
index_collation = index_member_collation(index, {:column, key})
Enum.find_value(conjuncts, fn
{:in, expr, list, false} when is_list(list) ->
case in_constraint(table, expr, list) do
{^key, exprs} ->
if not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation) do
exprs
end
_other ->
nil
end
term ->
member_or_literal_constraint(term, table, key, index_collation)
end)
end
defp member_in_constraint(_table, _index, {:expr, indexed_expr}, conjuncts) do
Enum.find_value(conjuncts, fn term ->
case term do
{:in, expr, list, false} when is_list(list) ->
if expression_equivalent?(expr, indexed_expr) and not explicit_collation_node?(list) and
Enum.all?(list, &constant_expr?/1) do
list
end
_other ->
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
{:ok, {query_expr, values}} <- or_literal_values(disjuncts),
true <- expression_equivalent?(query_expr, indexed_expr) do
Enum.map(values, &{:literal, &1})
else
_other -> nil
end
end
end)
end
defp member_in_constraint(_table, _index, _member, _conjuncts), do: nil
defp member_or_literal_constraint(term, table, key, index_collation) do
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
constraints <-
Enum.map(disjuncts, &member_or_literal_disjunct(table, key, index_collation, &1)),
true <- Enum.all?(constraints, &match?([_ | _], &1)) do
Enum.flat_map(constraints, & &1)
else
_other -> nil
end
end
defp member_or_literal_disjunct(table, key, index_collation, {:binary, :eq, left, right}) do
case equality_constraint(table, left, right) do
{^key, expr} ->
if collation_names_equal?(
equality_term_collation(table, key, left, right),
index_collation
) do
[expr]
end
_other ->
nil
end
end
defp member_or_literal_disjunct(table, key, index_collation, {:in, expr, list, false})
when is_list(list) do
case in_constraint(table, expr, list) do
{^key, exprs} ->
if not explicit_collation_node?(list) and
collation_names_equal?(in_term_collation(table, key, expr), index_collation) do
exprs
end
_other ->
nil
end
end
defp member_or_literal_disjunct(_table, _key, _index_collation, _term), do: nil
defp expression_equality_constraint(index, indexed_expr, conjuncts) do
Enum.find_value(conjuncts, fn
{:binary, :eq, left, right} ->
cond do
expression_equivalent?(left, indexed_expr) and constant_expr?(right) and
not explicit_collation_node?(right) ->
{index, indexed_expr, right}
expression_equivalent?(right, indexed_expr) and constant_expr?(left) and
not explicit_collation_node?(left) ->
{index, indexed_expr, left}
true ->
nil
end
_other ->
nil
end)
end
defp best_expression_in_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
case index_members(index) do
[{:expr, indexed_expr}] ->
expression_in_constraint(index, indexed_expr, conjuncts)
_other ->
nil
end
end)
end
defp expression_in_constraint(index, indexed_expr, conjuncts) do
Enum.find_value(conjuncts, fn
{:in, expr, list, false} when is_list(list) ->
if expression_equivalent?(expr, indexed_expr) and not explicit_collation_node?(list) and
Enum.all?(list, &constant_expr?/1) do
{index, indexed_expr, list}
end
_other ->
nil
end)
end
defp best_expression_or_literal_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
case index_members(index) do
[{:expr, indexed_expr}] ->
expression_or_literal_constraint(index, indexed_expr, conjuncts)
_other ->
nil
end
end)
end
defp expression_or_literal_constraint(index, indexed_expr, conjuncts) do
Enum.find_value(conjuncts, fn term ->
disjuncts = where_disjuncts(term)
with true <- multiple_terms?(disjuncts),
{:ok, {query_expr, values}} <- or_literal_values(disjuncts),
true <- expression_equivalent?(query_expr, indexed_expr) do
{index, indexed_expr, values}
else
_other -> nil
end
end)
end
defp best_expression_range_index(table, conjuncts) do
lookup_indexes(table)
|> Enum.find_value(fn index ->
case index_members(index) do
[{:expr, indexed_expr}] ->
case expression_range_constraints(indexed_expr, conjuncts) do
[] -> nil
bounds -> {index, indexed_expr, bounds}
end
_other ->
nil
end
end)
end
defp expression_range_constraints(indexed_expr, conjuncts) do
conjuncts
|> Enum.flat_map(fn
{:binary, op, left, {:literal, _value} = right} when op in [:lt, :le, :gt, :ge] ->
if expression_equivalent?(left, indexed_expr) and not explicit_collation_node?(right) do
[{op, right}]
else
[]
end
{:binary, op, {:literal, _value} = left, right} when op in [:lt, :le, :gt, :ge] ->
if expression_equivalent?(right, indexed_expr) and not explicit_collation_node?(left) do
[{flip_range_op(op), left}]
else
[]
end
{:between, expr, {:literal, _low} = low, {:literal, _high} = high, false} ->
if expression_equivalent?(expr, indexed_expr) do
[{:ge, low}, {:le, high}]
else
[]
end
_other ->
[]
end)
end
defp expression_equivalent?({:binary, :eq, left_a, right_a}, {:binary, :eq, left_b, right_b}) do
equivalent_binary_terms?(left_a, right_a, left_b, right_b)
end
defp expression_equivalent?({:binary, :ne, left_a, right_a}, {:binary, :ne, left_b, right_b}) do
equivalent_binary_terms?(left_a, right_a, left_b, right_b)
end
defp expression_equivalent?({:is, left_a, right_a}, {:is, left_b, right_b}) do
equivalent_binary_terms?(left_a, right_a, left_b, right_b)
end
defp expression_equivalent?({:is_not, left_a, right_a}, {:is_not, left_b, right_b}) do
equivalent_binary_terms?(left_a, right_a, left_b, right_b)
end
defp expression_equivalent?(left, right),
do: normalize_index_expr(left) == normalize_index_expr(right)
defp equivalent_binary_terms?(left_a, right_a, left_b, right_b) do
normalized_left_a = normalize_index_expr(left_a)
normalized_right_a = normalize_index_expr(right_a)
normalized_left_b = normalize_index_expr(left_b)
normalized_right_b = normalize_index_expr(right_b)
(normalized_left_a == normalized_left_b and normalized_right_a == normalized_right_b) or
(normalized_left_a == normalized_right_b and normalized_right_a == normalized_left_b)
end
defp normalize_index_expr({:column, qualifier, name}) do
{:column, normalize_identifier(qualifier), Table.key(name)}
end
defp normalize_index_expr({:function, name, args}) when is_list(args) do
{:function, Table.key(name), Enum.map(args, &normalize_index_expr/1)}
end
defp normalize_index_expr({:function, name, {:distinct, args}}) do
{:function, Table.key(name), {:distinct, Enum.map(args, &normalize_index_expr/1)}}
end
defp normalize_index_expr(tuple) when is_tuple(tuple) do
tuple
|> Tuple.to_list()
|> Enum.map(&normalize_index_expr/1)
|> List.to_tuple()
end
defp normalize_index_expr(list) when is_list(list), do: Enum.map(list, &normalize_index_expr/1)
defp normalize_index_expr(other), do: other
defp normalize_identifier(nil), do: nil
defp normalize_identifier(name), do: Table.key(name)
defp index_lookup_values(table, index, n_columns, where) do
constraints = equality_constraints(table, where)
values =
index.columns
|> Enum.take(n_columns)
|> Enum.map(fn key ->
case Map.fetch(constraints, key) do
{:ok, {:literal, value}} ->
column = Table.column(table, key)
Value.apply_affinity(value, column.affinity)
:error ->
:missing
end
end)
if :missing in values, do: :error, else: {:ok, values}
end
defp index_member_lookup_values(table, prefix) do
prefix
|> Enum.reduce_while({:ok, []}, fn {member, expr}, {:ok, acc} ->
case index_member_lookup_value(table, member, expr) do
{:ok, value} -> {:cont, {:ok, [value | acc]}}
:error -> {:halt, :error}
end
end)
|> case do
{:ok, values} -> {:ok, Enum.reverse(values)}
:error -> :error
end
end
defp index_member_lookup_value(table, {:column, key}, {:literal, value}) do
column = Table.column(table, key)
{:ok, Value.apply_affinity(value, column.affinity)}
end
defp index_member_lookup_value(_table, {:expr, _expr}, {:literal, value}), do: {:ok, value}
defp index_member_lookup_value(_table, _member, _expr), do: :error
defp index_member_range_lookup_bounds(table, member, bounds) do
bounds
|> Enum.reduce_while({:ok, []}, fn {op, expr}, {:ok, acc} ->
case index_member_lookup_value(table, member, expr) do
{:ok, value} -> {:cont, {:ok, [{op, value} | acc]}}
:error -> {:halt, :error}
end
end)
|> case do
{:ok, bounds} -> {:ok, Enum.reverse(bounds)}
:error -> :error
end
end
defp index_member_in_lookup_values(table, member, exprs) do
exprs
|> Enum.reduce_while({:ok, []}, fn expr, {:ok, acc} ->
case index_member_lookup_value(table, member, expr) do
{:ok, value} -> {:cont, {:ok, [value | acc]}}
:error -> {:halt, :error}
end
end)
|> case do
{:ok, values} -> {:ok, Enum.reverse(values)}
:error -> :error
end
end
defp expression_index_in_lookup_values(exprs) do
exprs
|> Enum.reduce_while({:ok, []}, fn
{:literal, value}, {:ok, acc} -> {:cont, {:ok, [value | acc]}}
_expr, _acc -> {:halt, :error}
end)
|> case do
{:ok, values} -> {:ok, Enum.reverse(values)}
:error -> :error
end
end
defp expression_index_range_lookup_bounds(bounds) do
bounds
|> Enum.reduce_while({:ok, []}, fn
{op, {:literal, value}}, {:ok, acc} -> {:cont, {:ok, [{op, value} | acc]}}
_bound, _acc -> {:halt, :error}
end)
|> case do
{:ok, bounds} -> {:ok, Enum.reverse(bounds)}
:error -> :error
end
end
defp equality_constraints(table, where) do
where
|> where_conjuncts()
|> Enum.reduce(%{}, fn
{:binary, :eq, left, right}, acc ->
case equality_constraint(table, left, right) do
nil ->
acc
{key, expr} ->
Map.put(acc, key, expr)
end
_other, acc ->
acc
end)
end
defp index_range_lookup_bounds(table, index, bounds) do
bounds
|> Enum.reduce_while({:ok, []}, fn {op, expr}, {:ok, acc} ->
case index_range_lookup_value(table, index, expr) do
{:ok, value} -> {:cont, {:ok, [{op, value} | acc]}}
:error -> {:halt, :error}
end
end)
|> case do
{:ok, bounds} -> {:ok, Enum.reverse(bounds)}
:error -> :error
end
end
defp index_range_lookup_value(table, index, {:literal, value}) do
key = List.first(index.columns)
column = Table.column(table, key)
{:ok, Value.apply_affinity(value, column.affinity)}
end
defp index_range_lookup_value(_table, _index, _expr), do: :error
defp index_in_lookup_values(table, index, exprs) do
exprs
|> Enum.reduce_while({:ok, []}, fn expr, {:ok, acc} ->
case index_range_lookup_value(table, index, expr) do
{:ok, value} -> {:cont, {:ok, [value | acc]}}
:error -> {:halt, :error}
end
end)
|> case do
{:ok, values} -> {:ok, Enum.reverse(values)}
:error -> :error
end
end
defp index_lookup_rowids(db, index, {:eq, values}) do
prefix_count = length(values)
collations = Map.get(index, :collations, []) |> Enum.take(prefix_count)
if direct_index_lookup?(index, values, collations) do
index
|> Map.get(:entries, %{})
|> Map.get(List.to_tuple(values), [])
|> Enum.sort()
else
index
|> Map.get(:entries, %{})
|> Enum.flat_map(fn {stored_values, rowids} ->
stored_prefix = stored_values |> Tuple.to_list() |> Enum.take(prefix_count)
if values_equal_with_collations?(db, stored_prefix, values, collations) do
rowids
else
[]
end
end)
|> Enum.uniq()
|> Enum.sort()
end
end
defp index_lookup_rowids(db, index, {:in, values}) do
values
|> Enum.flat_map(&index_lookup_rowids(db, index, {:eq, [&1]}))
|> Enum.uniq()
|> Enum.sort()
end
defp index_lookup_rowids(db, index, {:range, bounds}) do
case ordered_binary_range_rowids(index, bounds, 0) do
{:ok, rowids} ->
rowids
:error ->
collation = List.first(Map.get(index, :collations) || []) || :binary
index
|> Map.get(:entries, %{})
|> Enum.flat_map(fn {stored_values, rowids} ->
stored_value = stored_values |> Tuple.to_list() |> List.first()
if index_range_match?(db, stored_value, bounds, collation) do
rowids
else
[]
end
end)
|> Enum.uniq()
|> Enum.sort()
end
end
defp index_lookup_rowids(db, index, {:member_range, prefix_values, bounds}) do
prefix_count = length(prefix_values)
collations = Map.get(index, :collations, [])
prefix_collations = Enum.take(collations, prefix_count)
range_collation = Enum.at(collations, prefix_count) || :binary
index
|> Map.get(:entries, %{})
|> Enum.flat_map(fn {stored_values, rowids} ->
stored_values = Tuple.to_list(stored_values)
stored_prefix = Enum.take(stored_values, prefix_count)
stored_range_value = Enum.at(stored_values, prefix_count)
if values_equal_with_collations?(db, stored_prefix, prefix_values, prefix_collations) and
index_range_match?(db, stored_range_value, bounds, range_collation) do
rowids
else
[]
end
end)
|> Enum.uniq()
|> Enum.sort()
end
defp index_lookup_rowids(db, index, {:member_in, prefix_values, values}) do
values
|> Enum.flat_map(fn value ->
index_lookup_rowids(db, index, {:eq, prefix_values ++ [value]})
end)
|> Enum.uniq()
|> Enum.sort()
end
defp ordered_binary_range_rowids(index, bounds, member_index) do
with true <- binary_collation_index?(index),
ordered when is_tuple(ordered) <- Map.get(index, :ordered_entries),
{:ok, lower, upper} <- index_range_limits(bounds) do
start = ordered_range_start(ordered, lower, member_index)
ordered
|> collect_ordered_range(start, tuple_size(ordered), upper, member_index, [])
|> Enum.uniq()
|> Enum.sort()
|> then(&{:ok, &1})
else
_other -> :error
end
end
defp index_range_limits(bounds) do
Enum.reduce_while(bounds, {:ok, nil, nil}, fn
{_op, nil}, _acc ->
{:halt, :error}
{op, value}, {:ok, lower, upper} when op in [:gt, :ge] ->
{:cont, {:ok, strongest_lower_bound(lower, {value, op}), upper}}
{op, value}, {:ok, lower, upper} when op in [:lt, :le] ->
{:cont, {:ok, lower, strongest_upper_bound(upper, {value, op})}}
_other, _acc ->
{:halt, :error}
end)
end
defp strongest_lower_bound(nil, bound), do: bound
defp strongest_lower_bound({value, op} = current, {new_value, new_op} = new) do
case Value.compare(new_value, value) do
:gt -> new
:eq when new_op == :gt and op == :ge -> new
_other -> current
end
end
defp strongest_upper_bound(nil, bound), do: bound
defp strongest_upper_bound({value, op} = current, {new_value, new_op} = new) do
case Value.compare(new_value, value) do
:lt -> new
:eq when new_op == :lt and op == :le -> new
_other -> current
end
end
defp ordered_range_start(_ordered, nil, _member_index), do: 0
defp ordered_range_start(ordered, {value, op}, member_index) do
first_ordered_index(ordered, value, op == :ge, member_index, 0, tuple_size(ordered))
end
defp first_ordered_index(_ordered, _value, _include_equal?, _member_index, low, low), do: low
defp first_ordered_index(ordered, value, include_equal?, member_index, low, high) do
mid = div(low + high, 2)
{stored_values, _rowids} = elem(ordered, mid)
stored_value = elem(stored_values, member_index)
before_start? =
case Value.compare(stored_value, value) do
:lt -> true
:eq -> not include_equal?
:gt -> false
end
if before_start? do
first_ordered_index(ordered, value, include_equal?, member_index, mid + 1, high)
else
first_ordered_index(ordered, value, include_equal?, member_index, low, mid)
end
end
defp collect_ordered_range(_ordered, index, size, _upper, _member_index, acc)
when index >= size do
acc
end
defp collect_ordered_range(ordered, index, size, upper, member_index, acc) do
{stored_values, rowids} = elem(ordered, index)
stored_value = elem(stored_values, member_index)
cond do
is_nil(stored_value) ->
collect_ordered_range(ordered, index + 1, size, upper, member_index, acc)
upper_bound_exceeded?(stored_value, upper) ->
acc
true ->
collect_ordered_range(ordered, index + 1, size, upper, member_index, rowids ++ acc)
end
end
defp upper_bound_exceeded?(_stored_value, nil), do: false
defp upper_bound_exceeded?(stored_value, {value, op}) do
case Value.compare(stored_value, value) do
:gt -> true
:eq -> op == :lt
:lt -> false
end
end
defp direct_index_lookup?(index, values, collations) do
length(values) == length(index_members(index)) and
Enum.all?(collations, &binary_collation_name?/1)
end
defp binary_collation_name?(nil), do: true
defp binary_collation_name?(:binary), do: true
defp binary_collation_name?(name) when is_binary(name), do: String.downcase(name) == "binary"
defp binary_collation_name?(_name), do: false
defp index_range_match?(db, stored_value, bounds, collation) do
collation = normalize_collation!(collation, %{db: db})
Enum.all?(bounds, fn {op, value} ->
Value.compare_op(op, stored_value, value, collation) == true
end)
end
# SQLite prefers a unique index, then the longest usable equality prefix.
defp best_equality_index(table, eq_keys, where) do
lookup_indexes(table)
|> Enum.filter(fn index ->
prefix = Enum.take_while(index.columns, &MapSet.member?(eq_keys, &1))
prefix != [] and index_lookup_usable?(table, index, length(prefix), where)
end)
|> Enum.sort_by(fn index ->
prefix = Enum.take_while(index.columns, &MapSet.member?(eq_keys, &1))
{if(index.unique, do: 0, else: 1), -length(prefix)}
end)
|> List.first()
end
defp relation(_db, nil, _outer), do: {[], [[]]}
defp relation(db, {:table, {:schema, schema, name}, alias_name}, outer) do
ensure_table_schema!(db, schema, name)
key = Table.key(name)
if key in ["sqlite_schema", "sqlite_master"] do
relation_from_result(
["type", "name", "tbl_name", "rootpage", "sql"],
sqlite_schema_rows(db, schema),
[:text, :text, :text, :integer, :text],
alias_name || name
)
else
relation_named_table(db, Database.table_storage_key(schema, name), name, alias_name, outer)
end
end
defp relation(db, {:table, name, alias_name}, outer) do
key = Table.key(name)
table_key = relation_unqualified_table_key(db, name)
# Check materialized CTEs first, then pending (lazy), then tables, then views.
relation_named_table(db, key, table_key, name, alias_name, outer)
end
defp relation(db, {:subquery, select, alias_name}, outer) do
result = query_result(db, select, outer)
{keys, affinities} = subquery_column_meta(result.columns, result.affinities)
template_name =
if alias_name,
do: Table.key(alias_name),
else: "exsql_subquery_#{:erlang.phash2({keys, result.columns}) |> Integer.to_string()}"
columns = Enum.zip([keys, result.columns, affinities])
tmpl = %{
name: template_name,
source_name: alias_name,
columns: columns,
columns_by_key: index_columns(columns),
hidden: MapSet.new(),
row: %{},
rowid: nil,
has_rowid: false
}
rows = for row <- result.rows, do: [%{tmpl | row: Map.new(Enum.zip(keys, row))}]
{[tmpl], rows}
end
defp relation(db, {:grouped, source, alias_name}, outer) do
case relation(db, source, outer) do
{[tmpl], rows} when alias_name != nil ->
{[%{tmpl | name: Table.key(alias_name), source_name: alias_name}], rows}
relation ->
relation
end
end
# A table-valued function on the right of a join is lateral: its
# arguments may reference columns of the rows to its left, so it is
# re-evaluated per left row (`FROM t, json_each(t.doc)`).
defp relation(db, {:join, type, left, {:table_function, _, _, _} = tf, constraint}, outer) do
{ltmpls, lrows} = relation(db, left, outer)
{[rtmpl], _norows} =
table_function_relation(db, tf, %{db: db, frames: [], group: nil, outer: outer}, true)
rows =
Enum.flat_map(lrows, fn lframes ->
env = %{db: db, frames: lframes, group: nil, outer: outer}
{[_tmpl], rrows} = table_function_relation(db, tf, env, false)
matched =
for [rframe] <- rrows,
join_match?(db, constraint, [], lframes, rframe, outer),
do: lframes ++ [rframe]
case matched do
[] when type.left -> [lframes ++ [null_frame(rtmpl)]]
matched -> matched
end
end)
{ltmpls ++ [rtmpl], rows}
end
defp relation(db, {:table_function, _, _, _} = tf, outer) do
table_function_relation(db, tf, %{db: db, frames: [], group: nil, outer: outer}, false)
end
# Predicate pushdown: a base table wrapped with the WHERE conjuncts that
# reference only that table, so the nested-loop join sees a pre-filtered
# relation instead of the full table. The outer WHERE is re-applied later,
# so this only removes rows that could never have survived.
defp relation(db, {:prefiltered, src, preds}, outer) do
{tmpls, rows} = relation(db, src, outer)
filtered =
Enum.filter(rows, fn frames ->
env = %{db: db, frames: frames, group: nil, outer: outer}
Enum.all?(preds, &matches_where?(&1, env))
end)
{tmpls, filtered}
end
# Inner hash join: build a multimap of right rows keyed by the equi-join key,
# then probe once per left row. Replaces the O(n*m) nested loop for equality
# joins. Any extra (non-key) constraint is still checked per candidate, and
# the outer WHERE is re-applied downstream.
defp relation(db, {:hashjoin, type, left, right, constraint, equi}, outer) do
{ltmpls, lrows} = relation(db, left, outer)
{rtmpls, rrows} = relation(db, right, outer)
constraint = if constraint == nil, do: {:on, {:literal, true}}, else: constraint
using = using_columns(type, constraint, ltmpls, rtmpls)
rtmpls = Enum.map(rtmpls, &%{&1 | hidden: MapSet.union(&1.hidden, MapSet.new(using))})
rframes =
Enum.map(rrows, fn frames ->
frames
|> Enum.with_index()
|> Enum.map(fn {frame, index} -> %{frame | hidden: Enum.fetch!(rtmpls, index).hidden} end)
end)
{lexprs, rexprs} = Enum.unzip(equi)
build =
Enum.reduce(rframes, %{}, fn rframe, acc ->
case hash_join_key(rexprs, %{db: db, frames: rframe, group: nil, outer: outer}) do
:null -> acc
key -> Map.update(acc, key, [rframe], &[rframe | &1])
end
end)
rows =
Enum.flat_map(lrows, fn lframes ->
case hash_join_key(lexprs, %{db: db, frames: lframes, group: nil, outer: outer}) do
:null ->
[]
key ->
build
|> Map.get(key, [])
|> Enum.reverse()
|> Enum.filter(&join_match?(db, constraint, using, lframes, &1, outer))
|> Enum.map(&(lframes ++ &1))
end
end)
{ltmpls ++ rtmpls, rows}
end
# Pure cartesian (comma join with no ON/USING and no outer side): the cross
# product directly, skipping the per-pair predicate check, with_index, and
# matched-right bookkeeping the general path carries. The WHERE that selects
# rows is applied downstream.
defp relation(
db,
{:join, %{left: false, right: false, natural: false}, left, right, nil},
outer
) do
{ltmpls, lrows} = relation(db, left, outer)
{rtmpls, rrows} = relation(db, right, outer)
rows = for lframes <- lrows, rframe <- rrows, do: lframes ++ rframe
{ltmpls ++ rtmpls, rows}
end
defp relation(db, {:join, type, left, right, constraint}, outer) do
{ltmpls, lrows} = relation(db, left, outer)
{rtmpls, rrows} = relation(db, right, outer)
constraint =
if constraint == nil,
do: {:on, {:literal, true}},
else: constraint
using = using_columns(type, constraint, ltmpls, rtmpls)
rtmpls = Enum.map(rtmpls, &%{&1 | hidden: MapSet.union(&1.hidden, MapSet.new(using))})
rframes =
Enum.map(rrows, fn frames ->
Enum.with_index(frames)
|> Enum.map(fn {frame, index} ->
%{frame | hidden: Enum.fetch!(rtmpls, index).hidden}
end)
end)
null_right_rows = Enum.map(rtmpls, &null_frame/1)
{rows, matched_rights} =
Enum.map_reduce(lrows, MapSet.new(), fn lframes, matched_rights ->
matched =
for {rframe, right_index} <- Enum.with_index(rframes),
join_match?(db, constraint, using, lframes, rframe, outer),
do: {lframes ++ rframe, right_index}
rows =
case matched do
[] when type.left -> [lframes ++ null_right_rows]
[] -> []
matches -> Enum.map(matches, &elem(&1, 0))
end
matched_rights =
Enum.reduce(matched, matched_rights, fn {_row, right_index}, matched_rights ->
MapSet.put(matched_rights, right_index)
end)
{rows, matched_rights}
end)
|> then(fn {rows, matched_rights} -> {Enum.flat_map(rows, & &1), matched_rights} end)
right_rows =
if type.right do
for {rframe, right_index} <- Enum.with_index(rframes),
not MapSet.member?(matched_rights, right_index) do
right_unmatched_row(ltmpls, rframe, using)
end
else
[]
end
{ltmpls ++ rtmpls, rows ++ right_rows}
end
defp subquery_column_meta(columns, affinities) do
duplicate = %{}
{keys, _} =
columns
|> Enum.with_index()
|> Enum.map_reduce(duplicate, fn {name, index}, seen ->
base_key = Table.key(name)
n = Map.get(seen, base_key, 0) + 1
suffix = if n == 1, do: "", else: "__exsql#{n - 1}"
key = base_key <> suffix
affinity = Enum.at(affinities, index) || :blob
{{key, affinity}, Map.put(seen, base_key, n)}
end)
{Enum.map(keys, &elem(&1, 0)), Enum.map(keys, &elem(&1, 1))}
end
defp right_unmatched_row(ltmpls, rframes, using) when is_list(rframes) do
left_frames =
Enum.map(ltmpls, fn tmpl ->
frame = null_frame(tmpl)
row =
Enum.reduce(using, frame.row, fn key, row ->
if visible?(frame, key) do
Map.put(row, key, resolve_right_row_value(rframes, key))
else
row
end
end)
%{frame | row: row}
end)
left_frames ++ rframes
end
defp right_unmatched_row(ltmpls, rframe, using) when is_map(rframe),
do: right_unmatched_row(ltmpls, [rframe], using)
defp relation_named_table(db, key, name, alias_name, outer),
do: relation_named_table(db, key, key, name, alias_name, outer)
defp relation_named_table(db, key, table_key, name, alias_name, outer) do
cond do
Map.has_key?(db.ctes, key) ->
cte = Map.fetch!(db.ctes, key)
if Map.get(cte, :actual_count) != nil and cte.actual_count != length(cte.columns) do
fail("table #{name} has #{cte.actual_count} values for #{length(cte.columns)} columns")
end
relation_from_result(cte.columns, cte.rows, cte.affinities, alias_name || name)
key in ["sqlite_schema", "sqlite_master"] ->
relation_from_result(
["type", "name", "tbl_name", "rootpage", "sql"],
sqlite_schema_rows(db),
[:text, :text, :text, :integer, :text],
alias_name || name
)
key == "sqlite_sequence" and sqlite_sequence_exists?(db) ->
relation_from_result(
["name", "seq"],
sqlite_sequence_rows(db),
[:text, :integer],
alias_name || name
)
Map.has_key?(db.tables, table_key) ->
table = Map.fetch!(db.tables, table_key)
tmpl = table_frame(table, alias_name)
rows =
for {rowid, row} <- Table.scan_positional(table), do: [%{tmpl | row: row, rowid: rowid}]
{[tmpl], rows}
Map.has_key?(db.views, table_key) ->
view = Map.fetch!(db.views, table_key)
result = query_result(db, view.query, outer)
{columns, affinities} =
case view.columns do
nil ->
{result.columns, result.affinities}
col_names ->
if length(col_names) != length(result.columns) do
fail(
"expected #{length(col_names)} columns for '#{name}' but got #{length(result.columns)}"
)
end
affs = result.affinities ++ List.duplicate(:blob, length(col_names))
{col_names, Enum.take(affs, length(col_names))}
end
relation_from_result(columns, result.rows, affinities, alias_name || name)
true ->
fail("no such table: #{name}")
end
end
defp relation_unqualified_table_key(db, name) do
Enum.find_value(table_lookup_order(db), Table.key(name), fn schema ->
key = Database.table_storage_key(schema, name)
if Map.has_key?(db.tables, key) or Map.has_key?(db.views, key) do
key
else
nil
end
end)
end
defp table_frame(table, alias_name) do
columns = Table.frame_columns(table)
%{
name: Table.key(alias_name || table.name),
source_name: table.name,
columns: columns,
columns_by_key: index_columns(columns),
col_index: Table.column_index(table),
hidden: MapSet.new(),
row: %{},
rowid: nil,
has_rowid: not table.without_rowid
}
end
# A key => column-tuple map for O(1) `has_column?`/`frame_column`, instead of a
# linear `List.keymember?` scan of the columns list on every column access.
defp index_columns(columns), do: Map.new(columns, fn col -> {column_key(col), col} end)
defp null_frame(tmpl) do
%{tmpl | row: Map.new(tmpl.columns, fn column -> {column_key(column), nil} end), rowid: nil}
end
# Build frames from a pre-computed result set (used for views and CTEs).
# -- table-valued functions ------------------------------------------------------
@json_each_columns ~w(key value type atom id parent fullkey path)
defp table_function_relation(
_db,
{:table_function, name, args, alias_name},
env,
template_only?
) do
columns = table_function_columns!(name)
rows = if template_only?, do: [], else: table_function_rows(name, args, env)
relation_from_result(
columns,
rows,
List.duplicate(:blob, length(columns)),
alias_name || name
)
end
defp table_function_columns!(name) when name in ["json_each", "json_tree"],
do: @json_each_columns
defp table_function_columns!("pragma_table_info"), do: ~w(cid name type notnull dflt_value pk)
defp table_function_columns!("pragma_table_xinfo"),
do: ~w(cid name type notnull dflt_value pk hidden)
defp table_function_columns!("pragma_index_list"), do: ~w(seq name unique origin partial)
defp table_function_columns!("pragma_index_info"), do: ~w(seqno cid name)
defp table_function_columns!("pragma_foreign_key_list"),
do: ~w(id seq table from to on_update on_delete match)
defp table_function_columns!(name), do: fail("no such table: #{name}")
# `pragma_<name>(table)` table-valued functions reuse the corresponding
# PRAGMA's row builder, so they stay in sync with the statement form.
defp table_function_rows("pragma_table_info", args, env),
do: pragma_table_fn_rows(args, env, &table_info_rows(&1, false))
defp table_function_rows("pragma_table_xinfo", args, env),
do: pragma_table_fn_rows(args, env, &table_info_rows(&1, true))
defp table_function_rows("pragma_index_list", args, env),
do: pragma_table_fn_rows(args, env, &index_list_rows/1)
defp table_function_rows("pragma_foreign_key_list", args, env),
do: pragma_table_fn_rows(args, env, &foreign_key_list_rows/1)
defp table_function_rows("pragma_index_info", [arg | _], env) do
case pragma_find_index_owner(env.db, eval(arg, env)) do
{table, index} -> index_info_rows(table, index)
nil -> []
end
end
defp table_function_rows(name, args, env) do
{doc, path} =
case args do
[doc_expr] -> {eval(doc_expr, env), "$"}
[doc_expr, path_expr] -> {eval(doc_expr, env), eval(path_expr, env) || "$"}
_ -> fail("#{name}() requires 1 or 2 arguments")
end
if doc == nil do
[]
else
jv = json_parse!(doc)
steps = json_path!(path)
case Json.get(jv, steps) do
:missing ->
[]
{:ok, root} ->
case name do
"json_each" -> json_each_rows(root, path)
"json_tree" -> json_tree_rows(root, nil, path, path, nil, 1) |> elem(0)
end
end
end
end
defp pragma_table_fn_rows([arg | _], env, builder) do
case pragma_fetch_table(env.db, eval(arg, env)) do
{:ok, table} -> builder.(table)
:error -> []
end
end
defp pragma_table_fn_rows([], _env, _builder), do: []
defp json_each_rows({:object, pairs}, path) do
pairs
|> Enum.with_index(1)
|> Enum.map(fn {{key, jv}, id} ->
key = Json.object_key_text(key)
json_member_row(key, jv, id, nil, json_key_accessor(path, key), path)
end)
end
defp json_each_rows({:array, items}, path) do
items
|> Enum.with_index()
|> Enum.map(fn {jv, index} ->
json_member_row(index, jv, index + 1, nil, "#{path}[#{index}]", path)
end)
end
defp json_each_rows(scalar, path) do
[json_member_row(nil, scalar, 1, nil, path, path)]
end
# json_tree emits the value itself, then its descendants. The id column is
# an arbitrary unique integer, as documented for the SQLite originals.
defp json_tree_rows(jv, key, fullkey, path, parent_id, next_id) do
row = json_member_row(key, jv, next_id, parent_id, fullkey, path)
id = next_id
{child_rows, next_id} =
case jv do
{:object, pairs} ->
Enum.reduce(pairs, {[], next_id + 1}, fn {k, v}, {acc, n} ->
k = Json.object_key_text(k)
{rows, n} = json_tree_rows(v, k, json_key_accessor(fullkey, k), fullkey, id, n)
{acc ++ rows, n}
end)
{:array, items} ->
items
|> Enum.with_index()
|> Enum.reduce({[], next_id + 1}, fn {v, index}, {acc, n} ->
{rows, n} = json_tree_rows(v, index, "#{fullkey}[#{index}]", fullkey, id, n)
{acc ++ rows, n}
end)
_scalar ->
{[], next_id + 1}
end
{[row | child_rows], next_id}
end
defp json_member_row(key, jv, id, parent_id, fullkey, path) do
atom =
case jv do
{:array, _} -> nil
{:object, _} -> nil
scalar -> Json.to_sql(scalar)
end
[key, Json.to_sql(jv), Json.type_name(jv), atom, id, parent_id, fullkey, path]
end
defp json_key_accessor(path, key) do
if String.match?(key, ~r/^[A-Za-z_][A-Za-z0-9_]*$/) do
"#{path}.#{key}"
else
~s(#{path}."#{key}")
end
end
defp relation_from_result(columns, rows, affinities, alias_name) do
keys = Enum.map(columns, &Table.key/1)
affinities = affinities ++ List.duplicate(:blob, length(keys))
frame_columns = Enum.zip([keys, columns, Enum.take(affinities, length(keys))])
tmpl = %{
name: Table.key(alias_name),
source_name: alias_name,
columns: frame_columns,
columns_by_key: index_columns(frame_columns),
hidden: MapSet.new(),
row: %{},
rowid: nil,
has_rowid: false
}
rows = for row <- rows, do: [%{tmpl | row: Map.new(Enum.zip(keys, row))}]
{[tmpl], rows}
end
defp sqlite_schema_rows(db), do: sqlite_schema_rows(db, nil)
defp sqlite_schema_rows(db, schema) do
table_rows =
db.tables
|> Map.values()
|> Enum.filter(&sqlite_schema_matches?(&1.schema, schema))
|> Enum.sort_by(&Table.key(&1.name))
|> Enum.with_index(1)
|> Enum.flat_map(fn {table, rootpage} ->
table_row = ["table", table.name, table.name, rootpage, create_table_sql(table)]
autoindex_rows =
table.autoindexes
|> Enum.with_index(rootpage + 500)
|> Enum.map(fn {index, index_rootpage} ->
["index", index.name, table.name, index_rootpage, nil]
end)
index_rows =
table.indexes
|> Enum.with_index(rootpage + 1000)
|> Enum.map(fn {index, index_rootpage} ->
["index", index.name, table.name, index_rootpage, create_index_sql(table, index)]
end)
[table_row | autoindex_rows ++ index_rows]
end)
sequence_row =
if main_schema?(schema) and sqlite_sequence_exists?(db) do
[
[
"table",
"sqlite_sequence",
"sqlite_sequence",
0,
"CREATE TABLE sqlite_sequence(name,seq)"
]
]
else
[]
end
view_rows =
db.views
|> Map.values()
|> Enum.filter(&sqlite_schema_matches?(&1.schema, schema))
|> Enum.sort_by(&Table.key(&1.name))
|> Enum.map(fn view ->
["view", view.name, view.name, 0, create_view_sql(view)]
end)
trigger_rows =
db.triggers
|> Map.values()
|> Enum.filter(&sqlite_schema_matches?(&1.schema, schema))
|> Enum.sort_by(& &1.seq)
|> Enum.map(fn trigger ->
["trigger", trigger.name, trigger.table_name, 0, create_trigger_sql(trigger)]
end)
table_rows ++ sequence_row ++ view_rows ++ trigger_rows
end
defp sqlite_schema_matches?(object_schema, schema) do
Table.key(object_schema || "main") == Table.key(schema || "main")
end
# The stored SQL for a trigger is reconstructed from the parsed definition;
# body statements are not round-tripped to SQL text yet.
defp create_trigger_sql(trigger) do
timing =
case trigger.timing do
:before -> "BEFORE"
:after -> "AFTER"
:instead_of -> "INSTEAD OF"
end
event =
case {trigger.event, trigger.update_columns} do
{:update, columns} when is_list(columns) -> "UPDATE OF #{Enum.join(columns, ", ")}"
{event, _} -> event |> Atom.to_string() |> String.upcase()
end
"CREATE TRIGGER #{trigger.name} #{timing} #{event} ON #{trigger.table_name} " <>
"FOR EACH ROW BEGIN ... END"
end
defp sqlite_sequence_exists?(db),
do: Enum.any?(db.tables, fn {_key, table} -> table.autoincrement end)
defp ensure_sqlite_sequence_exists!(db) do
unless sqlite_sequence_exists?(db), do: fail("no such table: sqlite_sequence")
end
defp sqlite_sequence_table(db) do
rows =
db
|> sqlite_sequence_rows()
|> Enum.with_index(1)
|> Map.new(fn {[name, seq], rowid} -> {rowid, {name, seq}} end)
%Table{
name: "sqlite_sequence",
columns: [
%ColumnDef{name: "name", affinity: :text},
%ColumnDef{name: "seq", affinity: :integer}
],
rows: rows,
next_rowid: map_size(rows) + 1
}
end
defp put_sqlite_sequence(db, name, sequence, visible?) do
key = Table.key(name)
case Map.fetch(db.tables, key) do
{:ok, %{autoincrement: true} = table} ->
put_table(db, %{table | sequence: sequence, sequence_row: visible?})
_other ->
orphans =
if visible? do
Map.put(db.sqlite_sequence_orphans, key, {name, sequence})
else
Map.delete(db.sqlite_sequence_orphans, key)
end
%{db | sqlite_sequence_orphans: orphans}
end
end
defp sqlite_sequence_rows(db) do
table_rows =
db.tables
|> Map.values()
|> Enum.filter(&(&1.autoincrement and &1.sequence_row))
|> Enum.map(&{Table.key(&1.name), [&1.name, &1.sequence]})
orphan_rows =
db.sqlite_sequence_orphans
|> Enum.map(fn {key, {name, sequence}} -> {key, [name, sequence]} end)
(table_rows ++ orphan_rows)
|> Enum.sort_by(fn {key, _row} -> key end)
|> Enum.map(fn {_key, row} -> row end)
end
defp create_table_sql(table) do
definition_sql =
table.columns
|> Enum.map(&column_def_sql/1)
|> Kernel.++(Enum.map(table.foreign_keys, &table_foreign_key_sql(table, &1)))
|> Enum.join(", ")
options =
[
if(table.without_rowid, do: "WITHOUT ROWID"),
if(table.strict, do: "STRICT")
]
|> Enum.reject(&is_nil/1)
suffix = if options == [], do: "", else: " " <> Enum.join(options, ", ")
"CREATE TABLE #{table.name}(#{definition_sql})#{suffix}"
end
defp table_foreign_key_sql(table, {child_keys, parent_table, parent_keys, actions}) do
child_columns =
child_keys
|> Enum.map_join(", ", &display_column_name(table, &1))
references =
case parent_keys do
[] -> parent_table
keys -> "#{parent_table}(#{Enum.join(keys, ", ")})"
end
"FOREIGN KEY(#{child_columns}) REFERENCES #{references}#{fk_actions_sql(actions)}"
end
defp fk_actions_sql(actions) do
[
if(actions.on_delete != :no_action,
do: " ON DELETE #{fk_action_name(actions.on_delete)}"
),
if(actions.on_update != :no_action,
do: " ON UPDATE #{fk_action_name(actions.on_update)}"
),
if(actions.deferred, do: " DEFERRABLE INITIALLY DEFERRED")
]
|> Enum.reject(&is_nil/1)
|> Enum.join()
end
defp fk_action_name(:no_action), do: "NO ACTION"
defp fk_action_name(:restrict), do: "RESTRICT"
defp fk_action_name(:set_null), do: "SET NULL"
defp fk_action_name(:set_default), do: "SET DEFAULT"
defp fk_action_name(:cascade), do: "CASCADE"
defp column_def_sql(column) do
[
column.name,
column.declared_type,
generated_sql(column),
if(column.primary_key, do: "PRIMARY KEY"),
if(column.autoincrement, do: "AUTOINCREMENT"),
if(column.not_null, do: "NOT NULL"),
if(column.unique, do: "UNIQUE"),
column.default && "DEFAULT #{pragma_default(column.default)}",
column.collate && "COLLATE #{column.collate}",
references_sql(column)
]
|> Enum.reject(&is_nil/1)
|> Enum.join(" ")
end
defp generated_sql(%{generated: {kind, expr}}) do
"GENERATED ALWAYS AS (#{expr_name(expr)}) #{kind |> Atom.to_string() |> String.upcase()}"
end
defp generated_sql(_column), do: nil
defp references_sql(%{references: {table, [], actions}}),
do: "REFERENCES #{table}#{fk_actions_sql(actions)}"
defp references_sql(%{references: {table, columns, actions}}),
do: "REFERENCES #{table}(#{Enum.join(columns, ", ")})#{fk_actions_sql(actions)}"
defp references_sql(_column), do: nil
defp create_index_sql(table, index) do
unique = if index.unique, do: "UNIQUE ", else: ""
columns =
index
|> index_members()
|> Enum.map_join(", ", fn
{:column, key} -> display_column_name(table, key)
{:expr, expr} -> expr_name(expr)
end)
where = if index.where, do: " WHERE #{expr_name(index.where)}", else: ""
"CREATE #{unique}INDEX #{index.name} ON #{table.name}(#{columns})#{where}"
end
defp create_view_sql(view), do: "CREATE VIEW #{view.name} AS SELECT"
defp using_columns(%{natural: true}, _constraint, ltmpls, rtmpls) when is_list(rtmpls) do
rtmpls
|> Enum.flat_map(& &1.columns)
|> Enum.map(&column_key/1)
|> Enum.filter(fn key ->
Enum.any?(rtmpls, fn rtmpl ->
visible?(rtmpl, key) and not MapSet.member?(rtmpl.hidden, key)
end) and
Enum.any?(ltmpls, &visible?(&1, key))
end)
|> Enum.uniq()
end
defp using_columns(_type, {:using, names}, ltmpls, rtmpls) when is_list(rtmpls) do
Enum.map(names, fn name ->
key = Table.key(name)
right_visible = Enum.count(rtmpls, &visible?(&1, key))
if right_visible > 1 do
fail("ambiguous column name: #{name}")
end
if right_visible == 1 and Enum.any?(ltmpls, &visible?(&1, key)) do
key
else
fail("cannot join using column #{name} - column not present in both tables")
end
end)
end
defp using_columns(type, constraint, ltmpls, rtmpl) when is_map(rtmpl),
do: using_columns(type, constraint, ltmpls, [rtmpl])
defp using_columns(_type, _constraint, _ltmpls, _rtmpl), do: []
defp join_match?(db, constraint, using, lframes, rframe, outer) when is_map(rframe),
do: join_match?(db, constraint, using, lframes, [rframe], outer)
defp join_match?(db, constraint, using, lframes, rframes, outer) when is_list(rframes) do
case {constraint, using} do
{{:on, expr}, []} ->
env = %{db: db, frames: lframes ++ rframes, group: nil, outer: outer}
truth(expr, env) == true
{_, []} ->
true
{_, using} ->
left_env = %{db: db, frames: lframes, group: nil, outer: nil}
Enum.all?(using, fn key ->
{a, b} =
Value.comparison_coerce(
resolve_column(left_env, nil, key),
column_affinity(left_env, nil, key),
resolve_right_row_value(rframes, key),
resolve_right_frame_affinity(rframes, key)
)
Value.compare_op(:eq, a, b, column_collation(left_env, nil, key)) == true
end)
end
end
defp resolve_right_row_value([], _key), do: nil
defp resolve_right_row_value([frame | rest], key) do
if has_column?(frame, key) do
frame_cell(frame, key)
else
resolve_right_row_value(rest, key)
end
end
defp resolve_right_frame_affinity([], _key), do: nil
defp resolve_right_frame_affinity([frame | rest], key) do
if has_column?(frame, key) do
frame_affinity(frame, key)
else
resolve_right_frame_affinity(rest, key)
end
end
# Targets carry the folded column key (`{column, key}` | `:rowid`) so the
# per-row `insert_values/2` reuses it instead of re-folding `Table.key/1` for
# every column on every inserted row.
defp insert_targets(table, %Insert{columns: nil}) do
# Reuse the cached, pre-folded `frame_columns` keys (parallel to `columns`)
# rather than re-folding `Table.key(&1.name)` for every column on every insert.
table
|> Table.frame_columns()
|> Enum.zip(table.columns)
|> Enum.reject(fn {_fc, column} -> column.generated end)
|> Enum.map(fn {{key, _name, _aff, _coll}, column} -> {column, key} end)
end
defp insert_targets(table, %Insert{columns: names}) do
Enum.map(names, fn name ->
case Table.column(table, name) do
%{} = column ->
if column.generated do
fail("cannot INSERT into generated column \"#{column.name}\"")
end
{column, Table.key(column.name)}
nil ->
if Table.key(name) in @rowid_names and not table.without_rowid do
:rowid
else
fail("table #{table.name} has no column named #{name}")
end
end
end)
end
defp expand_columns(columns, templates) do
Enum.flat_map(columns, fn
:star ->
if templates == [], do: fail("no tables specified")
Enum.flat_map(templates, fn tmpl ->
for column <- tmpl.columns,
key = column_key(column),
display = column_display(column),
not MapSet.member?(tmpl.hidden, key) do
alias_name = if(Table.key(display) == key, do: nil, else: display)
{{:column, tmpl.name, key}, alias_name}
end
end)
{:qualified_star, table} ->
qkey = Table.key(table)
tmpl = Enum.find(templates, &(&1.name == qkey)) || fail("no such table: #{table}")
for column <- tmpl.columns,
key = column_key(column),
display = column_display(column) do
alias_name = if(Table.key(display) == key, do: nil, else: display)
{{:column, tmpl.name, key}, alias_name}
end
{expr, alias_name} ->
[{expr, alias_name}]
end)
end
# -- GROUP BY / HAVING ----------------------------------------------------------
defp grouped_envs(db, stmt, columns, templates, envs, outer) do
group_exprs =
stmt.group_by
|> Enum.with_index(1)
|> Enum.map(fn {expr, index} -> resolve_group_term(expr, index, columns, templates) end)
having = stmt.having && substitute_aliases(stmt.having, columns, templates)
template_env = %{db: db, frames: templates, group: nil, outer: outer}
group_collations = Enum.map(group_exprs, &expr_collation(&1, template_env))
grouped_raw = group_members(group_exprs, group_collations, envs)
grouped =
grouped_raw
|> Enum.map(fn members ->
frames =
case members do
[first | _] -> first.frames
[] -> []
end
%{db: db, frames: frames, group: members, outer: outer}
end)
|> Enum.filter(fn genv -> having == nil or truth(having, genv) == true end)
grouped
end
defp group_members([], _collations, envs) do
[envs]
end
defp group_members(group_exprs, group_collations, envs) do
envs
|> Enum.map(fn env -> {Enum.map(group_exprs, &eval(&1, env)), env} end)
|> Enum.sort(fn {a, _}, {b, _} -> compare_key_values(a, b, group_collations) != :gt end)
|> Enum.reduce([], fn
{key, env}, [{prev_key, members} | done] = acc ->
if compare_key_values(key, prev_key, group_collations) == :eq do
[{prev_key, [env | members]} | done]
else
[{key, [env]} | acc]
end
{key, env}, [] ->
[{key, [env]}]
end)
|> Enum.reverse()
|> Enum.map(fn {_key, members} -> Enum.reverse(members) end)
end
defp compare_key_values([], [], _collations), do: :eq
defp compare_key_values([], _right, _collations), do: :lt
defp compare_key_values(_left, [], _collations), do: :gt
defp compare_key_values([left | left_tail], [right | right_tail], [collation | collations]) do
case Value.compare(left, right, collation) do
:eq -> compare_key_values(left_tail, right_tail, collations)
other -> other
end
end
defp compare_key_values([left | left_tail], [right | right_tail], []) do
case Value.compare(left, right, :binary) do
:eq -> compare_key_values(left_tail, right_tail, [])
other -> other
end
end
defp compare_keys([], []), do: :eq
defp compare_keys([a | rest_a], [b | rest_b]) do
case Value.compare(a, b) do
:eq -> compare_keys(rest_a, rest_b)
other -> other
end
end
# A GROUP BY term may be a 1-based output column position, an output alias,
# or an expression over the source rows.
defp resolve_group_term({:literal, n}, index, columns, _templates) when is_integer(n) do
if n < 1 or n > length(columns) do
fail(
"#{ordinal(index)} GROUP BY term out of range - should be between 1 and #{length(columns)}"
)
end
columns |> Enum.at(n - 1) |> elem(0)
end
defp resolve_group_term(expr, _index, columns, templates),
do: substitute_aliases(expr, columns, templates)
defp ordinal(1), do: "1st"
defp ordinal(2), do: "2nd"
defp ordinal(3), do: "3rd"
defp ordinal(n), do: "#{n}th"
# Replaces references to output aliases (`GROUP BY x`, `HAVING y>=4`) with
# their expressions, unless the name is a real source column, which wins.
defp substitute_aliases({:column, nil, name} = expr, columns, templates) do
key = Table.key(name)
if Enum.any?(templates, &has_column?(&1, key)) do
expr
else
case Enum.find(columns, fn {_e, alias_name} ->
alias_name != nil and Table.key(alias_name) == key
end) do
{aliased, _} -> aliased
nil -> expr
end
end
end
defp substitute_aliases(expr, columns, templates) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.map(fn
element when is_tuple(element) ->
substitute_aliases(element, columns, templates)
elements when is_list(elements) ->
Enum.map(elements, &substitute_aliases(&1, columns, templates))
other ->
other
end)
|> List.to_tuple()
end
defp substitute_aliases(expr, _columns, _templates), do: expr
defp resolve_window_refs(columns, windows) do
Enum.map(columns, fn {expr, alias_name} ->
{resolve_window_refs_expr(expr, windows), alias_name}
end)
end
defp resolve_window_refs_expr({:window, name, args, {:ref, window_name}, filter}, windows) do
case Map.fetch(windows, window_name) do
{:ok, spec} -> {:window, name, args, spec, filter}
:error -> fail("no such window: #{window_name}")
end
end
defp resolve_window_refs_expr(expr, windows) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.map(fn
element when is_tuple(element) ->
resolve_window_refs_expr(element, windows)
elements when is_list(elements) ->
Enum.map(elements, &resolve_window_refs_expr(&1, windows))
other ->
other
end)
|> List.to_tuple()
end
defp resolve_window_refs_expr(expr, _windows), do: expr
# -- DISTINCT / ORDER BY / LIMIT ---------------------------------------------
defp distinct(projected, false, _columns, _env), do: projected
defp distinct(projected, true, columns, env) do
if columns == [] do
projected
else
collations = Enum.map(columns, &expr_collation(elem(&1, 0), env))
# When every column dedups under the default binary collation (the common
# case), a canonical `row_key/1` (the same key INTERSECT/EXCEPT use) gives
# O(1) membership — O(n) overall instead of the O(n²) all-pairs scan, which
# dominated DISTINCT over large result sets. Non-binary collations
# (NOCASE/RTRIM) keep the exact linear comparison path.
if Enum.all?(collations, &(&1 == :binary)) do
distinct_rows_hashed(projected, MapSet.new(), [])
else
distinct_rows(projected, collations, [])
end
end
end
defp distinct_rows_hashed([], _seen, acc), do: Enum.reverse(acc)
defp distinct_rows_hashed([{_env, row} = item | rest], seen, acc) do
key = row_key(row)
if MapSet.member?(seen, key) do
distinct_rows_hashed(rest, seen, acc)
else
distinct_rows_hashed(rest, MapSet.put(seen, key), [item | acc])
end
end
defp distinct_rows([], _collations, acc), do: Enum.reverse(acc)
defp distinct_rows([{env, row} | rest], collations, acc) do
if Enum.any?(acc, fn {_env_row, keep_row} ->
compare_key_values(row, keep_row, collations) == :eq
end) do
distinct_rows(rest, collations, acc)
else
distinct_rows(rest, collations, [{env, row} | acc])
end
end
defp order(projected, [], _columns, _names), do: projected
defp order([], _order_by, _columns, _names), do: []
# Decorate-sort-undecorate: evaluate each row's sort keys once (O(n)) instead
# of re-evaluating them inside every comparison (O(n log n)). Direction and
# collation are per-term and row-independent (collation is schema-derived), so
# they're resolved once up front. Enum.sort stays stable, preserving the
# original tie order.
defp order([{env0, _row0} | _] = projected, order_by, columns, names) do
term_meta =
Enum.map(order_by, fn {expr, direction} ->
{direction, order_collation(expr, env0, columns, names)}
end)
# Resolve each term to a per-row key extractor once (position → index, or
# output-column name → index, or fall back to evaluating the expression),
# rather than re-resolving the name (with a `downcase` + names search) for
# every row.
extractors =
Enum.map(order_by, fn {expr, _direction} -> order_key_extractor(expr, columns, names) end)
projected
|> Enum.map(fn {env, row} = item ->
{Enum.map(extractors, & &1.(env, row)), item}
end)
|> Enum.sort(&order_keys_before?(elem(&1, 0), elem(&2, 0), term_meta))
|> Enum.map(&elem(&1, 1))
end
defp order_keys_before?([a | as], [b | bs], [{direction, collation} | rest]) do
case Value.compare(a, b, collation) do
:eq -> order_keys_before?(as, bs, rest)
:lt -> direction == :asc
:gt -> direction == :desc
end
end
defp order_keys_before?([], [], []), do: true
defp sort_envs_by_terms(envs, []), do: envs
defp sort_envs_by_terms(envs, terms) do
Enum.sort(envs, fn env_a, env_b ->
compare_term_values(terms, env_a, env_b)
end)
end
defp sort_indexed_envs_by_terms(indexed_envs, []), do: indexed_envs
defp sort_indexed_envs_by_terms(indexed_envs, terms) do
Enum.sort(indexed_envs, fn {env_a, _}, {env_b, _} ->
compare_term_values(terms, env_a, env_b)
end)
end
defp compare_term_values(terms, env_a, env_b) do
terms
|> Enum.reduce_while(true, fn {expr, direction}, _ ->
case Value.compare(eval(expr, env_a), eval(expr, env_b), expr_collation(expr, env_a)) do
:eq -> {:cont, true}
:lt -> {:halt, direction == :asc}
:gt -> {:halt, direction == :desc}
end
end)
end
# ORDER BY terms may be 1-based output column positions, output aliases, or
# arbitrary expressions over the source row (including aggregates, in
# grouped queries).
defp order_key_extractor({:collate, expr, _name}, columns, names),
do: order_key_extractor(expr, columns, names)
defp order_key_extractor({:literal, n}, columns, _names) when is_integer(n) do
if n < 1 or n > length(columns), do: fail("ORDER BY term out of range: #{n}")
fn _env, row -> Enum.at(row, n - 1) end
end
defp order_key_extractor({:column, nil, name} = expr, _columns, names) do
lowered = Table.key(name)
case Enum.find_index(names, &(Table.key(&1) == lowered)) do
nil -> fn env, _row -> eval(expr, env) end
index -> fn _env, row -> Enum.at(row, index) end
end
end
defp order_key_extractor(expr, _columns, _names), do: fn env, _row -> eval(expr, env) end
defp order_collation({:collate, {:literal, _n}, name}, env, _columns, _names),
do: normalize_collation!(name, env)
defp order_collation({:literal, n}, env, columns, _names) when is_integer(n) do
case Enum.at(columns, n - 1) do
{expr, _alias} -> expr_collation(expr, env)
nil -> :binary
end
end
defp order_collation({:column, nil, name} = expr, env, _columns, names) do
lowered = String.downcase(name)
case Enum.find_index(names, &(String.downcase(&1) == lowered)) do
nil -> expr_collation(expr, env)
_index -> expr_collation(expr, env)
end
end
defp order_collation(expr, env, _columns, _names), do: expr_collation(expr, env)
defp clamp(rows, _db, nil, _offset), do: rows
defp clamp(rows, db, limit_expr, offset_expr) do
limit = int_clause(limit_expr, db, "LIMIT")
offset = if offset_expr, do: int_clause(offset_expr, db, "OFFSET"), else: 0
rows = Enum.drop(rows, max(offset, 0))
if limit < 0, do: rows, else: Enum.take(rows, limit)
end
defp int_clause(expr, db, clause) do
case eval(expr, constant_env(db)) do
n when is_integer(n) -> n
_ -> fail("#{clause} must be an integer")
end
end
# -- environments ------------------------------------------------------------------
defp constant_env(db), do: %{db: db, frames: [], group: nil, outer: nil}
defp table_env(db, table, rowid, row) do
frame = %{table_frame(table, nil) | row: row, rowid: rowid}
%{db: db, frames: [frame], group: nil, outer: nil}
end
defp matches_where?(nil, _env), do: true
defp matches_where?(expr, env), do: truth(expr, env) == true
# Pre-resolves plain column references in a single-table scan WHERE to
# `{:fast_column, key, affinity, collation}` (see the eval/expr_affinity/
# expr_collation clauses). Done once per query so the per-row filter skips the
# `String.downcase` + frame visibility search and the affinity/collation
# re-resolution that `resolve_column/3` and `comparison_operands/3` otherwise
# repeat for every row. Only single-frame scans, only columns that
# unambiguously resolve to that frame, and only through row-local predicate
# nodes — never into subqueries (different scope) or function args (kept
# simple); anything not rewritten still evaluates correctly via the normal path.
defp precompile_scan_where(nil, _db, _templates, _outer), do: nil
defp precompile_scan_where(where, db, [template] = templates, outer) do
visible =
template.columns
|> Enum.map(&elem(&1, 0))
|> MapSet.new()
|> MapSet.difference(template.hidden)
env = %{db: db, frames: templates, group: nil, outer: outer}
rewritten =
where
|> rewrite_scan_columns(template.name, visible, env)
|> rewrite_outer_frame_columns(outer_frame_lookup(db, outer))
if expr_node_count(rewritten, 0) >= 12 do
simplify_where_filter(rewritten)
else
rewritten
end
end
defp precompile_scan_where(where, db, templates, outer) when is_list(templates) do
where
|> rewrite_frame_columns(frame_column_lookup(db, templates))
|> rewrite_outer_frame_columns(outer_frame_lookup(db, outer))
end
defp precompile_scan_where(where, _db, _templates, _outer), do: where
# Pre-resolves projection/hash-key column references against the already
# planned frame layout. This is separate from precompile_scan_where/3 because
# projection expressions must not receive WHERE-only boolean simplifications.
defp precompile_scan_columns(columns, db, templates) do
lookup = frame_column_lookup(db, templates)
Enum.map(columns, fn {expr, alias} ->
{rewrite_frame_columns(expr, lookup), alias}
end)
end
defp frame_column_lookup(db, templates) do
templates
|> Enum.with_index()
|> Enum.reduce(%{db: db, unqualified: %{}, qualified: %{}}, fn {template, index}, acc ->
template.columns
|> Enum.reject(fn column -> MapSet.member?(template.hidden, column_key(column)) end)
|> Enum.reduce(acc, fn column, acc ->
key = column_key(column)
affinity = column_affinity_meta(column)
collation = column_collation_meta(column) |> normalize_collation!(%{db: db})
compiled = {:fast_frame_column, index, key, affinity, collation}
qualified_key = {template.name, key}
%{
acc
| unqualified: put_unqualified_column(acc.unqualified, key, compiled),
qualified: Map.put(acc.qualified, qualified_key, compiled)
}
end)
end)
end
defp put_unqualified_column(columns, key, compiled) do
case Map.fetch(columns, key) do
:error -> Map.put(columns, key, compiled)
{:ok, _existing} -> Map.put(columns, key, :ambiguous)
end
end
defp outer_frame_lookup(_db, nil), do: nil
defp outer_frame_lookup(_db, %{group: group}) when group != nil, do: nil
defp outer_frame_lookup(db, %{frames: frames}), do: frame_column_lookup(db, frames)
# Compiles a (already column-pre-resolved) scan predicate into a closure that
# avoids the per-row AST re-dispatch and the per-comparison affinity/collation
# function calls — those are baked in at compile time. Only the common
# row-local boolean/comparison/IN shapes over static-affinity, binary-collation
# operands are compiled; anything else (bare columns, non-binary collation,
# subqueries, …) falls back to the tree walker, so results are identical.
# Returns `(env -> bool)` matching `matches_where?/2`, or nil for no filter.
# A WHERE built entirely from literals and pure operators (no column/row, no
# subquery, no function) evaluates to the same boolean for every row, so the
# scan can fold it once. Returns `true`/`false`/`nil` for a constant predicate,
# `:dynamic` otherwise. Conservative allowlist — anything outside it (column
# refs, subqueries, functions like `random()`) is treated as dynamic.
defp constant_filter_value(where, db) when not is_nil(where) do
if const_predicate?(where), do: Value.truthy(eval(where, constant_env(db))), else: :dynamic
end
defp constant_filter_value(_where, _db), do: :dynamic
defp const_predicate?({:literal, _value}), do: true
defp const_predicate?({:not, e}), do: const_predicate?(e)
defp const_predicate?({:negate, e}), do: const_predicate?(e)
defp const_predicate?({:bitnot, e}), do: const_predicate?(e)
defp const_predicate?({:collate, e, _name}), do: const_predicate?(e)
defp const_predicate?({:cast, e, _aff}), do: const_predicate?(e)
defp const_predicate?({:is, l, r}), do: const_predicate?(l) and const_predicate?(r)
defp const_predicate?({:is_not, l, r}), do: const_predicate?(l) and const_predicate?(r)
defp const_predicate?({:between, e, lo, hi, _neg}),
do: const_predicate?(e) and const_predicate?(lo) and const_predicate?(hi)
defp const_predicate?({:binary, op, l, r})
when op in [
:eq,
:ne,
:lt,
:le,
:gt,
:ge,
:and,
:or,
:add,
:sub,
:mul,
:div,
:mod,
:bitand,
:bitor,
:shl,
:shr,
:concat
],
do: const_predicate?(l) and const_predicate?(r)
defp const_predicate?(_other), do: false
defp compile_scan_filter(nil), do: nil
defp compile_scan_filter(where) do
# Closures only pay off on large predicates; on a small one the extra call
# indirection costs more than the per-row AST dispatch it removes. Below the
# threshold, fall back to evaluating the (already column-pre-resolved) tree.
if System.get_env("EXSQL_NO_CLOSURE") == nil and expr_node_count(where, 0) >= 2 do
pred = compile_bool(where)
fn env -> pred.(env) == true end
else
fn env -> matches_where?(where, env) end
end
end
# A precompiled predicate is "row-local" when evaluating it touches only the
# current frame's columns — no `env.db`, `env.outer`, or other frames — so it
# can be filtered against a minimal `%{group:, frames:}` map. Conservative: an
# allowlist of node types, anything else (functions, LIKE/GLOB which read
# `db.case_sensitive_like`, subqueries, unresolved or cross-frame columns)
# falls back to the full-env path.
defp row_local?({:fast_column, _key, _aff, _coll}), do: true
defp row_local?({:literal, _value}), do: true
defp row_local?({:not, e}), do: row_local?(e)
defp row_local?({:negate, e}), do: row_local?(e)
defp row_local?({:bitnot, e}), do: row_local?(e)
defp row_local?({:cast, e, _aff}), do: row_local?(e)
defp row_local?({:collate, e, _name}), do: row_local?(e)
defp row_local?({:is, l, r}), do: row_local?(l) and row_local?(r)
defp row_local?({:is_not, l, r}), do: row_local?(l) and row_local?(r)
defp row_local?({:between, e, lo, hi, _neg}),
do: row_local?(e) and row_local?(lo) and row_local?(hi)
defp row_local?({:in, e, list, _neg}) when is_list(list),
do: row_local?(e) and Enum.all?(list, &row_local?/1)
defp row_local?({:in_cached, e, _members, _aff, _neg}), do: row_local?(e)
defp row_local?({:in_membership, e, _membership}), do: row_local?(e)
defp row_local?({:binary, op, l, r})
when op in [
:eq,
:ne,
:lt,
:le,
:gt,
:ge,
:and,
:or,
:add,
:sub,
:mul,
:div,
:mod,
:bitand,
:bitor,
:shl,
:shr,
:concat
],
do: row_local?(l) and row_local?(r)
defp row_local?(_other), do: false
# Boolean context (the WHERE filter and AND/OR/NOT operands): builds a closure
# returning `boolean()|nil` directly, skipping the `bool_value/1` (→ 0/1) wrap
# and `truthy/1` unwrap that the value-context `compile_pred/1` would pay for
# every row. `compiled_comparison/5` already yields a raw boolean.
defp compile_bool({:binary, :and, left, right}) do
l = compile_bool(left)
r = compile_bool(right)
fn env -> Value.sql_and(l.(env), r.(env)) end
end
defp compile_bool({:binary, :or, left, right} = node) do
# `col = v1 OR col = v2 OR …` (same column, all literal RHS) is exactly
# `col IN (v1, v2, …)` — route it through the IN membership compiler (O(1)
# set probe per row) instead of evaluating a chain of comparisons. Big win
# for an unindexed column (the indexed case already takes the index via
# or_access_path; this only changes the residual per-row filter).
case or_chain_to_in(node) do
{:ok, col_node, literals} ->
bool_fallback({:in, col_node, literals, false})
:no ->
l = compile_bool(left)
r = compile_bool(right)
fn env -> Value.sql_or(l.(env), r.(env)) end
end
end
defp compile_bool({:not, expr}) do
c = compile_bool(expr)
fn env -> Value.sql_not(c.(env)) end
end
defp compile_bool({:binary, op, left, right} = node)
when op in [:eq, :ne, :lt, :le, :gt, :ge] do
aff_l = static_affinity(left)
aff_r = static_affinity(right)
if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do
compiled_comparison(op, left, aff_l, right, aff_r)
else
bool_fallback(node)
end
end
defp compile_bool({:between, expr, low, high, negated} = node) do
aff_e = static_affinity(expr)
aff_lo = static_affinity(low)
aff_hi = static_affinity(high)
if aff_e && aff_lo && aff_hi && binary_collation?(expr) && binary_collation?(low) &&
binary_collation?(high) do
compiled_between(expr, aff_e, low, aff_lo, high, aff_hi, negated)
else
bool_fallback(node)
end
end
defp compile_bool({:is, left, right} = node), do: compiled_is(:eq, left, right, node)
defp compile_bool({:is_not, left, right} = node), do: compiled_is(:ne, left, right, node)
defp compile_bool(node), do: bool_fallback(node)
# Recognizes `col = lit OR col = lit OR …` (≥2 disjuncts, all the same fast
# column against literals) and returns `{:ok, col_node, [literal_nodes]}` so it
# can be compiled as `col IN (…)`. `:no` for any other OR shape (mixed columns,
# non-literal RHS, nested non-equality), which keeps the plain OR path.
defp or_chain_to_in({:binary, :or, _left, _right} = node) do
leaves = or_leaves(node)
with [_, _ | _] <- leaves,
parsed when is_list(parsed) <- parse_eq_col_literals(leaves),
[{key, col_node, _lit} | _] <- parsed,
true <- Enum.all?(parsed, fn {k, _, _} -> k == key end) do
{:ok, col_node, Enum.map(parsed, fn {_, _, lit} -> lit end)}
else
_ -> :no
end
end
defp or_leaves({:binary, :or, left, right}), do: or_leaves(left) ++ or_leaves(right)
defp or_leaves(other), do: [other]
defp parse_eq_col_literals(leaves) do
Enum.reduce_while(leaves, [], fn leaf, acc ->
case eq_col_literal(leaf) do
{_key, _col, _lit} = parsed -> {:cont, [parsed | acc]}
:no -> {:halt, :no}
end
end)
|> case do
:no -> :no
list -> Enum.reverse(list)
end
end
defp eq_col_literal(
{:binary, :eq, {:fast_column, key, _aff, _coll} = col, {:literal, _} = lit}
),
do: {key, col, lit}
defp eq_col_literal(
{:binary, :eq, {:literal, _} = lit, {:fast_column, key, _aff, _coll} = col}
),
do: {key, col, lit}
defp eq_col_literal(_other), do: :no
# `IS` / `IS NOT` (incl. `IS NULL`) was falling through to the per-row `eval`
# path (`comparison_operands` → per-row `static_affinity`/`comparison_coerce`).
# When affinities and collations are statically resolvable — the same gate the
# `=`/`<` comparisons use — precompile operands once and compare per row,
# mirroring `comparison_operands`' fast branch exactly (binary collation,
# affinity coercion folded in) but with `IS` semantics (`compare == :eq`,
# never NULL).
defp compiled_is(kind, left, right, node) do
aff_l = static_affinity(left)
aff_r = static_affinity(right)
if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do
{coerce_l, coerce_r} = coercion_pair(aff_l, aff_r)
build_is(kind, operand(left, coerce_l), operand(right, coerce_r))
else
bool_fallback(node)
end
end
defp build_is(kind, {:const, a}, {:const, b}) do
result = is_result(kind, a, b)
fn _env -> result end
end
defp build_is(kind, {:col, lk}, {:const, b}),
do: fn env -> is_result(kind, col_value(env, lk), b) end
defp build_is(kind, {:const, a}, {:col, rk}),
do: fn env -> is_result(kind, a, col_value(env, rk)) end
defp build_is(kind, {:col, lk}, {:col, rk}),
do: fn env -> is_result(kind, col_value(env, lk), col_value(env, rk)) end
defp build_is(kind, {:col, lk}, {:fun, rf}),
do: fn env -> is_result(kind, col_value(env, lk), rf.(env)) end
defp build_is(kind, {:fun, lf}, {:col, rk}),
do: fn env -> is_result(kind, lf.(env), col_value(env, rk)) end
defp build_is(kind, {:fun, lf}, {:const, b}),
do: fn env -> is_result(kind, lf.(env), b) end
defp build_is(kind, {:const, a}, {:fun, rf}),
do: fn env -> is_result(kind, a, rf.(env)) end
defp build_is(kind, {:fun, lf}, {:fun, rf}),
do: fn env -> is_result(kind, lf.(env), rf.(env)) end
defp is_result(:eq, a, b), do: Value.compare(a, b, :binary) == :eq
defp is_result(:ne, a, b), do: Value.compare(a, b, :binary) != :eq
# `col BETWEEN lo AND hi` desugars to `col >= lo AND col <= hi`. When `col` is a
# bare fast column (no per-operand coercion) and the bounds reduce to constants
# — the dominant shape — read the column *once* per row rather than once for
# each comparison. Anything else falls back to the two-comparison form.
defp compiled_between(expr, aff_e, low, aff_lo, high, aff_hi, negated) do
{ce_lo, c_lo} = coercion_pair(aff_e, aff_lo)
{ce_hi, c_hi} = coercion_pair(aff_e, aff_hi)
e_lo = operand(expr, ce_lo)
e_hi = operand(expr, ce_hi)
lo = operand(low, c_lo)
hi = operand(high, c_hi)
case {e_lo, e_hi, lo, hi} do
{{:col, key}, {:col, key}, {:const, a}, {:const, b}} ->
base = fn env ->
v = col_value(env, key)
Value.sql_and(
Value.compare_op(:ge, v, a, :binary),
Value.compare_op(:le, v, b, :binary)
)
end
if negated, do: fn env -> Value.sql_not(base.(env)) end, else: base
_ ->
ge = build_compare(:ge, e_lo, lo)
le = build_compare(:le, e_hi, hi)
if negated do
fn env -> Value.sql_not(Value.sql_and(ge.(env), le.(env))) end
else
fn env -> Value.sql_and(ge.(env), le.(env)) end
end
end
end
# Non-boolean or non-fast nodes: evaluate as a value, then reduce to a boolean
# exactly as the old `truthy(pred) == true` filter did.
defp bool_fallback(node) do
c = compile_pred(node)
fn env -> Value.truthy(c.(env)) end
end
defp expr_node_count(_expr, acc) when acc >= 12, do: acc
defp expr_node_count(tuple, acc) when is_tuple(tuple) do
Enum.reduce(Tuple.to_list(tuple), acc + 1, &expr_node_count/2)
end
defp expr_node_count(list, acc) when is_list(list) do
Enum.reduce(list, acc, &expr_node_count/2)
end
defp expr_node_count(_other, acc), do: acc
defp simplify_where_filter({:binary, :and, _left, _right} = expr) do
terms = expr |> where_conjuncts() |> Enum.map(&simplify_where_filter/1)
cond do
Enum.any?(terms, &false_filter?/1) ->
false_filter()
contradictory_conjuncts?(terms) ->
false_filter()
true ->
terms
|> Enum.reject(&true_filter?/1)
|> then(&rebuild_boolean_filter(:and, &1))
end
end
defp simplify_where_filter({:binary, :or, _left, _right} = expr) do
terms = expr |> where_disjuncts() |> Enum.map(&simplify_where_filter/1)
cond do
Enum.any?(terms, &true_filter?/1) ->
true_filter()
true ->
terms
|> Enum.reject(&false_filter?/1)
|> then(&rebuild_boolean_filter(:or, &1))
end
end
defp simplify_where_filter({:between, _expr, {:literal, low}, {:literal, high}, false} = expr)
when is_number(low) and is_number(high) do
if Value.compare(low, high) == :gt, do: false_filter(), else: expr
end
defp simplify_where_filter(other), do: other
defp false_filter, do: {:literal, 0}
defp true_filter, do: {:literal, 1}
defp false_filter?({:literal, value}), do: Value.truthy(value) == false
defp false_filter?(_expr), do: false
defp true_filter?({:literal, value}), do: Value.truthy(value) == true
defp true_filter?(_expr), do: false
defp rebuild_boolean_filter(:and, []), do: true_filter()
defp rebuild_boolean_filter(:or, []), do: false_filter()
defp rebuild_boolean_filter(_op, [term]), do: term
defp rebuild_boolean_filter(op, [term | terms]),
do: Enum.reduce(terms, term, &{:binary, op, &2, &1})
defp contradictory_conjuncts?(terms) do
terms
|> Enum.flat_map(&where_column_constraints/1)
|> Enum.group_by(fn {key, _constraint} -> key end, fn {_key, constraint} -> constraint end)
|> Enum.any?(fn {_key, constraints} -> contradictory_column_constraints?(constraints) end)
end
defp where_column_constraints({:binary, op, left, right})
when op in [:eq, :lt, :le, :gt, :ge] do
cond do
column_numeric_literal?(left, right) ->
[{fast_column_key(left), comparison_constraint(op, literal_value(right))}]
column_numeric_literal?(right, left) ->
[{fast_column_key(right), comparison_constraint(flip_range_op(op), literal_value(left))}]
true ->
[]
end
end
defp where_column_constraints({:is, column, {:literal, nil}}) do
if fast_column?(column), do: [{fast_column_key(column), :null}], else: []
end
defp where_column_constraints({:is_not, column, {:literal, nil}}) do
if fast_column?(column), do: [{fast_column_key(column), :not_null}], else: []
end
defp where_column_constraints({:between, column, {:literal, low}, {:literal, high}, false})
when is_number(low) and is_number(high) do
if fast_column?(column) do
[
{fast_column_key(column), {:range, :ge, low}},
{fast_column_key(column), {:range, :le, high}}
]
else
[]
end
end
defp where_column_constraints(_term), do: []
defp column_numeric_literal?(column, {:literal, value}),
do: fast_column?(column) and is_number(value)
defp column_numeric_literal?(_column, _literal), do: false
defp fast_column?({:fast_column, _key, _affinity, _collation}), do: true
defp fast_column?(_expr), do: false
defp fast_column_key({:fast_column, key, _affinity, _collation}), do: key
defp literal_value({:literal, value}), do: value
defp comparison_constraint(:eq, value), do: {:eq, value}
defp comparison_constraint(op, value), do: {:range, op, value}
defp contradictory_column_constraints?(constraints) do
null? = :null in constraints
not_null? = :not_null in constraints
comparisons? =
Enum.any?(constraints, &match?({:eq, _value}, &1)) or
Enum.any?(constraints, &match?({:range, _op, _value}, &1))
cond do
null? and (not_null? or comparisons?) ->
true
contradictory_equalities?(constraints) ->
true
equality_outside_range?(constraints) ->
true
contradictory_ranges?(constraints) ->
true
true ->
false
end
end
defp contradictory_equalities?(constraints) do
constraints
|> Enum.flat_map(fn
{:eq, value} -> [value]
_constraint -> []
end)
|> case do
[] ->
false
[first | rest] ->
Enum.any?(rest, &(Value.compare(&1, first) != :eq))
end
end
defp equality_outside_range?(constraints) do
equalities =
Enum.flat_map(constraints, fn
{:eq, value} -> [value]
_constraint -> []
end)
ranges =
Enum.flat_map(constraints, fn
{:range, op, value} -> [{op, value}]
_constraint -> []
end)
Enum.any?(equalities, fn equality ->
Enum.any?(ranges, fn {op, value} -> Value.compare_op(op, equality, value) != true end)
end)
end
defp contradictory_ranges?(constraints) do
constraints
|> Enum.reduce({nil, nil}, fn
{:range, op, value}, {lower, upper} when op in [:gt, :ge] ->
{strongest_lower_bound(lower, {value, op}), upper}
{:range, op, value}, {lower, upper} when op in [:lt, :le] ->
{lower, strongest_upper_bound(upper, {value, op})}
_constraint, acc ->
acc
end)
|> incompatible_range_bounds?()
end
defp incompatible_range_bounds?({nil, _upper}), do: false
defp incompatible_range_bounds?({_lower, nil}), do: false
defp incompatible_range_bounds?({{lower_value, lower_op}, {upper_value, upper_op}}) do
case Value.compare(lower_value, upper_value) do
:gt -> true
:eq -> lower_op == :gt or upper_op == :lt
:lt -> false
end
end
defp compile_pred({:binary, :and, left, right}) do
l = compile_pred(left)
r = compile_pred(right)
fn env -> bool_value(Value.sql_and(Value.truthy(l.(env)), Value.truthy(r.(env)))) end
end
defp compile_pred({:binary, :or, left, right}) do
l = compile_pred(left)
r = compile_pred(right)
fn env -> bool_value(Value.sql_or(Value.truthy(l.(env)), Value.truthy(r.(env)))) end
end
defp compile_pred({:not, expr}) do
c = compile_pred(expr)
fn env -> bool_value(Value.sql_not(Value.truthy(c.(env)))) end
end
defp compile_pred({:binary, op, left, right} = node)
when op in [:eq, :ne, :lt, :le, :gt, :ge] do
aff_l = static_affinity(left)
aff_r = static_affinity(right)
if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do
cmp = compiled_comparison(op, left, aff_l, right, aff_r)
fn env -> bool_value(cmp.(env)) end
else
fn env -> eval(node, env) end
end
end
# `expr BETWEEN low AND high` lowers to two comparisons. Compiling it (rather
# than falling through to the generic per-row `eval`) lets each comparison
# bake its affinity coercion in once — the hot path for the `index/between/*`
# corpus files.
defp compile_pred({:between, expr, low, high, negated} = node) do
aff_e = static_affinity(expr)
aff_lo = static_affinity(low)
aff_hi = static_affinity(high)
if aff_e && aff_lo && aff_hi && binary_collation?(expr) && binary_collation?(low) &&
binary_collation?(high) do
ge = compiled_comparison(:ge, expr, aff_e, low, aff_lo)
le = compiled_comparison(:le, expr, aff_e, high, aff_hi)
if negated do
fn env -> bool_value(Value.sql_not(Value.sql_and(ge.(env), le.(env)))) end
else
fn env -> bool_value(Value.sql_and(ge.(env), le.(env))) end
end
else
fn env -> eval(node, env) end
end
end
defp compile_pred({:in, expr, list, negated} = node) when is_list(list) do
aff = static_affinity(expr)
if aff && binary_collation?(expr) do
c = compile_pred(expr)
case literal_values(list) do
{:ok, values} ->
membership = compile_in_membership(values, aff, :binary, :blob, negated)
fn env -> compiled_in_membership(c.(env), membership) end
:error ->
members = Enum.map(list, &compile_pred/1)
fn env ->
in_membership(c.(env), aff, :binary, Enum.map(members, & &1.(env)), :blob, negated)
end
end
else
fn env -> eval(node, env) end
end
end
defp compile_pred({:in_cached, expr, members, rhs_affinity, negated} = node) do
aff = static_affinity(expr)
if aff && binary_collation?(expr) do
c = compile_pred(expr)
membership = compile_in_membership(members, aff, :binary, rhs_affinity, negated)
fn env -> compiled_in_membership(c.(env), membership) end
else
fn env -> eval(node, env) end
end
end
defp compile_pred({:in_membership, expr, membership}) do
c = compile_pred(expr)
fn env -> compiled_in_membership(c.(env), membership) end
end
defp compile_pred({:fast_column, key, _affinity, _collation}) do
fn
%{group: nil, frames: [frame]} -> frame_cell(frame, key)
env -> resolve_column(env, nil, key)
end
end
defp compile_pred({:fast_frame_column, index, key, _affinity, _collation}) do
fn env -> fast_frame_column(env, index, key) end
end
defp compile_pred({:fast_outer_frame_column, index, key, _affinity, _collation}) do
fn env -> fast_outer_frame_column(env, index, key) end
end
defp compile_pred({:literal, value}), do: fn _env -> value end
defp compile_pred(expr), do: fn env -> eval(expr, env) end
# Builds a closure computing `compare_op(op, left, right, :binary)` with §4.2
# comparison affinity applied. The affinities are static, so the coercion
# branch is decided **once** here instead of per row (as
# `Value.comparison_coerce/4` would). Returns the raw `boolean()|nil` result
# (callers wrap with `bool_value/1`). Mirrors `comparison_coerce/4` exactly.
defp compiled_comparison(op, left, aff_l, right, aff_r) do
{coerce_l, coerce_r} = coercion_pair(aff_l, aff_r)
build_compare(op, operand(left, coerce_l), operand(right, coerce_r))
end
# Splits the comparison-affinity rule into the per-operand coercion each side
# needs (`nil` = none), so the left/right coercions can be applied independently
# (e.g. fused once across both bounds of a BETWEEN).
defp coercion_pair(aff_l, aff_r) do
case coercion(aff_l, aff_r) do
:none -> {nil, nil}
:right_num -> {nil, :numeric}
:left_num -> {:numeric, nil}
:right_text -> {nil, :text}
:left_text -> {:text, nil}
end
end
# Classifies an operand as a compile-time `{:const, value}` (literals, with any
# affinity coercion folded in once) or a `{:fun, closure}` evaluated per row.
defp operand({:literal, value}, nil), do: {:const, value}
defp operand({:literal, value}, affinity), do: {:const, Value.apply_affinity(value, affinity)}
# An uncoerced column reads inline (`col_value/2`) in the comparison closure
# below, saving the separate `fast_column` closure call per row.
defp operand({:fast_column, key, _aff, _coll}, nil), do: {:col, key}
defp operand(node, nil), do: {:fun, compile_pred(node)}
defp operand(node, affinity) do
f = compile_pred(node)
{:fun, fn env -> Value.apply_affinity(f.(env), affinity) end}
end
# `fast_column` read, matching its eval clauses exactly. The hot single-frame
# path inlines the row map lookup instead of calling `Map.get/2` (which itself
# delegates to `Map.get/3`) — two fewer function calls on a per-row-per-column
# hot path (tens of millions of calls in scan-heavy queries).
defp col_value(%{group: nil, frames: [frame]}, key), do: frame_cell(frame, key)
defp col_value(env, key), do: resolve_column(env, nil, key)
# Reads a column from a frame, handling both representations: cold frames
# (joins built from maps, views, null-frames) carry a `key => value` map row;
# the hot single-table scan carries a positional tuple read via `col_index`.
defp frame_cell(frame, key) do
case frame.row do
%{^key => value} ->
value
row when is_tuple(row) ->
case frame.col_index do
%{^key => pos} -> ExSQL.Table.cell(row, pos)
_ -> nil
end
_ ->
nil
end
end
# Widen a frame's row to a full `key => value` map (for DML/returning paths
# that consume whole rows). Map rows pass through; tuple rows are rebuilt.
defp frame_row_map(%{row: row}) when is_map(row), do: row
defp frame_row_map(%{row: row, col_index: ci}),
do: Map.new(ci, fn {k, pos} -> {k, ExSQL.Table.cell(row, pos)} end)
# Specialized per the shape of each side, so a constant operand (the common
# `column <op> literal`) is captured directly and a column read is inlined,
# rather than each going through a closure every row.
defp build_compare(op, {:const, a}, {:const, b}) do
result = Value.compare_op(op, a, b, :binary)
fn _env -> result end
end
defp build_compare(op, {:col, lk}, {:const, b}),
do: fn env -> Value.compare_op(op, col_value(env, lk), b, :binary) end
defp build_compare(op, {:const, a}, {:col, rk}),
do: fn env -> Value.compare_op(op, a, col_value(env, rk), :binary) end
defp build_compare(op, {:col, lk}, {:col, rk}),
do: fn env -> Value.compare_op(op, col_value(env, lk), col_value(env, rk), :binary) end
defp build_compare(op, {:col, lk}, {:fun, rf}),
do: fn env -> Value.compare_op(op, col_value(env, lk), rf.(env), :binary) end
defp build_compare(op, {:fun, lf}, {:col, rk}),
do: fn env -> Value.compare_op(op, lf.(env), col_value(env, rk), :binary) end
defp build_compare(op, {:fun, lf}, {:const, b}),
do: fn env -> Value.compare_op(op, lf.(env), b, :binary) end
defp build_compare(op, {:const, a}, {:fun, rf}),
do: fn env -> Value.compare_op(op, a, rf.(env), :binary) end
defp build_compare(op, {:fun, lf}, {:fun, rf}),
do: fn env -> Value.compare_op(op, lf.(env), rf.(env), :binary) end
defp coercion(aff_l, aff_r) do
num_l = aff_l in [:integer, :real, :numeric]
num_r = aff_r in [:integer, :real, :numeric]
cond do
num_l and not num_r -> :right_num
num_r and not num_l -> :left_num
aff_l == :text and aff_r != :text -> :right_text
aff_r == :text and aff_l != :text -> :left_text
true -> :none
end
end
# Affinity computable without a row (matches expr_affinity/2): nil only for a
# bare {:column,…} that was not pre-resolved, which forces the eval fallback.
defp static_affinity({:fast_column, _key, affinity, _collation}), do: affinity
defp static_affinity({:fast_frame_column, _index, _key, affinity, _collation}), do: affinity
defp static_affinity({:fast_outer_frame_column, _index, _key, affinity, _collation}),
do: affinity
defp static_affinity({:cast, _expr, affinity}), do: affinity
defp static_affinity({:collate, expr, _name}), do: static_affinity(expr)
defp static_affinity({:column, _qualifier, _name}), do: nil
defp static_affinity(_expr), do: :blob
# True when an operand's comparison collation is provably binary (the default):
# a pre-resolved column carrying :binary, a literal, or any expression with no
# COLLATE and no bare column. Conservative — anything else returns false and
# the comparison takes the eval fallback (which resolves collation per row).
defp binary_collation?({:fast_column, _key, _affinity, collation}), do: collation == :binary
defp binary_collation?({:fast_frame_column, _index, _key, _affinity, collation}),
do: collation == :binary
defp binary_collation?({:fast_outer_frame_column, _index, _key, _affinity, collation}),
do: collation == :binary
defp binary_collation?({:literal, _value}), do: true
defp binary_collation?({:column, _qualifier, _name}), do: false
defp binary_collation?({:collate, _expr, _name}), do: false
defp binary_collation?({:cast, expr, _affinity}), do: binary_collation?(expr)
defp binary_collation?(expr) when is_tuple(expr) do
expr |> Tuple.to_list() |> Enum.all?(&binary_collation_part?/1)
end
defp binary_collation?(_expr), do: true
defp binary_collation_part?(element) when is_tuple(element), do: binary_collation?(element)
defp binary_collation_part?(elements) when is_list(elements),
do: Enum.all?(elements, &binary_collation_part?/1)
defp binary_collation_part?(_element), do: true
defp rewrite_scan_columns({:column, qualifier, name} = col, frame_name, visible, env) do
key = Table.key(name)
if (qualifier == nil or Table.key(qualifier) == frame_name) and MapSet.member?(visible, key) do
{:fast_column, key, expr_affinity(col, env), expr_collation(col, env)}
else
col
end
end
defp rewrite_scan_columns({:binary, op, l, r}, fname, vis, env) do
{:binary, op, rewrite_scan_columns(l, fname, vis, env),
rewrite_scan_columns(r, fname, vis, env)}
end
defp rewrite_scan_columns({:not, e}, fname, vis, env),
do: {:not, rewrite_scan_columns(e, fname, vis, env)}
defp rewrite_scan_columns({:negate, e}, fname, vis, env),
do: {:negate, rewrite_scan_columns(e, fname, vis, env)}
defp rewrite_scan_columns({:bitnot, e}, fname, vis, env),
do: {:bitnot, rewrite_scan_columns(e, fname, vis, env)}
defp rewrite_scan_columns({:cast, e, affinity}, fname, vis, env),
do: {:cast, rewrite_scan_columns(e, fname, vis, env), affinity}
defp rewrite_scan_columns({:collate, e, n}, fname, vis, env),
do: {:collate, rewrite_scan_columns(e, fname, vis, env), n}
defp rewrite_scan_columns({:is, l, r}, fname, vis, env),
do: {:is, rewrite_scan_columns(l, fname, vis, env), rewrite_scan_columns(r, fname, vis, env)}
defp rewrite_scan_columns({:is_not, l, r}, fname, vis, env),
do:
{:is_not, rewrite_scan_columns(l, fname, vis, env),
rewrite_scan_columns(r, fname, vis, env)}
defp rewrite_scan_columns({:between, e, lo, hi, neg}, fname, vis, env) do
{:between, rewrite_scan_columns(e, fname, vis, env),
rewrite_scan_columns(lo, fname, vis, env), rewrite_scan_columns(hi, fname, vis, env), neg}
end
defp rewrite_scan_columns({:in, e, list, neg}, fname, vis, env) when is_list(list) do
{:in, rewrite_scan_columns(e, fname, vis, env),
Enum.map(list, &rewrite_scan_columns(&1, fname, vis, env)), neg}
end
# `IN (uncorrelated subquery)`: evaluate the subquery once here (compile time)
# instead of for every scanned row. Only provably self-contained single-table
# subqueries are hoisted — see hoist_in_subquery/2 — so correctness is
# unaffected; a correlated or complex subquery is left as a normal IN.
defp rewrite_scan_columns({:in, e, {:select, %Select{} = select}, neg}, fname, vis, env) do
rewritten = rewrite_scan_columns(e, fname, vis, env)
case hoist_in_subquery(select, env.db) do
{:ok, members, rhs_affinity} -> in_membership_expr(rewritten, members, rhs_affinity, neg)
:no -> {:in, rewritten, {:select, select}, neg}
end
end
defp rewrite_scan_columns({:subquery, %Select{} = select} = expr, _fname, _vis, env) do
select
|> hoist_scalar_subquery(env.db, expr)
|> compile_correlated_subquery(select, env.db, frame_column_lookup(env.db, env.frames))
end
defp rewrite_scan_columns({:exists, %Select{} = select} = expr, _fname, _vis, env) do
select
|> hoist_exists_subquery(env.db, expr)
|> compile_correlated_subquery(select, env.db, frame_column_lookup(env.db, env.frames))
end
defp rewrite_scan_columns({:like, e, p, esc, neg}, fname, vis, env) do
{:like, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(p, fname, vis, env),
esc, neg}
end
defp rewrite_scan_columns({:glob, e, p, neg}, fname, vis, env),
do:
{:glob, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(p, fname, vis, env),
neg}
defp rewrite_scan_columns({:regexp, e, p, neg}, fname, vis, env) do
{:regexp, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(p, fname, vis, env),
neg}
end
defp rewrite_scan_columns(other, _fname, _vis, _env), do: other
defp rewrite_frame_columns({:column, nil, name} = column, lookup) do
case Map.get(lookup.unqualified, Table.key(name)) do
{:fast_frame_column, _index, _key, _affinity, _collation} = compiled -> compiled
_missing_or_ambiguous -> column
end
end
defp rewrite_frame_columns({:column, qualifier, name} = column, lookup) do
case Map.get(lookup.qualified, {Table.key(qualifier), Table.key(name)}) do
{:fast_frame_column, _index, _key, _affinity, _collation} = compiled -> compiled
nil -> column
end
end
defp rewrite_frame_columns({:subquery, %Select{} = select} = expr, %{db: db} = lookup) do
select
|> hoist_scalar_subquery(db, expr)
|> compile_correlated_subquery(select, db, lookup)
end
defp rewrite_frame_columns({:exists, %Select{} = select} = expr, %{db: db} = lookup) do
select
|> hoist_exists_subquery(db, expr)
|> compile_correlated_subquery(select, db, lookup)
end
defp rewrite_frame_columns({:window, _name, _args, _spec, _filter} = expr, _lookup), do: expr
defp rewrite_frame_columns(%_struct{} = term, _lookup), do: term
defp rewrite_frame_columns(tuple, lookup) when is_tuple(tuple) do
tuple
|> Tuple.to_list()
|> Enum.map(&rewrite_frame_columns(&1, lookup))
|> List.to_tuple()
end
defp rewrite_frame_columns(list, lookup) when is_list(list) do
Enum.map(list, &rewrite_frame_columns(&1, lookup))
end
defp rewrite_frame_columns(other, _lookup), do: other
defp rewrite_outer_frame_columns(expr, nil), do: expr
defp rewrite_outer_frame_columns({:column, nil, _name} = column, _lookup), do: column
defp rewrite_outer_frame_columns({:column, qualifier, name} = column, lookup) do
case Map.get(lookup.qualified, {Table.key(qualifier), Table.key(name)}) do
{:fast_frame_column, index, key, affinity, collation} ->
{:fast_outer_frame_column, index, key, affinity, collation}
nil ->
column
end
end
defp rewrite_outer_frame_columns(tuple, lookup) when is_tuple(tuple) do
tuple
|> Tuple.to_list()
|> Enum.map(&rewrite_outer_frame_columns(&1, lookup))
|> List.to_tuple()
end
defp rewrite_outer_frame_columns(list, lookup) when is_list(list) do
Enum.map(list, &rewrite_outer_frame_columns(&1, lookup))
end
defp rewrite_outer_frame_columns(other, _lookup), do: other
# Evaluates an IN-subquery once if it is provably uncorrelated: a single plain
# base table in FROM and every referenced column is an unqualified column of
# that table (or a rowid name), with no nested subquery. Anything qualified,
# multi-table, or containing a subquery is declined (`:no`) and left to run
# per-row — conservative, so a correlated subquery is never wrongly cached.
defp hoist_in_subquery(%Select{} = select, db) do
with {:ok, local_columns} <- subquery_local_columns(select, db),
true <- all_refs_local?(select, local_columns) do
result = query_result(db, select, nil)
{:ok, Enum.map(result.rows, &hd/1), List.first(result.affinities) || :blob}
else
_ -> :no
end
end
defp in_membership_expr(expr, members, rhs_affinity, negated) do
case static_affinity(expr) do
nil ->
{:in_cached, expr, members, rhs_affinity, negated}
affinity ->
if binary_collation?(expr) do
{:in_membership, expr,
compile_in_membership(members, affinity, :binary, rhs_affinity, negated)}
else
{:in_cached, expr, members, rhs_affinity, negated}
end
end
end
defp compile_correlated_subquery(
{:subquery, %Select{}},
%Select{columns: [{{:function, "count", :star}, nil}]} = select,
db,
outer_lookup
) do
case correlated_table_filter(select, db, outer_lookup) do
{:ok, table_key, template, filter} -> {:correlated_count, table_key, template, filter}
:no -> {:subquery, select}
end
end
defp compile_correlated_subquery({:exists, %Select{}}, %Select{} = select, db, outer_lookup) do
case correlated_table_filter(select, db, outer_lookup) do
{:ok, table_key, template, filter} -> {:correlated_exists, table_key, template, filter}
:no -> {:exists, select}
end
end
defp compile_correlated_subquery(expr, _select, _db, _outer_lookup), do: expr
defp correlated_table_filter(
%Select{
from: {:table, name, alias_name},
group_by: [],
having: nil,
windows: windows,
order_by: [],
limit: nil,
offset: nil,
distinct: false
} = select,
db,
outer_lookup
)
when windows == %{} do
table_key = table_source_key(name)
case plain_table(db, table_key) do
%Table{} = table ->
template = table_frame(table, alias_name)
filter = correlated_table_filter(select.where, db, template, outer_lookup)
{:ok, table_key, template, filter}
nil ->
:no
end
end
defp correlated_table_filter(_select, _db, _outer_lookup), do: :no
defp correlated_table_filter(nil, _db, _template, _outer_lookup), do: nil
defp correlated_table_filter(where, db, template, outer_lookup) do
where
|> precompile_scan_where(db, [template], nil)
|> rewrite_outer_frame_columns(outer_lookup)
|> compile_scan_filter()
end
defp hoist_scalar_subquery(%Select{} = select, db, fallback) do
if uncorrelated_subquery?(select, db) do
db
|> query_result(select, nil)
|> scalar_subquery_literal()
else
fallback
end
end
defp scalar_subquery_literal(%Result{rows: [[value | _] | _]}), do: {:literal, value}
defp scalar_subquery_literal(%Result{rows: []}), do: {:literal, nil}
defp hoist_exists_subquery(%Select{} = select, db, fallback) do
if uncorrelated_subquery?(select, db) do
{:literal, bool_value(query_result(db, select, nil).rows != [])}
else
fallback
end
end
defp uncorrelated_subquery?(%Select{} = select, db) do
with {:ok, local_columns} <- subquery_local_columns(select, db) do
all_refs_local?(select, local_columns)
else
_ -> false
end
end
# Union of column keys of every plain single-table FROM anywhere in the
# subquery tree (the subquery's own FROM plus any nested subqueries'). Bails on
# a join/subquery/non-plain FROM (more complex scoping than we analyze here).
defp subquery_local_columns(%Select{from: from} = select, db) do
with {:ok, base} <- from_table_columns(from, db) do
select
|> nested_selects()
|> Enum.reduce_while({:ok, base}, fn sub, {:ok, acc} ->
case subquery_local_columns(sub, db) do
{:ok, cols} -> {:cont, {:ok, MapSet.union(acc, cols)}}
:error -> {:halt, :error}
end
end)
end
end
defp from_table_columns({:table, name, _alias}, db) do
case plain_table(db, table_source_key(name)) do
%Table{} = table -> {:ok, MapSet.new(table.columns, &Table.key(&1.name))}
_ -> :error
end
end
defp from_table_columns(_other, _db), do: :error
defp nested_selects(%Select{} = select) do
select |> select_scope_exprs() |> Enum.flat_map(&collect_nested_selects/1)
end
defp collect_nested_selects({:select, %Select{} = s}), do: [s | nested_selects(s)]
defp collect_nested_selects({:subquery, %Select{} = s}), do: [s | nested_selects(s)]
defp collect_nested_selects({:subquery, %Select{} = s, _alias}), do: [s | nested_selects(s)]
defp collect_nested_selects({:scalar_subquery, %Select{} = s}), do: [s | nested_selects(s)]
defp collect_nested_selects({:exists, %Select{} = s}), do: [s | nested_selects(s)]
defp collect_nested_selects(tuple) when is_tuple(tuple),
do: tuple |> Tuple.to_list() |> Enum.flat_map(&collect_nested_selects/1)
defp collect_nested_selects(list) when is_list(list),
do: Enum.flat_map(list, &collect_nested_selects/1)
defp collect_nested_selects(_other), do: []
defp select_scope_exprs(%Select{} = s) do
column_exprs =
Enum.flat_map(s.columns, fn
{expr, _alias} -> [expr]
_star -> []
end)
order_exprs = Enum.map(s.order_by, &elem(&1, 0))
Enum.reject([s.where, s.having | column_exprs] ++ order_exprs ++ s.group_by, &is_nil/1)
end
# Every column the subquery (and its nested subqueries) references is an
# unqualified column of one of the subquery-tree's own tables — i.e. it never
# reaches the outer query, so the subquery is uncorrelated and safe to hoist.
# A column whose name is absent from the union must resolve to the outer scope
# (correlated) and forces `:no`; a qualified column is declined conservatively.
defp all_refs_local?(%Select{} = select, union) do
select |> select_scope_exprs() |> Enum.all?(&expr_refs_local?(&1, union))
end
defp expr_refs_local?(%Select{} = sub, union), do: all_refs_local?(sub, union)
defp expr_refs_local?({:column, nil, name}, union) do
key = Table.key(name)
MapSet.member?(union, key) or key in @rowid_names
end
defp expr_refs_local?({:column, _qualifier, _name}, _union), do: false
defp expr_refs_local?(tuple, union) when is_tuple(tuple) do
tuple |> Tuple.to_list() |> Enum.all?(&expr_refs_local?(&1, union))
end
defp expr_refs_local?(list, union) when is_list(list) do
Enum.all?(list, &expr_refs_local?(&1, union))
end
defp expr_refs_local?(_other, _union), do: true
# -- column resolution ----------------------------------------------------------
#
# Unqualified names search the visible columns of every frame (USING/NATURAL
# join columns are hidden on the right side, so they resolve to the left
# table); qualified names pick a frame by alias. Either falls back to the
# outer query's environment, which is what makes subqueries correlated.
defp resolve_column(env, nil, name) do
key = Table.key(name)
case Enum.filter(env.frames, &visible?(&1, key)) do
[frame] ->
frame_cell(frame, key)
[_ | _] ->
fail("ambiguous column name: #{name}")
[] ->
rowid_frames =
if key in @rowid_names, do: Enum.filter(env.frames, & &1.has_rowid), else: []
case rowid_frames do
[frame] -> frame.rowid
[_ | _] -> fail("ambiguous column name: #{name}")
[] when env.outer != nil -> eval({:column, nil, name}, env.outer)
[] -> fail("no such column: #{name}")
end
end
end
defp resolve_column(env, qualifier, name) do
qkey = Table.key(qualifier)
case Enum.find(env.frames, &(&1.name == qkey)) do
nil ->
if env.outer != nil do
eval({:column, qualifier, name}, env.outer)
else
fail("no such column: #{qualifier}.#{name}")
end
frame ->
key = Table.key(name)
cond do
has_column?(frame, key) -> frame_cell(frame, key)
key in @rowid_names and frame.has_rowid -> frame.rowid
true -> fail("no such column: #{qualifier}.#{name}")
end
end
end
defp visible?(frame, key), do: has_column?(frame, key) and not MapSet.member?(frame.hidden, key)
defp has_column?(%{columns_by_key: columns}, key), do: Map.has_key?(columns, key)
defp has_column?(frame, key), do: List.keymember?(frame.columns, key, 0)
defp column_key({key, _display, _affinity}), do: key
defp column_key({key, _display, _affinity, _collation}), do: key
defp column_display({_key, display, _affinity}), do: display
defp column_display({_key, display, _affinity, _collation}), do: display
defp column_affinity_meta({_key, _display, affinity}), do: affinity
defp column_affinity_meta({_key, _display, affinity, _collation}), do: affinity
defp column_collation_meta({_key, _display, _affinity}), do: nil
defp column_collation_meta({_key, _display, _affinity, collation}), do: collation
# -- expression affinity ----------------------------------------------------------
#
# Mirrors sqlite3ExprAffinity: column references carry their column's
# declared affinity, CAST its target, rowid INTEGER, everything else none
# (which BLOB also means). Used to apply comparison affinity (§4.2).
defp expr_affinity({:column, qualifier, name}, env) do
case env.group do
[first | _] -> expr_affinity({:column, qualifier, name}, first)
[] -> :blob
nil -> column_affinity(env, qualifier, name)
end
end
defp expr_affinity({:fast_column, _key, affinity, _collation}, _env), do: affinity
defp expr_affinity({:fast_frame_column, _index, _key, affinity, _collation}, _env),
do: affinity
defp expr_affinity({:fast_outer_frame_column, _index, _key, affinity, _collation}, _env),
do: affinity
defp expr_affinity({:collate, expr, _name}, env), do: expr_affinity(expr, env)
defp expr_affinity({:cast, _expr, affinity}, _env), do: affinity
defp expr_affinity(_expr, _env), do: :blob
defp column_affinity(env, nil, name) do
key = Table.key(name)
case Enum.filter(env.frames, &visible?(&1, key)) do
[frame] ->
frame_affinity(frame, key)
[_ | _] ->
:blob
[] ->
cond do
key in @rowid_names and Enum.any?(env.frames, & &1.has_rowid) -> :integer
env.outer != nil -> expr_affinity({:column, nil, name}, env.outer)
true -> :blob
end
end
end
defp column_affinity(env, qualifier, name) do
qkey = Table.key(qualifier)
case Enum.find(env.frames, &(&1.name == qkey)) do
nil ->
if env.outer != nil, do: expr_affinity({:column, qualifier, name}, env.outer), else: :blob
frame ->
key = Table.key(name)
cond do
has_column?(frame, key) -> frame_affinity(frame, key)
key in @rowid_names and frame.has_rowid -> :integer
true -> :blob
end
end
end
defp frame_affinity(frame, key) do
frame
|> frame_column(key)
|> column_affinity_meta()
end
defp expr_collation({:collate, _expr, name}, env), do: normalize_collation!(name, env)
defp expr_collation({:fast_column, _key, _affinity, collation}, _env), do: collation
defp expr_collation({:fast_frame_column, _index, _key, _affinity, collation}, _env),
do: collation
defp expr_collation({:fast_outer_frame_column, _index, _key, _affinity, collation}, _env),
do: collation
defp expr_collation({:column, qualifier, name}, env) do
case env.group do
[first | _] -> expr_collation({:column, qualifier, name}, first)
[] -> :binary
nil -> column_collation(env, qualifier, name)
end
end
defp expr_collation(expr, env) when is_tuple(expr) do
explicit_collation(expr, env) || inherited_collation(expr, env) || :binary
end
defp expr_collation(_expr, _env), do: :binary
defp explicit_collation({:collate, _expr, name}, env), do: normalize_collation!(name, env)
defp explicit_collation({:fast_column, _key, _affinity, _collation}, _env), do: nil
defp explicit_collation({:fast_frame_column, _index, _key, _affinity, _collation}, _env),
do: nil
defp explicit_collation({:fast_outer_frame_column, _index, _key, _affinity, _collation}, _env),
do: nil
defp explicit_collation(expr, env) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.find_value(fn
element when is_tuple(element) -> explicit_collation(element, env)
elements when is_list(elements) -> Enum.find_value(elements, &explicit_collation(&1, env))
_ -> nil
end)
end
defp explicit_collation(_expr, _env), do: nil
defp inherited_collation({:cast, expr, _affinity}, env), do: expr_collation(expr, env)
defp inherited_collation({:binary, :concat, left, _right}, env), do: expr_collation(left, env)
defp inherited_collation(_expr, _env), do: nil
defp column_collation(env, nil, name) do
key = Table.key(name)
case Enum.filter(env.frames, &visible?(&1, key)) do
[frame] ->
frame_collation(frame, key, env)
[] when env.outer != nil ->
expr_collation({:column, nil, name}, env.outer)
_ ->
:binary
end
end
defp column_collation(env, qualifier, name) do
qkey = Table.key(qualifier)
case Enum.find(env.frames, &(&1.name == qkey)) do
nil ->
if env.outer != nil,
do: expr_collation({:column, qualifier, name}, env.outer),
else: :binary
frame ->
key = Table.key(name)
if has_column?(frame, key), do: frame_collation(frame, key, env), else: :binary
end
end
defp frame_collation(frame, key, env) do
frame
|> frame_column(key)
|> column_collation_meta()
|> normalize_collation!(env)
end
defp frame_column(%{columns_by_key: columns}, key), do: Map.fetch!(columns, key)
defp frame_column(frame, key), do: List.keyfind(frame.columns, key, 0)
defp normalize_collation!(nil, _env), do: :binary
# Already-normalized built-in collations (the overwhelming majority, e.g. the
# atom stored on every index) short-circuit the `to_string/1` + Unicode
# `String.downcase/1` the general clause would otherwise pay — which on a
# per-index-entry path (range scans) was a measurable hot spot.
defp normalize_collation!(:binary, _env), do: :binary
defp normalize_collation!(:nocase, _env), do: :nocase
defp normalize_collation!(:rtrim, _env), do: :rtrim
defp normalize_collation!(name, env) do
case String.downcase(to_string(name)) do
"binary" -> :binary
"nocase" -> :nocase
"rtrim" -> :rtrim
other -> custom_collation!(env.db, other)
end
end
defp custom_collation!(db, name) do
case Database.fetch_collation(db, name) do
{:ok, %{callback: callback}} ->
{:custom, name, fn a, b -> call_collation(callback, name, a, b) end}
:error ->
fail("no such collation sequence: #{name}")
end
end
defp call_collation(callback, name, a, b) do
callback
|> apply([a, b])
|> normalize_collation_result(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined collation #{name} raised: #{Exception.message(e)}")
end
defp normalize_collation_result({:ok, result}, name),
do: normalize_collation_result(result, name)
defp normalize_collation_result({:error, message}, name),
do: fail("user-defined collation #{name} error: #{message}")
defp normalize_collation_result(:lt, _name), do: :lt
defp normalize_collation_result(:eq, _name), do: :eq
defp normalize_collation_result(:gt, _name), do: :gt
defp normalize_collation_result(result, _name) when is_integer(result) and result < 0, do: :lt
defp normalize_collation_result(0, _name), do: :eq
defp normalize_collation_result(result, _name) when is_integer(result) and result > 0, do: :gt
defp normalize_collation_result(_result, name),
do: fail("user-defined collation #{name} returned unsupported value")
# Evaluates both sides of a comparison and applies comparison affinity.
defp comparison_operands(left, right, env) do
aff_l = static_affinity(left)
aff_r = static_affinity(right)
if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do
{a, b} = Value.comparison_coerce(eval(left, env), aff_l, eval(right, env), aff_r)
{a, b, :binary}
else
comparison_operands_dynamic(left, right, env)
end
end
defp comparison_operands_dynamic(left, right, env) do
{a, b} =
Value.comparison_coerce(
eval(left, env),
expr_affinity(left, env),
eval(right, env),
expr_affinity(right, env)
)
{a, b, comparison_collation(left, right, env)}
end
defp comparison_collation(left, right, env) do
case explicit_collation(left, env) do
nil ->
case explicit_collation(right, env) do
nil ->
case expr_collation(left, env) do
:binary -> expr_collation(right, env)
collation -> collation
end
collation ->
collation
end
collation ->
collation
end
end
defp ensure_raise_allowed!(env) do
if env.db.active_triggers == [] do
fail("RAISE() may only be used within a trigger-program")
end
end
# -- expression evaluation ------------------------------------------------------------
#
# Booleans are SQL values: comparisons yield 1/0/NULL, matching SQLite where
# any expression result is a storage-class value.
defp eval({:literal, value}, _env), do: value
# A parameter that was never bound evaluates to NULL, as in the C API.
defp eval({:param, _index, _raw}, _env), do: nil
# RAISE() inside a trigger program: IGNORE abandons the row operation
# (caught by the trigger runner); the other forms abort with the message.
defp eval({:raise, :ignore}, env) do
ensure_raise_allowed!(env)
throw(:raise_ignore)
end
defp eval({:raise, _kind, message}, env) do
ensure_raise_allowed!(env)
fail(Value.to_text(eval(message, env)))
end
# `->` returns JSON text, `->>` the SQL value; the right side may be a
# full `$..` path, a bare object key, or an array index.
defp eval({:json_arrow, left, right}, env) do
json_arrow_value(eval(left, env), eval(right, env), &json_subtype/1)
end
defp eval({:json_arrow_text, left, right}, env) do
json_arrow_value(eval(left, env), eval(right, env), &Json.to_sql/1)
end
defp eval({:function, "changes", []}, env), do: env.db.changes
defp eval({:function, "total_changes", []}, env), do: env.db.total_changes
defp eval({:function, "last_insert_rowid", []}, env), do: env.db.last_insert_rowid
defp eval({:column, qualifier, name}, env) do
case env.group do
# A bare column in an aggregate query takes its value from an
# arbitrary row; SQLite uses the last visited, we use the first.
[first | _] -> eval({:column, qualifier, name}, first)
[] -> nil
nil -> resolve_column(env, qualifier, name)
end
end
# A `{:column, …}` pre-resolved by `precompile_scan_where/3` to a direct row
# lookup of the single scan frame, carrying its precomputed affinity and
# collation. Equivalent to the `[frame]` branch of `resolve_column/3`, but
# skips the per-row `String.downcase` + frame visibility search. The second
# head is a defensive fallback for any unexpected env shape.
defp eval({:fast_column, key, _affinity, _collation}, %{group: nil, frames: [frame]}) do
frame_cell(frame, key)
end
defp eval({:fast_column, key, _affinity, _collation}, env) do
resolve_column(env, nil, key)
end
defp eval({:fast_frame_column, index, key, _affinity, _collation}, env) do
fast_frame_column(env, index, key)
end
defp eval({:fast_outer_frame_column, index, key, _affinity, _collation}, env) do
fast_outer_frame_column(env, index, key)
end
defp eval({:binary, :and, left, right}, env) do
bool_value(Value.sql_and(truth(left, env), truth(right, env)))
end
defp eval({:binary, :or, left, right}, env) do
bool_value(Value.sql_or(truth(left, env), truth(right, env)))
end
defp eval({:binary, op, left, right}, env) when op in [:eq, :ne, :lt, :le, :gt, :ge] do
{a, b, collation} = comparison_operands(left, right, env)
bool_value(Value.compare_op(op, a, b, collation))
end
defp eval({:binary, op, left, right}, env) when op in [:add, :sub, :mul, :div, :mod] do
Value.arithmetic(op, eval(left, env), eval(right, env))
end
defp eval({:binary, op, left, right}, env) when op in [:bitand, :bitor, :shl, :shr] do
Value.bitwise(op, eval(left, env), eval(right, env))
end
defp eval({:binary, :concat, left, right}, env) do
Value.concat(eval(left, env), eval(right, env))
end
defp eval({:bitnot, expr}, env), do: Value.bitnot(eval(expr, env))
defp eval({:cast, expr, affinity}, env), do: Value.cast(eval(expr, env), affinity)
defp eval({:collate, expr, _name}, env), do: eval(expr, env)
defp eval({:not, expr}, env), do: bool_value(Value.sql_not(truth(expr, env)))
defp eval({:negate, expr}, env) do
case eval(expr, env) do
nil ->
nil
n when is_integer(n) ->
# Negating the smallest 64-bit integer overflows into REAL.
if Value.out_of_int64_range?(-n), do: -n * 1.0, else: -n
n when is_float(n) ->
-n
other ->
Value.arithmetic(:sub, 0, other)
end
end
# IS / IS NOT never return NULL: NULL IS NULL is true.
defp eval({:is, left, right}, env) do
{a, b, collation} = comparison_operands(left, right, env)
bool_value(Value.compare(a, b, collation) == :eq)
end
defp eval({:is_not, left, right}, env) do
{a, b, collation} = comparison_operands(left, right, env)
bool_value(Value.compare(a, b, collation) != :eq)
end
# IN comparisons use the affinity of the left side and (for subqueries)
# the result column; affinities of columns inside an expression list are
# ignored, as in SQLite.
defp eval({:in, expr, {:select, select}, negated}, env) do
result = query_result(env.db, select, env)
members = Enum.map(result.rows, &hd/1)
rhs_affinity = List.first(result.affinities) || :blob
in_membership(
eval(expr, env),
expr_affinity(expr, env),
expr_collation(expr, env),
members,
rhs_affinity,
negated
)
end
# An `IN (uncorrelated subquery)` whose member set + result affinity were
# evaluated once by `hoist_in_subquery/2` (see `rewrite_scan_columns/4`);
# equivalent to the clause above but without re-running the subquery per row.
defp eval({:in_cached, expr, members, rhs_affinity, negated}, env) do
in_membership(
eval(expr, env),
expr_affinity(expr, env),
expr_collation(expr, env),
members,
rhs_affinity,
negated
)
end
defp eval({:in, expr, list, negated}, env) when is_list(list) do
members = Enum.map(list, &eval(&1, env))
in_membership(
eval(expr, env),
expr_affinity(expr, env),
expr_collation(expr, env),
members,
:blob,
negated
)
end
defp eval({:in_membership, expr, membership}, env) do
compiled_in_membership(eval(expr, env), membership)
end
defp eval({:between, expr, low, high, negated}, env) do
{value_low, low_value, low_collation} = comparison_operands(expr, low, env)
{value_high, high_value, high_collation} = comparison_operands(expr, high, env)
result =
Value.sql_and(
Value.compare_op(:ge, value_low, low_value, low_collation),
Value.compare_op(:le, value_high, high_value, high_collation)
)
bool_value(if negated, do: Value.sql_not(result), else: result)
end
defp eval({:like, expr, pattern, escape, negated}, env) do
result =
if escape == nil do
Value.like(eval(expr, env), eval(pattern, env), env.db.case_sensitive_like)
else
case eval(escape, env) do
nil ->
nil
escape_value ->
Value.like(
eval(expr, env),
eval(pattern, env),
like_escape(escape_value),
env.db.case_sensitive_like
)
end
end
bool_value(if negated, do: Value.sql_not(result), else: result)
end
defp eval({:glob, expr, pattern, negated}, env) do
result = Value.glob(eval(expr, env), eval(pattern, env))
bool_value(if negated, do: Value.sql_not(result), else: result)
end
defp eval({:regexp, expr, pattern, negated}, env) do
result = regexp_match(eval(expr, env), eval(pattern, env))
bool_value(if negated, do: Value.sql_not(result), else: result)
end
defp eval({:match, _expr, _pattern, _negated}, _env) do
fail("unable to use function MATCH in the requested context")
end
# A scalar subquery's value is the first column of its first row; an empty
# result is NULL.
defp eval({:subquery, select}, env) do
case query_result(env.db, select, env).rows do
[[value | _] | _] -> value
[] -> nil
end
end
defp eval({:exists, select}, env) do
bool_value(query_result(env.db, select, env).rows != [])
end
defp eval({:correlated_count, table_key, template, filter}, env) do
env.db.tables
|> Map.fetch!(table_key)
|> Table.scan()
|> Enum.count(fn {rowid, row} ->
correlated_table_match?(env, template, filter, rowid, row)
end)
end
defp eval({:correlated_exists, table_key, template, filter}, env) do
exists? =
env.db.tables
|> Map.fetch!(table_key)
|> Table.scan()
|> Enum.any?(fn {rowid, row} ->
correlated_table_match?(env, template, filter, rowid, row)
end)
bool_value(exists?)
end
defp eval({:case, nil, branches, else_expr}, env) do
Enum.find_value(branches, fn {when_expr, then_expr} ->
if truth(when_expr, env) == true, do: {:matched, eval(then_expr, env)}
end)
|> case do
{:matched, value} -> value
nil -> if else_expr, do: eval(else_expr, env)
end
end
defp eval({:case, operand, branches, else_expr}, env) do
Enum.find_value(branches, fn {when_expr, then_expr} ->
{value, when_value, collation} = comparison_operands(operand, when_expr, env)
if Value.compare_op(:eq, value, when_value, collation) == true,
do: {:matched, eval(then_expr, env)}
end)
|> case do
{:matched, result} -> result
nil -> if else_expr, do: eval(else_expr, env)
end
end
defp eval({:function, name, args}, env) do
cond do
name in @window_functions ->
fail("misuse of window function #{name}()")
aggregate_call?(env.db, name, args) ->
case env.group do
group when is_list(group) -> aggregate(name, args, group, env)
nil -> fail("misuse of aggregate function #{name}()")
end
args == :star ->
fail("wrong use of '*' with function #{name}()")
true ->
scalar(env, name, Enum.map(args, &eval(&1, env)))
end
end
defp eval({:filter_function, name, args, filter}, env) do
cond do
aggregate_call?(env.db, name, args) ->
case env.group do
group when is_list(group) ->
group = Enum.filter(group, &(truth(filter, &1) == true))
aggregate(name, args, group, env)
nil ->
fail("misuse of aggregate function #{name}()")
end
true ->
fail("FILTER may not be used with non-aggregate #{name}()")
end
end
defp eval({:window, name, _args, _spec, _filter} = expr, env) do
case Map.fetch(Map.get(env, :windows, %{}), expr) do
{:ok, value} -> value
:error -> fail("misuse of window function #{name}()")
end
end
defp correlated_table_match?(_outer, _template, nil, _rowid, _row), do: true
defp correlated_table_match?(outer, template, filter, rowid, row) do
filter.(%{
db: outer.db,
frames: [%{template | row: row, rowid: rowid}],
group: nil,
outer: outer
})
end
defp in_membership(value, affinity, collation, members, rhs_affinity, negated) do
match? = fn member ->
{a, b} = Value.comparison_coerce(value, affinity, member, rhs_affinity)
Value.compare_op(:eq, a, b, collation) == true
end
membership =
cond do
value == nil and members != [] -> nil
Enum.any?(members, match?) -> true
Enum.any?(members, &is_nil/1) -> nil
true -> false
end
bool_value(if negated, do: Value.sql_not(membership), else: membership)
end
defp compile_in_membership(members, affinity, :binary, rhs_affinity, negated) do
{member_keys, numeric_member_keys, has_null?} =
Enum.reduce(members, {MapSet.new(), MapSet.new(), false}, fn
nil, {member_keys, numeric_member_keys, _has_null?} ->
{member_keys, numeric_member_keys, true}
member, {member_keys, numeric_member_keys, has_null?} ->
{_left, coerced_member} = Value.comparison_coerce(nil, affinity, member, rhs_affinity)
numeric_member_keys =
if number_value?(coerced_member),
do: MapSet.put(numeric_member_keys, numeric_value_key(coerced_member)),
else: numeric_member_keys
{MapSet.put(member_keys, value_key(coerced_member)), numeric_member_keys, has_null?}
end)
%{
affinity: affinity,
rhs_affinity: rhs_affinity,
member_keys: member_keys,
numeric_member_keys: numeric_member_keys,
numeric_probe?: numeric_probe_fast?(affinity, rhs_affinity),
has_members?: members != [],
has_null?: has_null?,
negated?: negated
}
end
defp compiled_in_membership(value, membership) do
result =
cond do
value == nil and membership.has_members? ->
nil
value == nil ->
false
membership.numeric_probe? and number_value?(value) and
MapSet.member?(membership.numeric_member_keys, numeric_value_key(value)) ->
true
membership.numeric_probe? and number_value?(value) ->
if membership.has_null?, do: nil, else: false
MapSet.member?(
membership.member_keys,
value |> in_lookup_value(membership.affinity, membership.rhs_affinity) |> value_key()
) ->
true
membership.has_null? ->
nil
true ->
false
end
bool_value(if membership.negated?, do: Value.sql_not(result), else: result)
end
defp numeric_probe_fast?(_affinity, :text), do: false
defp numeric_probe_fast?(_affinity, _rhs_affinity), do: true
defp number_value?(value), do: is_integer(value) or is_float(value)
defp numeric_value_key(value) when is_integer(value), do: value
defp numeric_value_key(value) when is_float(value) do
truncated = trunc(value)
if truncated == value, do: truncated, else: value
end
defp in_lookup_value(value, affinity, rhs_affinity) do
{coerced_value, _member} = Value.comparison_coerce(value, affinity, nil, rhs_affinity)
coerced_value
end
defp truth(expr, env), do: expr |> eval(env) |> Value.truthy()
defp like_escape(value) do
text = Value.to_text(value)
if String.length(text) == 1 do
text
else
fail("ESCAPE expression must be a single character")
end
end
defp bool_value(nil), do: nil
defp bool_value(true), do: 1
defp bool_value(false), do: 0
defp fast_frame_column(%{frames: frames}, index, key) do
frame = :lists.nth(index + 1, frames)
frame_cell(frame, key)
end
defp fast_outer_frame_column(%{outer: outer}, index, key) when is_map(outer) do
fast_frame_column(outer, index, key)
end
# -- scalar functions --------------------------------------------------------------
defp scalar(env, name, args) do
case Database.fetch_scalar_function(env.db, name, length(args)) do
{:ok, function} ->
call_scalar_function(function, Enum.map(args, &sql_value/1))
:error ->
if (Database.scalar_function_exists?(env.db, name) or
Database.aggregate_function_exists?(env.db, name)) and
not (Map.has_key?(@scalar_arity, name) or name in @aggregate_functions) do
fail("wrong number of arguments to function #{name}()")
else
builtin_scalar(env, name, args)
end
end
end
defp builtin_scalar(env, "like", [pattern, v]),
do: bool_value(Value.like(v, pattern, env.db.case_sensitive_like))
defp builtin_scalar(_env, "like", [_pattern, _v, nil]), do: nil
defp builtin_scalar(env, "like", [pattern, v, escape]),
do: bool_value(Value.like(v, pattern, like_escape(escape), env.db.case_sensitive_like))
defp builtin_scalar(_env, name, args), do: scalar(name, args)
defp call_scalar_function(%{name: name, callback: callback}, args) do
callback
|> apply(args)
|> normalize_scalar_function_result(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined function #{name}() raised: #{Exception.message(e)}")
end
defp normalize_scalar_function_result({:ok, value}, name),
do: normalize_scalar_function_result(value, name)
defp normalize_scalar_function_result({:error, message}, name),
do: fail("user-defined function #{name}() error: #{message}")
defp normalize_scalar_function_result(nil, _name), do: nil
defp normalize_scalar_function_result(value, _name) when is_integer(value), do: value
defp normalize_scalar_function_result(value, _name) when is_float(value), do: value
defp normalize_scalar_function_result(value, _name) when is_binary(value), do: value
defp normalize_scalar_function_result({:blob, value}, _name) when is_binary(value),
do: {:blob, value}
defp normalize_scalar_function_result(_value, name),
do: fail("user-defined function #{name}() returned unsupported value")
# -- JSON functions (json1) ---------------------------------------------------
@jsonb_magic "ExSQL.JSONB\0"
defp scalar("json", [nil]), do: nil
defp scalar("json", [v]), do: v |> json_parse!() |> json_subtype()
defp scalar("jsonb", [nil]), do: nil
defp scalar("jsonb", [v]), do: v |> json_parse!() |> jsonb_blob()
defp scalar("json_valid", [nil]), do: nil
defp scalar("json_valid", [{:blob, _}]), do: 0
defp scalar("json_valid", [v]) do
if match?({:ok, _}, Json.parse(Value.to_text(v))), do: 1, else: 0
end
defp scalar("json_valid", [v, flags]) do
flags = json_valid_flags!(flags)
cond do
v == nil ->
nil
match?({:blob, _}, v) ->
jsonb_valid(v, flags)
Bitwise.band(flags, 0b0001) != 0 and match?({:ok, _}, Json.parse(Value.to_text(v))) ->
1
Bitwise.band(flags, 0b0010) != 0 ->
if match?({:ok, _}, Json.parse_json5(Value.to_text(v))), do: 1, else: 0
Bitwise.band(flags, 0b0001) != 0 ->
if match?({:ok, _}, Json.parse(Value.to_text(v))), do: 1, else: 0
true ->
0
end
end
defp scalar("json_quote", [v]), do: v |> json_from_sql!() |> Json.render()
defp scalar("json_array", args), do: json_subtype({:array, Enum.map(args, &json_from_sql!/1)})
defp scalar("jsonb_array", args), do: jsonb_blob({:array, Enum.map(args, &json_from_sql!/1)})
defp scalar("json_object", args) do
json_object(args)
|> json_subtype()
end
defp scalar("jsonb_object", args) do
args
|> json_object()
|> jsonb_blob()
end
defp scalar("json_extract", [nil | _paths]), do: nil
defp scalar("json_extract", [v, path]) do
case Json.get(json_parse!(v), json_path!(path)) do
{:ok, found} -> json_to_sql(found)
:missing -> nil
end
end
defp scalar("json_extract", [v | paths]) do
jv = json_parse!(v)
items =
Enum.map(paths, fn path ->
case Json.get(jv, json_path!(path)) do
{:ok, found} -> found
:missing -> :null
end
end)
json_subtype({:array, items})
end
defp scalar("jsonb_extract", [nil | _paths]), do: nil
defp scalar("jsonb_extract", [v, path]) do
case Json.get(json_parse!(v), json_path!(path)) do
{:ok, {:array, _items} = found} -> jsonb_blob(found)
{:ok, {:object, _pairs} = found} -> jsonb_blob(found)
{:ok, found} -> Json.to_sql(found)
:missing -> nil
end
end
defp scalar("jsonb_extract", [v | paths]) do
jv = json_parse!(v)
items =
Enum.map(paths, fn path ->
case Json.get(jv, json_path!(path)) do
{:ok, found} -> found
:missing -> :null
end
end)
jsonb_blob({:array, items})
end
defp scalar("json_type", [nil]), do: nil
defp scalar("json_type", [v]), do: v |> json_parse!() |> Json.type_name()
defp scalar("json_type", [nil, _path]), do: nil
defp scalar("json_type", [v, path]) do
case Json.get(json_parse!(v), json_path!(path)) do
{:ok, found} -> Json.type_name(found)
:missing -> nil
end
end
defp scalar("json_array_length", [v]), do: scalar("json_array_length", [v, "$"])
defp scalar("json_array_length", [nil, _path]), do: nil
defp scalar("json_array_length", [v, path]) do
case Json.get(json_parse!(v), json_path!(path)) do
{:ok, {:array, items}} -> length(items)
{:ok, _other} -> 0
:missing -> nil
end
end
defp scalar("json_insert", [v | pairs]), do: json_write_pairs(v, pairs, :insert)
defp scalar("json_replace", [v | pairs]), do: json_write_pairs(v, pairs, :replace)
defp scalar("json_set", [v | pairs]), do: json_write_pairs(v, pairs, :set)
defp scalar("jsonb_insert", [v | pairs]), do: jsonb_write_pairs(v, pairs, :insert)
defp scalar("jsonb_replace", [v | pairs]), do: jsonb_write_pairs(v, pairs, :replace)
defp scalar("jsonb_set", [v | pairs]), do: jsonb_write_pairs(v, pairs, :set)
defp scalar("json_patch", [target, patch]) do
if target == nil or patch == nil do
nil
else
json_subtype(Json.merge_patch(json_parse!(target), json_parse!(patch)))
end
end
defp scalar("jsonb_patch", [target, patch]) do
if target == nil or patch == nil do
nil
else
jsonb_blob(Json.merge_patch(json_parse!(target), json_parse!(patch)))
end
end
defp scalar("json_pretty", [v]), do: scalar("json_pretty", [v, nil])
defp scalar("json_pretty", [nil, _indent]), do: nil
defp scalar("json_pretty", [v, indent]) do
indent = if is_nil(indent), do: " ", else: Value.to_text(indent)
v |> json_parse!() |> Json.pretty(indent)
end
defp scalar("json_remove", [nil | _paths]), do: nil
defp scalar("json_remove", [v | paths]) do
paths
|> Enum.reduce(json_parse!(v), fn path, jv -> Json.remove(jv, json_path!(path)) end)
|> json_subtype()
end
defp scalar("jsonb_remove", [nil | _paths]), do: nil
defp scalar("jsonb_remove", [v | paths]) do
paths
|> Enum.reduce(json_parse!(v), fn path, jv -> Json.remove(jv, json_path!(path)) end)
|> jsonb_blob()
end
defp scalar("abs", [nil]), do: nil
defp scalar("abs", [-9_223_372_036_854_775_808]), do: fail("integer overflow")
defp scalar("abs", [v]) when is_integer(v) or is_float(v), do: abs(v)
defp scalar("abs", [v]), do: scalar("abs", [Value.apply_affinity(v, :numeric)])
defp scalar("lower", [nil]), do: nil
defp scalar("lower", [v]), do: v |> Value.to_text() |> ascii_case(?A..?Z, 32)
defp scalar("upper", [nil]), do: nil
defp scalar("upper", [v]), do: v |> Value.to_text() |> ascii_case(?a..?z, -32)
defp scalar("length", [nil]), do: nil
defp scalar("length", [{:blob, b}]), do: byte_size(b)
defp scalar("length", [v]), do: v |> Value.to_text() |> String.length()
defp scalar("typeof", [v]), do: v |> Value.type_of() |> Atom.to_string()
defp scalar("coalesce", args) when length(args) >= 2, do: Enum.find(args, &(not is_nil(&1)))
defp scalar("ifnull", [a, b]), do: if(is_nil(a), do: b, else: a)
defp scalar("nullif", [a, b]) do
if Value.compare_op(:eq, a, b) == true, do: nil, else: a
end
defp scalar("round", [v]), do: scalar("round", [v, 0])
defp scalar("round", [nil, _]), do: nil
defp scalar("round", [v, digits]) when is_integer(digits) do
case Value.apply_affinity(v, :real) do
f when is_float(f) -> Float.round(f, max(digits, 0))
_ -> 0.0
end
end
defp scalar("pi", []), do: :math.pi()
defp scalar("ceil", [v]), do: math_rounding(v, &Float.ceil/1)
defp scalar("ceiling", [v]), do: scalar("ceil", [v])
defp scalar("floor", [v]), do: math_rounding(v, &Float.floor/1)
defp scalar("trunc", [v]), do: math_rounding(v, &trunc/1)
defp scalar("mod", [a, b]), do: math_binary(a, b, &math_mod/2)
defp scalar("pow", [a, b]), do: math_binary(a, b, &:math.pow/2)
defp scalar("power", args), do: scalar("pow", args)
defp scalar("sqrt", [v]),
do: math_unary(v, fn x -> if x < 0.0, do: nil, else: :math.sqrt(x) end)
defp scalar("exp", [v]), do: math_unary(v, &:math.exp/1)
defp scalar("ln", [v]), do: math_unary(v, fn x -> positive_math(x, &:math.log/1) end)
defp scalar("log10", [v]), do: math_unary(v, fn x -> positive_math(x, &:math.log10/1) end)
defp scalar("log2", [v]),
do: math_unary(v, fn x -> positive_math(x, fn x -> :math.log(x) / :math.log(2) end) end)
defp scalar("log", [v]), do: scalar("log10", [v])
defp scalar("log", [base, v]) do
math_binary(base, v, fn base, x ->
if base <= 0.0 or base == 1.0 or x <= 0.0 do
nil
else
:math.log(x) / :math.log(base)
end
end)
end
defp scalar("radians", [v]), do: math_unary(v, &(&1 * :math.pi() / 180.0))
defp scalar("degrees", [v]), do: math_unary(v, &(&1 * 180.0 / :math.pi()))
defp scalar("sin", [v]), do: math_unary(v, &:math.sin/1)
defp scalar("cos", [v]), do: math_unary(v, &:math.cos/1)
defp scalar("tan", [v]), do: math_unary(v, &:math.tan/1)
defp scalar("asin", [v]),
do: math_unary(v, fn x -> range_math(x, -1.0, 1.0, &:math.asin/1) end)
defp scalar("acos", [v]),
do: math_unary(v, fn x -> range_math(x, -1.0, 1.0, &:math.acos/1) end)
defp scalar("atan", [v]), do: math_unary(v, &:math.atan/1)
defp scalar("atan2", [a, b]), do: math_binary(a, b, &:math.atan2/2)
defp scalar("sinh", [v]), do: math_unary(v, &:math.sinh/1)
defp scalar("cosh", [v]), do: math_unary(v, &:math.cosh/1)
defp scalar("tanh", [v]), do: math_unary(v, &:math.tanh/1)
defp scalar("asinh", [v]),
do: math_unary(v, fn x -> :math.log(x + :math.sqrt(x * x + 1.0)) end)
defp scalar("acosh", [v]),
do:
math_unary(v, fn x -> if x < 1.0, do: nil, else: :math.log(x + :math.sqrt(x * x - 1.0)) end)
defp scalar("atanh", [v]),
do:
math_unary(v, fn x ->
if x <= -1.0 or x >= 1.0, do: nil, else: 0.5 * :math.log((1.0 + x) / (1.0 - x))
end)
defp scalar("substr", [v, start]), do: scalar("substr", [v, start, nil])
defp scalar("substr", [nil, _, _]), do: nil
defp scalar("substr", [v, start, len]) when is_integer(start) do
text = Value.to_text(v)
size = String.length(text)
explicit_len = if is_integer(len) and len >= 0, do: len, else: if(is_integer(len), do: 0)
# SQLite is 1-based. A negative start counts back from the end; a start at or
# before position 1 leaves "empty" leading positions that still consume the
# requested length (`substr('hello',0,2)` is `'h'`, not `'he'`).
{from, count} =
cond do
start > 0 ->
{start - 1, explicit_len || size}
start == 0 ->
{0, (explicit_len && max(explicit_len - 1, 0)) || size}
true ->
p1 = size + start
cond do
p1 >= 0 -> {p1, explicit_len || size}
is_integer(len) -> {0, max((explicit_len || 0) + p1, 0)}
true -> {0, size}
end
end
String.slice(text, from, count)
end
defp scalar("replace", [a, b, c]) do
if is_nil(a) or is_nil(b) or is_nil(c) do
nil
else
text = Value.to_text(a)
pattern = Value.to_text(b)
# SQLite leaves the string unchanged for an empty pattern.
if pattern == "", do: text, else: String.replace(text, pattern, Value.to_text(c))
end
end
defp scalar(name, [v]) when name in ["trim", "ltrim", "rtrim"], do: scalar(name, [v, " "])
defp scalar(name, [v, chars]) when name in ["trim", "ltrim", "rtrim"] do
if is_nil(v) or is_nil(chars) do
nil
else
text = Value.to_text(v)
set = chars |> Value.to_text() |> String.graphemes() |> MapSet.new()
case name do
"trim" -> text |> trim_chars(set) |> reverse_text() |> trim_chars(set) |> reverse_text()
"ltrim" -> trim_chars(text, set)
"rtrim" -> text |> reverse_text() |> trim_chars(set) |> reverse_text()
end
end
end
defp scalar("instr", [a, b]) do
if is_nil(a) or is_nil(b) do
nil
else
haystack = Value.to_text(a)
needle = Value.to_text(b)
cond do
needle == "" ->
1
true ->
case String.split(haystack, needle, parts: 2) do
[prefix, _] -> String.length(prefix) + 1
[_] -> 0
end
end
end
end
defp scalar("hex", [nil]), do: ""
defp scalar("hex", [{:blob, b}]), do: Base.encode16(b)
defp scalar("hex", [v]), do: v |> Value.to_text() |> Base.encode16()
defp scalar("unhex", [v]), do: scalar("unhex", [v, ""])
defp scalar("unhex", [nil, _ignore]), do: nil
defp scalar("unhex", [_v, nil]), do: nil
defp scalar("unhex", [v, ignore]) do
ignored = ignore |> Value.to_text() |> String.graphemes() |> MapSet.new()
hex =
v
|> Value.to_text()
|> String.graphemes()
|> Enum.reject(&MapSet.member?(ignored, &1))
|> Enum.join()
with 0 <- rem(String.length(hex), 2),
{:ok, bytes} <- Base.decode16(hex, case: :mixed) do
{:blob, bytes}
else
_ -> nil
end
end
defp scalar("quote", [nil]), do: "NULL"
defp scalar("quote", [{:blob, b}]), do: "X'" <> Base.encode16(b) <> "'"
defp scalar("quote", [v]) when is_binary(v), do: "'" <> String.replace(v, "'", "''") <> "'"
defp scalar("quote", [v]), do: Value.to_text(v)
defp scalar(name, [format | args]) when name in ["printf", "format"] do
sqlite_format(format, args)
end
defp scalar("random", []) do
<<value::signed-64>> = :crypto.strong_rand_bytes(8)
value
end
defp scalar("randomblob", [n]) do
size =
case Value.cast(n, :integer) do
n when is_integer(n) and n > 0 -> n
_ -> 1
end
{:blob, :crypto.strong_rand_bytes(size)}
end
defp scalar("sqlite_version", []), do: @sqlite_version
defp scalar("char", args) do
Enum.map_join(args, fn
codepoint when is_integer(codepoint) and codepoint > 0 -> <<codepoint::utf8>>
_ -> ""
end)
end
defp scalar("unicode", [nil]), do: nil
defp scalar("unicode", [v]) do
case Value.to_text(v) do
<<codepoint::utf8, _::binary>> -> codepoint
_ -> nil
end
end
defp scalar("sign", [v]) do
case Value.apply_affinity(v, :numeric) do
n when is_integer(n) or is_float(n) ->
cond do
n > 0 -> 1
n < 0 -> -1
true -> 0
end
_ ->
nil
end
end
defp scalar("iif", [a, b, c]), do: if(Value.truthy(a) == true, do: b, else: c)
defp scalar("zeroblob", [n]) when is_integer(n),
do: {:blob, :binary.copy(<<0>>, max(n, 0))}
defp scalar("octet_length", [nil]), do: nil
defp scalar("octet_length", [{:blob, b}]), do: byte_size(b)
defp scalar("octet_length", [v]), do: v |> Value.to_text() |> byte_size()
defp scalar("concat", args) when args != [],
do: args |> Enum.reject(&is_nil/1) |> Enum.map_join(&Value.to_text/1)
defp scalar("concat_ws", [nil | _rest]), do: nil
defp scalar("concat_ws", [separator | rest]) when rest != [] do
rest
|> Enum.reject(&is_nil/1)
|> Enum.map_join(Value.to_text(separator), &Value.to_text/1)
end
defp scalar("glob", [pattern, v]), do: bool_value(Value.glob(v, pattern))
defp scalar("regexp", [pattern, v]), do: bool_value(regexp_match(v, pattern))
defp scalar("match", [_pattern, _v]) do
fail("unable to use function MATCH in the requested context")
end
defp scalar("substring", args), do: scalar("substr", args)
# Multi-argument min/max are scalar functions; one-argument are aggregates.
defp scalar(name, args) when name in ["min", "max"] and length(args) >= 2 do
if Enum.any?(args, &is_nil/1) do
nil
else
comparator =
case name do
"min" -> fn a, b -> Value.compare(a, b) != :gt end
"max" -> fn a, b -> Value.compare(a, b) != :lt end
end
Enum.reduce(args, fn x, best -> if comparator.(x, best), do: x, else: best end)
end
end
# Date/time functions delegate to ExSQL.DateTime (mirrors date.c).
defp scalar("date", args), do: DateTime.date(args)
defp scalar("time", args), do: DateTime.time(args)
defp scalar("timediff", args), do: DateTime.timediff(args)
defp scalar("datetime", args), do: DateTime.datetime(args)
defp scalar("julianday", args), do: DateTime.julianday(args)
defp scalar("unixepoch", args), do: DateTime.unixepoch(args)
defp scalar("strftime", args), do: DateTime.strftime(args)
defp scalar(name, _args) do
if Map.has_key?(@scalar_arity, name) or name in @aggregate_functions do
fail("wrong number of arguments to function #{name}()")
else
fail("no such function: #{name}")
end
end
# -- JSON helpers ---------------------------------------------------------------
defp json_subtype(jv), do: {:json, Json.render(jv)}
defp jsonb_blob(jv), do: {:blob, Json.to_sqlite_jsonb(jv)}
defp json_aggregate_result("json_group_array", jv), do: json_subtype(jv)
defp json_aggregate_result("json_group_object", jv), do: json_subtype(jv)
defp json_aggregate_result("jsonb_group_array", jv), do: jsonb_blob(jv)
defp json_aggregate_result("jsonb_group_object", jv), do: jsonb_blob(jv)
defp json_to_sql({:array, _items} = jv), do: json_subtype(jv)
defp json_to_sql({:object, _pairs} = jv), do: json_subtype(jv)
defp json_to_sql(jv), do: Json.to_sql(jv)
defp json_parse!({:blob, @jsonb_magic <> text}), do: json_parse!(text)
defp json_parse!({:blob, blob}) do
case Json.parse_sqlite_jsonb(blob) do
{:ok, jv} -> jv
:error -> fail("malformed JSON")
end
end
defp json_parse!({:json, text}), do: json_parse!(text)
defp json_parse!(v) do
case Json.parse_json5(Value.to_text(v)) do
{:ok, jv} -> jv
:error -> fail("malformed JSON")
end
end
defp json_valid_flags!(flags) do
flags = Value.cast(flags, :integer)
if is_integer(flags) and flags in 1..15 do
flags
else
fail("FLAGS parameter to json_valid() must be between 1 and 15")
end
end
defp json_path!(path) when is_binary(path) do
case Json.parse_path(path) do
{:ok, steps} -> steps
:error -> fail("bad JSON path: '#{path}'")
end
end
defp json_path!(path), do: fail("bad JSON path: '#{Value.to_text(path)}'")
# An SQL value used as a JSON ingredient (json_array, json_set values, ...).
defp json_from_sql!(nil), do: :null
defp json_from_sql!(n) when is_number(n), do: n
defp json_from_sql!({:json, text}), do: json_parse!(text)
defp json_from_sql!({:blob, @jsonb_magic <> text}), do: json_parse!(text)
defp json_from_sql!({:blob, blob}) do
case Json.parse_sqlite_jsonb(blob) do
{:ok, jv} -> jv
:error -> fail("JSON cannot hold BLOB values")
end
end
defp json_from_sql!(s) when is_binary(s), do: s
defp jsonb_valid({:blob, @jsonb_magic <> _text}, flags),
do: bool_value(Bitwise.band(flags, 0b1100) != 0)
defp jsonb_valid({:blob, blob}, flags) do
strict? = Bitwise.band(flags, 0b1000) != 0 and Json.sqlite_jsonb_strict?(blob)
superficial? = Bitwise.band(flags, 0b0100) != 0 and Json.sqlite_jsonb_superficial?(blob)
bool_value(strict? or superficial?)
end
defp json_object(args) do
if rem(length(args), 2) != 0 do
fail("json_object() requires an even number of arguments")
end
pairs =
args
|> Enum.chunk_every(2)
|> Enum.map(fn [key, value] ->
unless is_binary(key), do: fail("json_object() labels must be TEXT")
{key, json_from_sql!(value)}
end)
{:object, pairs}
end
defp json_write_pairs(nil, _pairs, _mode), do: nil
defp json_write_pairs(v, pairs, mode) do
if rem(length(pairs), 2) != 0 do
fail("json_#{mode}() needs an odd number of arguments")
end
pairs
|> Enum.chunk_every(2)
|> Enum.reduce(json_parse!(v), fn [path, value], jv ->
Json.write(jv, json_path!(path), json_from_sql!(value), mode)
end)
|> json_subtype()
end
defp jsonb_write_pairs(nil, _pairs, _mode), do: nil
defp jsonb_write_pairs(v, pairs, mode) do
if rem(length(pairs), 2) != 0 do
fail("jsonb_#{mode}() needs an odd number of arguments")
end
pairs
|> Enum.chunk_every(2)
|> Enum.reduce(json_parse!(v), fn [path, value], jv ->
Json.write(jv, json_path!(path), json_from_sql!(value), mode)
end)
|> jsonb_blob()
end
defp json_arrow_value(json, path, _render) when is_nil(json) or is_nil(path), do: nil
defp json_arrow_value(json, path, render) do
case Json.get(json_parse!(json), json_arrow_path!(path)) do
{:ok, found} -> render.(found)
:missing -> nil
end
end
defp json_arrow_path!(path) do
cond do
is_integer(path) -> [{:index, path}]
is_binary(path) and String.starts_with?(path, "$") -> json_path!(path)
is_binary(path) -> [{:key, path}]
true -> fail("bad JSON path: '#{Value.to_text(path)}'")
end
end
defp math_unary(value, fun) do
case math_number(value) do
nil -> nil
number -> safe_math(fn -> fun.(number * 1.0) end)
end
end
defp math_binary(a, b, fun) do
with a when is_number(a) <- math_number(a),
b when is_number(b) <- math_number(b) do
safe_math(fn -> fun.(a * 1.0, b * 1.0) end)
else
_ -> nil
end
end
defp math_rounding(value, fun) do
case math_number(value) do
nil ->
nil
number when is_integer(number) ->
number
number ->
safe_math(fn -> fun.(number * 1.0) end)
end
end
defp math_number(nil), do: nil
defp math_number(value) when is_integer(value) or is_float(value), do: value
defp math_number(value) do
case Value.apply_affinity(value, :numeric) do
number when is_integer(number) or is_float(number) -> number
_ -> nil
end
end
defp positive_math(value, fun) do
if value <= 0.0, do: nil, else: fun.(value)
end
defp range_math(value, min, max, fun) do
if value < min or value > max, do: nil, else: fun.(value)
end
defp math_mod(_a, b) when b == 0.0, do: nil
defp math_mod(a, b), do: a - trunc(a / b) * b
defp regexp_match(nil, _pattern), do: nil
defp regexp_match(_value, nil), do: nil
defp regexp_match(value, pattern) do
case Regex.compile(Value.to_text(pattern)) do
{:ok, regex} -> Regex.match?(regex, Value.to_text(value))
{:error, _reason} -> nil
end
end
defp safe_math(fun) do
case fun.() do
nil -> nil
result when is_number(result) -> result
_ -> nil
end
rescue
ArithmeticError -> nil
ErlangError -> nil
end
defp trim_chars(text, set) do
text |> String.graphemes() |> Enum.drop_while(&MapSet.member?(set, &1)) |> Enum.join()
end
defp reverse_text(text), do: String.reverse(text)
# SQLite's printf/format is its own formatter, not libc's. This covers the
# SQL-visible core used by func.test: strings, SQL quoting, numeric bases,
# floating output, width/precision, dynamic * width/precision, and %c repeats.
defp sqlite_format(nil, _args), do: nil
defp sqlite_format(format, args), do: format |> Value.to_text() |> format_chunks(args, [])
defp format_chunks("", _args, acc), do: acc |> Enum.reverse() |> IO.iodata_to_binary()
defp format_chunks(<<"%%", rest::binary>>, args, acc),
do: format_chunks(rest, args, ["%" | acc])
defp format_chunks(<<"%", rest::binary>>, args, acc) do
{spec, rest, args} = parse_format_spec(rest, args)
{piece, args} = format_piece(spec, args)
format_chunks(rest, args, [piece | acc])
end
defp format_chunks(<<char::utf8, rest::binary>>, args, acc),
do: format_chunks(rest, args, [<<char::utf8>> | acc])
defp parse_format_spec(text, args) do
{flags, text} = take_format_flags(text, MapSet.new())
{width, flags, text, args} = take_format_width(text, flags, args)
{precision, text, args} = take_format_precision(text, args)
text = drop_format_length(text)
case text do
<<conv::utf8, rest::binary>> ->
{%{flags: flags, width: width, precision: precision, conv: conv}, rest, args}
"" ->
{%{flags: flags, width: width, precision: precision, conv: ?%}, "", args}
end
end
defp take_format_flags(<<flag::utf8, rest::binary>>, flags)
when flag in [?-, ?+, ?\s, ?0, ?#, ?,, ?!] do
take_format_flags(rest, MapSet.put(flags, flag))
end
defp take_format_flags(text, flags), do: {flags, text}
defp take_format_width(<<"*", rest::binary>>, flags, [arg | args]) do
case format_int(arg) do
width when width < 0 -> {abs(width), MapSet.put(flags, ?-), rest, args}
width -> {width, flags, rest, args}
end
end
defp take_format_width(<<"*", rest::binary>>, flags, []), do: {0, flags, rest, []}
defp take_format_width(text, flags, args) do
{digits, rest} = take_digits(text, "")
width = if digits == "", do: nil, else: String.to_integer(digits)
{width, flags, rest, args}
end
defp take_format_precision(<<".*", rest::binary>>, [arg | args]) do
precision = format_int(arg)
{if(precision < 0, do: nil, else: precision), rest, args}
end
defp take_format_precision(<<".*", rest::binary>>, []), do: {0, rest, []}
defp take_format_precision(<<".", rest::binary>>, args) do
{digits, rest} = take_digits(rest, "")
{if(digits == "", do: 0, else: String.to_integer(digits)), rest, args}
end
defp take_format_precision(text, args), do: {nil, text, args}
defp take_digits(<<digit::utf8, rest::binary>>, acc) when digit in ?0..?9,
do: take_digits(rest, acc <> <<digit::utf8>>)
defp take_digits(text, acc), do: {acc, text}
defp drop_format_length(<<"ll", rest::binary>>), do: rest
defp drop_format_length(<<"l", rest::binary>>), do: rest
defp drop_format_length(text), do: text
defp format_piece(%{conv: conv} = spec, args) when conv in [?s, ?z] do
{arg, args} = next_format_arg(args)
arg
|> format_text()
|> limit_format_text(spec)
|> apply_format_width(spec)
|> then(&{&1, args})
end
defp format_piece(%{conv: ?q} = spec, args) do
{arg, args} = next_format_arg(args)
arg
|> format_text()
|> limit_format_text(spec)
|> escape_sql_quote()
|> apply_format_width(spec)
|> then(&{&1, args})
end
defp format_piece(%{conv: ?Q} = spec, args) do
{arg, args} = next_format_arg(args)
piece =
if is_nil(arg) do
"NULL"
else
"'" <>
(arg |> format_text() |> limit_format_text(spec) |> escape_sql_quote()) <> "'"
end
{apply_format_width(piece, spec), args}
end
defp format_piece(%{conv: ?j} = spec, args) do
{arg, args} = next_format_arg(args)
piece = if is_nil(arg), do: "", else: arg |> format_text() |> json_format_escape(spec)
{apply_format_width(piece, spec), args}
end
defp format_piece(%{conv: ?J} = spec, args) do
{arg, args} = next_format_arg(args)
piece =
if is_nil(arg) do
"null"
else
"\"" <> (arg |> format_text() |> json_format_escape(spec)) <> "\""
end
{apply_format_width(piece, spec), args}
end
defp format_piece(%{conv: ?w} = spec, args) do
{arg, args} = next_format_arg(args)
piece =
arg
|> format_text()
|> limit_format_text(spec)
|> String.replace("\"", "\"\"")
{apply_format_width(piece, spec), args}
end
defp format_piece(%{conv: conv} = spec, args) when conv in [?d, ?i, ?u, ?x, ?X, ?o] do
{arg, args} = next_format_arg(args)
value = format_int(arg)
piece =
case conv do
?d -> signed_integer_piece(value, 10, false, spec)
?i -> signed_integer_piece(value, 10, false, spec)
?u -> unsigned_integer_piece(value, 10, false, spec)
?x -> unsigned_integer_piece(value, 16, false, spec)
?X -> unsigned_integer_piece(value, 16, true, spec)
?o -> unsigned_integer_piece(value, 8, false, spec)
end
{piece, args}
end
defp format_piece(%{conv: conv} = spec, args) when conv in [?f, ?e, ?E, ?g, ?G] do
{arg, args} = next_format_arg(args)
precision = format_float_precision(spec)
value = format_float(arg)
piece =
case conv do
?f ->
:io_lib.format(format_control(".#{precision}f"), [value])
?e ->
value |> scientific_piece(precision) |> maybe_alternate_float(spec) |> String.downcase()
?E ->
value |> scientific_piece(precision) |> maybe_alternate_float(spec) |> String.upcase()
?g ->
general_piece(value, precision, spec, false)
?G ->
general_piece(value, precision, spec, true)
end
|> IO.iodata_to_binary()
{piece |> signed_float_piece(value, spec) |> apply_numeric_width(spec, true), args}
end
defp format_piece(%{conv: ?c} = spec, args) do
{arg, args} = next_format_arg(args)
char =
case format_text(arg) do
<<codepoint::utf8, _::binary>> -> <<codepoint::utf8>>
"" -> <<0>>
end
piece = String.duplicate(char, spec.precision || 1)
{apply_format_width(piece, spec), args}
end
defp format_piece(%{conv: conv} = spec, args) do
{arg, args} = next_format_arg(args)
piece = "%" <> <<conv::utf8>> <> format_text(arg)
{apply_format_width(piece, spec), args}
end
defp next_format_arg([arg | args]), do: {arg, args}
defp next_format_arg([]), do: {nil, []}
defp format_text(nil), do: ""
defp format_text(value), do: Value.to_text(value)
defp limit_format_text(text, %{precision: nil}), do: text
defp limit_format_text(text, %{precision: precision, flags: flags}) do
if MapSet.member?(flags, ?!) do
limit_format_text_characters(text, precision)
else
limit_format_text_bytes(text, precision)
end
end
defp limit_format_text_characters(text, precision) do
text |> String.graphemes() |> Enum.take(max(precision, 0)) |> Enum.join()
end
defp limit_format_text_bytes(text, precision) do
limit = min(byte_size(text), max(precision, 0))
valid_utf8_prefix(text, limit)
end
defp valid_utf8_prefix(_text, limit) when limit <= 0, do: ""
defp valid_utf8_prefix(text, limit) do
prefix = binary_part(text, 0, limit)
if String.valid?(prefix) do
prefix
else
valid_utf8_prefix(text, limit - 1)
end
end
defp escape_sql_quote(text), do: String.replace(text, "'", "''")
defp json_format_escape(text, spec) do
text
|> limit_format_text(spec)
|> :unicode.characters_to_binary(:utf8, :utf8)
|> json_escape_bytes([])
end
defp json_escape_bytes(<<>>, acc), do: acc |> Enum.reverse() |> IO.iodata_to_binary()
defp json_escape_bytes(<<"\"", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\\"" | acc])
defp json_escape_bytes(<<"\\", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\\\" | acc])
defp json_escape_bytes(<<"\b", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\b" | acc])
defp json_escape_bytes(<<"\t", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\t" | acc])
defp json_escape_bytes(<<"\n", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\n" | acc])
defp json_escape_bytes(<<"\f", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\f" | acc])
defp json_escape_bytes(<<"\r", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\r" | acc])
defp json_escape_bytes(<<byte, rest::binary>>, acc) when byte <= 0x1F do
escape =
"\\u00" <>
(byte
|> Integer.to_string(16)
|> String.downcase()
|> String.pad_leading(2, "0"))
json_escape_bytes(rest, [escape | acc])
end
defp json_escape_bytes(<<char::utf8, rest::binary>>, acc),
do: json_escape_bytes(rest, [<<char::utf8>> | acc])
# SQLite's upper()/lower() fold only ASCII letters; non-ASCII (and the bytes of
# multi-byte UTF-8 sequences, all >= 0x80) pass through unchanged.
defp ascii_case(text, range, delta) do
for <<byte <- text>>, into: <<>> do
if byte in range, do: <<byte + delta>>, else: <<byte>>
end
end
defp format_int(nil), do: 0
defp format_int(value) do
case Value.cast(value, :integer) do
value when is_integer(value) -> value
_ -> 0
end
end
defp format_float(nil), do: 0.0
defp format_float(value) do
case Value.cast(value, :real) do
value when is_integer(value) -> value * 1.0
value when is_float(value) -> value
_ -> 0.0
end
end
defp format_float_precision(%{precision: nil}), do: 6
defp format_float_precision(%{precision: precision}), do: precision
defp signed_integer_piece(value, base, uppercase?, spec) do
sign =
cond do
value < 0 -> "-"
MapSet.member?(spec.flags, ?+) -> "+"
MapSet.member?(spec.flags, ?\s) -> " "
true -> ""
end
digits = abs(value) |> Integer.to_string(base) |> maybe_upcase(uppercase?)
digits = integer_precision(digits, spec.precision)
digits = maybe_group_integer(digits, base, spec)
apply_numeric_width(sign <> digits, spec)
end
defp unsigned_integer_piece(value, base, uppercase?, spec) do
value = if value < 0, do: value + 18_446_744_073_709_551_616, else: value
digits = value |> Integer.to_string(base) |> maybe_upcase(uppercase?)
prefix =
cond do
not MapSet.member?(spec.flags, ?#) -> ""
base == 16 and uppercase? -> "0X"
base == 16 -> "0x"
base == 8 -> "0"
true -> ""
end
digits = integer_precision(digits, spec.precision)
digits = maybe_group_integer(digits, base, spec)
apply_numeric_width(prefix <> digits, spec)
end
defp integer_precision(digits, nil), do: digits
defp integer_precision(digits, precision) do
String.duplicate("0", max(precision - String.length(digits), 0)) <> digits
end
defp maybe_upcase(text, true), do: String.upcase(text)
defp maybe_upcase(text, false), do: String.downcase(text)
defp signed_float_piece(piece, value, spec) when value >= 0 do
piece =
cond do
MapSet.member?(spec.flags, ?+) -> "+" <> piece
MapSet.member?(spec.flags, ?\s) -> " " <> piece
true -> piece
end
maybe_group_decimal(piece, spec)
end
defp signed_float_piece(piece, _value, spec), do: maybe_group_decimal(piece, spec)
defp maybe_group_integer(digits, 10, spec) do
if MapSet.member?(spec.flags, ?,), do: group_digits(digits), else: digits
end
defp maybe_group_integer(digits, _base, _spec), do: digits
defp maybe_group_decimal(piece, spec) do
if MapSet.member?(spec.flags, ?,) and spec.conv == ?f do
{prefix, rest} = numeric_prefix(piece)
[whole | tail] = String.split(rest, ".", parts: 2)
prefix <> group_digits(whole) <> if(tail == [], do: "", else: "." <> hd(tail))
else
piece
end
end
defp group_digits(digits) do
digits
|> String.reverse()
|> String.graphemes()
|> Enum.chunk_every(3)
|> Enum.map_join(",", &Enum.join/1)
|> String.reverse()
end
defp general_piece(value, precision, spec, uppercase?) do
safe_precision = if precision == 0, do: 1, else: precision
piece =
cond do
MapSet.member?(spec.flags, ?#) ->
hash_general_piece(value, precision, safe_precision)
MapSet.member?(spec.flags, ?!) ->
alternate_general_piece(value, max(precision, 1))
true ->
format_control(".#{safe_precision}g")
|> :io_lib.format([value])
|> IO.iodata_to_binary()
|> normalize_general_piece(precision)
end
if uppercase?, do: String.upcase(piece), else: piece
end
defp hash_general_piece(value, precision, safe_precision) do
format_control(".#{safe_precision}g")
|> :io_lib.format([value])
|> IO.iodata_to_binary()
|> normalize_general_piece(precision)
|> apply_hash_general_precision(precision)
end
defp apply_hash_general_precision(text, precision) do
if String.contains?(text, ["e", "E"]) do
apply_hash_scientific_precision(text, precision)
else
apply_hash_fixed_precision(text, precision)
end
end
defp apply_hash_scientific_precision(text, precision) do
Regex.replace(~r/^(.+?)([eE][+-]\d+)$/, text, fn _match, mantissa, exponent ->
cond do
precision == 0 and String.contains?(mantissa, ".") ->
[head, _ | _] = String.split(mantissa, ".", parts: 2)
head <> "." <> exponent
precision == 0 ->
mantissa <> "." <> exponent
String.contains?(mantissa, ".") ->
mantissa <> exponent
true ->
mantissa <> ".0" <> exponent
end
end)
end
defp apply_hash_fixed_precision(text, precision) do
{sign, rest} =
case text do
<<"-", value::binary>> -> {"-", value}
<<"+", value::binary>> -> {"+", value}
_ -> {"", text}
end
{int_part, frac_part} =
case String.split(rest, ".", parts: 2) do
[integer, fraction] -> {integer, fraction}
[integer] -> {integer, ""}
end
significant = hash_significant_digits(int_part, frac_part)
needed = max(precision - significant, 0)
padded_frac = frac_part <> String.duplicate("0", needed)
sign <> int_part <> "." <> padded_frac
end
defp hash_significant_digits(int_part, frac_part) do
if int_part != "0" do
String.length(int_part <> frac_part)
else
sig = String.trim_leading(frac_part, "0")
if sig == "", do: 1, else: String.length(sig)
end
end
defp scientific_piece(value, 0) do
format_control(".2e")
|> :io_lib.format([value])
|> IO.iodata_to_binary()
|> String.replace(~r/\.0(e[+-]\d+)$/i, "\\1")
|> pad_scientific_exponent()
end
defp scientific_piece(value, precision) do
format_control(".#{precision + 1}e")
|> :io_lib.format([value])
|> IO.iodata_to_binary()
|> pad_scientific_exponent()
end
defp maybe_alternate_float(piece, %{flags: flags, precision: precision}) do
cond do
MapSet.member?(flags, ?!) ->
if precision == 0 do
force_scientific_exclamation(piece)
else
force_decimal_point(piece)
end
MapSet.member?(flags, ?#) ->
force_scientific_hash(piece)
true ->
piece
end
end
defp force_scientific_hash(piece) do
case Regex.run(~r/^(.+?)([eE][+-]\d+)$/, piece) do
[_, mantissa, exponent] ->
if String.contains?(mantissa, ".") do
[head, _ | _] = String.split(mantissa, ".", parts: 2)
head <> "." <> exponent
else
mantissa <> "." <> exponent
end
nil ->
piece
end
end
defp force_scientific_exclamation(piece) do
case Regex.run(~r/^(.+?)([eE][+-]\d+)$/, piece) do
[_, mantissa, exponent] ->
if String.contains?(mantissa, ".") do
[head, _ | _] = String.split(mantissa, ".", parts: 2)
head <> ".0" <> exponent
else
mantissa <> ".0" <> exponent
end
nil ->
piece
end
end
defp alternate_general_piece(value, precision) do
value
|> Float.to_string()
|> normalize_general_piece(precision)
|> force_decimal_point()
end
defp normalize_general_piece(piece, precision) do
case Regex.run(~r/^([+-]?)(\d+(?:\.\d+)?)[eE]([+-]?\d+)$/, piece) do
[_, sign, significand, exponent_text] ->
exponent = String.to_integer(exponent_text)
if exponent >= -4 and exponent < precision do
sign <> expand_scientific(significand, exponent)
else
sign <> trim_general_mantissa(significand) <> "e" <> signed_exponent(exponent)
end
nil ->
piece
end
end
defp expand_scientific(significand, exponent) do
{whole, fractional} =
case String.split(significand, ".", parts: 2) do
[whole, fractional] -> {whole, fractional}
[whole] -> {whole, ""}
end
digits = whole <> fractional
decimal_position = String.length(whole) + exponent
cond do
decimal_position <= 0 ->
"0." <> String.duplicate("0", -decimal_position) <> digits
decimal_position >= String.length(digits) ->
digits <> String.duplicate("0", decimal_position - String.length(digits))
true ->
{left, right} = String.split_at(digits, decimal_position)
left <> "." <> right
end
|> trim_general_mantissa()
end
defp trim_general_mantissa(text) do
if String.contains?(text, ".") do
text
|> String.trim_trailing("0")
|> String.trim_trailing(".")
else
text
end
end
defp force_decimal_point(piece) do
case Regex.run(~r/^(.+?)([eE][+-]\d+)$/, piece) do
[_, mantissa, exponent] -> force_decimal_point(mantissa) <> exponent
nil -> if String.contains?(piece, "."), do: piece, else: piece <> ".0"
end
end
defp signed_exponent(exponent) do
sign = if exponent < 0, do: "-", else: "+"
digits = exponent |> abs() |> Integer.to_string() |> String.pad_leading(2, "0")
sign <> digits
end
defp pad_scientific_exponent(text) do
Regex.replace(~r/e([+-])(\d)$/, text, fn _match, sign, digit -> "e#{sign}0#{digit}" end)
end
defp format_control(options), do: ("~" <> options) |> String.to_charlist()
# The `0` flag zero-pads to the field width. For integer conversions an
# explicit precision suppresses it (C printf: precision sets the minimum digit
# count); for float conversions (`zero_with_precision?` = true) precision is
# the fraction width, so the flag still applies — e.g. `%05.2f` of 3.14 is
# `03.14`, not ` 3.14`.
defp apply_numeric_width(piece, spec, zero_with_precision? \\ false) do
if MapSet.member?(spec.flags, ?0) and not MapSet.member?(spec.flags, ?-) and
is_integer(spec.width) and (spec.precision == nil or zero_with_precision?) do
pad_numeric_zero(piece, spec.width)
else
apply_format_width(piece, spec)
end
end
defp pad_numeric_zero(piece, width) do
size = String.length(piece)
if size >= width do
piece
else
{head, rest} = numeric_prefix(piece)
head <> String.duplicate("0", width - size) <> rest
end
end
defp numeric_prefix("-" <> rest), do: {"-", rest}
defp numeric_prefix("+" <> rest), do: {"+", rest}
defp numeric_prefix(" " <> rest), do: {" ", rest}
defp numeric_prefix("0x" <> rest), do: {"0x", rest}
defp numeric_prefix("0X" <> rest), do: {"0X", rest}
defp numeric_prefix(rest), do: {"", rest}
defp apply_format_width(piece, %{width: width, flags: flags}) when is_integer(width) do
size =
if MapSet.member?(flags, ?!) do
String.length(piece)
else
byte_size(piece)
end
if size >= width do
piece
else
padding = String.duplicate(" ", width - size)
if MapSet.member?(flags, ?-) do
piece <> padding
else
padding <> piece
end
end
end
defp apply_format_width(piece, _spec), do: piece
# -- aggregate functions --------------------------------------------------------------
defp aggregate_call?(_db, name, :star), do: name == "count"
defp aggregate_call?(db, name, {:distinct, args}), do: aggregate_call?(db, name, args)
defp aggregate_call?(db, name, args) do
is_list(args) and
(match?({:ok, _}, Database.fetch_aggregate_function(db, name, length(args))) or
(name in @aggregate_functions and not (name in ["min", "max"] and length(args) != 1)))
end
defp contains_aggregate?({:window, _name, _args, _spec, _filter}, _db), do: false
defp contains_aggregate?({:filter_function, name, args, filter}, db) do
aggregate_call?(db, name, args) or contains_aggregate?(filter, db) or
(is_list(args) and Enum.any?(args, &contains_aggregate?(&1, db)))
end
defp contains_aggregate?({:function, name, {:distinct, args}}, db) do
aggregate_call?(db, name, {:distinct, args}) or Enum.any?(args, &contains_aggregate?(&1, db))
end
defp contains_aggregate?({:function, name, args}, db) do
aggregate_call?(db, name, args) or
(is_list(args) and Enum.any?(args, &contains_aggregate?(&1, db)))
end
defp contains_aggregate?(expr, db) when is_tuple(expr) do
expr
|> Tuple.to_list()
|> Enum.any?(fn
# Subquery ASTs are structs (maps), so this walk never descends into
# them — an aggregate inside a subquery belongs to the subquery.
sub when is_tuple(sub) -> contains_aggregate?(sub, db)
subs when is_list(subs) -> Enum.any?(subs, &(is_tuple(&1) and contains_aggregate?(&1, db)))
_ -> false
end)
end
defp contains_aggregate?(_expr, _db), do: false
defp aggregate("count", :star, group, _env), do: length(group)
defp aggregate(name, {:distinct, args}, group, env) do
if length(args) != 1 do
fail("DISTINCT aggregates must have exactly one argument")
end
[arg] = args
distinct_group =
group
|> Enum.map(fn member -> {eval(arg, member), member} end)
|> Enum.reject(fn {value, _member} -> is_nil(value) end)
|> Enum.uniq_by(fn {value, _member} -> value end)
|> Enum.map(fn {_value, member} -> member end)
aggregate(name, args, distinct_group, env)
end
defp aggregate(name, args, group, env) when is_list(args) do
case Database.fetch_aggregate_function(env.db, name, length(args)) do
{:ok, %{kind: :incremental_window} = function} ->
call_incremental_window_aggregate(function, aggregate_rows_with_nulls(args, group))
{:ok, function} ->
call_aggregate_function(function, aggregate_rows(args, group))
:error ->
built_in_aggregate(name, args, group, env)
end
end
defp aggregate_rows(args, group) do
group
|> Enum.map(fn member -> Enum.map(args, &eval(&1, member)) end)
|> Enum.reject(&Enum.any?(&1, fn value -> is_nil(value) end))
end
defp aggregate_rows_with_nulls(args, group) do
Enum.map(group, fn member -> Enum.map(args, &eval(&1, member)) end)
end
defp call_aggregate_function(%{name: name, callback: callback}, rows) do
callback
|> apply([rows])
|> normalize_aggregate_function_result(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined aggregate #{name}() raised: #{Exception.message(e)}")
end
defp normalize_aggregate_function_result({:ok, value}, name),
do: normalize_aggregate_function_result(value, name)
defp normalize_aggregate_function_result({:error, message}, name),
do: fail("user-defined aggregate #{name}() error: #{message}")
defp normalize_aggregate_function_result(nil, _name), do: nil
defp normalize_aggregate_function_result(value, _name) when is_integer(value), do: value
defp normalize_aggregate_function_result(value, _name) when is_float(value), do: value
defp normalize_aggregate_function_result(value, _name) when is_binary(value), do: value
defp normalize_aggregate_function_result({:blob, value}, _name) when is_binary(value),
do: {:blob, value}
defp normalize_aggregate_function_result(_value, name),
do: fail("user-defined aggregate #{name}() returned unsupported value")
defp call_incremental_window_aggregate(function, rows) do
state = call_incremental_window_init(function)
state =
Enum.reduce(rows, state, fn args, state ->
call_incremental_window_update(function, :step, state, args)
end)
call_incremental_window_final(function, state)
end
defp call_incremental_window_init(%{name: name, callback: %{init: init}}) do
init
|> apply([])
|> normalize_incremental_window_state(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined window function #{name}() raised: #{Exception.message(e)}")
end
defp call_incremental_window_update(
%{name: name, callback: callbacks},
callback_name,
state,
args
) do
callbacks
|> Map.fetch!(callback_name)
|> apply([state, args])
|> normalize_incremental_window_state(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined window function #{name}() raised: #{Exception.message(e)}")
end
defp call_incremental_window_value(%{name: name, callback: %{value: value}}, state) do
value
|> apply([state])
|> normalize_aggregate_function_result(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined window function #{name}() raised: #{Exception.message(e)}")
end
defp call_incremental_window_final(%{name: name, callback: %{final: final}}, state) do
final
|> apply([state])
|> normalize_aggregate_function_result(name)
rescue
e in Error ->
reraise e, __STACKTRACE__
e ->
fail("user-defined window function #{name}() raised: #{Exception.message(e)}")
end
defp normalize_incremental_window_state({:ok, state}, name),
do: normalize_incremental_window_state(state, name)
defp normalize_incremental_window_state({:error, message}, name),
do: fail("user-defined window function #{name}() error: #{message}")
defp normalize_incremental_window_state(state, _name), do: state
defp built_in_aggregate("count", [], group, _env), do: length(group)
defp built_in_aggregate(name, [arg | rest], group, env) do
values = for member <- group, value = eval(arg, member), not is_nil(value), do: value
case name do
"count" ->
length(values)
"sum" ->
if values == [], do: nil, else: numeric_sum(values)
"total" ->
1.0 * (values |> Enum.map(&to_num/1) |> Enum.sum())
"avg" ->
if values == [],
do: nil,
else: (values |> Enum.map(&to_num/1) |> Enum.sum()) / length(values)
"min" ->
Enum.min(values, fn a, b -> Value.compare(a, b) != :gt end, fn -> nil end)
"max" ->
Enum.max(values, fn a, b -> Value.compare(a, b) != :lt end, fn -> nil end)
name when name in ["group_concat", "string_agg"] ->
separator =
case rest do
[sep_expr] ->
case eval(sep_expr, env) do
nil -> ","
value -> Value.to_text(value)
end
[] ->
","
end
if values == [], do: nil, else: Enum.map_join(values, separator, &Value.to_text/1)
# JSON aggregates keep NULL members, unlike the filtered `values`.
name when name in ["json_group_array", "jsonb_group_array"] ->
members = for member <- group, do: member |> then(&eval(arg, &1)) |> json_from_sql!()
json_aggregate_result(name, {:array, members})
name when name in ["json_group_object", "jsonb_group_object"] ->
case rest do
[value_expr] ->
pairs =
for member <- group do
key = eval(arg, member)
{Value.to_text(key || ""), json_from_sql!(eval(value_expr, member))}
end
json_aggregate_result(name, {:object, pairs})
_ ->
fail("wrong number of arguments to function #{name}()")
end
end
end
defp built_in_aggregate(name, _args, _group, _env),
do: fail("wrong number of arguments to function #{name}()")
defp numeric_sum(values) do
nums = Enum.map(values, &to_num/1)
if Enum.all?(nums, &is_integer/1) do
sum = Enum.sum(nums)
# sum() over integers errors on 64-bit overflow rather than going REAL.
if Value.out_of_int64_range?(sum), do: fail("integer overflow")
sum
else
nums |> Enum.map(&(&1 * 1.0)) |> Enum.sum()
end
end
# Numeric coercion for aggregation. Unlike NUMERIC affinity this must not
# demote integral floats: sum() over a REAL column stays REAL (ticket #2251).
defp to_num(v) when is_integer(v) or is_float(v), do: v
defp to_num(v) do
case Value.apply_affinity(v, :numeric) do
n when is_integer(n) or is_float(n) -> n
_ -> 0
end
end
# -- naming -------------------------------------------------------------------------
# Output column name for an expression without an alias. SQLite uses the
# original SQL text; we render an equivalent form.
defp result_column_name(_db, _templates, {_expr, alias_name}) when is_binary(alias_name),
do: alias_name
defp result_column_name(
%{full_column_names: true},
templates,
{{:column, qualifier, name}, nil}
) do
case result_column_source(templates, qualifier, name) do
nil -> expr_name({:column, qualifier, name})
source -> "#{source}.#{name}"
end
end
defp result_column_name(
%{short_column_names: true},
_templates,
{{:column, _qualifier, name}, nil}
),
do: name
defp result_column_name(_db, _templates, {expr, nil}), do: expr_name(expr)
defp result_column_source(templates, nil, name) do
key = Table.key(name)
case Enum.filter(templates, &visible?(&1, key)) do
[frame] -> frame.source_name || frame.name
_ -> nil
end
end
defp result_column_source(templates, qualifier, name) do
qkey = Table.key(qualifier)
key = Table.key(name)
case Enum.find(templates, &(&1.name == qkey)) do
%{source_name: source_name} = frame when source_name != nil ->
if has_column?(frame, key), do: source_name, else: nil
frame when is_map(frame) ->
if has_column?(frame, key), do: frame.name, else: nil
nil ->
nil
end
end
defp expr_name({:param, _index, raw}), do: raw
defp expr_name({:column, nil, name}), do: name
defp expr_name({:column, table, name}), do: "#{table}.#{name}"
defp expr_name({:literal, nil}), do: "NULL"
defp expr_name({:literal, {:blob, b}}), do: "x'#{Base.encode16(b)}'"
defp expr_name({:literal, v}) when is_binary(v), do: "'#{v}'"
defp expr_name({:literal, v}), do: Value.to_text(v)
defp expr_name({:function, name, :star}), do: "#{name}(*)"
defp expr_name({:function, name, {:distinct, args}}),
do: "#{name}(DISTINCT #{Enum.map_join(args, ", ", &expr_name/1)})"
defp expr_name({:function, name, args}),
do: "#{name}(#{Enum.map_join(args, ", ", &expr_name/1)})"
defp expr_name({:window, name, :star, spec, nil}), do: "#{name}(*) OVER #{window_name(spec)}"
defp expr_name({:window, name, args, spec, nil}),
do: "#{name}(#{Enum.map_join(args, ", ", &expr_name/1)}) OVER #{window_name(spec)}"
defp expr_name({:window, name, args, spec, filter}) do
"#{name}(#{Enum.map_join(args, ", ", &expr_name/1)}) FILTER (WHERE #{expr_name(filter)}) OVER #{window_name(spec)}"
end
defp expr_name({:binary, op, left, right}),
do: "#{expr_name(left)} #{op_text(op)} #{expr_name(right)}"
defp expr_name({:collate, expr, name}), do: "#{expr_name(expr)} COLLATE #{name}"
defp expr_name({:negate, expr}), do: "-#{expr_name(expr)}"
defp expr_name({:not, expr}), do: "NOT #{expr_name(expr)}"
defp expr_name(_expr), do: "expr"
defp window_name({:ref, name}), do: name
defp window_name(%{partition_by: [], order_by: []}), do: "()"
defp window_name(%{partition_by: partition_by, order_by: order_by}) do
parts = []
parts =
if partition_by == [],
do: parts,
else: ["PARTITION BY #{Enum.map_join(partition_by, ", ", &expr_name/1)}" | parts]
parts =
if order_by == [],
do: parts,
else: ["ORDER BY #{Enum.map_join(order_by, ", ", &order_expr_name/1)}" | parts]
"(" <> (parts |> Enum.reverse() |> Enum.join(" ")) <> ")"
end
defp order_expr_name({expr, :asc}), do: expr_name(expr)
defp order_expr_name({expr, :desc}), do: "#{expr_name(expr)} DESC"
# Renders an expression compactly (no spaces around operators) for CHECK
# constraint error messages, matching SQLite's behavior of using the raw SQL.
defp check_text({:column, nil, name}), do: name
defp check_text({:column, table, name}), do: "#{table}.#{name}"
defp check_text({:literal, nil}), do: "NULL"
defp check_text({:literal, {:blob, b}}), do: "x'#{Base.encode16(b)}'"
defp check_text({:literal, v}) when is_binary(v), do: "'#{v}'"
defp check_text({:literal, v}), do: Value.to_text(v)
defp check_text({:function, name, :star}), do: "#{name}(*)"
defp check_text({:function, name, args}),
do: "#{name}(#{Enum.map_join(args, ",", &check_text/1)})"
defp check_text({:binary, op, left, right}),
do: "#{check_text(left)}#{op_text(op)}#{check_text(right)}"
defp check_text({:negate, expr}), do: "-#{check_text(expr)}"
defp check_text({:not, expr}), do: "NOT #{check_text(expr)}"
defp check_text(expr), do: expr_name(expr)
defp op_text(op) do
%{
eq: "=",
ne: "<>",
lt: "<",
le: "<=",
gt: ">",
ge: ">=",
add: "+",
sub: "-",
mul: "*",
div: "/",
mod: "%",
concat: "||",
and: "AND",
or: "OR"
}[op]
end
# -- helpers -------------------------------------------------------------------------
# -- CTE resolution ----------------------------------------------------------
#
# Non-recursive: evaluate each CTE in order (materialized), shadowing tables
# and prior CTEs. The results are stored in db.ctes.
#
# Recursive: the standard queue algorithm from https://sqlite.org/lang_with.html
# — UNION ALL appends all new rows, UNION only appends rows not yet seen.
# A runaway guard stops at 1_000_000 rows.
@recursive_row_cap 1_000_000
defp resolve_ctes(db, cte_defs, _recursive, outer_limit) do
# Validate for duplicate CTE names
names = Enum.map(cte_defs, &Table.key(&1.name))
case Enum.find(Enum.zip(names, cte_defs), fn {key, _} ->
Enum.count(names, &(&1 == key)) > 1
end) do
{_, cte} -> fail("duplicate WITH table name: #{cte.name}")
nil -> :ok
end
# Effective row cap: use outer LIMIT if given, otherwise the global cap.
row_cap = outer_limit || @recursive_row_cap
# Build an index of all CTE defs for forward reference resolution.
defs_by_key = Map.new(cte_defs, &{Table.key(&1.name), &1})
# Evaluate each CTE in declaration order, supporting forward references.
# Cycle detection uses a MapSet of keys currently being evaluated.
{db, _} =
Enum.reduce(cte_defs, {db, MapSet.new()}, fn cte_def, {db, in_progress} ->
key = Table.key(cte_def.name)
if Map.has_key?(db.ctes, key) do
# Already evaluated (e.g. pulled in as a forward reference)
{db, in_progress}
else
evaluate_cte(db, cte_def, key, defs_by_key, in_progress, row_cap)
end
end)
db
end
# Evaluate a single CTE, resolving forward references as needed.
defp evaluate_cte(db, cte_def, key, defs_by_key, in_progress, row_cap) do
if MapSet.member?(in_progress, key) do
fail("circular reference: #{cte_def.name}")
end
in_progress = MapSet.put(in_progress, key)
if recursive_cte?(cte_def.query, key, db) do
db = resolve_single_recursive_cte(db, cte_def, key, row_cap)
{db, in_progress}
else
# Check for self-references via subqueries (IN, EXISTS, scalar) — these are
# circular references, not recursion.
if query_references_key?(cte_def.query, key) do
fail("circular reference: #{cte_def.name}")
end
# Resolve any forward references in this CTE's query before evaluating.
# When query_result runs into an undefined CTE name, it will raise "no such table".
# We intercept by pre-loading forward references here.
{db, in_progress} =
resolve_forward_refs(db, cte_def.query, key, defs_by_key, in_progress, row_cap)
# For a compound query with declared columns: evaluate only the seed first to check
# column count against declared columns. This ensures the column count error takes
# priority over the compound-width-mismatch error when both would apply.
# (Non-compound or no declared columns: skip this pre-check.)
case {cte_def.columns, cte_def.query} do
{cols, %Compound{left: seed_query}} when cols != nil ->
seed_result = query_result(db, seed_query, nil)
if length(seed_result.columns) != length(cols) do
fail(
"table #{cte_def.name} has #{length(seed_result.columns)} values for #{length(cols)} columns"
)
end
_ ->
:ok
end
result = query_result(db, cte_def.query, nil)
{columns, affinities, actual_count} = apply_cte_columns(cte_def, result, result.columns)
cte = %{
columns: columns,
rows: result.rows,
affinities: affinities,
actual_count: actual_count
}
{%{db | ctes: Map.put(db.ctes, key, cte)}, in_progress}
end
end
# Walk a query's FROM clauses and pre-evaluate any CTEs that are referenced
# but not yet in db.ctes.
defp resolve_forward_refs(
db,
%Compound{left: l, right: r},
self_key,
defs,
in_progress,
row_cap
) do
{db, in_progress} = resolve_forward_refs(db, l, self_key, defs, in_progress, row_cap)
resolve_forward_refs(db, r, self_key, defs, in_progress, row_cap)
end
defp resolve_forward_refs(db, %Select{from: from}, self_key, defs, in_progress, row_cap) do
resolve_forward_refs_from(db, from, self_key, defs, in_progress, row_cap)
end
defp resolve_forward_refs(db, _query, _self_key, _defs, in_progress, _row_cap),
do: {db, in_progress}
defp resolve_forward_refs_from(db, nil, _self_key, _defs, in_progress, _row_cap),
do: {db, in_progress}
defp resolve_forward_refs_from(db, {:table, name, _alias}, self_key, defs, in_progress, row_cap) do
ref_key = table_source_key(name)
cond do
# Already resolved or is a table/view or the CTE itself
Map.has_key?(db.ctes, ref_key) or Map.has_key?(db.tables, ref_key) ->
{db, in_progress}
ref_key == self_key ->
# Self-reference is allowed only in recursive CTEs; if we're here,
# it will naturally cause "no such table" at eval time.
{db, in_progress}
# Circular reference: self_key is in in_progress, meaning the CTE identified
# by self_key references another CTE (ref_key) that is already being evaluated.
# Report the current CTE (self_key) as the source of the circular reference.
MapSet.member?(in_progress, ref_key) ->
self_def = Map.get(defs, self_key)
self_name = if self_def, do: self_def.name, else: name
fail("circular reference: #{self_name}")
Map.has_key?(defs, ref_key) ->
evaluate_cte(db, Map.fetch!(defs, ref_key), ref_key, defs, in_progress, row_cap)
true ->
{db, in_progress}
end
end
defp resolve_forward_refs_from(db, {:subquery, _, _}, _self_key, _defs, in_progress, _row_cap),
do: {db, in_progress}
defp resolve_forward_refs_from(db, {:join, _t, l, r, _c}, self_key, defs, in_progress, row_cap) do
{db, in_progress} = resolve_forward_refs_from(db, l, self_key, defs, in_progress, row_cap)
resolve_forward_refs_from(db, r, self_key, defs, in_progress, row_cap)
end
# Check if a query directly references the CTE key in FROM (not inside a subquery).
# A self-reference via EXISTS/IN subquery is a circular reference error, not recursion.
defp recursive_cte?(%Compound{left: left, right: right}, key, db) do
recursive_cte?(left, key, db) or recursive_cte?(right, key, db)
end
defp recursive_cte?(%Select{from: from}, key, _db), do: from_references?(from, key)
defp recursive_cte?(%Values{}, _key, _db), do: false
defp recursive_cte?(%With{}, _key, _db), do: false
defp from_references?(nil, _key), do: false
defp from_references?({:table, name, _alias}, key), do: table_source_key(name) == key
defp from_references?({:subquery, _, _}, _key), do: false
defp from_references?({:join, _type, left, right, _constraint}, key) do
from_references?(left, key) or from_references?(right, key)
end
# Check whether a query references `key` anywhere (including in subquery WHERE clauses).
# Used to detect self-referential CTEs that aren't direct FROM references (e.g. IN subqueries).
defp query_references_key?(%Compound{left: l, right: r}, key) do
query_references_key?(l, key) or query_references_key?(r, key)
end
defp query_references_key?(%Select{from: from, where: where, columns: cols}, key) do
from_references_deep?(from, key) or
expr_references_key?(where, key) or
Enum.any?(cols, &expr_references_key?(&1, key))
end
defp query_references_key?(%Values{}, _key), do: false
defp query_references_key?(%With{}, _key), do: false
defp query_references_key?(nil, _key), do: false
# Check FROM for key, including inside subqueries.
defp from_references_deep?(nil, _key), do: false
defp from_references_deep?({:table, name, _}, key), do: table_source_key(name) == key
defp from_references_deep?({:subquery, q, _}, key), do: query_references_key?(q, key)
defp from_references_deep?({:join, _, l, r, _}, key) do
from_references_deep?(l, key) or from_references_deep?(r, key)
end
# Check expressions for subquery references to key.
defp expr_references_key?(nil, _key), do: false
defp expr_references_key?(list, key) when is_list(list),
do: Enum.any?(list, &expr_references_key?(&1, key))
# IN with subquery: {:in, expr, {:select, query}, negated}
defp expr_references_key?({:in, expr, {:select, q}, _negated}, key) do
expr_references_key?(expr, key) or query_references_key?(q, key)
end
defp expr_references_key?({:in, expr, items, _negated}, key) when is_list(items) do
expr_references_key?(expr, key) or Enum.any?(items, &expr_references_key?(&1, key))
end
defp expr_references_key?({:exists, q}, key), do: query_references_key?(q, key)
defp expr_references_key?({:subquery, q}, key), do: query_references_key?(q, key)
defp expr_references_key?({op, l, r}, key) when is_atom(op) do
expr_references_key?(l, key) or expr_references_key?(r, key)
end
defp expr_references_key?({op, arg}, key) when is_atom(op) do
expr_references_key?(arg, key)
end
defp expr_references_key?(_other, _key), do: false
defp resolve_single_recursive_cte(db, cte_def, key, row_cap) do
# A recursive CTE is: initial UNION [ALL] recursive_part
# We require the top-level compound to be UNION or UNION ALL.
# If the CTE itself has ORDER BY/LIMIT, extract and apply those after expansion.
if contains_window?(cte_def.query) do
fail("cannot use window functions in recursive queries")
end
case cte_def.query do
%Compound{
op: op,
left: initial_query,
right: recursive_query,
order_by: order_by,
limit: limit,
offset: offset
}
when op in [:union, :union_all] ->
# Step 1: evaluate the initial (seed) query
initial_result = query_result(db, initial_query, nil)
{col_names, affinities, actual_count} =
apply_cte_columns(cte_def, initial_result, initial_result.columns)
# Validate column count against initial result's declared column count
if actual_count != nil and actual_count != length(col_names) do
fail(
"table #{cte_def.name} has #{actual_count} values for #{length(col_names)} columns"
)
end
# Validate column count match against the initial result
seed_rows = validate_cte_rows(initial_result.rows, col_names, cte_def.name)
# Set up the working set
all_rows = seed_rows
seen =
if op == :union,
do: MapSet.new(Enum.map(seed_rows, &:erlang.term_to_binary/1)),
else: nil
working = seed_rows
# Use the smaller of row_cap and CTE-level LIMIT (if present) as the expansion cap.
expansion_cap =
if limit != nil do
n = int_clause(limit, db, "LIMIT")
min(row_cap, n)
else
row_cap
end
# Iteratively expand (stop when no new rows or row_cap/limit reached)
{expanded_rows, _seen} =
iterate_recursive_cte(
db,
key,
col_names,
affinities,
cte_def,
recursive_query,
all_rows,
seen,
working,
op,
expansion_cap
)
# Apply ORDER BY / LIMIT from the CTE body if present
final_rows =
if order_by != [] or limit != nil do
expanded_rows
|> compound_order(order_by, col_names, [col_names])
|> clamp(%{db | ctes: %{}}, limit, offset)
else
expanded_rows
end
cte = %{columns: col_names, rows: final_rows, affinities: affinities, actual_count: nil}
%{db | ctes: Map.put(db.ctes, key, cte)}
_ ->
# No UNION/UNION ALL at top — treat as non-recursive
result = query_result(db, cte_def.query, nil)
{columns, affinities, actual_count} = apply_cte_columns(cte_def, result, result.columns)
cte = %{
columns: columns,
rows: result.rows,
affinities: affinities,
actual_count: actual_count
}
%{db | ctes: Map.put(db.ctes, key, cte)}
end
end
defp iterate_recursive_cte(
db,
key,
col_names,
affinities,
cte_def,
recursive_query,
all_rows,
seen,
working,
op,
row_cap
) do
if working == [] or length(all_rows) >= row_cap do
{all_rows, seen}
else
# Bind the working set as the current value of this CTE
working_cte = %{columns: col_names, rows: working, affinities: affinities}
db_step = %{db | ctes: Map.put(db.ctes, key, working_cte)}
step_result = query_result(db_step, recursive_query, nil)
new_rows = validate_cte_rows(step_result.rows, col_names, cte_def.name)
# Accumulate new rows for the working set (forward order via reversal)
# and append them to all_rows in forward order.
{next_working_rev, next_seen, added_rev} =
Enum.reduce(new_rows, {[], seen, []}, fn row, {working_acc, seen_acc, added_acc} ->
case op do
:union_all ->
{[row | working_acc], seen_acc, [row | added_acc]}
:union ->
bin = :erlang.term_to_binary(row)
if MapSet.member?(seen_acc, bin) do
{working_acc, seen_acc, added_acc}
else
{[row | working_acc], MapSet.put(seen_acc, bin), [row | added_acc]}
end
end
end)
next_working = Enum.reverse(next_working_rev)
next_all = all_rows ++ Enum.reverse(added_rev)
iterate_recursive_cte(
db,
key,
col_names,
affinities,
cte_def,
recursive_query,
next_all,
next_seen,
next_working,
op,
row_cap
)
end
end
defp validate_cte_rows(rows, col_names, cte_name) do
expected = length(col_names)
Enum.each(rows, fn row ->
got = length(row)
if got != expected do
fail("table #{cte_name} has #{got} values for #{expected} columns")
end
end)
rows
end
defp apply_cte_columns(cte_def, result, result_columns) do
case cte_def.columns do
nil ->
{result_columns, result.affinities, nil}
col_names ->
affs = result.affinities ++ List.duplicate(:blob, length(col_names))
# Return {declared_columns, affinities, actual_column_count}
# Validation happens when the CTE is actually accessed via relation/3
{col_names, Enum.take(affs, length(col_names)), length(result_columns)}
end
end
defp fetch_table!(db, name) do
ensure_schema_table_not_modified!(nil, name)
# DML against a view is an error
case dml_view(db, nil, name) do
{:ok, _view} ->
fail("cannot modify #{name} because it is a view")
:error ->
case Database.lookup_table(db, name) do
{:ok, table} -> table
{:error, message} -> fail(message)
end
end
end
defp fetch_table!(db, nil, name), do: fetch_table!(db, name)
defp fetch_table!(db, schema, name) do
ensure_schema_table_not_modified!(schema, name)
ensure_schema_exists!(db, schema)
case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do
{:ok, table} -> table
:error -> fail("no such table: #{name}")
end
end
defp ensure_schema_table_not_modified!(schema, name) do
key = Table.key(name)
if key in ["sqlite_schema", "sqlite_master"] do
fail("table #{schema_label(schema)} may not be modified")
end
end
defp schema_label("temp"), do: "sqlite_temp_master"
defp schema_label(_), do: "sqlite_master"
defp dml_view(db, nil, name), do: Database.lookup_view(db, name)
defp dml_view(db, schema, name), do: Database.fetch_view(db, schema, name)
defp put_table(db, %Table{} = table) do
Database.put_table(db, refresh_index_entries(db, table))
end
defp refresh_index_entries(db, %Table{} = table) do
indexes =
Enum.map(table.indexes, fn index ->
build_index_entries(db, table, index)
end)
autoindexes =
Enum.map(table.autoindexes, fn index ->
build_index_entries(db, table, index)
end)
%{table | indexes: indexes, autoindexes: autoindexes}
end
defp ensure_index_entries(db, %Table{} = table) do
if Enum.all?(lookup_indexes(table), &Map.has_key?(&1, :entries)) do
table
else
refresh_index_entries(db, table)
end
end
defp indexes_have_entries?(%Table{} = table) do
Enum.all?(lookup_indexes(table), &Map.has_key?(&1, :entries))
end
# Adds one freshly-inserted row's entry to each index when entries are already
# materialized, keeping them current row-by-row; a no-op otherwise (the final
# store rebuilds). Lets a bulk insert avoid the per-row full rebuild and lets
# the unique-conflict check use the O(1) entry lookup.
defp maybe_add_index_entries(db, %Table{} = table, rowid) do
if indexes_have_entries?(table), do: add_index_entries(db, table, [rowid]), else: table
end
# Adds the given rowids' entries to every index, mirroring build_index_entries/3
# per-row logic (partial-index WHERE check, member values, sorted rowid lists),
# but only for those rowids rather than rescanning the whole table.
defp add_index_entries(db, %Table{} = table, rowids) do
%{
table
| indexes: Enum.map(table.indexes, &add_rowids_to_index(db, table, &1, rowids)),
autoindexes: Enum.map(table.autoindexes, &add_rowids_to_index(db, table, &1, rowids))
}
end
defp add_rowids_to_index(db, table, index, rowids) do
entries =
Enum.reduce(rowids, Map.fetch!(index, :entries), fn rowid, acc ->
# Read the raw stored tuple (no per-row widen to a map): `index_member_values`
# reads only the index's member columns, positionally. Widening here ran
# once per index per insert — O(columns) x N-indexes on every row.
case Map.get(table.rows, rowid) do
nil ->
acc
row ->
if index.where == nil or
row_matches_partial_index?(db, table, rowid, row, index.where) do
key = List.to_tuple(index_member_values(db, table, rowid, row, index))
# Prepend (O(1)); the per-key rowid list is sorted lazily at read
# (`index_lookup_rowids`) instead of re-sorting the whole list on
# every insert — that re-sort was O(n²) for low-cardinality indexes
# (each key accumulates O(n) rowids).
Map.update(acc, key, [rowid], &[rowid | &1])
else
acc
end
end
end)
index
|> Map.put(:entries, entries)
|> Map.delete(:ordered_entries)
end
# Removes the given deleted `{rowid, row}`s from each index's entries, when
# entries are materialized — the delete counterpart of add_index_entries/3, so
# a delete touches only the affected keys instead of rescanning the whole
# table to rebuild every index.
defp remove_index_entries(db, %Table{} = table, deleted_pairs) do
if indexes_have_entries?(table) do
%{
table
| indexes:
Enum.map(table.indexes, &remove_pairs_from_index(db, table, &1, deleted_pairs)),
autoindexes:
Enum.map(table.autoindexes, &remove_pairs_from_index(db, table, &1, deleted_pairs))
}
else
table
end
end
defp remove_pairs_from_index(db, table, index, deleted_pairs) do
# Group the removed rowids by their index key, then filter each affected
# key's list once against a set (O(k) per key) instead of `rowids -- [rowid]`
# per deleted pair (O(k²) when many rows share a low-cardinality key).
removed_by_key =
Enum.reduce(deleted_pairs, %{}, fn {rowid, row}, acc ->
if index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where) do
key = List.to_tuple(index_member_values(db, table, rowid, row, index))
Map.update(acc, key, [rowid], &[rowid | &1])
else
acc
end
end)
entries =
Enum.reduce(removed_by_key, Map.fetch!(index, :entries), fn {key, removed}, acc ->
case acc do
%{^key => rowids} ->
removed_set = MapSet.new(removed)
case Enum.reject(rowids, &MapSet.member?(removed_set, &1)) do
[] -> Map.delete(acc, key)
kept -> Map.put(acc, key, kept)
end
_ ->
acc
end
end)
index
|> Map.put(:entries, entries)
|> Map.delete(:ordered_entries)
end
defp build_index_entries(db, table, index) do
entries =
table
|> Table.scan()
|> Enum.reduce(%{}, fn {rowid, row}, entries ->
if index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where) do
values = index_member_values(db, table, rowid, row, index)
Map.update(entries, List.to_tuple(values), [rowid], fn rowids -> [rowid | rowids] end)
else
entries
end
end)
|> Map.new(fn {values, rowids} -> {values, Enum.sort(rowids)} end)
index
|> Map.put(:entries, entries)
|> maybe_put_ordered_entries(entries)
end
defp maybe_put_ordered_entries(index, entries) do
if binary_collation_index?(index) do
ordered =
entries
|> Enum.sort(fn {left, _left_rowids}, {right, _right_rowids} ->
compare_index_keys(left, right) != :gt
end)
|> List.to_tuple()
Map.put(index, :ordered_entries, ordered)
else
Map.delete(index, :ordered_entries)
end
end
defp compare_index_keys(left, right) do
left_values = Tuple.to_list(left)
right_values = Tuple.to_list(right)
left_values
|> Enum.zip(right_values)
|> Enum.reduce_while(:eq, fn {left_value, right_value}, :eq ->
case Value.compare(left_value, right_value) do
:eq -> {:cont, :eq}
other -> {:halt, other}
end
end)
end
defp ensure_unique_names(%CreateTable{} = stmt) do
if stmt.columns == [], do: fail("table #{stmt.name} must have at least one column")
duplicate =
stmt.columns
|> Enum.frequencies_by(&Table.key(&1.name))
|> Enum.find(fn {_name, count} -> count > 1 end)
case duplicate do
{name, _} -> fail("duplicate column name: #{name}")
nil -> :ok
end
# Count inline primary keys
inline_pk_count = Enum.count(stmt.columns, & &1.primary_key)
# Count table-level primary key constraints
table_pk_count = Enum.count(stmt.constraints, &match?({:primary_key, _, _}, &1))
total_pk = inline_pk_count + table_pk_count
case total_pk do
n when n > 1 -> fail("table #{stmt.name} has more than one primary key")
_ -> :ok
end
end
defp ensure_valid_autoincrement!(%CreateTable{} = stmt) do
case Enum.filter(stmt.columns, & &1.autoincrement) do
[] ->
:ok
[_column] when stmt.without_rowid ->
fail("AUTOINCREMENT not allowed on WITHOUT ROWID tables")
[column] when column.primary_key and column.affinity == :integer ->
:ok
[_ | _] ->
fail("AUTOINCREMENT is only allowed on an INTEGER PRIMARY KEY")
end
end
defp ensure_without_rowid_primary_key!(%CreateTable{without_rowid: false}), do: :ok
defp ensure_without_rowid_primary_key!(%CreateTable{} = stmt) do
has_primary_key? =
Enum.any?(stmt.columns, & &1.primary_key) or
Enum.any?(stmt.constraints, &match?({:primary_key, _, _}, &1))
unless has_primary_key? do
fail("PRIMARY KEY missing on table #{stmt.name}")
end
end
defp ensure_valid_strict_types!(%CreateTable{strict: false}), do: :ok
defp ensure_valid_strict_types!(%CreateTable{} = stmt) do
Enum.each(stmt.columns, fn column ->
case column.declared_type do
nil ->
fail("missing datatype for #{stmt.name}.#{column.name}")
type ->
unless String.upcase(type) in ["INT", "INTEGER", "REAL", "TEXT", "BLOB", "ANY"] do
fail(~s(unknown datatype for #{stmt.name}.#{column.name}: "#{type}"))
end
end
end)
end
# Raises if the existing table data already violates a new unique index.
defp check_unique_index_data!(db, table, index) do
rows =
table
|> Table.scan()
|> Enum.filter(fn {rowid, row} ->
index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where)
end)
# Build value tuples for each row, skipping rows with any NULL
value_tuples =
rows
|> Enum.map(fn {rowid, row} -> index_member_values(db, table, rowid, row, index) end)
|> Enum.reject(fn vals -> Enum.any?(vals, &is_nil/1) end)
# Check for duplicates using the index collations for type-correct equality.
Enum.reduce_while(value_tuples, [], fn tuple, seen ->
duplicate? =
Enum.any?(seen, fn existing ->
index_values_equal?(db, index, existing, tuple)
end)
if duplicate? do
fail(index_conflict_message(table, index))
else
{:cont, [tuple | seen]}
end
end)
end
@spec fail(String.t()) :: no_return()
defp fail(message), do: raise(Error, message: message)
# Conflict-clause failure: ABORT (the default) discards the statement's
# changes, FAIL keeps the rows already changed, ROLLBACK additionally
# rolls back and closes the enclosing transaction (or acts as ABORT when
# there is none), as in SQLite's conflict-resolution algorithms.
defp conflict_fail!(db, table, on_conflict, message) do
case on_conflict do
:fail ->
raise(Error, message: message, db: put_table(db, table))
:rollback ->
case List.last(db.txn_stack) do
nil ->
fail(message)
{_kind, snapshot} ->
db = %{db | txn_stack: [], defer_foreign_keys: false}
raise(Error, message: message, db: Database.restore_schema(db, snapshot))
end
_abort ->
fail(message)
end
end
end