defmodule Mix.Tasks.Exsql.Slt do
@shortdoc "Runs sqllogictest corpus files against ExSQL"
@moduledoc """
Runs SQLite's sqllogictest conformance corpus against ExSQL and reports a
score.
mix exsql.slt path/to/select1.test path/to/random/expr
mix exsql.slt --sample 25 /Users/you/src/sqllogictest/test
mix exsql.slt --limit-files 10 --limit-records 500 --timings test
Directories are searched recursively for `*.test` files. `--sample N`
takes every Nth file of each directory, for a quick representative pass
over the full 600-file corpus. `--limit-files N` caps the expanded file
list, `--limit-records N` caps each file's parsed records, `--timeout-ms N`
aborts a slow file, `--slow-ms N` labels slow files, `--timings` prints
per-file elapsed time, and `--verbose` prints failure details.
"""
use Mix.Task
@impl true
def run(argv) do
{opts, paths} =
OptionParser.parse!(argv,
strict: [
sample: :integer,
verbose: :boolean,
limit_files: :integer,
limit_records: :integer,
timeout_ms: :integer,
slow_ms: :integer,
timings: :boolean
]
)
opts = validate_opts(opts)
if paths == [] do
Mix.raise(
"usage: mix exsql.slt [--sample N] [--limit-files N] [--limit-records N] " <>
"[--timeout-ms N] [--slow-ms N] [--timings] [--verbose] <file-or-directory>..."
)
end
Mix.Task.run("compile")
files =
paths
|> Enum.flat_map(&expand/1)
|> sample(opts[:sample])
|> limit_files(opts[:limit_files])
started = System.monotonic_time(:millisecond)
totals =
Enum.reduce(files, empty_totals(), fn file, totals ->
case timed_run_file(file, opts) do
{:ok, stats, elapsed} ->
print_stats(file, stats, elapsed, opts)
%{
ok: totals.ok + stats.ok,
fail: totals.fail + stats.fail,
skip: totals.skip + stats.skip,
aborted: totals.aborted + if(stats.aborted, do: 1, else: 0),
timed_out: totals.timed_out,
files: totals.files + 1
}
{:timeout, elapsed} ->
Mix.shell().info(
"#{Path.relative_to_cwd(file)}: TIMEOUT after #{format_elapsed(elapsed)}"
)
%{totals | timed_out: totals.timed_out + 1, files: totals.files + 1}
end
end)
elapsed = System.monotonic_time(:millisecond) - started
ran = totals.ok + totals.fail
score = if ran > 0, do: Float.round(totals.ok * 100 / ran, 2), else: 0.0
Mix.shell().info("""
#{totals.files} files, #{ran} records run in #{format_elapsed(elapsed)} \
(#{totals.skip} skipped, #{totals.aborted} files aborted, #{totals.timed_out} files timed out)
ok: #{totals.ok} fail: #{totals.fail} score: #{score}%
""")
end
defp empty_totals, do: %{ok: 0, fail: 0, skip: 0, aborted: 0, timed_out: 0, files: 0}
defp validate_opts(opts) do
validate_positive(opts, :sample)
Enum.each([:limit_files, :limit_records], &validate_non_negative(opts, &1))
Enum.each([:timeout_ms, :slow_ms], &validate_positive(opts, &1))
opts
end
defp validate_non_negative(opts, key) do
value = opts[key]
if is_integer(value) and value < 0 do
Mix.raise("--#{switch_name(key)} must be greater than or equal to 0")
end
end
defp validate_positive(opts, key) do
value = opts[key]
if is_integer(value) and value <= 0 do
Mix.raise("--#{switch_name(key)} must be greater than 0")
end
end
defp switch_name(key), do: key |> Atom.to_string() |> String.replace("_", "-")
defp timed_run_file(file, opts) do
started = System.monotonic_time(:millisecond)
fun = fn -> ExSQL.SqlLogicTest.run_file(file, max_records: opts[:limit_records]) end
result =
case opts[:timeout_ms] do
nil ->
{:ok, fun.()}
timeout ->
task = Task.async(fun)
case Task.yield(task, timeout) || Task.shutdown(task, :brutal_kill) do
{:ok, stats} -> {:ok, stats}
nil -> :timeout
end
end
elapsed = System.monotonic_time(:millisecond) - started
case result do
{:ok, stats} -> {:ok, stats, elapsed}
:timeout -> {:timeout, elapsed}
end
end
defp print_stats(file, stats, elapsed, opts) do
status = if stats.aborted, do: " ABORTED", else: ""
timing = timing_suffix(elapsed, opts)
Mix.shell().info(
"#{Path.relative_to_cwd(file)}: #{stats.ok} ok, #{stats.fail} fail, " <>
"#{stats.skip} skip#{status}#{timing}"
)
if opts[:verbose] do
Enum.each(stats.failures, fn failure ->
Mix.shell().info(" line #{failure.line}: #{failure.detail}")
Mix.shell().info(" #{String.replace(failure.sql, "\n", " ")}")
end)
end
end
defp timing_suffix(elapsed, opts) do
slow? = opts[:slow_ms] != nil and elapsed >= opts[:slow_ms]
timings? = opts[:timings] == true
cond do
timings? and slow? -> " (#{format_elapsed(elapsed)}, SLOW)"
timings? -> " (#{format_elapsed(elapsed)})"
slow? -> " SLOW #{format_elapsed(elapsed)}"
true -> ""
end
end
defp expand(path) do
cond do
File.dir?(path) -> path |> Path.join("**/*.test") |> Path.wildcard() |> Enum.sort()
File.exists?(path) -> [path]
true -> Mix.raise("no such file or directory: #{path}")
end
end
defp sample(files, nil), do: files
defp sample(files, n), do: files |> Enum.take_every(n)
defp limit_files(files, nil), do: files
defp limit_files(files, n), do: Enum.take(files, n)
defp format_elapsed(ms) when ms < 1000, do: "#{ms}ms"
defp format_elapsed(ms), do: "#{Float.round(ms / 1000, 2)}s"
end