Skip to main content

lib/mix/tasks/exsql.slt.ex

defmodule Mix.Tasks.Exsql.Slt do
  @shortdoc "Runs sqllogictest corpus files against ExSQL"

  @moduledoc """
  Runs SQLite's sqllogictest conformance corpus against ExSQL and reports a
  score.

      mix exsql.slt path/to/select1.test path/to/random/expr
      mix exsql.slt --sample 25 /Users/you/src/sqllogictest/test
      mix exsql.slt --limit-files 10 --limit-records 500 --timings test

  Directories are searched recursively for `*.test` files. `--sample N`
  takes every Nth file of each directory, for a quick representative pass
  over the full 600-file corpus. `--limit-files N` caps the expanded file
  list, `--limit-records N` caps each file's parsed records, `--timeout-ms N`
  aborts a slow file, `--slow-ms N` labels slow files, `--timings` prints
  per-file elapsed time, and `--verbose` prints failure details.
  """

  use Mix.Task

  @impl true
  def run(argv) do
    {opts, paths} =
      OptionParser.parse!(argv,
        strict: [
          sample: :integer,
          verbose: :boolean,
          limit_files: :integer,
          limit_records: :integer,
          timeout_ms: :integer,
          slow_ms: :integer,
          timings: :boolean
        ]
      )

    opts = validate_opts(opts)

    if paths == [] do
      Mix.raise(
        "usage: mix exsql.slt [--sample N] [--limit-files N] [--limit-records N] " <>
          "[--timeout-ms N] [--slow-ms N] [--timings] [--verbose] <file-or-directory>..."
      )
    end

    Mix.Task.run("compile")

    files =
      paths
      |> Enum.flat_map(&expand/1)
      |> sample(opts[:sample])
      |> limit_files(opts[:limit_files])

    started = System.monotonic_time(:millisecond)

    totals =
      Enum.reduce(files, empty_totals(), fn file, totals ->
        case timed_run_file(file, opts) do
          {:ok, stats, elapsed} ->
            print_stats(file, stats, elapsed, opts)

            %{
              ok: totals.ok + stats.ok,
              fail: totals.fail + stats.fail,
              skip: totals.skip + stats.skip,
              aborted: totals.aborted + if(stats.aborted, do: 1, else: 0),
              timed_out: totals.timed_out,
              files: totals.files + 1
            }

          {:timeout, elapsed} ->
            Mix.shell().info(
              "#{Path.relative_to_cwd(file)}: TIMEOUT after #{format_elapsed(elapsed)}"
            )

            %{totals | timed_out: totals.timed_out + 1, files: totals.files + 1}
        end
      end)

    elapsed = System.monotonic_time(:millisecond) - started
    ran = totals.ok + totals.fail
    score = if ran > 0, do: Float.round(totals.ok * 100 / ran, 2), else: 0.0

    Mix.shell().info("""

    #{totals.files} files, #{ran} records run in #{format_elapsed(elapsed)} \
    (#{totals.skip} skipped, #{totals.aborted} files aborted, #{totals.timed_out} files timed out)
    ok: #{totals.ok}  fail: #{totals.fail}  score: #{score}%
    """)
  end

  defp empty_totals, do: %{ok: 0, fail: 0, skip: 0, aborted: 0, timed_out: 0, files: 0}

  defp validate_opts(opts) do
    validate_positive(opts, :sample)
    Enum.each([:limit_files, :limit_records], &validate_non_negative(opts, &1))
    Enum.each([:timeout_ms, :slow_ms], &validate_positive(opts, &1))
    opts
  end

  defp validate_non_negative(opts, key) do
    value = opts[key]

    if is_integer(value) and value < 0 do
      Mix.raise("--#{switch_name(key)} must be greater than or equal to 0")
    end
  end

  defp validate_positive(opts, key) do
    value = opts[key]

    if is_integer(value) and value <= 0 do
      Mix.raise("--#{switch_name(key)} must be greater than 0")
    end
  end

  defp switch_name(key), do: key |> Atom.to_string() |> String.replace("_", "-")

  defp timed_run_file(file, opts) do
    started = System.monotonic_time(:millisecond)
    fun = fn -> ExSQL.SqlLogicTest.run_file(file, max_records: opts[:limit_records]) end

    result =
      case opts[:timeout_ms] do
        nil ->
          {:ok, fun.()}

        timeout ->
          task = Task.async(fun)

          case Task.yield(task, timeout) || Task.shutdown(task, :brutal_kill) do
            {:ok, stats} -> {:ok, stats}
            nil -> :timeout
          end
      end

    elapsed = System.monotonic_time(:millisecond) - started

    case result do
      {:ok, stats} -> {:ok, stats, elapsed}
      :timeout -> {:timeout, elapsed}
    end
  end

  defp print_stats(file, stats, elapsed, opts) do
    status = if stats.aborted, do: " ABORTED", else: ""
    timing = timing_suffix(elapsed, opts)

    Mix.shell().info(
      "#{Path.relative_to_cwd(file)}: #{stats.ok} ok, #{stats.fail} fail, " <>
        "#{stats.skip} skip#{status}#{timing}"
    )

    if opts[:verbose] do
      Enum.each(stats.failures, fn failure ->
        Mix.shell().info("    line #{failure.line}: #{failure.detail}")
        Mix.shell().info("      #{String.replace(failure.sql, "\n", " ")}")
      end)
    end
  end

  defp timing_suffix(elapsed, opts) do
    slow? = opts[:slow_ms] != nil and elapsed >= opts[:slow_ms]
    timings? = opts[:timings] == true

    cond do
      timings? and slow? -> " (#{format_elapsed(elapsed)}, SLOW)"
      timings? -> " (#{format_elapsed(elapsed)})"
      slow? -> " SLOW #{format_elapsed(elapsed)}"
      true -> ""
    end
  end

  defp expand(path) do
    cond do
      File.dir?(path) -> path |> Path.join("**/*.test") |> Path.wildcard() |> Enum.sort()
      File.exists?(path) -> [path]
      true -> Mix.raise("no such file or directory: #{path}")
    end
  end

  defp sample(files, nil), do: files
  defp sample(files, n), do: files |> Enum.take_every(n)

  defp limit_files(files, nil), do: files
  defp limit_files(files, n), do: Enum.take(files, n)

  defp format_elapsed(ms) when ms < 1000, do: "#{ms}ms"
  defp format_elapsed(ms), do: "#{Float.round(ms / 1000, 2)}s"
end