defmodule Mix.Tasks.Github.Gen do
@moduledoc """
Generates the full `Noizu.Github` REST client from the OpenAPI description in
`docs/github-api/api.github.com.json`.
Emits, following the project's existing conventions:
* one `Noizu.Github.Api.<Category>` module per spec category, with one
function per operation, dispatching through `Noizu.Github.api_call/5`
* one struct module `Noizu.Github.<Schema>` per object schema, with a
permissive `from_json/2`
* typed list wrappers `Noizu.Github.Collection.<Item>` plus the generic
`Noizu.Github.Collection` / `Noizu.Github.Raw` fallbacks
Usage:
mix github.gen
All generated files carry a "do not edit" banner and live under `lib/api/`.
Re-run after updating the vendored spec.
"""
use Mix.Task
@shortdoc "Generate the Noizu.Github client from the OpenAPI spec"
@spec_path "docs/github-api/api.github.com.json"
@api_root "lib/api"
@structs_dir "lib/api/structs"
@banner "# Generated by `mix github.gen` from docs/github-api/api.github.com.json.\n# Do not edit by hand; re-run the task instead.\n"
@impl true
def run(_args) do
spec = load_spec()
schemas = spec["components"]["schemas"] || %{}
params = spec["components"]["parameters"] || %{}
ctx = build_context(schemas, params)
clean_output()
struct_count = emit_structs(ctx)
emit_runtime_wrappers()
{wrapper_keys, op_count, cat_count, skipped} = emit_api_modules(spec, ctx)
emit_collection_wrappers(wrapper_keys, ctx)
Mix.shell().info("""
github.gen complete:
categories: #{cat_count}
operations: #{op_count}
struct schemas: #{struct_count}
list wrappers: #{MapSet.size(wrapper_keys)}
skipped operations:#{length(skipped)}
""")
unless skipped == [] do
Mix.shell().info("Skipped operations:\n" <> Enum.map_join(skipped, "\n", &(" - " <> &1)))
end
end
# ----------------------------------------------------------------------------
# Spec loading
# ----------------------------------------------------------------------------
defp load_spec do
path = Path.join(File.cwd!(), @spec_path)
unless File.exists?(path) do
Mix.raise("OpenAPI spec not found at #{@spec_path}")
end
path |> File.read!() |> Jason.decode!()
end
# ----------------------------------------------------------------------------
# Context: deterministic name maps + structability for every schema
# ----------------------------------------------------------------------------
defp build_context(schemas, params) do
# First pass: deterministic module name per schema key, resolving collisions.
{modules, _} =
schemas
|> Map.keys()
|> Enum.sort()
|> Enum.reduce({%{}, MapSet.new()}, fn key, {acc, used} ->
base = "Noizu.Github." <> camelize(key)
name = dedupe(base, used)
{Map.put(acc, key, name), MapSet.put(used, name)}
end)
structable =
schemas
|> Enum.filter(fn {_k, schema} -> structable?(schema, schemas) end)
|> Enum.map(&elem(&1, 0))
|> MapSet.new()
%{
schemas: schemas,
params: params,
modules: modules,
structable: structable
}
end
defp dedupe(base, used) do
if MapSet.member?(used, base) do
Enum.reduce_while(1..1000, nil, fn i, _ ->
cand = base <> Integer.to_string(i)
if MapSet.member?(used, cand), do: {:cont, nil}, else: {:halt, cand}
end)
else
base
end
end
# A schema is "structable" (gets its own struct) when it resolves to an object
# with named properties (directly, or via allOf composition).
defp structable?(schema, schemas, seen \\ MapSet.new())
defp structable?(%{"$ref" => ref}, schemas, seen) do
key = ref_key(ref)
if MapSet.member?(seen, key) do
false
else
case schemas[key] do
nil -> false
s -> structable?(s, schemas, MapSet.put(seen, key))
end
end
end
defp structable?(schema, schemas, seen) when is_map(schema) do
cond do
is_map(schema["properties"]) and map_size(schema["properties"]) > 0 -> true
is_list(schema["allOf"]) -> Enum.any?(schema["allOf"], &structable?(&1, schemas, seen))
true -> false
end
end
defp structable?(_, _, _), do: false
# ----------------------------------------------------------------------------
# Output cleanup
# ----------------------------------------------------------------------------
defp clean_output do
root = Path.join(File.cwd!(), @api_root)
if File.dir?(root) do
File.rm_rf!(root)
end
File.mkdir_p!(Path.join(File.cwd!(), @structs_dir))
end
# ----------------------------------------------------------------------------
# Struct emission
# ----------------------------------------------------------------------------
defp emit_structs(ctx) do
ctx.structable
|> Enum.sort()
|> Enum.reduce(0, fn key, count ->
schema = ctx.schemas[key]
module = ctx.modules[key]
props = collect_properties(schema, ctx.schemas)
if map_size(props) == 0 do
count
else
code = struct_module(module, props, ctx)
write_struct_file(key, code)
count + 1
end
end)
end
# Merge properties across allOf composition (deduped by name).
defp collect_properties(schema, schemas, seen \\ MapSet.new())
defp collect_properties(%{"$ref" => ref}, schemas, seen) do
key = ref_key(ref)
if MapSet.member?(seen, key) do
%{}
else
collect_properties(schemas[key] || %{}, schemas, MapSet.put(seen, key))
end
end
defp collect_properties(schema, schemas, seen) when is_map(schema) do
direct = schema["properties"] || %{}
composed =
(schema["allOf"] || [])
|> Enum.reduce(%{}, fn member, acc ->
Map.merge(acc, collect_properties(member, schemas, seen))
end)
Map.merge(composed, direct)
end
defp collect_properties(_, _, _), do: %{}
defp struct_module(module, props, ctx) do
field_names = props |> Map.keys() |> Enum.sort()
defstruct_fields =
field_names
|> Enum.map(&(" " <> atom_literal(&1)))
|> Enum.join(",\n")
assignments =
field_names
|> Enum.map(fn name -> " " <> field_assignment(name, props[name], ctx) end)
|> Enum.join(",\n")
"""
#{@banner}
defmodule #{module} do
@moduledoc false
defstruct [
#{defstruct_fields}
]
def from_json(json, headers \\\\ [])
def from_json(nil, _headers), do: nil
def from_json(list, headers) when is_list(list) do
Enum.map(list, &from_json(&1, headers))
end
def from_json(json, _headers) when is_map(json) do
%__MODULE__{
#{assignments}
}
end
def from_json(other, _headers), do: other
end
"""
end
# Emit a single `field: <decoder>` line based on the property schema.
defp field_assignment(name, prop, ctx) do
getter = "Map.get(json, #{atom_literal(name)})"
key = key_literal(name)
decoder =
case decode_kind(prop, ctx) do
{:ref, module} -> "#{module}.from_json(#{getter})"
{:list, module} -> "Enum.map(#{getter} || [], &#{module}.from_json(&1))"
:passthrough -> getter
end
"#{key} #{decoder}"
end
# Decide how a property value should be decoded.
defp decode_kind(%{"$ref" => ref}, ctx) do
key = ref_key(ref)
if MapSet.member?(ctx.structable, key) do
{:ref, ctx.modules[key]}
else
:passthrough
end
end
defp decode_kind(%{"type" => "array", "items" => %{"$ref" => ref}}, ctx) do
key = ref_key(ref)
if MapSet.member?(ctx.structable, key) do
{:list, ctx.modules[key]}
else
:passthrough
end
end
# allOf wrapping a single ref (common nullable pattern).
defp decode_kind(%{"allOf" => [%{"$ref" => ref}]}, ctx) do
key = ref_key(ref)
if MapSet.member?(ctx.structable, key) do
{:ref, ctx.modules[key]}
else
:passthrough
end
end
defp decode_kind(_prop, _ctx), do: :passthrough
defp write_struct_file(key, code) do
path = Path.join([File.cwd!(), @structs_dir, snake(key) <> ".ex"])
write!(path, code)
end
# ----------------------------------------------------------------------------
# Runtime wrappers: generic Collection + Raw
# ----------------------------------------------------------------------------
defp emit_runtime_wrappers do
raw = """
#{@banner}
defmodule Noizu.Github.Raw do
@moduledoc \"\"\"
Generic passthrough result for endpoints without a dedicated schema
(inline objects, unions, empty `204` bodies). Carries the decoded body in
`:data` and any pagination links in `:links`.
\"\"\"
defstruct [:data, :links]
def from_json(json, headers \\\\ [])
def from_json(json, headers) do
%__MODULE__{data: json, links: Noizu.Github.extract_links(headers)}
end
end
"""
collection = """
#{@banner}
defmodule Noizu.Github.Collection do
@moduledoc \"\"\"
Generic list result. Wraps a bare array (or a `{total_count, items}` search
envelope) of untyped items together with pagination links.
\"\"\"
defstruct [:items, :complete, :total, :links]
def from_json(json, headers \\\\ [])
def from_json(list, headers) when is_list(list) do
%__MODULE__{
items: list,
complete: true,
total: length(list),
links: Noizu.Github.extract_links(headers)
}
end
def from_json(%{items: items} = env, headers) when is_list(items) do
%__MODULE__{
items: items,
complete: !Map.get(env, :incomplete_results, false),
total: Map.get(env, :total_count, length(items)),
links: Noizu.Github.extract_links(headers)
}
end
def from_json(other, headers) do
%__MODULE__{items: other, links: Noizu.Github.extract_links(headers)}
end
end
"""
write!(Path.join([File.cwd!(), @structs_dir, "raw.ex"]), raw)
write!(Path.join([File.cwd!(), @structs_dir, "collection.ex"]), collection)
end
defp emit_collection_wrappers(wrapper_keys, ctx) do
dir = Path.join([File.cwd!(), @structs_dir, "collection"])
File.mkdir_p!(dir)
Enum.each(wrapper_keys, fn key ->
item = ctx.modules[key]
module = "Noizu.Github.Collection." <> camelize(key)
code = """
#{@banner}
defmodule #{module} do
@moduledoc false
defstruct [:items, :complete, :total, :links]
def from_json(json, headers \\\\ [])
def from_json(list, headers) when is_list(list) do
%__MODULE__{
items: Enum.map(list, &#{item}.from_json(&1)),
complete: true,
total: length(list),
links: Noizu.Github.extract_links(headers)
}
end
def from_json(%{items: items} = env, headers) when is_list(items) do
%__MODULE__{
items: Enum.map(items, &#{item}.from_json(&1)),
complete: !Map.get(env, :incomplete_results, false),
total: Map.get(env, :total_count, length(items)),
links: Noizu.Github.extract_links(headers)
}
end
def from_json(other, headers) do
%__MODULE__{items: other, links: Noizu.Github.extract_links(headers)}
end
end
"""
write!(Path.join(dir, snake(key) <> ".ex"), code)
end)
end
# ----------------------------------------------------------------------------
# API module emission
# ----------------------------------------------------------------------------
defp emit_api_modules(spec, ctx) do
operations = collect_operations(spec, ctx)
by_category = Enum.group_by(operations, & &1.category)
wrapper_keys =
operations
|> Enum.flat_map(fn op -> if op.wrapper_key, do: [op.wrapper_key], else: [] end)
|> MapSet.new()
skipped =
operations
|> Enum.filter(& &1.skipped)
|> Enum.map(& &1.operation_id)
Enum.each(by_category, fn {category, ops} ->
emit_category(category, ops)
end)
op_count = Enum.count(operations, &(not &1.skipped))
{wrapper_keys, op_count, map_size(by_category), skipped}
end
defp collect_operations(spec, ctx) do
methods = ~w(get put post delete patch)
for {path, item} <- spec["paths"],
{method, op} <- item,
method in methods,
is_map(op) do
path_level_params = item["parameters"] || []
build_operation(path, method, op, path_level_params, ctx)
end
|> Enum.reject(&is_nil/1)
end
defp build_operation(path, method, op, path_level_params, ctx) do
operation_id = op["operationId"]
if is_nil(operation_id) do
nil
else
{category, verb} = split_operation_id(operation_id)
all_params =
(path_level_params ++ (op["parameters"] || []))
|> Enum.map(&resolve_param(&1, ctx))
|> Enum.reject(&is_nil/1)
path_params = Enum.filter(all_params, &(&1["in"] == "path"))
query_params = Enum.filter(all_params, &(&1["in"] == "query"))
{result_model, wrapper_key} = result_model(op, ctx)
%{
operation_id: operation_id,
category: category,
function: function_name(verb),
method: method,
path: path,
path_params: path_params,
query_params: query_params,
has_body: not is_nil(op["requestBody"]),
result_model: result_model,
wrapper_key: wrapper_key,
summary: op["summary"],
doc_url: get_in(op, ["externalDocs", "url"]),
skipped: false
}
end
end
defp resolve_param(%{"$ref" => ref}, ctx) do
key = ref_key(ref)
ctx.params[key]
end
defp resolve_param(param, _ctx) when is_map(param), do: param
defp resolve_param(_, _), do: nil
# Choose the success result model + (optional) list wrapper item key.
defp result_model(op, ctx) do
responses = op["responses"] || %{}
status =
["200", "201", "202", "203", "204"]
|> Enum.find(fn s -> Map.has_key?(responses, s) end)
schema =
status &&
get_in(responses, [status, "content", "application/json", "schema"])
classify_schema(schema, ctx)
end
defp classify_schema(nil, _ctx), do: {"Noizu.Github.Raw", nil}
defp classify_schema(%{"$ref" => ref}, ctx) do
key = ref_key(ref)
if MapSet.member?(ctx.structable, key) do
{ctx.modules[key], nil}
else
{"Noizu.Github.Raw", nil}
end
end
defp classify_schema(%{"type" => "array", "items" => %{"$ref" => ref}}, ctx) do
key = ref_key(ref)
if MapSet.member?(ctx.structable, key) do
{"Noizu.Github.Collection." <> camelize(key), key}
else
{"Noizu.Github.Collection", nil}
end
end
defp classify_schema(%{"type" => "array"}, _ctx), do: {"Noizu.Github.Collection", nil}
defp classify_schema(_schema, _ctx), do: {"Noizu.Github.Raw", nil}
defp emit_category(category, ops) do
module = "Noizu.Github.Api." <> camelize(category)
dir = Path.join([File.cwd!(), @api_root, snake(category)])
File.mkdir_p!(dir)
# Guard against duplicate function/arity within a category.
{functions, _used} =
ops
|> Enum.sort_by(& &1.operation_id)
|> Enum.map_reduce(MapSet.new(), fn op, used ->
{fun, name} = render_function(op, used)
{fun, MapSet.put(used, name)}
end)
body = Enum.join(functions, "\n")
code = """
#{@banner}
defmodule #{module} do
@moduledoc \"\"\"
GitHub `#{category}` API.
\"\"\"
import Noizu.Github
#{body}
end
"""
write!(Path.join(dir, snake(category) <> ".ex"), code)
end
defp render_function(op, used) do
# Positional path params (snake), excluding owner/repo which come from options.
positional =
op.path_params
|> Enum.map(& &1["name"])
|> Enum.reject(&(&1 in ["owner", "repo"]))
|> Enum.map(&snake/1)
has_owner = path_has?(op.path, "owner")
has_repo = path_has?(op.path, "repo")
write? = op.method in ["post", "put", "patch"] or (op.method == "delete" and op.has_body)
arg_list =
positional ++
if(write?, do: ["body"], else: []) ++
["options \\\\ nil"]
base_name = unique_function(op.function, length(arg_list_atoms(positional, write?)), used)
signature = "#{base_name}(#{Enum.join(arg_list, ", ")})"
owner_repo =
[
has_owner && " owner = repo_owner(options)",
has_repo && " repo = repo_name(options)"
]
|> Enum.filter(& &1)
|> Enum.join("\n")
url_expr = build_url(op.path, op.query_params)
body_expr = if write?, do: "body", else: "%{}"
doc = function_doc(op)
lines =
[
doc,
" def #{signature} do",
owner_repo != "" && owner_repo,
" url = #{url_expr}",
" body = #{body_expr}",
" api_call(:#{op.method}, url, body, #{op.result_model}, options)",
" end"
]
|> Enum.filter(& &1)
|> Enum.join("\n")
{lines <> "\n", base_name}
end
defp arg_list_atoms(positional, write?), do: positional ++ if(write?, do: ["body"], else: [])
defp unique_function(name, _arity, used) do
if MapSet.member?(used, name) do
Enum.reduce_while(1..1000, name, fn i, _ ->
cand = "#{name}_v#{i}"
if MapSet.member?(used, cand), do: {:cont, cand}, else: {:halt, cand}
end)
else
name
end
end
defp path_has?(path, param), do: String.contains?(path, "{#{param}}")
# Build the URL expression: github_base() <> "interpolated path" <> query string.
defp build_url(path, query_params) do
interpolated =
Regex.replace(~r/\{([^}]+)\}/, path, fn _, name ->
var = if name in ["owner", "repo"], do: name, else: snake(name)
"\#{#{var}}"
end)
base = "github_base() <> \"#{interpolated}\""
if query_params == [] do
base
else
fields =
query_params
|> Enum.map(&"get_field(#{atom_literal(&1["name"])}, options, nil)")
|> Enum.join(",\n ")
"""
(
query = [
#{fields}
]
|> Enum.filter(& &1)
qs = if query == [], do: "", else: "?" <> Enum.join(query, "&")
#{base} <> qs
)\
"""
end
end
defp function_doc(op) do
summary = op.summary || op.operation_id
url = op.doc_url
doc =
if url do
"#{summary}\n\n @see #{url}"
else
summary
end
" @doc \"\"\"\n #{doc}\n \"\"\""
end
# ----------------------------------------------------------------------------
# Naming helpers
# ----------------------------------------------------------------------------
defp split_operation_id(id) do
case String.split(id, "/", parts: 2) do
[category, verb] -> {category, verb}
[single] -> {"misc", single}
end
end
defp function_name(verb) do
name = snake(verb)
if Regex.match?(~r/^[a-z_]/, name), do: name, else: "op_" <> name
end
defp ref_key(ref), do: ref |> String.split("/") |> List.last()
# Write generated source, formatting it first so re-runs stay clean. Falls
# back to the raw source if formatting fails for any reason.
defp write!(path, code) do
formatted =
try do
Code.format_string!(code) |> IO.iodata_to_binary() |> Kernel.<>("\n")
rescue
_ -> code
end
File.write!(path, formatted)
end
defp camelize(str) do
str
|> String.split(~r/[^a-zA-Z0-9]+/, trim: true)
|> Enum.map_join("", &capitalize_word/1)
end
defp capitalize_word(<<first::utf8, rest::binary>>),
do: String.upcase(<<first::utf8>>) <> rest
defp capitalize_word(""), do: ""
defp snake(str) do
str
|> String.replace(~r/[^a-zA-Z0-9]+/, "_")
|> String.replace(~r/([a-z0-9])([A-Z])/, "\\1_\\2")
|> String.downcase()
|> String.trim("_")
end
# Render `name:` for a key literal (struct field assignment / map key).
defp key_literal(name) do
if valid_identifier?(name), do: "#{name}:", else: "#{inspect(name)}:"
end
# Render `:name` atom literal (defstruct entry / Map.get key).
defp atom_literal(name) do
if valid_identifier?(name), do: ":#{name}", else: ":#{inspect(name)}"
end
defp valid_identifier?(name), do: Regex.match?(~r/^[a-z_][a-zA-Z0-9_]*$/, name)
end