defmodule Wiki.Action.Session do
@moduledoc """
This module provides a struct for holding private connection state and accumulated results.
## Fields
- `result` - Map with recursively merged values from all requests made using this session.
- `state` - Cache for session state and accumulation.
"""
@type client :: Tesla.Client.t()
@type result :: map
@type state :: keyword
@type t :: %__MODULE__{
__client__: client,
result: result,
state: keyword
}
defstruct __client__: nil,
result: %{},
state: []
end
defmodule Wiki.Action do
@moduledoc """
Adapter to the MediaWiki [Action API](https://www.mediawiki.org/wiki/Special:MyLanguage/API:Main_page)
Most commands return a pipeable Wiki.Action.Session, which can be reused repeatedly.
Anonymous requests,
```elixir
"dewiki"
|> Wiki.Site.get()
|> Wiki.Action.new()
|> Wiki.Action.get(
action: :query,
meta: :siteinfo,
siprop: :statistics
)
# %Wiki.Action.Session{
# __client__: %Tesla.Client{
# adapter: {Tesla.Adapter.Hackney, :call, [[]]},
# fun: nil,
# post: [],
# pre: [
# {Tesla.Middleware.BaseUrl, :call, ["https://de.wikipedia.org/w/api.php"]},
# {Tesla.Middleware.Compression, :call, [[format: "gzip"]]},
# {Wiki.StatefulClient.CookieJar, :call, [[]]},
# {Tesla.Middleware.FormUrlencoded, :call, [[]]},
# {Tesla.Middleware.Headers, :call,
# [[{"user-agent", "wiki_elixir/0.2.2 (spam@ludd.net)"}]]},
# {Tesla.Middleware.JSON, :call, [[]]}
# ]
# },
# result: %{
# "batchcomplete" => true,
# "query" => %{
# "statistics" => %{
# "activeusers" => 19393,
# "admins" => 188,
# "articles" => 2583636,
# "edits" => 211249646,
# "images" => 130213,
# "jobs" => 0,
# "pages" => 7164514,
# "queued-massmessages" => 0,
# "users" => 3716049
# }
# }
# },
# state: [
# cookies: %{
# "GeoIP" => "DE:BE:Berlin:52.57:13.42:v4",
# "WMF-Last-Access" => "04-Jun-2021",
# "WMF-Last-Access-Global" => "04-Jun-2021"
# }
# ]
# }
```
Commands can be pipelined while accumulating results, and logged-in user permissions
delegated by supplying a [bot password](https://www.mediawiki.org/wiki/Manual:Bot_passwords).
```elixir
"enwiki"
|> Wiki.Site.get()
|> Wiki.Action.new(
accumulate: true
)
|> Wiki.Action.authenticate(
Application.get_env(:example_app, :bot_username),
Application.get_env(:example_app, :bot_password)
)
|> Wiki.Action.get(
action: :query,
meta: :tokens,
type: :csrf
)
|> (&Wiki.Action.post(&1,
action: :edit,
title: "Sandbox",
assert: :user,
token: &1.result["query"]["tokens"]["csrftoken"],
appendtext: "~~~~ was here."
)).()
|> Map.get(:result)
|> IO.inspect()
```
Streaming results from multiple requests using continuation,
```elixir
"https://de.wikipedia.org/w/api.php"
|> Wiki.Action.new()
|> Wiki.Action.stream(
action: :query,
list: :recentchanges,
rclimit: 5
)
|> Stream.take(10)
|> Enum.flat_map(fn response -> response["query"]["recentchanges"] end)
|> Enum.map(fn rc -> rc["timestamp"] <> " " <> rc["title"] end)
|> IO.inspect()
```
"""
alias Wiki.Action.Session
alias Wiki.Site
alias Wiki.Util
@doc """
Create a new client session
## Arguments
- `site` - Site.Spec or raw `api.php` endpoint for the wiki you will connect
to. For example, "https://en.wikipedia.org/w/api.php".
- `opts`
- `:accumulate` - Merge results from each step of a pipeline, rather than overwriting with the latest response.
"""
@spec new(String.t() | Site.Spec.t(), keyword) :: Session.t()
def new(site, opts \\ [])
def new(%Site.Spec{} = site, opts) do
site
|> Site.action_api()
|> new(opts)
end
def new(url, opts) do
# TODO: This belongs in client/1, maybe pass options through?
middleware =
if opts[:accumulate] do
[Wiki.StatefulClient.CumulativeResult]
else
[]
end ++
[{Tesla.Middleware.BaseUrl, url}]
%Session{
__client__: client(middleware)
}
end
@doc """
Make requests to authenticate a client session. This should only be done using
a [bot username and password](https://www.mediawiki.org/wiki/Manual:Bot_passwords),
which can be created for any normal user account.
## Arguments
- `session` - Base session pointing to a wiki.
- `username` - Bot username, may be different than the final logged-in username.
- `password` - Bot password. Protect this string, it allows others to take on-wiki actions on your behalf.
## Return value
Authenticated session object.
"""
@spec authenticate(Session.t(), String.t(), String.t()) :: Session.t()
def authenticate(session, username, password) do
session
|> get(
action: :query,
meta: :tokens,
type: :login
)
|> (&post(&1,
action: :login,
lgname: username,
lgpassword: password,
lgtoken: &1.result["query"]["tokens"]["logintoken"]
)).()
end
@doc """
Make an API GET request
## Arguments
- `session` - `Wiki.Action.Session` object.
- `params` - Keyword list of query parameters as atoms or strings.
- `opts` - Options to pass to the adapter.
## Return value
Session object with its `.result` populated.
"""
@spec get(Session.t(), keyword, keyword) :: Session.t()
def get(session, params, opts \\ []),
do: request!(session, :get, opts ++ [query: normalize_params(params)])
@doc """
Make an API POST request.
## Arguments
- `session` - `Wiki.Action.Session` object. If credentials are required for this
action, you should have created this object with the `authenticate/3` function.
- `params` - Keyword list of query parameters as atoms or strings.
- `opts` - Options to pass to the adapter.
## Return value
Session object with a populated `:result` attribute.
"""
@spec post(Session.t(), keyword, keyword) :: Session.t()
def post(session, params, opts \\ []),
do: request!(session, :post, opts ++ [body: normalize_params(params)])
@doc """
Make a GET request and follow continuations until exhausted or the stream is closed.
## Arguments
- `session` - `Wiki.Action.Session` object.
- `params` - Keyword list of query parameters as atoms or strings.
## Return value
Enumerable `Stream`, where each returned chunk is a raw result map, possibly
containing multiple records. This corresponds to `session.result` from the other
entry points.
"""
@spec stream(Session.t(), keyword) :: Enumerable.t()
def stream(session, params) do
Stream.resource(
fn -> {session, :start} end,
fn
{prev, :start} ->
do_stream_get(prev, params)
{prev, :cont} ->
case prev.result do
%{"continue" => continue} -> do_stream_get(prev, params ++ Map.to_list(continue))
_ -> {:halt, nil}
end
end,
fn _ -> nil end
)
end
defp do_stream_get(session, params) do
next = get(session, params)
{[next.result], {next, :cont}}
end
@spec request!(Session.t(), :get | :post, keyword) :: Session.t()
defp request!(session, method, opts) do
# TODO: This can be extracted into a generic StatefulAdapter now.
opts = [opts: session.state] ++ opts ++ [method: method]
result = Tesla.request!(session.__client__, opts)
assert_success(result)
%Session{
__client__: session.__client__,
result: result.body,
state: Keyword.delete(result.opts, :opts)
}
end
@spec normalize_params(keyword) :: keyword
defp normalize_params(params) do
defaults = [
format: :json,
formatversion: 2
]
(defaults ++ params)
|> remove_boolean_false()
|> pipe_lists()
|> Enum.sort()
|> Enum.dedup()
end
defp remove_boolean_false(params) do
params
|> Enum.filter(fn {_, v} -> v not in [false, nil] end)
end
defp pipe_lists(params) do
params
|> Enum.map(fn
{k, v} when is_list(v) -> {k, pipe_list(v)}
entry -> entry
end)
end
defp pipe_list(values) do
if Enum.any?(values, fn v -> String.contains?(to_string(v), "|") end) do
# Use a special join character because pipe would conflict with the value.
unit_separator = "\x1f"
Enum.join([""] ++ values, unit_separator)
else
Enum.join(values, "|")
end
end
defp assert_success(result) do
cond do
result.status < 200 or result.status >= 300 ->
raise "Error received with HTTP status #{result.status}"
result.body in [nil, "", %{}] ->
raise "Empty response"
not is_map(result.body) ->
# TODO: If we test for format=json in the request, we can assert a malformed response.
nil
error = result.body["error"] ->
raise summarize_legacy_error(error)
errors = result.body["errors"] ->
raise summarize_new_error(errors)
true ->
nil
end
end
defp summarize_legacy_error(error) do
error["info"] ||
error["code"] ||
"Unknown error (legacy format)"
end
defp summarize_new_error(errors) do
case(List.first(errors)) do
%{"text" => text} -> text
%{"html" => html} -> html
%{"key" => key, "params" => params} -> [key, params] |> List.flatten() |> Enum.join("-")
%{"code" => code} -> code
_ -> "unknown"
end
end
@spec client(list) :: Tesla.Client.t()
defp client(extra) do
middleware =
extra ++
[
{Tesla.Middleware.Compression, format: "gzip"},
Wiki.StatefulClient.CookieJar,
Tesla.Middleware.FormUrlencoded,
{Tesla.Middleware.Headers,
[
{"user-agent", Util.user_agent()}
]},
Tesla.Middleware.JSON
# Debugging only:
# Tesla.Middleware.Logger
]
Tesla.client(middleware, Util.default_adapter())
end
end
defmodule Wiki.StatefulClient.CookieJar do
@moduledoc false
@behaviour Tesla.Middleware
@impl true
def call(env, next, _opts) do
cookie_header =
case env.opts[:cookies] do
nil -> []
cookies -> [{"cookie", serialize_cookies(cookies)}]
end
env =
env
|> Tesla.put_headers(cookie_header)
with {:ok, env} <- Tesla.run(env, next) do
cookies =
env
|> Tesla.get_headers("set-cookie")
|> extract_cookies()
|> update_cookies(env.opts[:cookies])
env =
env
|> Tesla.put_opt(:cookies, cookies)
{:ok, env}
end
end
@spec update_cookies(map, map) :: map
defp update_cookies(new_cookies, old_cookies) do
case old_cookies do
nil -> new_cookies
_ -> Map.merge(old_cookies, new_cookies)
end
end
@spec extract_cookies(Keyword.t()) :: map
defp extract_cookies(headers) do
headers
|> Enum.map(&SetCookie.parse/1)
|> Enum.into(%{}, fn %{key: k, value: v} -> {k, v} end)
end
@spec serialize_cookies(map) :: String.t()
defp serialize_cookies(cookies) do
cookies
|> Enum.map_join("; ", fn {key, value} -> key <> "=" <> value end)
end
end
defmodule Wiki.StatefulClient.CumulativeResult do
@moduledoc false
@behaviour Tesla.Middleware
@impl true
def call(env, next, _opts) do
with {:ok, env} <- Tesla.run(env, next) do
accumulated = recursive_merge(env.opts[:accumulated_result] || %{}, env.body)
{:ok,
Tesla.put_opt(env, :accumulated_result, accumulated)
|> Tesla.put_body(accumulated)}
end
end
@spec recursive_merge(map, map) :: map
defp recursive_merge(%{} = v1, %{} = v2), do: Map.merge(v1, v2, &recursive_merge/3)
# TODO: _key can be dropped
@spec recursive_merge(String.t(), map | String.t(), map | String.t()) :: map
defp recursive_merge(_key, v1, v2)
defp recursive_merge(_key, %{} = v1, %{} = v2), do: recursive_merge(v1, v2)
defp recursive_merge(_key, v1, v2) when is_list(v1) and is_list(v2), do: v1 ++ v2
defp recursive_merge(_key, v1, v2) when v1 == v2, do: v1
end
