-module(lightspeed@ops@failover_harness).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/lightspeed/ops/failover_harness.gleam").
-export([run_scenario/1, run_matrix/0, scenario_label/1, pass_fail_label/1, signature/1, scenario/1, deterministic/1, outcomes/1, failed_scenarios/1, nondeterministic_failures/1, report_signature/1, snapshot_signature/0, snapshot_report_markdown/0]).
-export_type([scenario/0, scenario_outcome/0, report/0]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(" Deterministic failover certification harness for M27.\n").
-type scenario() :: supported_backends |
ownership_fencing |
crash_restart_failover |
resume_policy_matrix |
reconnect_continuity_slo.
-type scenario_outcome() :: {scenario_outcome,
scenario(),
boolean(),
boolean(),
binary()}.
-type report() :: {report, list(scenario_outcome()), integer(), integer()}.
-file("src/lightspeed/ops/failover_harness.gleam", 470).
-spec count_nondeterministic(list(scenario_outcome())) -> integer().
count_nondeterministic(Outcomes) ->
case Outcomes of
[] ->
0;
[Outcome | Rest] ->
case erlang:element(4, Outcome) of
true ->
count_nondeterministic(Rest);
false ->
1 + count_nondeterministic(Rest)
end
end.
-file("src/lightspeed/ops/failover_harness.gleam", 459).
-spec count_failed(list(scenario_outcome())) -> integer().
count_failed(Outcomes) ->
case Outcomes of
[] ->
0;
[Outcome | Rest] ->
case erlang:element(3, Outcome) of
true ->
count_failed(Rest);
false ->
1 + count_failed(Rest)
end
end.
-file("src/lightspeed/ops/failover_harness.gleam", 488).
-spec join_with(binary(), list(binary())) -> binary().
join_with(Separator, Values) ->
case Values of
[] ->
<<""/utf8>>;
[Value] ->
Value;
[Value@1 | Rest] ->
<<<<Value@1/binary, Separator/binary>>/binary,
(join_with(Separator, Rest))/binary>>
end.
-file("src/lightspeed/ops/failover_harness.gleam", 449).
-spec entries_all_met(list({boolean(), binary()})) -> boolean().
entries_all_met(Entries) ->
case Entries of
[] ->
true;
[Entry | Rest] ->
{Met, _} = Entry,
Met andalso entries_all_met(Rest)
end.
-file("src/lightspeed/ops/failover_harness.gleam", 360).
-spec evaluate_reconnect_continuity_slo() -> {boolean(), binary()}.
evaluate_reconnect_continuity_slo() ->
Entries = gleam@list:map(
lightspeed@cluster@durable_session:supported_backends(),
fun(Backend) ->
Base = begin
_pipe = lightspeed@cluster@durable_session:start(
<<"s-4"/utf8>>,
<<"node-a"/utf8>>,
<<"/chat"/utf8>>,
Backend,
rehydrate,
0,
25,
1
),
_pipe@1 = lightspeed@cluster@durable_session:append_counter_delta(
_pipe,
1,
2
),
_pipe@2 = lightspeed@cluster@durable_session:append_counter_delta(
_pipe@1,
1,
3
),
_pipe@3 = lightspeed@cluster@durable_session:append_counter_delta(
_pipe@2,
1,
4
),
_pipe@4 = lightspeed@cluster@durable_session:crash(
_pipe@3,
<<"boom"/utf8>>
),
lightspeed@cluster@durable_session:restart(_pipe@4, 200)
end,
Recovered = case lightspeed@cluster@durable_session:request_takeover(
Base,
<<"node-b"/utf8>>,
2,
lightspeed@cluster@durable_session:expected_token(
<<"s-4"/utf8>>,
<<"node-b"/utf8>>,
2
),
210
) of
{ok, Updated} ->
Updated;
{error, _} ->
Base
end,
Reconnected = lightspeed@cluster@durable_session:reconnect(
Recovered,
rehydrate,
<<"/chat"/utf8>>,
220
),
Continuity = lightspeed@cluster@durable_session:continuity_report(
Reconnected,
200,
220
),
{erlang:element(4, Continuity),
<<<<(lightspeed@cluster@durable_session:backend_label(Backend))/binary,
":"/utf8>>/binary,
(lightspeed@cluster@durable_session:continuity_signature(
Continuity
))/binary>>}
end
),
Passed = entries_all_met(Entries),
Signatures = gleam@list:map(
Entries,
fun(Entry) ->
{_, Signature} = Entry,
Signature
end
),
{Passed, join_with(<<";"/utf8>>, Signatures)}.
-file("src/lightspeed/ops/failover_harness.gleam", 322).
-spec evaluate_resume_policy_matrix() -> {boolean(), binary()}.
evaluate_resume_policy_matrix() ->
Baseline = begin
_pipe = lightspeed@cluster@durable_session:start(
<<"s-3"/utf8>>,
<<"node-a"/utf8>>,
<<"/todos"/utf8>>,
{journal_only, <<"journal"/utf8>>},
resume,
0,
25,
0
),
lightspeed@cluster@durable_session:append_counter_delta(_pipe, 2, 6)
end,
Resumed = lightspeed@cluster@durable_session:reconnect(
Baseline,
resume,
<<"/todos"/utf8>>,
40
),
Rehydrated = lightspeed@cluster@durable_session:reconnect(
Baseline,
rehydrate,
<<"/todos"/utf8>>,
40
),
Remounted = lightspeed@cluster@durable_session:reconnect(
Baseline,
remount,
<<"/todos"/utf8>>,
40
),
Resumed_counter = erlang:element(
4,
lightspeed@cluster@durable_session:snapshot(Resumed)
),
Rehydrated_counter = erlang:element(
4,
lightspeed@cluster@durable_session:snapshot(Rehydrated)
),
Remounted_counter = erlang:element(
4,
lightspeed@cluster@durable_session:snapshot(Remounted)
),
Passed = ((Resumed_counter =:= 2) andalso (Rehydrated_counter =:= 2))
andalso (Remounted_counter =:= 0),
{Passed,
<<<<<<<<<<"resume="/utf8,
(erlang:integer_to_binary(Resumed_counter))/binary>>/binary,
"|rehydrate="/utf8>>/binary,
(erlang:integer_to_binary(Rehydrated_counter))/binary>>/binary,
"|remount="/utf8>>/binary,
(erlang:integer_to_binary(Remounted_counter))/binary>>}.
-file("src/lightspeed/ops/failover_harness.gleam", 267).
-spec evaluate_crash_restart_failover() -> {boolean(), binary()}.
evaluate_crash_restart_failover() ->
Durable = begin
_pipe = lightspeed@cluster@durable_session:start(
<<"s-2"/utf8>>,
<<"node-a"/utf8>>,
<<"/counter"/utf8>>,
{snapshot_journal, <<"snap"/utf8>>, <<"journal"/utf8>>, 2},
rehydrate,
0,
25,
1
),
_pipe@1 = lightspeed@cluster@durable_session:append_counter_delta(
_pipe,
3,
5
),
_pipe@2 = lightspeed@cluster@durable_session:crash(
_pipe@1,
<<"boom"/utf8>>
),
lightspeed@cluster@durable_session:restart(_pipe@2, 100)
end,
Recovered = case lightspeed@cluster@durable_session:request_takeover(
Durable,
<<"node-b"/utf8>>,
2,
lightspeed@cluster@durable_session:expected_token(
<<"s-2"/utf8>>,
<<"node-b"/utf8>>,
2
),
110
) of
{ok, Updated} ->
Updated;
{error, _} ->
Durable
end,
Reconnected = lightspeed@cluster@durable_session:reconnect(
Recovered,
rehydrate,
<<"/counter"/utf8>>,
118
),
Continuity = lightspeed@cluster@durable_session:continuity_report(
Reconnected,
100,
118
),
Snapshot = lightspeed@cluster@durable_session:snapshot(Reconnected),
Passed = ((erlang:element(4, Snapshot) =:= 3) andalso (lightspeed@cluster@durable_session:lifecycle_label(
erlang:element(5, Snapshot)
)
=:= <<"live"/utf8>>))
andalso erlang:element(4, Continuity),
{Passed,
<<<<(lightspeed@cluster@durable_session:signature(Reconnected))/binary,
"|continuity="/utf8>>/binary,
(lightspeed@cluster@durable_session:continuity_signature(Continuity))/binary>>}.
-file("src/lightspeed/ops/failover_harness.gleam", 440).
-spec ownership_result_label(
{ok, lightspeed@cluster@durable_session:durable_session()} |
{error, lightspeed@cluster@durable_session:ownership_error()}
) -> binary().
ownership_result_label(Value) ->
case Value of
{ok, _} ->
<<"ok"/utf8>>;
{error, Error} ->
lightspeed@cluster@durable_session:ownership_error_label(Error)
end.
-file("src/lightspeed/ops/failover_harness.gleam", 194).
-spec evaluate_ownership_fencing() -> {boolean(), binary()}.
evaluate_ownership_fencing() ->
Baseline = lightspeed@cluster@durable_session:start(
<<"s-1"/utf8>>,
<<"node-a"/utf8>>,
<<"/counter"/utf8>>,
{snapshot_journal, <<"snap"/utf8>>, <<"journal"/utf8>>, 2},
rehydrate,
0,
25,
1
),
Takeover_token = lightspeed@cluster@durable_session:expected_token(
<<"s-1"/utf8>>,
<<"node-b"/utf8>>,
2
),
Takeover = lightspeed@cluster@durable_session:request_takeover(
Baseline,
<<"node-b"/utf8>>,
2,
Takeover_token,
10
),
{Takeover_label, Takeover_signature} = case Takeover of
{ok, Updated} ->
{<<"ok"/utf8>>,
lightspeed@cluster@durable_session:fence_signature(
lightspeed@cluster@durable_session:fence(Updated)
)};
{error, Error} ->
{lightspeed@cluster@durable_session:ownership_error_label(Error),
<<"none"/utf8>>}
end,
Stale_label = case Takeover of
{ok, Updated@1} ->
_pipe = lightspeed@cluster@durable_session:request_takeover(
Updated@1,
<<"node-a"/utf8>>,
1,
lightspeed@cluster@durable_session:expected_token(
<<"s-1"/utf8>>,
<<"node-a"/utf8>>,
1
),
11
),
ownership_result_label(_pipe);
{error, Error@1} ->
lightspeed@cluster@durable_session:ownership_error_label(Error@1)
end,
Split_label = case Takeover of
{ok, Updated@2} ->
_pipe@1 = lightspeed@cluster@durable_session:request_takeover(
Updated@2,
<<"node-c"/utf8>>,
2,
lightspeed@cluster@durable_session:expected_token(
<<"s-1"/utf8>>,
<<"node-c"/utf8>>,
2
),
12
),
ownership_result_label(_pipe@1);
{error, Error@2} ->
lightspeed@cluster@durable_session:ownership_error_label(Error@2)
end,
Passed = ((Takeover_label =:= <<"ok"/utf8>>) andalso (Stale_label =:= <<"stale_owner:2:1"/utf8>>))
andalso (Split_label =:= <<"split_brain:node-b:node-c:2"/utf8>>),
{Passed,
<<<<<<<<<<<<<<"takeover="/utf8, Takeover_label/binary>>/binary,
"|fence="/utf8>>/binary,
Takeover_signature/binary>>/binary,
"|stale="/utf8>>/binary,
Stale_label/binary>>/binary,
"|split="/utf8>>/binary,
Split_label/binary>>}.
-file("src/lightspeed/ops/failover_harness.gleam", 420).
-spec backends_valid(
list(lightspeed@cluster@durable_session:persistence_backend())
) -> boolean().
backends_valid(Backends) ->
case Backends of
[] ->
true;
[Backend | Rest] ->
Contract = lightspeed@cluster@durable_session:start(
<<"s-check"/utf8>>,
<<"node-a"/utf8>>,
<<"/counter"/utf8>>,
Backend,
rehydrate,
0,
25,
1
),
lightspeed@cluster@durable_session:valid(Contract) andalso backends_valid(
Rest
)
end.
-file("src/lightspeed/ops/failover_harness.gleam", 186).
-spec evaluate_supported_backends() -> {boolean(), binary()}.
evaluate_supported_backends() ->
Backends = lightspeed@cluster@durable_session:supported_backends(),
Labels = gleam@list:map(
Backends,
fun lightspeed@cluster@durable_session:backend_label/1
),
Passed = (erlang:length(Backends) =:= 3) andalso backends_valid(Backends),
{Passed, join_with(<<","/utf8>>, Labels)}.
-file("src/lightspeed/ops/failover_harness.gleam", 176).
-spec evaluate(scenario()) -> {boolean(), binary()}.
evaluate(Scenario) ->
case Scenario of
supported_backends ->
evaluate_supported_backends();
ownership_fencing ->
evaluate_ownership_fencing();
crash_restart_failover ->
evaluate_crash_restart_failover();
resume_policy_matrix ->
evaluate_resume_policy_matrix();
reconnect_continuity_slo ->
evaluate_reconnect_continuity_slo()
end.
-file("src/lightspeed/ops/failover_harness.gleam", 57).
?DOC(" Run one scenario twice and require deterministic parity.\n").
-spec run_scenario(scenario()) -> scenario_outcome().
run_scenario(Scenario) ->
{First_passed, First_signature} = evaluate(Scenario),
{Second_passed, Second_signature} = evaluate(Scenario),
Deterministic = (First_passed =:= Second_passed) andalso (First_signature
=:= Second_signature),
Passed = (First_passed andalso Second_passed) andalso Deterministic,
{scenario_outcome, Scenario, Passed, Deterministic, First_signature}.
-file("src/lightspeed/ops/failover_harness.gleam", 38).
?DOC(" Run all M27 scenarios.\n").
-spec run_matrix() -> report().
run_matrix() ->
Outcomes = begin
_pipe = [supported_backends,
ownership_fencing,
crash_restart_failover,
resume_policy_matrix,
reconnect_continuity_slo],
gleam@list:map(_pipe, fun run_scenario/1)
end,
{report, Outcomes, count_failed(Outcomes), count_nondeterministic(Outcomes)}.
-file("src/lightspeed/ops/failover_harness.gleam", 73).
?DOC(" Scenario label.\n").
-spec scenario_label(scenario()) -> binary().
scenario_label(Scenario) ->
case Scenario of
supported_backends ->
<<"supported_backends"/utf8>>;
ownership_fencing ->
<<"ownership_fencing"/utf8>>;
crash_restart_failover ->
<<"crash_restart_failover"/utf8>>;
resume_policy_matrix ->
<<"resume_policy_matrix"/utf8>>;
reconnect_continuity_slo ->
<<"reconnect_continuity_slo"/utf8>>
end.
-file("src/lightspeed/ops/failover_harness.gleam", 84).
?DOC(" Stable pass/fail label.\n").
-spec pass_fail_label(scenario_outcome()) -> binary().
pass_fail_label(Outcome) ->
case erlang:element(3, Outcome) of
true ->
<<"pass"/utf8>>;
false ->
<<"fail"/utf8>>
end.
-file("src/lightspeed/ops/failover_harness.gleam", 92).
?DOC(" Scenario signature.\n").
-spec signature(scenario_outcome()) -> binary().
signature(Outcome) ->
erlang:element(5, Outcome).
-file("src/lightspeed/ops/failover_harness.gleam", 97).
?DOC(" Scenario accessor.\n").
-spec scenario(scenario_outcome()) -> scenario().
scenario(Outcome) ->
erlang:element(2, Outcome).
-file("src/lightspeed/ops/failover_harness.gleam", 102).
?DOC(" Scenario determinism accessor.\n").
-spec deterministic(scenario_outcome()) -> boolean().
deterministic(Outcome) ->
erlang:element(4, Outcome).
-file("src/lightspeed/ops/failover_harness.gleam", 107).
?DOC(" Report outcomes.\n").
-spec outcomes(report()) -> list(scenario_outcome()).
outcomes(Report) ->
erlang:element(2, Report).
-file("src/lightspeed/ops/failover_harness.gleam", 112).
?DOC(" Failed scenario count.\n").
-spec failed_scenarios(report()) -> integer().
failed_scenarios(Report) ->
erlang:element(3, Report).
-file("src/lightspeed/ops/failover_harness.gleam", 117).
?DOC(" Nondeterministic scenario count.\n").
-spec nondeterministic_failures(report()) -> integer().
nondeterministic_failures(Report) ->
erlang:element(4, Report).
-file("src/lightspeed/ops/failover_harness.gleam", 481).
-spec bool_label(boolean()) -> binary().
bool_label(Value) ->
case Value of
true ->
<<"true"/utf8>>;
false ->
<<"false"/utf8>>
end.
-file("src/lightspeed/ops/failover_harness.gleam", 122).
?DOC(" Stable report signature.\n").
-spec report_signature(report()) -> binary().
report_signature(Report) ->
Entries = gleam@list:map(
erlang:element(2, Report),
fun(Outcome) ->
<<<<<<<<<<<<(scenario_label(erlang:element(2, Outcome)))/binary,
"="/utf8>>/binary,
(pass_fail_label(Outcome))/binary>>/binary,
":deterministic="/utf8>>/binary,
(bool_label(erlang:element(4, Outcome)))/binary>>/binary,
":"/utf8>>/binary,
(erlang:element(5, Outcome))/binary>>
end
),
join_with(<<";"/utf8>>, Entries).
-file("src/lightspeed/ops/failover_harness.gleam", 138).
?DOC(" Deterministic snapshot signature for fixture drift gates.\n").
-spec snapshot_signature() -> binary().
snapshot_signature() ->
<<<<<<"m27.snapshot.v"/utf8, (erlang:integer_to_binary(1))/binary>>/binary,
"|"/utf8>>/binary,
(report_signature(run_matrix()))/binary>>.
-file("src/lightspeed/ops/failover_harness.gleam", 146).
?DOC(" Deterministic markdown report for fixture scripts.\n").
-spec snapshot_report_markdown() -> binary().
snapshot_report_markdown() ->
Report = run_matrix(),
Failed = failed_scenarios(Report),
Nondeterministic = nondeterministic_failures(Report),
Status = case (Failed =:= 0) andalso (Nondeterministic =:= 0) of
true ->
<<"OK"/utf8>>;
false ->
<<"FAIL"/utf8>>
end,
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"# Failover Fixture Report\n\n"/utf8,
"snapshot_version: "/utf8>>/binary,
(erlang:integer_to_binary(
1
))/binary>>/binary,
"\n"/utf8>>/binary,
"status: "/utf8>>/binary,
Status/binary>>/binary,
"\n"/utf8>>/binary,
"failed_scenarios: "/utf8>>/binary,
(erlang:integer_to_binary(
Failed
))/binary>>/binary,
"\n"/utf8>>/binary,
"nondeterministic_failures: "/utf8>>/binary,
(erlang:integer_to_binary(Nondeterministic))/binary>>/binary,
"\n\n"/utf8>>/binary,
"snapshot_signature: "/utf8>>/binary,
(snapshot_signature())/binary>>/binary,
"\n\n"/utf8>>/binary,
"report_signature: "/utf8>>/binary,
(report_signature(Report))/binary>>/binary,
"\n"/utf8>>.
