-module(tomlet@lexer).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/tomlet/lexer.gleam").
-export([lex/1]).
-export_type([spanned/0]).

-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.

?MODULEDOC(false).

-type spanned() :: {spanned, william:token(), integer()}.

-file("src/tomlet/lexer.gleam", 20).
?DOC(false).
-spec annotate(list(william:token()), integer(), list(spanned())) -> list(spanned()).
annotate(Tokens, Offset, Acc) ->
    case Tokens of
        [] ->
            lists:reverse(Acc);

        [Token | Rest] ->
            Width = erlang:byte_size(william:to_source([Token])),
            annotate(Rest, Offset + Width, [{spanned, Token, Offset} | Acc])
    end.

-file("src/tomlet/lexer.gleam", 15).
?DOC(false).
-spec lex(binary()) -> list(spanned()).
lex(Input) ->
    Tokens = william:tokenise(william:new(), Input),
    annotate(Tokens, 0, []).