Skip to main content

src/william.gleam 

import gleam/list
import gleam_community/ansi
import houdini
import splitter.{type Splitter}

/// HighlightTokens stringify, classify and flatten TOML tokens into groups.
pub type HighlightToken {
  HighlightError(str: String)
  HighlightKey(str: String)
  HighlightTable(str: String)
  HighlightString(str: String)
  HighlightNumber(str: String)
  HighlightDateTime(str: String)
  HighlightLiteral(str: String)
  HighlightOperator(str: String)
  HighlightPunctuation(str: String)
  HighlightComment(str: String)
  HighlightWhitespace(str: String)
}

type HighlightContext {
  Normal
  TableHeader
}

/// Parse TOML source code into a list of `HighlightToken` using default
/// settings.
///
/// The resulting token list can be highlighted by using functions like
/// `to_ansi`, `to_html`, or by pattern matching on it yourself.
pub fn highlight(source: String) -> List(HighlightToken) {
  to_highlight(tokenise(new(), source))
}

/// Convert a list of lexer tokens into a list of `HighlightToken`.
pub fn to_highlight(tokens: List(Token)) -> List(HighlightToken) {
  let tokens = highlight_loop(tokens, Normal, [])

  list.reverse(tokens)
}

fn highlight_loop(
  tokens: List(Token),
  context: HighlightContext,
  acc: List(HighlightToken),
) -> List(HighlightToken) {
  case context, tokens {
    _, [] -> acc
    _, [OpenTable, ..tokens] ->
      highlight_loop(tokens, TableHeader, [HighlightPunctuation("["), ..acc])
    _, [OpenArrayTable, ..tokens] ->
      highlight_loop(tokens, TableHeader, [HighlightPunctuation("[["), ..acc])
    TableHeader, [CloseTable, ..tokens] ->
      highlight_loop(tokens, Normal, [HighlightPunctuation("]"), ..acc])
    TableHeader, [CloseArrayTable, ..tokens] ->
      highlight_loop(tokens, Normal, [HighlightPunctuation("]]"), ..acc])
    TableHeader, [BareKey(name), ..tokens] ->
      highlight_loop(tokens, TableHeader, [HighlightTable(name), ..acc])
    TableHeader, [String(delimiter:, value:), ..tokens] -> {
      let delim = string_delimiter(delimiter)
      let table = HighlightTable(delim <> value <> delim)

      highlight_loop(tokens, TableHeader, [table, ..acc])
    }
    Normal, [String(delimiter:, value:), Equal, ..tokens] -> {
      let delim = string_delimiter(delimiter)
      let key = HighlightKey(delim <> value <> delim)

      highlight_loop([Equal, ..tokens], Normal, [key, ..acc])
    }
    Normal, [String(delimiter:, value:), Whitespace(ws), Equal, ..tokens] -> {
      let delim = string_delimiter(delimiter)
      let key = HighlightKey(delim <> value <> delim)

      highlight_loop([Whitespace(ws), Equal, ..tokens], Normal, [key, ..acc])
    }
    Normal, [String(delimiter:, value:), Dot, ..tokens] -> {
      let delim = string_delimiter(delimiter)
      let key = HighlightKey(delim <> value <> delim)

      highlight_loop([Dot, ..tokens], Normal, [key, ..acc])
    }
    Normal, [String(delimiter:, value:), Whitespace(ws), Dot, ..tokens] -> {
      let delim = string_delimiter(delimiter)
      let key = HighlightKey(delim <> value <> delim)

      highlight_loop([Whitespace(ws), Dot, ..tokens], Normal, [key, ..acc])
    }
    _, [token, ..tokens] ->
      highlight_loop(tokens, context, [highlight_normal_token(token), ..acc])
  }
}

/// Format a list of `HighlightToken` into ANSI highlighting for display in a
/// terminal.
pub fn to_ansi(tokens: List(HighlightToken)) -> String {
  use acc, token <- list.fold(tokens, "")

  let highlighted = case token {
    HighlightComment(str:) -> ansi.italic(ansi.gray(str))
    HighlightDateTime(str:) -> ansi.blue(str)
    HighlightError(str:) -> ansi.bg_bright_red(ansi.white(str))
    HighlightKey(str:) -> ansi.yellow(str)
    HighlightLiteral(str:) -> ansi.green(str)
    HighlightNumber(str:) -> ansi.green(str)
    HighlightOperator(str:) -> ansi.magenta(str)
    HighlightPunctuation(str:) -> str
    HighlightString(str:) -> ansi.green(str)
    HighlightTable(str:) -> ansi.cyan(str)
    HighlightWhitespace(str:) -> str
  }

  acc <> highlighted
}

/// Format a list of `HighlightToken` as HTML.
///
/// Each non-whitespace token is wrapped inside a `<span>` tag with a class
/// indicating the type.
///
/// Class names are based on [`contour`](https://hexdocs.pm/contour):
///
/// | Token       | CSS class      |
/// | ----------- | -------------- |
/// | Comment     | hl-comment     |
/// | Date/time   | hl-function    |
/// | Error       | hl-error       |
/// | Key         | hl-attribute   |
/// | Literal     | hl-literal     |
/// | Number      | hl-number      |
/// | Operator    | hl-operator    |
/// | Punctuation | hl-punctuation |
/// | String      | hl-string      |
/// | Table name  | hl-module      |
/// | Whitespace  | no class       |
///
/// Place the output within
/// `<pre><code class="language-toml">...</code></pre>` and add styling for
/// these CSS classes to get highlighting on your website. Here's some CSS you
/// could use:
///
/// ```css
/// pre code {
///   .hl-comment     { color: #d4d4d4; font-style: italic }
///   .hl-function    { color: #9ce7ff }
///   .hl-attribute   { color: #ffd596 }
///   .hl-operator    { color: #ffaff3 }
///   .hl-string      { color: #c8ffa7 }
///   .hl-number      { color: #c8ffa7 }
///   .hl-literal     { color: #c8ffa7 }
///   .hl-module      { color: #ffddfa }
///   .hl-punctuation { color: inherit }
///   .hl-error       { background: red; color: white }
/// }
/// ```
pub fn to_html(tokens: List(HighlightToken)) -> String {
  use acc, token <- list.fold(tokens, "")

  let highlighted = case token {
    HighlightComment(str:) -> span("hl-comment", str)
    HighlightDateTime(str:) -> span("hl-function", str)
    HighlightError(str:) -> span("hl-error", str)
    HighlightKey(str:) -> span("hl-attribute", str)
    HighlightLiteral(str:) -> span("hl-literal", str)
    HighlightNumber(str:) -> span("hl-number", str)
    HighlightOperator(str:) -> span("hl-operator", str)
    HighlightPunctuation(str:) -> span("hl-punctuation", str)
    HighlightString(str:) -> span("hl-string", str)
    HighlightTable(str:) -> span("hl-module", str)
    HighlightWhitespace(str:) -> str
  }

  acc <> highlighted
}

fn highlight_normal_token(token: Token) -> HighlightToken {
  case token {
    BareKey(name:) -> HighlightKey(name)
    Boolean(value:) -> HighlightLiteral(value)
    CloseArrayTable -> HighlightPunctuation("]]")
    CloseBrace -> HighlightPunctuation("}")
    CloseBracket -> HighlightPunctuation("]")
    CloseTable -> HighlightPunctuation("]")
    Comma -> HighlightPunctuation(",")
    Comment(str:) -> HighlightComment(str)
    DateTime(value:) -> HighlightDateTime(value)
    Dot -> HighlightOperator(".")
    EndOfLine(str:) -> HighlightWhitespace(str)
    Equal -> HighlightOperator("=")
    Float(value:) -> HighlightNumber(value)
    Integer(base: _, value:) -> HighlightNumber(value)
    InvalidNumber(str:) -> HighlightError(str)
    OpenArrayTable -> HighlightPunctuation("[[")
    OpenBrace -> HighlightPunctuation("{")
    OpenBracket -> HighlightPunctuation("[")
    OpenTable -> HighlightPunctuation("[")
    String(delimiter:, value:) ->
      HighlightString(
        string_delimiter(delimiter) <> value <> string_delimiter(delimiter),
      )
    Unexpected(str:) -> HighlightError(str)
    UnterminatedString(delimiter:, value:) ->
      HighlightError(string_delimiter(delimiter) <> value)
    Whitespace(str:) -> HighlightWhitespace(str)
  }
}

fn span(class: String, text: String) -> String {
  "<span class=\"" <> class <> "\">" <> houdini.escape(text) <> "</span>"
}

/// Convert a list of tokens back into their source representation.
pub fn to_source(tokens: List(Token)) -> String {
  use acc, token <- list.fold(tokens, "")
  acc <> token_source(token)
}

fn token_source(token: Token) -> String {
  case token {
    BareKey(name:) -> name
    Boolean(value:) -> value
    CloseArrayTable -> "]]"
    CloseBrace -> "}"
    CloseBracket -> "]"
    CloseTable -> "]"
    Comma -> ","
    Comment(str:) -> str
    DateTime(value:) -> value
    Dot -> "."
    EndOfLine(str:) -> str
    Equal -> "="
    Float(value:) -> value
    Integer(base: _, value:) -> value
    InvalidNumber(str:) -> str
    OpenArrayTable -> "[["
    OpenBrace -> "{"
    OpenBracket -> "["
    OpenTable -> "["
    String(delimiter:, value:) ->
      string_delimiter(delimiter) <> value <> string_delimiter(delimiter)
    Unexpected(str:) -> str
    UnterminatedString(delimiter:, value:) ->
      string_delimiter(delimiter) <> value
    Whitespace(str:) -> str
  }
}

/// A token useful for parsing or processing TOML source code.
pub type Token {
  // Whitespace
  EndOfLine(str: String)
  Whitespace(str: String)
  Comment(str: String)

  // Keys and values
  BareKey(name: String)
  String(delimiter: StringDelimiter, value: String)
  Boolean(value: String)
  Integer(base: Base, value: String)
  Float(value: String)
  DateTime(value: String)

  // Operators and punctuation
  Equal
  Dot
  Comma
  OpenTable
  CloseTable
  OpenArrayTable
  CloseArrayTable
  OpenBracket
  CloseBracket
  OpenBrace
  CloseBrace

  // Errors
  Unexpected(str: String)
  InvalidNumber(str: String)
  UnterminatedString(delimiter: StringDelimiter, value: String)
}

pub type Base {
  Binary
  Octal
  Decimal
  Hexadecimal
}

pub type StringDelimiter {
  BasicString
  LiteralString
  MultilineBasicString
  MultilineLiteralString
}

type Mode {
  KeyMode
  ValueMode
  AfterValueMode
  TableMode
}

type Container {
  ArrayContainer
  InlineTableContainer
}

pub opaque type Lexer {
  Lexer(
    skip_whitespace: Bool,
    skip_comments: Bool,
    line: Splitter,
    value_atom: Splitter,
    unexpected: Splitter,
    basic_string: Splitter,
    literal_string: Splitter,
    multiline_basic_string: Splitter,
    multiline_literal_string: Splitter,
  )
}

/// Create a new Lexer with default settings.
///
/// Lexers can be cached and reused to parse many source files efficiently.
pub fn new() -> Lexer {
  Lexer(
    skip_whitespace: False,
    skip_comments: False,
    line: splitter.new(["\r\n", "\n", "\r"]),
    value_atom: splitter.new([
      "\r\n",
      "\n",
      "\r",
      " ",
      "\t",
      "#",
      ",",
      "]",
      "}",
      "[",
      "{",
      "=",
      "\"",
      "'",
    ]),
    unexpected: splitter.new(["\r\n", "\n", "\r", " ", "\t", "#", ",", "]", "}"]),
    basic_string: splitter.new(["\r\n", "\n", "\r", "\\", "\""]),
    literal_string: splitter.new(["\r\n", "\n", "\r", "'"]),
    multiline_basic_string: splitter.new([
      "\"\"\"\"\"",
      "\"\"\"\"",
      "\"\"\"",
      "\\",
    ]),
    multiline_literal_string: splitter.new(["'''''", "''''", "'''"]),
  )
}

/// Skip whitespace and end-of-line tokens in the resulting token list.
pub fn ignore_whitespace(lexer: Lexer) -> Lexer {
  Lexer(..lexer, skip_whitespace: True)
}

/// Skip comment tokens in the resulting token list.
pub fn ignore_comments(lexer: Lexer) -> Lexer {
  Lexer(..lexer, skip_comments: True)
}

/// Parse TOML source code into a list of lexer tokens.
pub fn tokenise(lexer: Lexer, input: String) -> List(Token) {
  case input {
    "\u{FEFF}" <> rest -> {
      let acc = emit(lexer, Whitespace("\u{FEFF}"), [])
      start(lexer, rest, acc, KeyMode, [], True)
    }
    _ -> start(lexer, input, [], KeyMode, [], True)
  }
}

fn start(
  lexer: Lexer,
  input: String,
  acc: List(Token),
  mode: Mode,
  containers: List(Container),
  at_line_start: Bool,
) -> List(Token) {
  case input {
    "" -> list.reverse(acc)

    "\r\n" <> rest -> {
      let acc = emit(lexer, EndOfLine("\r\n"), acc)
      let mode = reset_mode_after_newline(mode, containers)
      start(lexer, rest, acc, mode, containers, True)
    }

    "\n" <> rest -> {
      let acc = emit(lexer, EndOfLine("\n"), acc)
      let mode = reset_mode_after_newline(mode, containers)
      start(lexer, rest, acc, mode, containers, True)
    }

    "\r" <> rest -> {
      let acc = emit(lexer, EndOfLine("\r"), acc)
      let mode = reset_mode_after_newline(mode, containers)
      start(lexer, rest, acc, mode, containers, True)
    }

    " " <> rest -> {
      let #(token, rest) = whitespace(rest, " ")
      let acc = emit(lexer, token, acc)
      start(lexer, rest, acc, mode, containers, at_line_start)
    }

    "\t" <> rest -> {
      let #(token, rest) = whitespace(rest, "\t")
      let acc = emit(lexer, token, acc)
      start(lexer, rest, acc, mode, containers, at_line_start)
    }

    "#" <> rest -> {
      let #(token, rest) = comment(lexer, rest)
      let acc = emit(lexer, token, acc)
      start(lexer, rest, acc, mode, containers, at_line_start)
    }

    "[[" <> rest -> {
      case at_line_start, containers {
        True, [] -> {
          let acc = emit(lexer, OpenArrayTable, acc)
          start(lexer, rest, acc, TableMode, containers, False)
        }

        _, _ -> {
          let acc = emit(lexer, OpenBracket, acc)
          let input = "[" <> rest
          let containers = [ArrayContainer, ..containers]
          start(lexer, input, acc, ValueMode, containers, False)
        }
      }
    }

    "[" <> rest -> {
      case at_line_start, containers {
        True, [] -> {
          let acc = emit(lexer, OpenTable, acc)
          start(lexer, rest, acc, TableMode, containers, False)
        }

        _, _ -> {
          let acc = emit(lexer, OpenBracket, acc)
          let containers = [ArrayContainer, ..containers]
          start(lexer, rest, acc, ValueMode, containers, False)
        }
      }
    }

    "]]" <> rest -> {
      case mode {
        TableMode -> {
          let acc = emit(lexer, CloseArrayTable, acc)
          start(lexer, rest, acc, AfterValueMode, containers, False)
        }

        _ -> {
          let acc = emit(lexer, CloseBracket, acc)
          let acc = emit(lexer, CloseBracket, acc)
          let containers = close_array(close_array(containers))
          start(lexer, rest, acc, AfterValueMode, containers, False)
        }
      }
    }

    "]" <> rest -> {
      case mode {
        TableMode -> {
          let acc = emit(lexer, CloseTable, acc)
          start(lexer, rest, acc, AfterValueMode, containers, False)
        }

        _ -> {
          let acc = emit(lexer, CloseBracket, acc)
          let containers = close_array(containers)
          start(lexer, rest, acc, AfterValueMode, containers, False)
        }
      }
    }

    "{" <> rest -> {
      let acc = emit(lexer, OpenBrace, acc)
      let containers = [InlineTableContainer, ..containers]
      start(lexer, rest, acc, KeyMode, containers, False)
    }

    "}" <> rest -> {
      let acc = emit(lexer, CloseBrace, acc)
      let containers = close_inline_table(containers)
      start(lexer, rest, acc, AfterValueMode, containers, False)
    }

    "," <> rest -> {
      let acc = emit(lexer, Comma, acc)
      start(lexer, rest, acc, mode_after_comma(containers), containers, False)
    }

    "=" <> rest -> {
      let acc = emit(lexer, Equal, acc)
      start(lexer, rest, acc, ValueMode, containers, False)
    }

    "." <> rest -> {
      case mode {
        KeyMode | TableMode -> {
          let acc = emit(lexer, Dot, acc)
          start(lexer, rest, acc, mode, containers, False)
        }

        ValueMode -> value_token(lexer, input, acc, containers)
        AfterValueMode -> unexpected_token(lexer, input, acc, containers)
      }
    }

    "\"\"\"" <> rest -> {
      let #(token, rest) = multiline_basic_string(lexer, rest, "")
      let mode = mode_after_string(mode)
      start(lexer, rest, emit(lexer, token, acc), mode, containers, False)
    }

    "\"" <> rest -> {
      let #(token, rest) = basic_string(lexer, rest, "")
      let mode = mode_after_string(mode)
      start(lexer, rest, emit(lexer, token, acc), mode, containers, False)
    }

    "'''" <> rest -> {
      let #(token, rest) = multiline_literal_string(lexer, rest, "")
      let mode = mode_after_string(mode)
      start(lexer, rest, emit(lexer, token, acc), mode, containers, False)
    }

    "'" <> rest -> {
      let #(token, rest) = literal_string(lexer, rest, "")
      let mode = mode_after_string(mode)
      start(lexer, rest, emit(lexer, token, acc), mode, containers, False)
    }

    _ -> {
      case mode {
        KeyMode | TableMode -> key_token(lexer, input, acc, mode, containers)
        ValueMode -> value_token(lexer, input, acc, containers)
        AfterValueMode -> unexpected_token(lexer, input, acc, containers)
      }
    }
  }
}

fn key_token(
  lexer: Lexer,
  input: String,
  acc: List(Token),
  mode: Mode,
  containers: List(Container),
) {
  let #(key, rest) = bare_key(input, "")
  case key {
    "" -> unexpected_token(lexer, input, acc, containers)
    _ -> {
      let acc = emit(lexer, BareKey(key), acc)
      start(lexer, rest, acc, mode, containers, False)
    }
  }
}

fn value_token(
  lexer: Lexer,
  input: String,
  acc: List(Token),
  containers: List(Container),
) {
  let #(token, rest) = value(lexer, input)
  start(lexer, rest, emit(lexer, token, acc), AfterValueMode, containers, False)
}

fn unexpected_token(
  lexer: Lexer,
  input: String,
  acc: List(Token),
  containers: List(Container),
) {
  let #(token, rest) = unexpected(lexer, input, "")
  start(lexer, rest, emit(lexer, token, acc), AfterValueMode, containers, False)
}

fn emit(lexer: Lexer, token: Token, acc: List(Token)) -> List(Token) {
  case token {
    Whitespace(_) | EndOfLine(_) ->
      case lexer.skip_whitespace {
        True -> acc
        False -> [token, ..acc]
      }

    Comment(_) ->
      case lexer.skip_comments {
        True -> acc
        False -> [token, ..acc]
      }

    _ -> [token, ..acc]
  }
}

fn reset_mode_after_newline(mode: Mode, containers: List(Container)) -> Mode {
  case containers {
    [] -> KeyMode
    _ -> mode
  }
}

fn mode_after_string(mode: Mode) -> Mode {
  case mode {
    ValueMode -> AfterValueMode
    _ -> mode
  }
}

fn close_array(containers: List(Container)) -> List(Container) {
  case containers {
    [ArrayContainer, ..containers] -> containers
    _ -> containers
  }
}

fn close_inline_table(containers: List(Container)) -> List(Container) {
  case containers {
    [InlineTableContainer, ..containers] -> containers
    _ -> containers
  }
}

fn mode_after_comma(containers: List(Container)) -> Mode {
  case containers {
    [InlineTableContainer, ..] -> KeyMode
    [ArrayContainer, ..] -> ValueMode
    [] -> ValueMode
  }
}

fn whitespace(input: String, acc: String) -> #(Token, String) {
  case input {
    " " <> rest -> whitespace(rest, acc <> " ")
    "\t" <> rest -> whitespace(rest, acc <> "\t")
    _ -> #(Whitespace(acc), input)
  }
}

fn comment(lexer: Lexer, input: String) -> #(Token, String) {
  let #(body, rest) = splitter.split_before(lexer.line, input)
  #(Comment("#" <> body), rest)
}

fn bare_key(input: String, acc: String) -> #(String, String) {
  case input {
    "A" <> rest -> bare_key(rest, acc <> "A")
    "B" <> rest -> bare_key(rest, acc <> "B")
    "C" <> rest -> bare_key(rest, acc <> "C")
    "D" <> rest -> bare_key(rest, acc <> "D")
    "E" <> rest -> bare_key(rest, acc <> "E")
    "F" <> rest -> bare_key(rest, acc <> "F")
    "G" <> rest -> bare_key(rest, acc <> "G")
    "H" <> rest -> bare_key(rest, acc <> "H")
    "I" <> rest -> bare_key(rest, acc <> "I")
    "J" <> rest -> bare_key(rest, acc <> "J")
    "K" <> rest -> bare_key(rest, acc <> "K")
    "L" <> rest -> bare_key(rest, acc <> "L")
    "M" <> rest -> bare_key(rest, acc <> "M")
    "N" <> rest -> bare_key(rest, acc <> "N")
    "O" <> rest -> bare_key(rest, acc <> "O")
    "P" <> rest -> bare_key(rest, acc <> "P")
    "Q" <> rest -> bare_key(rest, acc <> "Q")
    "R" <> rest -> bare_key(rest, acc <> "R")
    "S" <> rest -> bare_key(rest, acc <> "S")
    "T" <> rest -> bare_key(rest, acc <> "T")
    "U" <> rest -> bare_key(rest, acc <> "U")
    "V" <> rest -> bare_key(rest, acc <> "V")
    "W" <> rest -> bare_key(rest, acc <> "W")
    "X" <> rest -> bare_key(rest, acc <> "X")
    "Y" <> rest -> bare_key(rest, acc <> "Y")
    "Z" <> rest -> bare_key(rest, acc <> "Z")
    "a" <> rest -> bare_key(rest, acc <> "a")
    "b" <> rest -> bare_key(rest, acc <> "b")
    "c" <> rest -> bare_key(rest, acc <> "c")
    "d" <> rest -> bare_key(rest, acc <> "d")
    "e" <> rest -> bare_key(rest, acc <> "e")
    "f" <> rest -> bare_key(rest, acc <> "f")
    "g" <> rest -> bare_key(rest, acc <> "g")
    "h" <> rest -> bare_key(rest, acc <> "h")
    "i" <> rest -> bare_key(rest, acc <> "i")
    "j" <> rest -> bare_key(rest, acc <> "j")
    "k" <> rest -> bare_key(rest, acc <> "k")
    "l" <> rest -> bare_key(rest, acc <> "l")
    "m" <> rest -> bare_key(rest, acc <> "m")
    "n" <> rest -> bare_key(rest, acc <> "n")
    "o" <> rest -> bare_key(rest, acc <> "o")
    "p" <> rest -> bare_key(rest, acc <> "p")
    "q" <> rest -> bare_key(rest, acc <> "q")
    "r" <> rest -> bare_key(rest, acc <> "r")
    "s" <> rest -> bare_key(rest, acc <> "s")
    "t" <> rest -> bare_key(rest, acc <> "t")
    "u" <> rest -> bare_key(rest, acc <> "u")
    "v" <> rest -> bare_key(rest, acc <> "v")
    "w" <> rest -> bare_key(rest, acc <> "w")
    "x" <> rest -> bare_key(rest, acc <> "x")
    "y" <> rest -> bare_key(rest, acc <> "y")
    "z" <> rest -> bare_key(rest, acc <> "z")
    "0" <> rest -> bare_key(rest, acc <> "0")
    "1" <> rest -> bare_key(rest, acc <> "1")
    "2" <> rest -> bare_key(rest, acc <> "2")
    "3" <> rest -> bare_key(rest, acc <> "3")
    "4" <> rest -> bare_key(rest, acc <> "4")
    "5" <> rest -> bare_key(rest, acc <> "5")
    "6" <> rest -> bare_key(rest, acc <> "6")
    "7" <> rest -> bare_key(rest, acc <> "7")
    "8" <> rest -> bare_key(rest, acc <> "8")
    "9" <> rest -> bare_key(rest, acc <> "9")
    "-" <> rest -> bare_key(rest, acc <> "-")
    "_" <> rest -> bare_key(rest, acc <> "_")
    _ -> #(acc, input)
  }
}

fn value(lexer: Lexer, input: String) -> #(Token, String) {
  case input {
    "true" <> rest -> literal_or_unexpected(lexer, rest, Boolean("true"), input)
    "false" <> rest ->
      literal_or_unexpected(lexer, rest, Boolean("false"), input)
    "inf" <> rest -> literal_or_unexpected(lexer, rest, Float("inf"), input)
    "+inf" <> rest -> literal_or_unexpected(lexer, rest, Float("+inf"), input)
    "-inf" <> rest -> literal_or_unexpected(lexer, rest, Float("-inf"), input)
    "nan" <> rest -> literal_or_unexpected(lexer, rest, Float("nan"), input)
    "+nan" <> rest -> literal_or_unexpected(lexer, rest, Float("+nan"), input)
    "-nan" <> rest -> literal_or_unexpected(lexer, rest, Float("-nan"), input)
    "+0b" <> _ | "+0o" <> _ | "+0x" <> _ -> invalid_number(lexer, input, "")
    "-0b" <> _ | "-0o" <> _ | "-0x" <> _ -> invalid_number(lexer, input, "")
    "+" <> rest ->
      case take_digit(rest) {
        Ok(#(digit, rest)) -> decimal(lexer, rest, "+" <> digit, 1, False)
        Error(_) -> invalid_number(lexer, rest, "+")
      }
    "-" <> rest ->
      case take_digit(rest) {
        Ok(#(digit, rest)) -> decimal(lexer, rest, "-" <> digit, 1, False)
        Error(_) -> invalid_number(lexer, rest, "-")
      }
    "." <> _ -> invalid_number(lexer, input, "")
    "0b" <> rest -> binary(lexer, rest, "0b", True)
    "0o" <> rest -> octal(lexer, rest, "0o", True)
    "0x" <> rest -> hexadecimal(lexer, rest, "0x", True)
    "0" <> rest -> decimal(lexer, rest, "0", 1, True)
    "1" <> rest -> decimal(lexer, rest, "1", 1, True)
    "2" <> rest -> decimal(lexer, rest, "2", 1, True)
    "3" <> rest -> decimal(lexer, rest, "3", 1, True)
    "4" <> rest -> decimal(lexer, rest, "4", 1, True)
    "5" <> rest -> decimal(lexer, rest, "5", 1, True)
    "6" <> rest -> decimal(lexer, rest, "6", 1, True)
    "7" <> rest -> decimal(lexer, rest, "7", 1, True)
    "8" <> rest -> decimal(lexer, rest, "8", 1, True)
    "9" <> rest -> decimal(lexer, rest, "9", 1, True)
    _ -> unexpected(lexer, input, "")
  }
}

fn literal_or_unexpected(
  lexer: Lexer,
  rest: String,
  token: Token,
  original: String,
) -> #(Token, String) {
  case is_value_terminator(rest) {
    True -> #(token, rest)
    False -> unexpected(lexer, original, "")
  }
}

fn unexpected(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  let #(unexpected, rest) = splitter.split_before(lexer.unexpected, input)
  #(Unexpected(acc <> unexpected), rest)
}

fn basic_string(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  let #(before, delimiter, rest) = splitter.split(lexer.basic_string, input)
  case delimiter {
    "\"" -> #(String(BasicString, acc <> before), rest)
    "\r\n" | "\n" | "\r" -> #(
      UnterminatedString(BasicString, acc <> before),
      delimiter <> rest,
    )
    "\\" -> basic_string_escape(lexer, rest, acc <> before <> "\\")
    "" -> #(UnterminatedString(BasicString, acc <> before), "")
    _ -> #(UnterminatedString(BasicString, acc <> before <> delimiter), rest)
  }
}

fn basic_string_escape(
  lexer: Lexer,
  input: String,
  acc: String,
) -> #(Token, String) {
  case input {
    "" -> #(UnterminatedString(BasicString, acc), "")
    "\r\n" <> _ | "\n" <> _ | "\r" <> _ -> #(
      UnterminatedString(BasicString, acc),
      input,
    )
    "\"" <> rest -> basic_string(lexer, rest, acc <> "\"")
    "\\" <> rest -> basic_string(lexer, rest, acc <> "\\")
    _ -> {
      let #(escaped, delimiter, rest) =
        splitter.split(lexer.basic_string, input)
      case escaped, delimiter {
        "", "" -> #(UnterminatedString(BasicString, acc), "")
        _, "" -> #(UnterminatedString(BasicString, acc <> escaped), "")
        "", _ -> basic_string(lexer, rest, acc <> delimiter)
        _, _ -> basic_string(lexer, delimiter <> rest, acc <> escaped)
      }
    }
  }
}

fn literal_string(
  lexer: Lexer,
  input: String,
  acc: String,
) -> #(Token, String) {
  let #(before, delimiter, rest) = splitter.split(lexer.literal_string, input)
  case delimiter {
    "'" -> #(String(LiteralString, acc <> before), rest)
    "\r\n" | "\n" | "\r" -> #(
      UnterminatedString(LiteralString, acc <> before),
      delimiter <> rest,
    )
    "" -> #(UnterminatedString(LiteralString, acc <> before), "")
    _ -> #(UnterminatedString(LiteralString, acc <> before <> delimiter), rest)
  }
}

fn multiline_basic_string(
  lexer: Lexer,
  input: String,
  acc: String,
) -> #(Token, String) {
  let #(before, delimiter, rest) =
    splitter.split(lexer.multiline_basic_string, input)
  case delimiter {
    "\"\"\"\"\"" -> #(
      String(MultilineBasicString, acc <> before <> "\"\""),
      rest,
    )
    "\"\"\"\"" -> #(String(MultilineBasicString, acc <> before <> "\""), rest)
    "\"\"\"" -> #(String(MultilineBasicString, acc <> before), rest)
    "\\" -> multiline_basic_string_escape(lexer, rest, acc <> before <> "\\")
    "" -> #(UnterminatedString(MultilineBasicString, acc <> before), "")
    _ -> #(
      UnterminatedString(MultilineBasicString, acc <> before <> delimiter),
      rest,
    )
  }
}

fn multiline_basic_string_escape(
  lexer: Lexer,
  input: String,
  acc: String,
) -> #(Token, String) {
  case input {
    "" -> #(UnterminatedString(MultilineBasicString, acc), "")
    "\"" <> rest -> multiline_basic_string(lexer, rest, acc <> "\"")
    "\\" <> rest -> multiline_basic_string(lexer, rest, acc <> "\\")
    _ -> {
      let #(escaped, delimiter, rest) =
        splitter.split(lexer.multiline_basic_string, input)
      case escaped, delimiter {
        "", "" -> #(UnterminatedString(MultilineBasicString, acc), "")
        _, "" -> #(UnterminatedString(MultilineBasicString, acc <> escaped), "")
        "", _ -> multiline_basic_string(lexer, rest, acc <> delimiter)
        _, _ -> multiline_basic_string(lexer, delimiter <> rest, acc <> escaped)
      }
    }
  }
}

fn multiline_literal_string(
  lexer: Lexer,
  input: String,
  acc: String,
) -> #(Token, String) {
  let #(before, delimiter, rest) =
    splitter.split(lexer.multiline_literal_string, input)
  case delimiter {
    "'''''" -> #(String(MultilineLiteralString, acc <> before <> "''"), rest)
    "''''" -> #(String(MultilineLiteralString, acc <> before <> "'"), rest)
    "'''" -> #(String(MultilineLiteralString, acc <> before), rest)
    "" -> #(UnterminatedString(MultilineLiteralString, acc <> before), "")
    _ -> #(
      UnterminatedString(MultilineLiteralString, acc <> before <> delimiter),
      rest,
    )
  }
}

fn binary(
  lexer: Lexer,
  input: String,
  acc: String,
  last_was_underscore: Bool,
) -> #(Token, String) {
  case input {
    "0" <> rest -> binary(lexer, rest, acc <> "0", False)
    "1" <> rest -> binary(lexer, rest, acc <> "1", False)
    "_" <> rest if !last_was_underscore -> {
      let acc = acc <> "_"
      binary(lexer, rest, acc, True)
    }
    _ -> based_integer_end(lexer, input, acc, Binary, last_was_underscore)
  }
}

fn octal(
  lexer: Lexer,
  input: String,
  acc: String,
  last_was_underscore: Bool,
) -> #(Token, String) {
  case input {
    "0" <> rest -> octal(lexer, rest, acc <> "0", False)
    "1" <> rest -> octal(lexer, rest, acc <> "1", False)
    "2" <> rest -> octal(lexer, rest, acc <> "2", False)
    "3" <> rest -> octal(lexer, rest, acc <> "3", False)
    "4" <> rest -> octal(lexer, rest, acc <> "4", False)
    "5" <> rest -> octal(lexer, rest, acc <> "5", False)
    "6" <> rest -> octal(lexer, rest, acc <> "6", False)
    "7" <> rest -> octal(lexer, rest, acc <> "7", False)
    "_" <> rest if !last_was_underscore -> {
      let acc = acc <> "_"
      octal(lexer, rest, acc, True)
    }
    _ -> based_integer_end(lexer, input, acc, Octal, last_was_underscore)
  }
}

fn hexadecimal(
  lexer: Lexer,
  input: String,
  acc: String,
  last_was_underscore: Bool,
) -> #(Token, String) {
  case input {
    "0" <> rest -> hexadecimal(lexer, rest, acc <> "0", False)
    "1" <> rest -> hexadecimal(lexer, rest, acc <> "1", False)
    "2" <> rest -> hexadecimal(lexer, rest, acc <> "2", False)
    "3" <> rest -> hexadecimal(lexer, rest, acc <> "3", False)
    "4" <> rest -> hexadecimal(lexer, rest, acc <> "4", False)
    "5" <> rest -> hexadecimal(lexer, rest, acc <> "5", False)
    "6" <> rest -> hexadecimal(lexer, rest, acc <> "6", False)
    "7" <> rest -> hexadecimal(lexer, rest, acc <> "7", False)
    "8" <> rest -> hexadecimal(lexer, rest, acc <> "8", False)
    "9" <> rest -> hexadecimal(lexer, rest, acc <> "9", False)
    "A" <> rest -> hexadecimal(lexer, rest, acc <> "A", False)
    "B" <> rest -> hexadecimal(lexer, rest, acc <> "B", False)
    "C" <> rest -> hexadecimal(lexer, rest, acc <> "C", False)
    "D" <> rest -> hexadecimal(lexer, rest, acc <> "D", False)
    "E" <> rest -> hexadecimal(lexer, rest, acc <> "E", False)
    "F" <> rest -> hexadecimal(lexer, rest, acc <> "F", False)
    "a" <> rest -> hexadecimal(lexer, rest, acc <> "a", False)
    "b" <> rest -> hexadecimal(lexer, rest, acc <> "b", False)
    "c" <> rest -> hexadecimal(lexer, rest, acc <> "c", False)
    "d" <> rest -> hexadecimal(lexer, rest, acc <> "d", False)
    "e" <> rest -> hexadecimal(lexer, rest, acc <> "e", False)
    "f" <> rest -> hexadecimal(lexer, rest, acc <> "f", False)
    "_" <> rest if !last_was_underscore -> {
      let acc = acc <> "_"
      hexadecimal(lexer, rest, acc, True)
    }
    _ -> based_integer_end(lexer, input, acc, Hexadecimal, last_was_underscore)
  }
}

fn based_integer_end(
  lexer: Lexer,
  input: String,
  acc: String,
  base: Base,
  last_was_underscore: Bool,
) -> #(Token, String) {
  case last_was_underscore {
    False -> {
      let to_token = fn(value) { Integer(base, value) }
      number_end(lexer, input, acc, to_token)
    }
    True -> invalid_number(lexer, input, acc)
  }
}

fn decimal(
  lexer: Lexer,
  input: String,
  acc: String,
  digits: Int,
  date_time: Bool,
) -> #(Token, String) {
  case input {
    "0" <> rest -> decimal(lexer, rest, acc <> "0", digits + 1, date_time)
    "1" <> rest -> decimal(lexer, rest, acc <> "1", digits + 1, date_time)
    "2" <> rest -> decimal(lexer, rest, acc <> "2", digits + 1, date_time)
    "3" <> rest -> decimal(lexer, rest, acc <> "3", digits + 1, date_time)
    "4" <> rest -> decimal(lexer, rest, acc <> "4", digits + 1, date_time)
    "5" <> rest -> decimal(lexer, rest, acc <> "5", digits + 1, date_time)
    "6" <> rest -> decimal(lexer, rest, acc <> "6", digits + 1, date_time)
    "7" <> rest -> decimal(lexer, rest, acc <> "7", digits + 1, date_time)
    "8" <> rest -> decimal(lexer, rest, acc <> "8", digits + 1, date_time)
    "9" <> rest -> decimal(lexer, rest, acc <> "9", digits + 1, date_time)
    "_0" <> rest -> decimal(lexer, rest, acc <> "_0", digits + 1, False)
    "_1" <> rest -> decimal(lexer, rest, acc <> "_1", digits + 1, False)
    "_2" <> rest -> decimal(lexer, rest, acc <> "_2", digits + 1, False)
    "_3" <> rest -> decimal(lexer, rest, acc <> "_3", digits + 1, False)
    "_4" <> rest -> decimal(lexer, rest, acc <> "_4", digits + 1, False)
    "_5" <> rest -> decimal(lexer, rest, acc <> "_5", digits + 1, False)
    "_6" <> rest -> decimal(lexer, rest, acc <> "_6", digits + 1, False)
    "_7" <> rest -> decimal(lexer, rest, acc <> "_7", digits + 1, False)
    "_8" <> rest -> decimal(lexer, rest, acc <> "_8", digits + 1, False)
    "_9" <> rest -> decimal(lexer, rest, acc <> "_9", digits + 1, False)
    ".0" <> rest -> float(lexer, rest, acc <> ".0")
    ".1" <> rest -> float(lexer, rest, acc <> ".1")
    ".2" <> rest -> float(lexer, rest, acc <> ".2")
    ".3" <> rest -> float(lexer, rest, acc <> ".3")
    ".4" <> rest -> float(lexer, rest, acc <> ".4")
    ".5" <> rest -> float(lexer, rest, acc <> ".5")
    ".6" <> rest -> float(lexer, rest, acc <> ".6")
    ".7" <> rest -> float(lexer, rest, acc <> ".7")
    ".8" <> rest -> float(lexer, rest, acc <> ".8")
    ".9" <> rest -> float(lexer, rest, acc <> ".9")
    "e+" <> rest -> exponent_start(lexer, input, rest, acc, "e+")
    "e-" <> rest -> exponent_start(lexer, input, rest, acc, "e-")
    "e" <> rest -> exponent_start(lexer, input, rest, acc, "e")
    "E+" <> rest -> exponent_start(lexer, input, rest, acc, "E+")
    "E-" <> rest -> exponent_start(lexer, input, rest, acc, "E-")
    "E" <> rest -> exponent_start(lexer, input, rest, acc, "E")
    ":" <> rest if date_time && digits == 2 ->
      time_minutes(lexer, rest, acc <> ":", False)
    "-" <> rest if date_time && digits == 4 -> date(lexer, rest, acc <> "-")
    _ -> {
      let to_token = fn(value) { Integer(Decimal, value) }
      number_end(lexer, input, acc, to_token)
    }
  }
}

fn float(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  case input {
    "0" <> rest -> float(lexer, rest, acc <> "0")
    "1" <> rest -> float(lexer, rest, acc <> "1")
    "2" <> rest -> float(lexer, rest, acc <> "2")
    "3" <> rest -> float(lexer, rest, acc <> "3")
    "4" <> rest -> float(lexer, rest, acc <> "4")
    "5" <> rest -> float(lexer, rest, acc <> "5")
    "6" <> rest -> float(lexer, rest, acc <> "6")
    "7" <> rest -> float(lexer, rest, acc <> "7")
    "8" <> rest -> float(lexer, rest, acc <> "8")
    "9" <> rest -> float(lexer, rest, acc <> "9")
    "_0" <> rest -> float(lexer, rest, acc <> "_0")
    "_1" <> rest -> float(lexer, rest, acc <> "_1")
    "_2" <> rest -> float(lexer, rest, acc <> "_2")
    "_3" <> rest -> float(lexer, rest, acc <> "_3")
    "_4" <> rest -> float(lexer, rest, acc <> "_4")
    "_5" <> rest -> float(lexer, rest, acc <> "_5")
    "_6" <> rest -> float(lexer, rest, acc <> "_6")
    "_7" <> rest -> float(lexer, rest, acc <> "_7")
    "_8" <> rest -> float(lexer, rest, acc <> "_8")
    "_9" <> rest -> float(lexer, rest, acc <> "_9")
    "e+" <> rest -> exponent_start(lexer, input, rest, acc, "e+")
    "e-" <> rest -> exponent_start(lexer, input, rest, acc, "e-")
    "e" <> rest -> exponent_start(lexer, input, rest, acc, "e")
    "E+" <> rest -> exponent_start(lexer, input, rest, acc, "E+")
    "E-" <> rest -> exponent_start(lexer, input, rest, acc, "E-")
    "E" <> rest -> exponent_start(lexer, input, rest, acc, "E")
    _ -> number_end(lexer, input, acc, Float)
  }
}

fn exponent_start(
  lexer: Lexer,
  before_marker: String,
  input: String,
  acc: String,
  marker: String,
) -> #(Token, String) {
  case input {
    "0" <> rest -> exponent(lexer, rest, acc <> marker <> "0")
    "1" <> rest -> exponent(lexer, rest, acc <> marker <> "1")
    "2" <> rest -> exponent(lexer, rest, acc <> marker <> "2")
    "3" <> rest -> exponent(lexer, rest, acc <> marker <> "3")
    "4" <> rest -> exponent(lexer, rest, acc <> marker <> "4")
    "5" <> rest -> exponent(lexer, rest, acc <> marker <> "5")
    "6" <> rest -> exponent(lexer, rest, acc <> marker <> "6")
    "7" <> rest -> exponent(lexer, rest, acc <> marker <> "7")
    "8" <> rest -> exponent(lexer, rest, acc <> marker <> "8")
    "9" <> rest -> exponent(lexer, rest, acc <> marker <> "9")
    _ -> invalid_number(lexer, before_marker, acc)
  }
}

fn exponent(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  case input {
    "0" <> rest -> exponent(lexer, rest, acc <> "0")
    "1" <> rest -> exponent(lexer, rest, acc <> "1")
    "2" <> rest -> exponent(lexer, rest, acc <> "2")
    "3" <> rest -> exponent(lexer, rest, acc <> "3")
    "4" <> rest -> exponent(lexer, rest, acc <> "4")
    "5" <> rest -> exponent(lexer, rest, acc <> "5")
    "6" <> rest -> exponent(lexer, rest, acc <> "6")
    "7" <> rest -> exponent(lexer, rest, acc <> "7")
    "8" <> rest -> exponent(lexer, rest, acc <> "8")
    "9" <> rest -> exponent(lexer, rest, acc <> "9")
    "_0" <> rest -> exponent(lexer, rest, acc <> "_0")
    "_1" <> rest -> exponent(lexer, rest, acc <> "_1")
    "_2" <> rest -> exponent(lexer, rest, acc <> "_2")
    "_3" <> rest -> exponent(lexer, rest, acc <> "_3")
    "_4" <> rest -> exponent(lexer, rest, acc <> "_4")
    "_5" <> rest -> exponent(lexer, rest, acc <> "_5")
    "_6" <> rest -> exponent(lexer, rest, acc <> "_6")
    "_7" <> rest -> exponent(lexer, rest, acc <> "_7")
    "_8" <> rest -> exponent(lexer, rest, acc <> "_8")
    "_9" <> rest -> exponent(lexer, rest, acc <> "_9")
    _ -> number_end(lexer, input, acc, Float)
  }
}

fn date(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  case take_two_digits(input) {
    Ok(#(month, "-" <> rest)) ->
      case take_two_digits(rest) {
        Ok(#(day, rest)) -> date_tail(lexer, rest, acc <> month <> "-" <> day)
        _ -> invalid_number(lexer, input, acc)
      }
    _ -> invalid_number(lexer, input, acc)
  }
}

fn date_tail(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  case input {
    "T" <> rest -> date_time(lexer, rest, acc <> "T")
    "t" <> rest -> date_time(lexer, rest, acc <> "t")
    " " <> rest ->
      case take_two_digits(rest) {
        Ok(#(hour, ":" <> rest)) ->
          time_minutes(lexer, rest, acc <> " " <> hour <> ":", True)
        _ -> number_end(lexer, input, acc, DateTime)
      }
    _ -> number_end(lexer, input, acc, DateTime)
  }
}

fn date_time(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  case take_two_digits(input) {
    Ok(#(hour, ":" <> rest)) ->
      time_minutes(lexer, rest, acc <> hour <> ":", True)
    _ -> invalid_number(lexer, input, acc)
  }
}

fn time_minutes(
  lexer: Lexer,
  input: String,
  acc: String,
  allow_offset: Bool,
) -> #(Token, String) {
  case take_two_digits(input) {
    Ok(#(minute, rest)) -> {
      let acc = acc <> minute
      case rest {
        ":" <> rest -> time_seconds(lexer, rest, acc <> ":", allow_offset)
        _ -> date_time_end(lexer, rest, acc, allow_offset)
      }
    }
    _ -> invalid_number(lexer, input, acc)
  }
}

fn time_seconds(
  lexer: Lexer,
  input: String,
  acc: String,
  allow_offset: Bool,
) -> #(Token, String) {
  case take_two_digits(input) {
    Ok(#(second, "." <> rest)) ->
      fraction(lexer, rest, acc <> second <> ".", allow_offset)
    Ok(#(second, rest)) ->
      date_time_end(lexer, rest, acc <> second, allow_offset)
    _ -> invalid_number(lexer, input, acc)
  }
}

fn fraction(
  lexer: Lexer,
  input: String,
  acc: String,
  allow_offset: Bool,
) -> #(Token, String) {
  case input {
    "0" <> rest -> fraction_digits(lexer, rest, acc <> "0", allow_offset)
    "1" <> rest -> fraction_digits(lexer, rest, acc <> "1", allow_offset)
    "2" <> rest -> fraction_digits(lexer, rest, acc <> "2", allow_offset)
    "3" <> rest -> fraction_digits(lexer, rest, acc <> "3", allow_offset)
    "4" <> rest -> fraction_digits(lexer, rest, acc <> "4", allow_offset)
    "5" <> rest -> fraction_digits(lexer, rest, acc <> "5", allow_offset)
    "6" <> rest -> fraction_digits(lexer, rest, acc <> "6", allow_offset)
    "7" <> rest -> fraction_digits(lexer, rest, acc <> "7", allow_offset)
    "8" <> rest -> fraction_digits(lexer, rest, acc <> "8", allow_offset)
    "9" <> rest -> fraction_digits(lexer, rest, acc <> "9", allow_offset)
    _ -> invalid_number(lexer, input, acc)
  }
}

fn fraction_digits(
  lexer: Lexer,
  input: String,
  acc: String,
  allow_offset: Bool,
) -> #(Token, String) {
  case input {
    "0" <> rest -> fraction_digits(lexer, rest, acc <> "0", allow_offset)
    "1" <> rest -> fraction_digits(lexer, rest, acc <> "1", allow_offset)
    "2" <> rest -> fraction_digits(lexer, rest, acc <> "2", allow_offset)
    "3" <> rest -> fraction_digits(lexer, rest, acc <> "3", allow_offset)
    "4" <> rest -> fraction_digits(lexer, rest, acc <> "4", allow_offset)
    "5" <> rest -> fraction_digits(lexer, rest, acc <> "5", allow_offset)
    "6" <> rest -> fraction_digits(lexer, rest, acc <> "6", allow_offset)
    "7" <> rest -> fraction_digits(lexer, rest, acc <> "7", allow_offset)
    "8" <> rest -> fraction_digits(lexer, rest, acc <> "8", allow_offset)
    "9" <> rest -> fraction_digits(lexer, rest, acc <> "9", allow_offset)
    _ -> date_time_end(lexer, input, acc, allow_offset)
  }
}

fn date_time_end(
  lexer: Lexer,
  input: String,
  acc: String,
  allow_offset: Bool,
) -> #(Token, String) {
  case input {
    "Z" <> rest if allow_offset -> number_end(lexer, rest, acc <> "Z", DateTime)
    "z" <> rest if allow_offset -> number_end(lexer, rest, acc <> "z", DateTime)
    "+" <> rest if allow_offset -> offset(lexer, rest, acc <> "+")
    "-" <> rest if allow_offset -> offset(lexer, rest, acc <> "-")
    _ -> number_end(lexer, input, acc, DateTime)
  }
}

fn offset(lexer: Lexer, input: String, acc: String) -> #(Token, String) {
  case take_two_digits(input) {
    Ok(#(hour, ":" <> rest)) ->
      case take_two_digits(rest) {
        Ok(#(minute, rest)) ->
          number_end(lexer, rest, acc <> hour <> ":" <> minute, DateTime)
        _ -> invalid_number(lexer, input, acc)
      }
    _ -> invalid_number(lexer, input, acc)
  }
}

fn take_two_digits(input: String) -> Result(#(String, String), Nil) {
  case take_digit(input) {
    Ok(#(first, rest)) ->
      case take_digit(rest) {
        Ok(#(second, rest)) -> Ok(#(first <> second, rest))
        Error(_) -> Error(Nil)
      }
    Error(_) -> Error(Nil)
  }
}

fn take_digit(input: String) -> Result(#(String, String), Nil) {
  case input {
    "0" <> rest -> Ok(#("0", rest))
    "1" <> rest -> Ok(#("1", rest))
    "2" <> rest -> Ok(#("2", rest))
    "3" <> rest -> Ok(#("3", rest))
    "4" <> rest -> Ok(#("4", rest))
    "5" <> rest -> Ok(#("5", rest))
    "6" <> rest -> Ok(#("6", rest))
    "7" <> rest -> Ok(#("7", rest))
    "8" <> rest -> Ok(#("8", rest))
    "9" <> rest -> Ok(#("9", rest))
    _ -> Error(Nil)
  }
}

fn number_end(
  lexer: Lexer,
  input: String,
  acc: String,
  token: fn(String) -> Token,
) -> #(Token, String) {
  case is_value_terminator(input) {
    True -> #(token(acc), input)
    False -> invalid_number(lexer, input, acc)
  }
}

fn invalid_number(
  lexer: Lexer,
  input: String,
  acc: String,
) -> #(Token, String) {
  let #(unexpected, rest) = splitter.split_before(lexer.value_atom, input)
  #(InvalidNumber(acc <> unexpected), rest)
}

fn is_value_terminator(input: String) -> Bool {
  case input {
    "" | " " <> _ | "\t" <> _ | "\r\n" <> _ | "\n" <> _ | "\r" <> _ -> True
    "#" <> _ | "," <> _ | "]" <> _ | "}" <> _ -> True
    _ -> False
  }
}

fn string_delimiter(delimiter: StringDelimiter) -> String {
  case delimiter {
    BasicString -> "\""
    LiteralString -> "'"
    MultilineBasicString -> "\"\"\""
    MultilineLiteralString -> "'''"
  }
}