lib/bumblebee/text/albert_tokenizer.ex

defmodule Bumblebee.Text.AlbertTokenizer do
  @moduledoc """
  ALBERT tokenizer.
  """

  import Bumblebee.Shared

  tokenizer_impl(
    special_tokens: %{
      bos: "[CLS]",
      eos: "[SEP]",
      unk: "<unk>",
      sep: "[SEP]",
      pad: "<pad>",
      cls: "[CLS]",
      mask: "[MASK]"
    }
  )
end