defmodule Bumblebee.Vision do
@moduledoc """
High-level tasks related to vision.
"""
@typedoc """
A term representing an image.
Either `Nx.Tensor` or a struct implementing `Nx.Container` and
resolving to a tensor, with the following properties:
* HWC order
* RGB color channels
* alpha channel may be present, but it's usually stripped out
* integer type (`:s` or `:u`)
"""
@type image :: Nx.Container.t()
@type image_classification_input :: image()
@type image_classification_output :: %{predictions: list(image_classification_prediction())}
@type image_classification_prediction :: %{score: number(), label: String.t()}
@doc """
Builds serving for image classification.
The serving accepts `t:image_classification_input/0` and returns
`t:image_classification_output/0`. A list of inputs is also supported.
## Options
* `:top_k` - the number of top predictions to include in the output. If
the configured value is higher than the number of labels, all
labels are returned. Defaults to `5`
* `:compile` - compiles all computations for predefined input shapes
during serving initialization. Should be a keyword list with the
following keys:
* `:batch_size` - the maximum batch size of the input. Inputs
are optionally padded to always match this batch size
It is advised to set this option in production and also configure
a defn compiler using `:defn_options` to maximally reduce inference
time.
* `:scores_function` - the function to use for converting logits to
scores. Should be one of `:softmax`, `:sigmoid`, or `:none`.
Defaults to `:softmax`
* `:defn_options` - the options for JIT compilation. Defaults to `[]`
* `:preallocate_params` - when `true`, explicitly allocates params
on the device configured by `:defn_options`. You may want to set
this option when using partitioned serving, to allocate params
on each of the devices. Defaults to `false`
## Examples
{:ok, resnet} = Bumblebee.load_model({:hf, "microsoft/resnet-50"})
{:ok, featurizer} = Bumblebee.load_featurizer({:hf, "microsoft/resnet-50"})
serving = Bumblebee.Vision.image_classification(resnet, featurizer)
image = StbImage.read_file!(path)
Nx.Serving.run(serving, image)
#=> %{
#=> predictions: [
#=> %{label: "Egyptian cat", score: 0.979233980178833},
#=> %{label: "tabby, tabby cat", score: 0.00679466687142849},
#=> %{label: "tiger cat", score: 0.005290505941957235},
#=> %{label: "lynx, catamount", score: 0.004550771787762642},
#=> %{label: "Siamese cat, Siamese", score: 1.1611092486418784e-4}
#=> ]
#=> }
"""
@spec image_classification(
Bumblebee.model_info(),
Bumblebee.Featurizer.t(),
keyword()
) :: Nx.Serving.t()
defdelegate image_classification(model_info, featurizer, opts \\ []),
to: Bumblebee.Vision.ImageClassification
@type image_to_text_input :: image()
@type image_to_text_output :: %{results: list(image_to_text_result())}
@type image_to_text_result :: %{text: String.t()}
@doc """
Builds serving for image-to-text generation.
The serving accepts `t:image_to_text_input/0` and returns
`t:image_to_text_output/0`. A list of inputs is also supported.
## Options
* `:seed` - random seed to use when sampling. By default the current
timestamp is used
* `:compile` - compiles all computations for predefined input shapes
during serving initialization. Should be a keyword list with the
following keys:
* `:batch_size` - the maximum batch size of the input. Inputs
are optionally padded to always match this batch size
It is advised to set this option in production and also configure
a defn compiler using `:defn_options` to maximally reduce inference
time.
* `:defn_options` - the options for JIT compilation. Defaults to `[]`
* `:preallocate_params` - when `true`, explicitly allocates params
on the device configured by `:defn_options`. You may want to set
this option when using partitioned serving, to allocate params
on each of the devices. Defaults to `false`
## Examples
{:ok, blip} = Bumblebee.load_model({:hf, "Salesforce/blip-image-captioning-base"})
{:ok, featurizer} = Bumblebee.load_featurizer({:hf, "Salesforce/blip-image-captioning-base"})
{:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "Salesforce/blip-image-captioning-base"})
{:ok, generation_config} =
Bumblebee.load_generation_config({:hf, "Salesforce/blip-image-captioning-base"})
serving =
Bumblebee.Vision.image_to_text(blip, featurizer, tokenizer, generation_config,
defn_options: [compiler: EXLA]
)
image = StbImage.read_file!(path)
Nx.Serving.run(serving, image)
#=> %{results: [%{text: "a cat sitting on a chair"}]}
"""
@spec image_to_text(
Bumblebee.model_info(),
Bumblebee.Featurizer.t(),
Bumblebee.Tokenizer.t(),
Bumblebee.Text.GenerationConfig.t(),
keyword()
) :: Nx.Serving.t()
defdelegate image_to_text(model_info, featurizer, tokenizer, generation_config, opts \\ []),
to: Bumblebee.Vision.ImageToText
@type image_embedding_input :: image()
@type image_embedding_output :: %{embedding: Nx.Tensor.t()}
@doc """
Builds serving for image embeddings.
The serving accepts `t:image_embedding_input/0` and returns
`t:image_embedding_output/0`. A list of inputs is also supported.
## Options
* `:output_attribute` - the attribute of the model output map to
retrieve. When the output is a single tensor (rather than a map),
this option is ignored. Defaults to `:pooled_state`
* `:embedding_processor` - a post-processing step to apply to the
embedding. Supported values: `:l2_norm`. By default the output is
returned as is
* `:compile` - compiles all computations for predefined input shapes
during serving initialization. Should be a keyword list with the
following keys:
* `:batch_size` - the maximum batch size of the input. Inputs
are optionally padded to always match this batch size
It is advised to set this option in production and also configure
a defn compiler using `:defn_options` to maximally reduce inference
time.
* `:defn_options` - the options for JIT compilation. Defaults to `[]`
* `:preallocate_params` - when `true`, explicitly allocates params
on the device configured by `:defn_options`. You may want to set
this option when using partitioned serving, to allocate params
on each of the devices. Defaults to `false`
## Examples
{:ok, clip} =
Bumblebee.load_model({:hf, "openai/clip-vit-base-patch32"},
module: Bumblebee.Vision.ClipVision
)
{:ok, featurizer} = Bumblebee.load_featurizer({:hf, "openai/clip-vit-base-patch32"})
serving = Bumblebee.Vision.ImageEmbedding.image_embedding(clip, featurizer)
image = StbImage.read_file!(path)
Nx.Serving.run(serving, image)
#=> %{
#=> embedding: #Nx.Tensor<
#=> f32[768]
#=> [-0.43403682112693787, 0.09786412119865417, -0.7233262062072754, -0.7707743644714355, 0.5550824403762817, -0.8923342227935791, 0.2687447965145111, 0.9633643627166748, 0.3520320951938629, 0.43195801973342896, 2.1438512802124023, -0.6542983651161194, -1.9736307859420776, 0.1611439287662506, 0.24555791914463043, 0.16985465586185455, 0.9012499451637268, 1.0657984018325806, 1.087411642074585, -0.5864712595939636, 0.3314521908760071, 0.8396108150482178, 0.3906593322753906, 0.13463366031646729, 0.2605385184288025, -0.07457947731018066, 0.4735124707221985, -0.41367805004119873, 0.18244807422161102, 1.4741417169570923, -5.807061195373535, 0.38920706510543823, 0.057687126100063324, 0.060301072895526886, 0.9680367708206177, 0.9670255184173584, 1.3876476287841797, -0.15498873591423035, -0.969764232635498, -0.38127464056015015, 0.05450016260147095, 2.2317700386047363, -0.07926210761070251, -0.11876475065946579, -1.5408644676208496, 0.7505669593811035, 0.9280041456222534, -0.3571934103965759, -1.1390857696533203, ...]
#=> >
#=> }
"""
@spec image_embedding(
Bumblebee.model_info(),
Bumblebee.Featurizer.t(),
keyword()
) :: Nx.Serving.t()
defdelegate image_embedding(model_info, featurizer, opts \\ []),
to: Bumblebee.Vision.ImageEmbedding
end
