%% @author Marc Worrell
%% @copyright 2014-2025 Marc Worrell
%% @doc Discover metadata about an url. Follows redirects
%% and URL shorteners, and then fetches the data at the final URL
%% to inspect for metadata tags, content headers and the first part of the HTML.
%%
%% The returned opaque metadata can be questioned for properties using p/2.
%%
%% The Slackbot user-agent is used for fetching URLs so that the URL shorteners
%% return a location header and other sites are coerced to give correct metadata.
%%
%% Only the first MB of data is fetched, this prevents fetching large objects.
%% @end
%% Copyright 2014-2025 Marc Worrell
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
-module(z_url_metadata).
-author("Marc Worrell <marc@worrell.nl>").
-export([
fetch/1,
fetch/2,
html_meta/1,
p/2,
header/2,
filename/2
]).
-include("../include/z_url_metadata.hrl").
-type metadata() :: #url_metadata{}.
-type property() :: mime | mime_options | site_name | content_length |
url | canonical_url | short_url | final_url |
headers | title | h1 | summary | tags | filename |
mtitle | description | keywords | author | charset | language |
image | image_nav | thumbnail |
icon | icon_nav | icon_shortcut | icon_touch |
binary().
-export_type([ metadata/0 ]).
% User-agent used for metadata sniffing - we pretend to be Slack so
% that some websites with bot-protection allow us to sniff the metadata.
-define(USER_AGENT, <<"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)">>).
% Per default we fetch max 1MB of data to analyze.
% We need to fetch this much as (for example) Youtube adds a lot of css/scripts
% above the metadata of the page.
-define(FETCH_LENGTH, 1024*1024).
% Below this size an image is considered too small to be a representative image or icon.
-define(IMG_SMALL_SIZE, 16).
%% @doc Fetch metadata information for the URL
-spec fetch(binary()|string()) -> {ok, metadata()} | {error, term()}.
fetch(Url) ->
fetch(Url, []).
-spec fetch(binary()|string(), z_url_fetch:options()) -> {ok, metadata()} | {error, term()}.
fetch(Url, Options) ->
Options1 = case proplists:is_defined(max_length, Options) of
true -> Options;
false -> [ {max_length, ?FETCH_LENGTH} | Options ]
end,
Options2 = case proplists:is_defined(user_agent, Options1) of
true -> Options1;
false -> [ {user_agent, ?USER_AGENT} | Options1 ]
end,
case z_url_fetch:fetch_partial(Url, Options2) of
{ok, {FinalUrl, Headers, _Size, Data}} ->
{ok, partial_metadata(FinalUrl, Headers, Data)};
{error, _} = Error ->
Error
end.
%% @doc Fetch properties of the fetched metadata
-spec p(Property, Metadata) -> Value when
Property :: property() | [ property() ],
Metadata :: metadata(),
Value :: binary() | list( binary() ) | Headers | undefined,
Headers :: list({binary(), binary()}).
p(mime, MD) ->
MD#url_metadata.content_type;
p(mime_options, MD) ->
MD#url_metadata.content_type_options;
p(final_url, MD) ->
MD#url_metadata.final_url;
p(url, MD) ->
case p1([<<"og:url">>, <<"twitter:url">>, canonical_url, short_url], MD) of
undefined -> MD#url_metadata.final_url;
PrefUrl -> z_url:abs_link(PrefUrl, MD#url_metadata.final_url)
end;
p(site_name, MD) ->
case p1([<<"og:site_name">>, <<"twitter:site">>], MD) of
undefined ->
Url = case p1([canonical_url], MD) of
undefined -> MD#url_metadata.final_url;
Canonical -> Canonical
end,
case uri_string:parse(Url) of
#{ host := Host } ->
case unicode:characters_to_binary(Host) of
<<"www.", H/binary>> -> H;
H -> H
end;
{error, _, _} ->
undefined
end;
Sitename ->
Sitename
end;
p(content_length, MD) ->
MD#url_metadata.content_length;
p(headers, MD) ->
MD#url_metadata.headers;
p(title, MD) ->
case p1([<<"og:title">>, <<"twitter:title">>, mtitle, h1, title], MD) of
undefined -> p(filename, MD);
Title -> Title
end;
p(summary, MD) ->
p1([<<"og:description">>, <<"twitter:description">>, description], MD);
p(image, MD) ->
case MD#url_metadata.content_type of
<<"image/", _/binary>> ->
MD#url_metadata.final_url;
_ ->
Ps = case MD#url_metadata.is_index_page of
true ->
[<<"twitter:image:src">>, <<"twitter:image">>, <<"og:image">>,
image_nav, image];
false ->
[<<"twitter:image:src">>, <<"twitter:image">>, <<"og:image">>,
image, image_nav]
end,
case p1(Ps, MD) of
undefined -> undefined;
ImgSrc -> z_url:abs_link(ImgSrc, MD#url_metadata.final_url)
end
end;
p(icon, MD) ->
case MD#url_metadata.content_type of
<<"image/", _/binary>> ->
MD#url_metadata.final_url;
_ ->
Ps = case MD#url_metadata.is_index_page of
true ->
[image_nav, icon_touch, icon_shortcut, icon_fav];
false ->
[icon_touch, image_nav, icon_shortcut, icon_fav]
end,
case p1(Ps, MD) of
undefined -> undefined;
ImgSrc -> z_url:abs_link(ImgSrc, MD#url_metadata.final_url)
end
end;
p(tags, MD) ->
case p1([keywords], MD) of
undefined ->
% Check og tags? (youtube uses og:video:tag)
[];
KW ->
Ks = [ z_string:trim(K) || K <- binary:split(KW, <<",">>, [global]) ],
[ K || K <- Ks, K =/= <<>> ]
end;
p(filename, MD) ->
filename(MD#url_metadata.final_url, MD#url_metadata.headers);
p(Ks, MD) when is_list(Ks) ->
p1(Ks, MD);
p(K, MD) ->
p1([K], MD).
-spec header( binary() | string(), metadata() ) -> binary() | undefined.
header(H, #url_metadata{ headers = Hs }) ->
proplists:get_value(z_convert:to_binary(H), Hs).
-spec filename(binary()|string(), list()) -> binary() | undefined.
filename(Url, Hs) ->
case content_disp_filename(proplists:get_value(<<"content-disposition">>, Hs)) of
undefined -> basename(Url);
FN -> z_convert:to_binary(FN)
end.
%% ------------------------------------------------ Internal Functions ------------------------------------------------
%% Find the first defined property
p1([], _MD) ->
undefined;
p1([P|Ps], MD) ->
case proplists:get_value(P, MD#url_metadata.metadata) of
undefined -> p1(Ps, MD);
Value ->
case z_string:trim(Value) of
<<>> -> p1(Ps, MD);
Trimmed -> maybe_abs_link(is_link_property(P), Trimmed, MD#url_metadata.final_url)
end
end.
maybe_abs_link(false, Value, _FinalUrl) ->
Value;
maybe_abs_link(true, <<>>, _FinalUrl) ->
undefined;
maybe_abs_link(true, Value, FinalUrl) ->
Url1 = z_url:abs_link(Value, FinalUrl),
z_html:sanitize_uri(Url1).
is_link_property(canonical_url) -> true;
is_link_property(short_url) -> true;
is_link_property(image_nav) -> true;
is_link_property(image) -> true;
is_link_property(icon_nav) -> true;
is_link_property(icon_shortcut) -> true;
is_link_property(_) -> false.
content_disp_filename(undefined) ->
undefined;
content_disp_filename(Vs) ->
{_Disp, Options} = parse_header(Vs),
case proplists:get_value(<<"filename">>, Options) of
undefined -> undefined;
<<>> -> undefined;
FN -> FN
end.
basename(<<"data:", _/binary>>) ->
undefined;
basename("data:" ++ _) ->
undefined;
basename(Url) ->
#{ path := Path } = uri_string:parse( z_convert:to_binary(Url) ),
case Path of
<<>> -> undefined;
<<"/">> -> undefined;
_ ->
case lists:last( binary:split(Path, <<"/">>, [ global ]) ) of
<<>> ->
undefined;
Basename ->
% Perform percent-decode of the path
try
z_url:url_decode(Basename)
catch
_:_ ->
Basename
end
end
end.
%% ------------------------------------------------ From Mochiweb ------------------------------------------------
%% author Bob Ippolito <bob@mochimedia.com>
%% copyright 2007 Mochi Media, Inc.
%% @doc Parse a Content-Type like header, return the main Content-Type
%% and a property list of options.
-spec parse_header( binary() ) -> {binary(), [ {binary(), binary()} ]}.
parse_header(String) ->
%% TODO: This is exactly as broken as Python's cgi module.
%% Should parse properly like mochiweb_cookies.
[Type | Parts] = [z_string:trim(S) || S <- binary:split(String, <<";">>, [ global ])],
F = fun (S, Acc) ->
case binary:split(S, <<"=">>) of
[<<>>, _] ->
%% Skip anything with no name
Acc;
[_, <<>>] ->
%% Skip anything with no value
Acc;
[_] ->
Acc;
[Name, Value] ->
[{z_string:to_lower(z_string:trim(Name)),
unquote_header(z_string:trim(Value))} | Acc]
end
end,
{z_string:to_lower(Type), lists:foldr(F, [], Parts)}.
unquote_header(<<"\"", Rest/binary>>) ->
unquote_header(Rest, <<>>);
unquote_header(S) ->
S.
unquote_header(<<>>, Acc) ->
Acc;
unquote_header(<<"\"">>, Acc) ->
Acc;
unquote_header(<<$\\, C, Rest/binary>>, Acc) ->
unquote_header(Rest, <<Acc/binary, C>>);
unquote_header(<<C, Rest/binary>>, Acc) ->
unquote_header(Rest, <<Acc/binary, C>>).
%% -------------------------------------- Analyze fetched data -----------------------------------------
-record(ps, { in_nav = false }).
partial_metadata(Url, Hs, Data) ->
HsBin = [ {z_convert:to_binary(H), z_convert:to_binary(V)} || {H, V} <- Hs ],
{CT, CTOpts} = content_type(HsBin),
IsText = is_text(CT, Data),
IsHTML = IsText andalso is_html(CT),
Data1 = maybe_convert_utf8(IsText, IsHTML, proplists:get_value(<<"charset">>, CTOpts), Data),
#url_metadata{
final_url = z_convert:to_binary(Url),
content_type = CT,
content_type_options = CTOpts,
content_length = content_length(HsBin),
metadata = html_meta(IsHTML, Data1),
is_index_page = is_index_page(Url),
headers = HsBin,
partial_data = Data
}.
is_index_page(Url) ->
case uri_string:parse( z_convert:to_binary(Url) ) of
#{ query := _ } -> false;
#{ path := <<>> } -> true;
#{ path := <<"/">> } -> true;
#{ path := <<"/index.", _/binary>> } -> true;
#{ path := <<"/default.htm">> } -> true;
#{ path := <<"/Default.htm">> } -> true;
#{ path := <<"index.", _/binary>> } -> true;
#{ path := <<"default.htm">> } -> true;
#{ path := <<"Default.htm">> } -> true;
_ -> false
end.
html_meta(Data) ->
html_meta(true, Data).
html_meta(true, PartialData) ->
case parse(PartialData) of
{ok, Parsed} ->
lists:reverse(html(Parsed, [], #ps{}));
{error, _} ->
[]
end;
html_meta(false, _PartialData) ->
[].
parse(PartialData) when is_binary(PartialData) ->
parse_html(<<"<partial>", PartialData/binary, "</partial>">>);
parse(PartialData) when is_list(PartialData) ->
parse_html(iolist_to_binary([<<"<partial>">>, PartialData, <<"</partial>">>])).
parse_html(Html) ->
z_html_parse:parse(Html).
html([], MD, _P) ->
MD;
html([B|Es], MD, P) when is_binary(B) ->
html(Es, MD, P);
html([{comment, _}|Es], MD, P) ->
% <!-- ... -->
html(Es, MD, P);
html([{pi, _Xml, _Attrs}|Es], MD, P) ->
% <?xml version="1.0" encoding="UTF-8"?>
html(Es, MD, P);
html([Tag|Es], MD, P) ->
{MD1, P1} = tag(Tag, MD, P),
html(Es, MD1, P1);
html(Tag, MD, P) when is_tuple(Tag) ->
{MD1, _} = tag(Tag, MD, P),
MD1.
tag({<<"html">>, As, Es}, MD, P) ->
MD1 = case proplists:get_value(<<"lang">>, As) of
undefined -> MD;
Lang -> [{language, Lang} | MD]
end,
{html(Es, MD1, P), P};
tag({<<"meta">>, As, _}, MD, P) ->
Name = z_string:to_lower(proplists:get_value(<<"name">>, As)),
Property = proplists:get_value(<<"property">>, As),
HttpEquiv = proplists:get_value(<<"http-equiv">>, As),
Value = proplists:get_value(<<"value">>, As),
Content = proplists:get_value(<<"content">>, As, Value),
case first([Name, Property, HttpEquiv]) of
undefined ->
case proplists:get_value(<<"charset">>, As) of
undefined -> {MD, P};
Charset -> {[{charset,Charset} | MD], P}
end;
Prop ->
{meta_tag(Prop, Content, MD), P}
end;
tag({<<"title">>, _As, Es}, MD, P) ->
Text = z_string:trim(fetch_text(Es, <<>>)),
{[{title, Text} | MD], P};
tag({<<"link">>, As, _}, MD, P) ->
Name = z_string:to_lower(proplists:get_value(<<"rel">>, As)),
Content = proplists:get_value(<<"href">>, As),
{meta_link(Name, Content, As, MD), P};
tag({<<"img">>, As, _}, MD, P) ->
case proplists:get_value(<<"src">>, As, <<>>) of
<<>> ->
{MD, P};
Src ->
case is_img_allowed(Src, As) of
true ->
case P#ps.in_nav of
true -> {[{image_nav, Src} | MD], P};
false -> {[{image, Src} | MD], P}
end;
false ->
{MD, P}
end
end;
tag({<<"h1">>, _As, Es}, MD, #ps{in_nav=false} = P) ->
case proplists:is_defined(h1, MD) of
false ->
Text = z_string:trim(fetch_text(Es, <<>>)),
{[{h1, Text} | MD], P};
true ->
{MD, P}
end;
tag({<<"h1">>, _As, _Es}, MD, P) ->
{MD, P};
tag({<<"nav">>, _As, Es}, MD, P) ->
{html(Es, MD, P#ps{in_nav=true}), P};
tag({<<"header">>, _As, Es}, MD, P) ->
{html(Es, MD, P#ps{in_nav=true}), P};
tag({<<"footer">>, _As, Es}, MD, P) ->
{html(Es, MD, P#ps{in_nav=true}), P};
tag({<<"aside">>, _As, Es}, MD, P) ->
{html(Es, MD, P#ps{in_nav=true}), P};
tag({_Tag, As, Es}, MD, P) ->
Cs = split_class(proplists:get_value(<<"class">>, As)),
Id = proplists:get_value(<<"id">>, As),
case is_ads(Id, Cs) of
true ->
{MD, P};
false ->
{html(Es, MD, P#ps{in_nav = P#ps.in_nav orelse has_nav_class(Cs) orelse is_topbar_id(Id)}), P}
end.
meta_tag(_Name, undefined, MD) -> MD;
meta_tag(_Name, <<>>, MD) -> MD;
meta_tag(<<"og:", _/binary>> = OG, Content, MD) -> [{OG, Content}|MD];
meta_tag(<<"twitter:", _/binary>> = Tw, Content, MD) -> [{Tw, Content}|MD];
meta_tag(<<"title">>, Content, MD) -> [{mtitle, Content}|MD];
meta_tag(<<"keywords">>, Content, MD) -> [{keywords, Content}|MD];
meta_tag(<<"description">>, Content, MD) -> [{description, Content}|MD];
meta_tag(<<"author">>, Content, MD) -> [{author, Content}|MD];
meta_tag(<<"thumbnail">>, Content, MD) -> [{thumbnail, Content}|MD];
meta_tag(<<"content-type">>, Content, MD) -> [{content_type, Content}|MD];
meta_tag(_Name, _Content, MD) -> MD.
meta_link(_Name, undefined, _As, MD) -> MD;
meta_link(_Name, <<>>, _As, MD) -> MD;
meta_link(_Name, <<"undefined">>, _As, MD) -> MD; % Youtube...
meta_link(<<"canonical">>, Content, _As, MD) -> [{canonical_url, Content}|MD];
meta_link(<<"shortlink">>, Content, _As, MD) -> [{short_url, Content}|MD];
meta_link(<<"shorturl">>, Content, _As, MD) -> [{short_url, Content}|MD];
meta_link(<<"icon">>, Content, As, MD) ->
case proplists:is_defined(<<"mask">>, As) of
true -> MD;
false -> [{icon_fav, Content}|MD]
end;
meta_link(<<"shortcut icon">>, Content, _As, MD) -> [{icon_shortcut, Content}|MD];
meta_link(<<"apple-touch-icon">>, Content, _As, MD) -> [{icon_touch, Content}|MD];
meta_link(_Name, _Content, _As, MD) -> MD.
split_class(undefined) -> [];
split_class(Class) -> binary:split(Class, <<" ">>, [global]).
has_nav_class(Cs) ->
lists:any(fun is_nav_class/1, Cs).
is_nav_class(<<"nav", _/binary>>) -> true;
is_nav_class(<<"menu", _/binary>>) -> true;
is_nav_class(_) -> false.
is_topbar_id(<<"top">>) -> true;
is_topbar_id(<<"header", _/binary>>) -> true;
is_topbar_id(_) -> false.
is_ads(<<"ad">>, _Cs) -> true;
is_ads(<<"ads">>, _Cs) -> true;
is_ads(_, Cs) -> lists:any(fun is_ad_class/1, Cs).
is_ad_class(<<"ads">>) -> true;
is_ad_class(<<"ad">>) -> true;
is_ad_class(<<"deckad">>) -> true;
is_ad_class(_) -> false.
fetch_text(B, Acc) when is_binary(B) ->
<<Acc/binary, B/binary>>;
fetch_text({comment, _}, Acc) ->
Acc;
fetch_text({_Tag, _As, Es}, Acc) ->
fetch_text(Es, Acc);
fetch_text([], Acc) ->
Acc;
fetch_text([E|Es], Acc) ->
fetch_text(Es, fetch_text(E, Acc)).
first([]) -> undefined;
first([undefined|Rest]) -> first(Rest);
first([<<>>|Rest]) -> first(Rest);
first([X|_]) -> X.
is_html(<<"text/html">>) -> true;
is_html(<<"application/xhtml">>) -> true;
is_html(<<"application/xhtml+", _/binary>>) -> true;
is_html(_) -> false.
%% Some servers send us 'gzip', even when we ask for 'identity'
is_text(_CT, <<31, 198, Method, _/binary>>) when Method =< 8 ->
false;
is_text(CT, _Data) ->
is_text(CT).
is_text(<<"text/", _/binary>>) -> true;
is_text(<<"application/javascript">>) -> true;
is_text(<<"application/xhtml">>) -> true;
is_text(<<"application/xhtml+", _/binary>>) -> true;
is_text(_) -> false.
% Suppres tracking pixels and small images
is_img_allowed(<<>>, _As) ->
false;
is_img_allowed(Url, As) ->
not is_img_small(As)
andalso binary:match(Url, img_blocklist()) =:= nomatch.
% Images are considered small if their width or height is smaller than 16px
is_img_small(As) ->
is_small_size(proplists:get_value(<<"width">>, As))
orelse is_small_size(proplists:get_value(<<"height">>, As)).
is_small_size(undefined) -> false;
is_small_size(<<>>) -> false;
is_small_size(Size) ->
try
Sz = z_convert:to_integer(Size),
Sz =< ?IMG_SMALL_SIZE
catch
_:_ -> false
end.
% Add parts of image URLs to be suppressed
img_blocklist() -> [
<<"//www.facebook.com/tr?">>,
<<"//www.googleadservices.com/pagead/">>,
<<"futuresimple.com/api/v1/">>,
<<"tracking.cirrusinsight.com">>,
<<"list-manage.com/track">>,
<<"mjt.lu/oo">>,
<<"/1x1/">>
].
% Add nowarn because the iconv module is optional.
-dialyzer({[ nowarn_function ], maybe_convert_utf8/4}).
maybe_convert_utf8(true, IsHtml, Charset, Html) ->
CS1 = z_convert:to_list(
z_string:to_lower(
html_charset(IsHtml, Charset, Html))),
case is_utf8(CS1) of
true ->
Html;
false ->
try
case iconv:open(CS1, "UTF-8") of
{ok, C} ->
case iconv:conv(C, Html) of
{ok, Html1} ->
iconv:close(C),
Html1;
{error, _} ->
Html
end;
{error, _} ->
Html
end
catch
_:_ -> Html
end
end;
maybe_convert_utf8(false, _IsHtml, _Charset, Data) ->
Data.
is_utf8("utf-8") -> true;
is_utf8("utf8") -> true;
is_utf8(_) -> false.
html_charset(IsHtml, undefined, Text) ->
html_charset(IsHtml, <<"iso-8859-1">>, Text);
html_charset(true, Charset, Html) ->
meta_charset(Charset, Html);
html_charset(false, Charset, _Text) ->
Charset.
meta_charset(Ch, Html) ->
case re:run(Html, "<[mM][eE][tT][aA][^>]*[cC][hH][aA][rR][sS][eE][tT]\\s*=\\s*[\"']?([A-Za-z0-9_-]+)", [{capture,all_but_first,binary}]) of
{match, [CharSet|_]} -> CharSet;
_ -> Ch
end.
content_type(Hs) ->
case proplists:get_value(<<"content-type">>, Hs) of
undefined ->
{<<"application/octet-stream">>, []};
CT ->
{Mime, Options} = parse_header(CT),
{z_convert:to_binary(Mime), Options}
end.
content_length(Hs) ->
try
case proplists:get_value(<<"content-range">>, Hs) of
undefined ->
case proplists:get_value(<<"content-length">>, Hs) of
undefined -> undefined;
N -> binary_to_integer(N)
end;
<<"bytes ", Range/binary>> ->
Ts = binary:split(Range, <<"/">>, [global]),
binary_to_integer(lists:last(Ts))
end
catch
_:_ -> undefined
end.
%%
%% Tests
%%
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
simple_partial_metadata_test() ->
Url = "http://example.org",
Headers = [{"content-type", "text/html"}],
Data = <<"<html><head><title>Example</title><body></body></html>">>,
MD = partial_metadata(Url, Headers, Data),
?assertEqual(<<"http://example.org">>, MD#url_metadata.final_url),
?assertEqual(<<"text/html">>, MD#url_metadata.content_type),
?assertEqual([{title, <<"Example">>}], MD#url_metadata.metadata),
ok.
simple_html_meta_test() ->
Data = <<"<html><head><title>Example</title><body></body></html>">>,
?assertEqual([{title, <<"Example">>}], html_meta(Data)),
ok.
partial_unbalanced_tags_html_meta_test() ->
Data = <<"<head><meta name=\"description\" content=\"Example Content\"><title>Example</title>">>,
?assertEqual([{description, <<"Example Content">>},
{title, <<"Example">>}], html_meta(Data)),
ok.
partial_no_surrounding_tags_html_meta_test() ->
Data = <<"<meta name=\"description\" content=\"Example Content\"><title>Example</title>">>,
?assertEqual([{description, <<"Example Content">>},
{title, <<"Example">>}], html_meta(Data)),
ok.
partial_ampersant_in_html_meta_test() ->
Data = <<"<meta name=\"description\" content=\"Example & Stuff\"><title>Foo & Co</title>">>,
?assertEqual([{description, <<"Example & Stuff">>},
{title, <<"Foo & Co">>}], html_meta(Data)),
ok.
-endif.
