defmodule Mix.Tasks.Gettext.Merge do
use Mix.Task
@recursive true
@shortdoc "Merge template files into message files"
@moduledoc """
Merges PO/POT files with PO files.
This task is used when messages in the source code change: when they do,
`mix gettext.extract` is usually used to extract the new messages to POT
files. At this point, developers or translators can use this task to "sync"
the newly-updated POT files with the existing locale-specific PO files. All
the metadata for each message (like position in the source code, comments,
and so on) is taken from the newly-updated POT file; the only things taken
from the PO file are the actual translated strings.
#### Fuzzy Matching
Messages in the updated PO/POT file that have an exact match (a
message with the same `msgid`) in the old PO file are merged as described
above. When a message in the updated PO/POT files has no match in the old
PO file, Gettext attemps a **fuzzy match** for that message. For example, imagine
we have this POT file:
msgid "hello, world!"
msgstr ""
and we merge it with this PO file:
# No exclamation point here in the msgid
msgid "hello, world"
msgstr "ciao, mondo"
Since the two messages are similar, Gettext takes the `msgstr` from the
existing message over to the new message, which it however
marks as *fuzzy*:
#, fuzzy
msgid "hello, world!"
msgstr "ciao, mondo"
Generally, a `fuzzy` flag calls for review from a translator.
Fuzzy matching can be configured (for example, the threshold for message
similarity can be tweaked) or disabled entirely. Look at the
["Options" section](#module-options).
## Usage
```bash
mix gettext.merge OLD_FILE UPDATED_FILE [OPTIONS]
mix gettext.merge DIR [OPTIONS]
```
If two files are given as arguments, `OLD_FILE` must be a `.po` file and
`UPDATE_FILE` must be a `.po`/`.pot` file. The first one is the old PO file,
while the second one is the last generated one. They are merged and written
over the first file. For example:
```bash
mix gettext.merge priv/gettext/en/LC_MESSAGES/default.po priv/gettext/default.pot
```
If only one argument is given, then that argument must be a directory
containing Gettext messages (with `.pot` files at the root level alongside
locale directories - this is usually a "backend" directory used by a Gettext
backend, see `Gettext.Backend`). For example:
```bash
mix gettext.merge priv/gettext
```
If the `--locale LOCALE` option is given, then only the PO files in
`<DIR>/<LOCALE>/LC_MESSAGES` will be merged with the POT files in `DIR`. If no
options are given, then all the PO files for all locales under `DIR` are
merged with the POT files in `DIR`.
## Plural Forms
By default, Gettext will determine the number of plural forms for newly-generated messages
by checking the value of `nplurals` in the `Plural-Forms` header in the existing `.po` file. If
a `.po` file doesn't already exist and Gettext is creating a new one or if the `Plural-Forms`
header is not in the `.po` file, Gettext will use the number of plural forms that
the plural module (see `Gettext.Plural`) returns for the locale of the file being created.
The content of the `Plural-Forms` header can be forced through the `--plural-forms-header`
option (see below).
## Options
* `--locale` - a string representing a locale. If this is provided, then only the PO
files in `<DIR>/<LOCALE>/LC_MESSAGES` will be merged with the POT files in `DIR`. This
option can only be given when a single argument is passed to the task
(a directory).
* `--no-fuzzy` - don't perform fuzzy matching when merging files.
* `--fuzzy-threshold` - a float between `0` and `1` which represents the
minimum Jaro distance needed for two messages to be considered a fuzzy
match. Overrides the global `:fuzzy_threshold` option (see the docs for
`Gettext` for more information on this option).
* `--plural-forms` - (**deprecated in v0.22.0**) an integer strictly greater than `0`.
If this is passed, new messages in the target PO files will have this number of empty
plural forms. This is deprecated in favor of passing the `--plural-forms-header`,
which contains the whole plural-forms specification. See the "Plural forms" section above.
* `--plural-forms-header` - the content of the `Plural-Forms` header as a string.
If this is passed, new messages in the target PO files will use this content
to determine the number of plurals. See the ["Plural Forms" section](#module-plural-forms).
* `--on-obsolete` - controls what happens when **obsolete** messages are found.
If `mark_as_obsolete`, messages are kept and marked as obsolete.
If `delete`, obsolete messages are deleted. Defaults to `delete`.
* `--store-previous-message-on-fuzzy-match` - controls if the previous
messages are recorded on fuzzy matches. Is off by default.
"""
alias Expo.PO
alias Gettext.Merger
@default_fuzzy_threshold 0.8
@switches [
locale: :string,
fuzzy: :boolean,
fuzzy_threshold: :float,
plural_forms_header: :string,
on_obsolete: :string,
store_previous_message_on_fuzzy_match: :boolean
]
@impl true
def run(args) do
Mix.Task.run("loadpaths")
_ = Mix.Project.get!()
gettext_config = Mix.Project.config()[:gettext] || []
case OptionParser.parse!(args, switches: @switches) do
{opts, [po_file, reference_file]} ->
merge_two_files(po_file, reference_file, opts, gettext_config)
{opts, [messages_dir]} ->
merge_messages_dir(messages_dir, opts, gettext_config)
{_opts, _args} ->
Mix.raise(
"You can only pass one or two arguments to the \"gettext.merge\" task. " <>
"Use `mix help gettext.merge` to see the usage of this task"
)
end
Mix.Task.reenable("gettext.merge")
end
defp merge_two_files(po_file, reference_file, opts, gettext_config) do
merging_opts = validate_merging_opts!(opts, gettext_config)
if Path.extname(po_file) == ".po" and Path.extname(reference_file) in [".po", ".pot"] do
ensure_file_exists!(po_file)
ensure_file_exists!(reference_file)
locale = locale_from_path(po_file)
{contents, stats} =
merge_files(po_file, reference_file, locale, merging_opts, gettext_config)
write_file(po_file, contents, stats)
else
Mix.raise("Arguments must be a PO file and a PO/POT file")
end
end
defp merge_messages_dir(dir, opts, gettext_config) do
ensure_dir_exists!(dir)
merging_opts = validate_merging_opts!(opts, gettext_config)
if locale = opts[:locale] do
merge_locale_dir(dir, locale, merging_opts, gettext_config)
else
merge_all_locale_dirs(dir, merging_opts, gettext_config)
end
end
defp merge_locale_dir(pot_dir, locale, opts, gettext_config) do
locale_dir = locale_dir(pot_dir, locale)
create_missing_locale_dir(locale_dir)
merge_dirs(locale_dir, pot_dir, locale, opts, gettext_config)
end
defp merge_all_locale_dirs(pot_dir, opts, gettext_config) do
for locale <- File.ls!(pot_dir), File.dir?(Path.join(pot_dir, locale)) do
merge_dirs(locale_dir(pot_dir, locale), pot_dir, locale, opts, gettext_config)
end
end
def locale_dir(pot_dir, locale) do
Path.join([pot_dir, locale, "LC_MESSAGES"])
end
defp merge_dirs(po_dir, pot_dir, locale, opts, gettext_config) do
merger = fn pot_file ->
po_file = find_matching_po(pot_file, po_dir)
{contents, stats} = merge_or_create(pot_file, po_file, locale, opts, gettext_config)
write_file(po_file, contents, stats)
end
pot_dir
|> Path.join("*.pot")
|> Path.wildcard()
|> Task.async_stream(merger, ordered: false, timeout: :infinity)
|> Stream.run()
warn_for_po_without_pot(po_dir, pot_dir)
end
defp find_matching_po(pot_file, po_dir) do
domain = Path.basename(pot_file, ".pot")
Path.join(po_dir, "#{domain}.po")
end
defp merge_or_create(pot_file, po_file, locale, opts, gettext_config) do
if File.regular?(po_file) do
merge_files(po_file, pot_file, locale, opts, gettext_config)
else
{new_po, stats} = Merger.new_po_file(po_file, pot_file, locale, opts)
{new_po
|> Merger.prune_references(gettext_config)
|> PO.compose(), stats}
end
end
defp merge_files(po_file, pot_file, locale, opts, gettext_config) do
{merged, stats} =
Merger.merge(
PO.parse_file!(po_file),
PO.parse_file!(pot_file),
locale,
opts,
gettext_config
)
{merged
|> Merger.prune_references(gettext_config)
|> PO.compose(), stats}
end
defp write_file(path, contents, stats) do
File.mkdir_p!(Path.dirname(path))
File.write!(path, contents)
Mix.shell().info("Wrote #{path} (#{format_stats(stats)})")
end
# Warns for every PO file that has no matching POT file.
defp warn_for_po_without_pot(po_dir, pot_dir) do
po_dir
|> Path.join("*.po")
|> Path.wildcard()
|> Enum.reject(&po_has_matching_pot?(&1, pot_dir))
|> Enum.each(fn po_file ->
Mix.shell().info("Warning: PO file #{po_file} has no matching POT file in #{pot_dir}")
end)
end
defp po_has_matching_pot?(po_file, pot_dir) do
domain = Path.basename(po_file, ".po")
pot_path = Path.join(pot_dir, "#{domain}.pot")
File.exists?(pot_path)
end
defp ensure_file_exists!(path) do
unless File.regular?(path), do: Mix.raise("No such file: #{path}")
end
defp ensure_dir_exists!(path) do
unless File.dir?(path), do: Mix.raise("No such directory: #{path}")
end
defp create_missing_locale_dir(dir) do
unless File.dir?(dir) do
File.mkdir_p!(dir)
Mix.shell().info("Created directory #{dir}")
end
end
defp validate_merging_opts!(opts, gettext_config) do
opts =
opts
|> Keyword.take([
:fuzzy,
:fuzzy_threshold,
:plural_forms_header,
:on_obsolete,
:store_previous_message_on_fuzzy_match
])
|> Keyword.put_new(:store_previous_message_on_fuzzy_match, false)
|> Keyword.put_new(:fuzzy, true)
|> Keyword.put_new_lazy(:fuzzy_threshold, fn ->
gettext_config[:fuzzy_threshold] || @default_fuzzy_threshold
end)
|> Keyword.update(:on_obsolete, :delete, &cast_on_obsolete/1)
threshold = opts[:fuzzy_threshold]
unless threshold >= 0.0 and threshold <= 1.0 do
Mix.raise("The :fuzzy_threshold option must be a float >= 0.0 and <= 1.0")
end
opts
end
defp locale_from_path(path) do
parts = Path.split(path)
index = Enum.find_index(parts, &(&1 == "LC_MESSAGES"))
Enum.at(parts, index - 1)
end
defp format_stats(stats) do
pluralized = if stats.new == 1, do: "message", else: "messages"
"#{stats.new} new #{pluralized}, #{stats.removed} removed, " <>
"#{stats.exact_matches} unchanged, #{stats.fuzzy_matches} reworded (fuzzy), " <>
"#{stats.marked_as_obsolete} marked as obsolete"
end
defp cast_on_obsolete("delete" = _on_obsolete), do: :delete
defp cast_on_obsolete("mark_as_obsolete" = _on_obsolete), do: :mark_as_obsolete
defp cast_on_obsolete(on_obsolete) do
Mix.raise("""
An invalid value was provided for the option `on_obsolete`.
Value: #{inspect(on_obsolete)}
Valid Choices: "delete" / "mark_as_obsolete"
""")
end
end
