defmodule Nerves.Artifact.BuildRunners.Docker do
@moduledoc """
Produce an artifact for a package using Docker.
The Nerves Docker artifact build_runner will use docker to create the artifact
for the package. The output in Mix will be limited to the headlines from the
process and the full build log can be found in the file `build.log` located
root of the package path.
## Images
Docker containers will be created based off the image that is loaded.
By default, containers will use the default image
`nervesproject/nerves_system_br:latest`. Sometimes additional host tools
are required to build a package. Therefore, packages can provide their own
images by specifying it in the package config under `:build_runner_config`.
the file is specified as a tuple `{"path/to/Dockerfile", tag_name}`.
Example:
build_runner_config: [
docker: {"Dockerfile", "my_system:0.1.0"}
]
## Volumes and Cache
Nerves will mount several volumes to the container for use in building
the artifact.
Mounted from the host:
* `/nerves/env/<package.name>` - The package being built.
* `/nerves/env/platform` - The package platform package.
* `/nerves/host/artifacts` - The host artifact directory.
Nerves will also mount the host NERVES_DL_DIR to save downloaded assets the
build platform requires for producing the artifact.
This is mounted at `/nerves/dl`. This volume can significantly reduce build
times but has potential for corruption. If you suspect that your build is
failing due to a faulty downloaded cached data, you can manually mount
the offending container and remove the file from this location or delete the
entire directory.
Nerves uses a docker volume to attach the build files. The volume name is
defined as the package name and a unique id that is stored at
`ARTIFACT_DIR/.docker_id`. The build directory is mounted to the container at
`/nerves/build` and is configured as the current working directory.
## Cleanup
Periodically, you may want to destroy all unused volumes to clean up.
Please refer to the Docker documentation for more information on how to
do this.
When the build_runner is finished, the artifact is decompressed on the host at
the packages defined artifact directory.
"""
@behaviour Nerves.Artifact.BuildRunner
alias Nerves.Artifact
alias Nerves.Artifact.BuildRunners.Docker
import Docker.Utils
@version "~> 1.12 or ~> 1.12.0-rc2 or >= 17.0.0"
@working_dir "/nerves/build"
@doc """
Create an artifact for the package
Opts:
`make_args:` - Extra arguments to be passed to make.
For example:
You can configure the number of parallel jobs that buildroot
can use for execution. This is useful for situations where you may
have a machine with a lot of CPUs but not enough ram.
# mix.exs
defp nerves_package do
[
# ...
build_runner_opts: [make_args: ["PARALLEL_JOBS=8"]],
]
end
"""
@spec build(Nerves.Package.t(), Nerves.Package.t(), term) :: :ok
def build(pkg, _toolchain, opts) do
preflight(pkg)
{:ok, pid} = Nerves.Utils.Stream.start_link(file: build_log_path())
stream = IO.stream(pid, :line)
:ok = create_build(pkg, stream)
:ok = make(pkg, stream, opts)
Mix.shell().info("\n")
:ok = make_artifact(pkg, stream)
Mix.shell().info("\n")
{:ok, path} = copy_artifact(pkg, stream)
Mix.shell().info("\n")
_ = Nerves.Utils.Stream.stop(pid)
{:ok, path}
end
@spec archive(Nerves.Package.t(), Nerves.Package.t(), term) :: :ok
def archive(pkg, _toolchain, _opts) do
{:ok, pid} = Nerves.Utils.Stream.start_link(file: "archive.log")
stream = IO.stream(pid, :line)
make_artifact(pkg, stream)
copy_artifact(pkg, stream)
end
def clean(pkg) do
Docker.Volume.name(pkg)
|> Docker.Volume.delete()
Artifact.Cache.path(pkg)
|> File.rm_rf()
end
@doc """
Connect to a system configuration shell in a Docker container
"""
@spec system_shell(Nerves.Package.t()) :: :ok
def system_shell(pkg) do
preflight(pkg)
{_, image} = config(pkg)
platform_config = pkg.config[:platform_config][:defconfig]
defconfig = Path.join("/nerves/env/#{pkg.app}", platform_config)
initial_input = [
"echo Updating build directory.",
"echo This will take a while if it is the first time...",
"/nerves/env/platform/create-build.sh #{defconfig} #{@working_dir} >/dev/null"
]
mounts = Enum.join(mounts(pkg), " ")
ssh_agent = Enum.join(ssh_agent(), " ")
env_vars = Enum.join(env(), " ")
cmd =
"docker run --rm -it -w #{@working_dir} #{env_vars} #{mounts} #{ssh_agent} #{image} /bin/bash"
set_volume_permissions(pkg)
Mix.Nerves.Shell.open(cmd, initial_input)
end
defp preflight(pkg) do
Docker.Volume.id(pkg) || Docker.Volume.create_id(pkg)
name = Docker.Volume.name(pkg)
_ = host_check()
_ = config_check(pkg, name)
name
end
# Build Commands
defp create_build(pkg, stream) do
platform_config = pkg.config[:platform_config][:defconfig]
defconfig = Path.join("/nerves/env/#{pkg.app}", platform_config)
cmd = ["/nerves/env/platform/create-build.sh", defconfig, @working_dir]
shell_info("Starting Build... (this may take a while)")
run(pkg, cmd, stream)
end
defp make(pkg, stream, opts) do
make_args = Keyword.get(opts, :make_args, [])
run(pkg, ["make" | make_args], stream)
end
defp make_artifact(pkg, stream) do
name = Artifact.download_name(pkg)
shell_info("Creating artifact archive")
cmd = ["make", "system", "NERVES_ARTIFACT_NAME=#{name}"]
run(pkg, cmd, stream)
end
defp copy_artifact(pkg, stream) do
shell_info("Copying artifact archive to host")
name = Artifact.download_name(pkg) <> Artifact.ext(pkg)
cmd = ["cp", name, "/nerves/dl/#{name}"]
run(pkg, cmd, stream)
path = Artifact.download_path(pkg)
{:ok, path}
end
# Helpers
defp run(pkg, cmd, stream) do
set_volume_permissions(pkg)
{_dockerfile, image} = config(pkg)
args =
[
"run",
"--rm",
"-w=#{@working_dir}",
"-a",
"stdout",
"-a",
"stderr"
] ++ env() ++ mounts(pkg) ++ ssh_agent() ++ [image | cmd]
case Mix.Nerves.Utils.shell("docker", args, stream: stream) do
{_result, 0} ->
:ok
{_result, _} ->
Mix.raise("""
The Nerves Docker build_runner encountered an error while building:
-----
#{end_of_build_log()}
-----
See #{build_log_path()}.
""")
end
end
defp set_volume_permissions(pkg) do
{_dockerfile, image} = config(pkg)
# (chown)
# Set the permissions of the build volume
# to match those of the host user:group.
# (--rm)
# Remove the container when finished.
args =
[
"run",
"--rm",
"-w=#{@working_dir}"
] ++ env(:root) ++ mounts(pkg) ++ [image | ["chown", "#{uid()}:#{gid()}", @working_dir]]
case Mix.Nerves.Utils.shell("docker", args) do
{_result, 0} ->
:ok
{result, _} ->
Mix.raise("""
The Nerves Docker build_runner encountered an error while setting permissions:
#{inspect(result)}
""")
end
end
defp env(), do: env(uid(), gid())
defp env(:root), do: env(0, 0)
defp env(uid, gid) do
["--env", "UID=#{uid}", "--env", "GID=#{gid}"]
end
defp uid() do
{uid, _} = Nerves.Port.cmd("id", ["-u"])
String.trim(uid)
end
defp gid() do
{gid, _} = Nerves.Port.cmd("id", ["-g"])
String.trim(gid)
end
defp end_of_build_log() do
{lines, _rc} = Nerves.Port.cmd("tail", ["-16", build_log_path()])
lines
end
defp build_log_path() do
File.cwd!()
|> Path.join("build.log")
end
defp mounts(pkg) do
build_paths = build_paths(pkg)
build_volume = Docker.Volume.name(pkg)
download_dir = Nerves.Env.download_dir() |> Path.expand()
mounts = ["--env", "NERVES_BR_DL_DIR=/nerves/dl"]
mounts =
Enum.reduce(build_paths, mounts, fn {_, host, target}, acc ->
["--mount", "type=bind,src=#{host},target=#{target}" | acc]
end)
mounts = ["--mount", "type=bind,src=#{download_dir},target=/nerves/dl" | mounts]
["--mount", "type=volume,src=#{build_volume},target=#{@working_dir}" | mounts]
end
defp ssh_agent() do
ssh_auth_sock = System.get_env("SSH_AUTH_SOCK")
["-v", "#{ssh_auth_sock}:/ssh-agent", "-e", "SSH_AUTH_SOCK=/ssh-agent"]
end
defp build_paths(pkg) do
system_br = Nerves.Env.package(:nerves_system_br)
[
{:platform, system_br.path, "/nerves/env/platform"},
{:package, pkg.path, "/nerves/env/#{pkg.app}"}
]
end
defp host_check() do
try do
case Nerves.Port.cmd("docker", ["--version"]) do
{result, 0} ->
<<"Docker version ", vsn::binary>> = result
{:ok, requirement} = Version.parse_requirement(@version)
{:ok, vsn} = parse_docker_version(vsn)
unless Version.match?(vsn, requirement) do
error_invalid_version(vsn)
end
:ok
_ ->
error_not_installed()
end
rescue
ErlangError -> error_not_installed()
end
end
defp config_check(pkg, name) do
{dockerfile, tag} = config(pkg)
# Check for the Build Volume
unless Docker.Volume.exists?(name) do
Docker.Volume.create(name)
end
unless Docker.Image.exists?(tag) do
Docker.Image.pull(tag)
unless Docker.Image.exists?(tag) do
Docker.Image.create(dockerfile, tag)
end
end
:ok
end
defp config(pkg) do
{dockerfile, tag} =
(pkg.config[:build_runner_config] || [])
|> Keyword.get(:docker, default_docker_config())
dockerfile =
dockerfile
|> Path.relative_to_cwd()
|> Path.expand()
{dockerfile, tag}
end
defp default_docker_config() do
[platform] = Nerves.Env.packages_by_type(:system_platform)
dockerfile = Path.join(platform.path, "support/docker/#{platform.app}")
tag = "nervesproject/#{platform.app}:#{platform.version}"
{dockerfile, tag}
end
defp error_not_installed do
Mix.raise("""
Docker is not installed on your machine.
Please install docker #{@version} or later
""")
end
defp error_invalid_version(vsn) do
Mix.raise("""
Your version of docker: #{vsn}
does not meet the requirements: #{@version}
""")
end
def parse_docker_version(vsn) do
[vsn | _] = String.split(vsn, ",", parts: 2)
Regex.replace(~r/(\.|^)0+(?=\d)/, vsn, "\\1")
|> Version.parse()
end
end
