defmodule Membrane.RTP.OutboundTrackingSerializer do
@moduledoc """
Tracks statistics of outbound packets.
Besides tracking statistics, tracker can also serialize packet's header and payload stored inside an incoming buffer
into a proper RTP packet. When encountering header extensions, it remaps its identifiers from locally used extension
names to integer values expected by the receiver.
"""
use Membrane.Filter
require Membrane.Logger
require Membrane.TelemetryMetrics
alias Membrane.RTCP.TransportFeedbackPacket.NACK
alias Membrane.{Buffer, Payload, RemoteStream, RTCP, RTCPEvent, RTP, Time}
alias Membrane.RTCP.FeedbackPacket.{FIR, PLI}
alias Membrane.RTCP.TransportFeedbackPacket.NACK
alias Membrane.RTP.Session.SenderReport
def_input_pad :input, accepted_format: RTP, demand_mode: :auto
def_output_pad :output,
accepted_format: %RemoteStream{type: :packetized, content_format: RTP},
demand_mode: :auto
def_input_pad :rtcp_input,
availability: :on_request,
accepted_format: _any,
demand_mode: :auto
def_output_pad :rtcp_output,
availability: :on_request,
accepted_format: %RemoteStream{type: :packetized, content_format: RTCP},
demand_mode: :auto
def_options ssrc: [spec: RTP.ssrc_t()],
payload_type: [spec: RTP.payload_type_t()],
clock_rate: [spec: RTP.clock_rate_t()],
extension_mapping: [spec: RTP.SessionBin.rtp_extension_mapping_t()],
telemetry_label: [
spec: Membrane.TelemetryMetrics.label(),
default: []
]
@packet_sent_telemetry_event [Membrane.RTP, :packet, :sent]
@marker_sent_telemetry_event [Membrane.RTP, :rtp, :marker_sent]
@rtcp_sent_telemetry_event [Membrane.RTP, :rtcp, :sent]
@nack_received_telemetry_event [Membrane.RTP, :rtcp, :nack, :arrival]
@pli_received_telemetry_event [Membrane.RTP, :rtcp, :pli, :arrival]
@fir_received_telemetry_event [Membrane.RTP, :rtcp, :fir, :arrival]
@sender_report_sent_telemetry_event [Membrane.RTP, :rtcp, :sender_report, :sent]
@telemetry_events [
@packet_sent_telemetry_event,
@marker_sent_telemetry_event,
@rtcp_sent_telemetry_event,
@nack_received_telemetry_event,
@pli_received_telemetry_event,
@fir_received_telemetry_event,
@sender_report_sent_telemetry_event
]
defmodule State do
@moduledoc false
use Bunch.Access
alias Membrane.RTP
@type t :: %__MODULE__{
ssrc: RTP.ssrc_t(),
payload_type: RTP.payload_type_t(),
extension_mapping: RTP.SessionBin.rtp_extension_mapping_t(),
any_buffer_sent?: boolean(),
rtcp_output_pad: Membrane.Pad.ref_t() | nil,
stats_acc: %{}
}
defstruct ssrc: 0,
payload_type: 0,
extension_mapping: %{},
any_buffer_sent?: false,
rtcp_output_pad: nil,
stats_acc: %{
clock_rate: 0,
timestamp: 0,
rtp_timestamp: 0,
sender_packet_count: 0,
sender_octet_count: 0
}
end
@impl true
def handle_init(_ctx, options) do
for event <- @telemetry_events do
Membrane.TelemetryMetrics.register(event, options.telemetry_label)
end
state =
%State{}
|> put_in([:stats_acc, :clock_rate], options.clock_rate)
|> Map.merge(options |> Map.from_struct() |> Map.drop([:clock_rate]))
{[], state}
end
@impl true
def handle_pad_added(Pad.ref(:rtcp_input, _id), _ctx, state) do
{[], state}
end
@impl true
def handle_pad_added(
Pad.ref(:rtcp_output, _id) = pad,
%{playback: :playing},
%{rtcp_output_pad: nil} = state
) do
stream_format = %RemoteStream{type: :packetized, content_format: RTCP}
{[stream_format: {pad, stream_format}], %{state | rtcp_output_pad: pad}}
end
@impl true
def handle_pad_added(Pad.ref(:rtcp_output, _id) = pad, _ctx, %{rtcp_output_pad: nil} = state) do
{[], %{state | rtcp_output_pad: pad}}
end
@impl true
def handle_pad_added(Pad.ref(:rtcp_output, _id), _ctx, _state) do
raise "rtcp_output pad can get linked just once"
end
@impl true
def handle_stream_format(:input, _stream_format, _ctx, state) do
stream_format = %RemoteStream{type: :packetized, content_format: RTP}
{[stream_format: {:output, stream_format}], state}
end
@impl true
def handle_stream_format(_pad, _stream_format, _ctx, state) do
{[], state}
end
@impl true
def handle_event(
Pad.ref(:rtcp_input, _id),
%RTCPEvent{rtcp: %{payload: %PLI{}}},
_ctx,
state
) do
Membrane.TelemetryMetrics.execute(
@pli_received_telemetry_event,
%{},
%{},
state.telemetry_label
)
# PLI or FIR reaching OutboundTrackingSerializer means the receiving peer sent it
# We need to pass it to the sending peer's RTCP.Receiver (in StreamReceiveBin) to get translated again into FIR/PLI with proper SSRCs
# and then sent to the sender. So the KeyframeRequestEvent, like salmon, starts an upstream journey here trying to reach that peer.
{[event: {:input, %Membrane.KeyframeRequestEvent{}}], state}
end
@impl true
def handle_event(
Pad.ref(:rtcp_input, _id),
%RTCPEvent{rtcp: %{payload: %FIR{}}},
_ctx,
state
) do
Membrane.TelemetryMetrics.execute(
@fir_received_telemetry_event,
%{},
%{},
state.telemetry_label
)
{[event: {:input, %Membrane.KeyframeRequestEvent{}}], state}
end
@impl true
def handle_event(
Pad.ref(:rtcp_input, _id),
%RTCPEvent{rtcp: %{payload: %NACK{lost_packet_ids: ids}}},
_ctx,
state
) do
Membrane.TelemetryMetrics.execute(
@nack_received_telemetry_event,
%{},
%{},
state.telemetry_label
)
# The OutboundRetransmissionController is behind encryptor, so we need to send the event downstream to reach it
{[event: {:output, %Membrane.RTP.RetransmissionRequestEvent{packet_ids: ids}}], state}
end
@impl true
def handle_event(pad, event, ctx, state) do
super(pad, event, ctx, state)
end
@impl true
def handle_playing(_ctx, state) do
if state.rtcp_output_pad do
stream_format = %RemoteStream{type: :packetized, content_format: RTCP}
{[stream_format: {state.rtcp_output_pad, stream_format}], state}
else
{[], state}
end
end
@impl true
def handle_process(:input, %Buffer{} = buffer, _ctx, state) do
state = update_stats(buffer, state)
%{rtp: rtp_metadata} = buffer.metadata
supported_extensions = Map.keys(state.extension_mapping)
extensions =
rtp_metadata.extensions
|> Enum.filter(fn extension -> extension.identifier in supported_extensions end)
|> Enum.map(fn extension ->
%{extension | identifier: Map.fetch!(state.extension_mapping, extension.identifier)}
end)
header =
struct(RTP.Header, %{
rtp_metadata
| ssrc: state.ssrc,
payload_type: state.payload_type,
extensions: extensions
})
if header.marker do
Membrane.TelemetryMetrics.execute(
@marker_sent_telemetry_event,
%{},
%{},
state.telemetry_label
)
end
padding_size = Map.get(rtp_metadata, :padding_size, 0)
payload =
RTP.Packet.serialize(%RTP.Packet{header: header, payload: buffer.payload},
padding_size: padding_size
)
buffer = %Buffer{buffer | payload: payload}
Membrane.TelemetryMetrics.execute(
@packet_sent_telemetry_event,
%{bytes: byte_size(buffer.payload)},
%{},
state.telemetry_label
)
{[buffer: {:output, buffer}], %{state | any_buffer_sent?: true}}
end
@impl true
def handle_parent_notification(:send_stats, ctx, state) do
%{rtcp_output_pad: rtcp_output} = state
if rtcp_output && not ctx.pads[rtcp_output].end_of_stream? do
stats = get_stats(state)
actions =
%{state.ssrc => stats}
|> SenderReport.generate_report()
|> Enum.map(&Membrane.RTCP.Packet.serialize(&1))
|> Enum.map(&{:buffer, {rtcp_output, %Membrane.Buffer{payload: &1}}})
for _buffer_action <- actions do
Membrane.TelemetryMetrics.execute(
@rtcp_sent_telemetry_event,
%{},
%{},
state.telemetry_label
)
Membrane.TelemetryMetrics.execute(
@sender_report_sent_telemetry_event,
%{},
%{},
state.telemetry_label
)
end
{actions, %{state | any_buffer_sent?: false}}
else
{[], state}
end
end
defp get_stats(%State{any_buffer_sent?: false}), do: :no_stats
defp get_stats(%State{stats_acc: stats}), do: stats
defp update_stats(%Buffer{payload: payload, metadata: metadata}, state) do
%{
sender_octet_count: octet_count,
sender_packet_count: packet_count
} = state.stats_acc
updated_stats = %{
clock_rate: state.stats_acc.clock_rate,
sender_octet_count: octet_count + Payload.size(payload),
sender_packet_count: packet_count + 1,
timestamp: Time.vm_time(),
rtp_timestamp: metadata.rtp.timestamp
}
Map.put(state, :stats_acc, updated_stats)
end
end
