blob: 69ae90a37b55a04b253623d8aea59597bdb08c20 [file] [log] [blame]
// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <map>
#include <memory>
#include "base/callback_forward.h"
#include "base/macros.h"
#include "base/time/time.h"
#include "media/base/media_export.h"
#include "media/base/media_log.h"
#include "media/base/stream_parser.h"
#include "media/filters/chunk_demuxer.h"
#include "media/filters/source_buffer_parse_warnings.h"
namespace media {
class MseTrackBuffer;
// Helper class that implements Media Source Extension's coded frame processing
// algorithm.
class MEDIA_EXPORT FrameProcessor {
using UpdateDurationCB = base::RepeatingCallback<void(base::TimeDelta)>;
FrameProcessor(UpdateDurationCB update_duration_cb, MediaLog* media_log);
FrameProcessor(const FrameProcessor&) = delete;
FrameProcessor& operator=(const FrameProcessor&) = delete;
// This must be called exactly once, before doing any track buffer creation or
// frame processing.
void SetParseWarningCallback(SourceBufferParseWarningCB parse_warning_cb);
// Get/set the current append mode, which if true means "sequence" and if
// false means "segments".
// See
bool sequence_mode() { return sequence_mode_; }
void SetSequenceMode(bool sequence_mode);
// Processes buffers in |buffer_queue_map|.
// Returns true on success or false on failure which indicates decode error.
// |append_window_start| and |append_window_end| correspond to the MSE spec's
// similarly named source buffer attributes that are used in coded frame
// processing.
// Uses |*timestamp_offset| according to the coded frame processing algorithm,
// including updating it as required in 'sequence' mode frame processing.
bool ProcessFrames(const StreamParser::BufferQueueMap& buffer_queue_map,
base::TimeDelta append_window_start,
base::TimeDelta append_window_end,
base::TimeDelta* timestamp_offset);
// Signals the frame processor to update its group start timestamp to be
// |timestamp_offset| if it is in sequence append mode.
void SetGroupStartTimestampIfInSequenceMode(base::TimeDelta timestamp_offset);
// Adds a new track with unique track ID |id|.
// If |id| has previously been added, returns false to indicate error.
// Otherwise, returns true, indicating future ProcessFrames() will emit
// frames for the track |id| to |stream|.
bool AddTrack(StreamParser::TrackId id, ChunkDemuxerStream* stream);
// A map that describes how track ids changed between init segment. Maps the
// old track id for a new track id for the same track.
using TrackIdChanges = std::map<StreamParser::TrackId, StreamParser::TrackId>;
// Updates the internal mapping of TrackIds to track buffers. The input
// parameter |track_id_changes| maps old track ids to new ones. The track ids
// not present in the map must be assumed unchanged. Returns false if
// remapping failed.
bool UpdateTrackIds(const TrackIdChanges& track_id_changes);
// Sets the need random access point flag on all track buffers to true.
void SetAllTrackBuffersNeedRandomAccessPoint();
// Resets state for the coded frame processing algorithm as described in steps
// 2-5 of the MSE Reset Parser State algorithm described at
void Reset();
// Must be called when the audio config is updated. Used to manage when
// the preroll buffer is cleared and the allowed "fudge" factor between
// preroll buffers.
void OnPossibleAudioConfigUpdate(const AudioDecoderConfig& config);
friend class FrameProcessorTest;
// If |track_buffers_| contains |id|, returns a pointer to the associated
// MseTrackBuffer. Otherwise, returns NULL.
MseTrackBuffer* FindTrack(StreamParser::TrackId id);
// Signals all track buffers' streams that a coded frame group is starting
// with |start_dts| and |start_pts|.
void NotifyStartOfCodedFrameGroup(DecodeTimestamp start_dts,
base::TimeDelta start_pts);
// Helper that signals each track buffer to append any processed, but not yet
// appended, frames to its stream. Returns true on success, or false if one or
// more of the appends failed.
bool FlushProcessedFrames();
// Handles partial append window trimming of |buffer|. Returns true if the
// given |buffer| can be partially trimmed or have preroll added; otherwise,
// returns false.
// If |buffer| overlaps |append_window_start|, the portion of |buffer| before
// |append_window_start| will be marked for post-decode discard. Further, if
// |audio_preroll_buffer_| exists and abuts |buffer|, it will be set as
// preroll on |buffer| and |audio_preroll_buffer_| will be cleared. If the
// preroll buffer does not abut |buffer|, it will be discarded unused.
// Likewise, if |buffer| overlaps |append_window_end|, the portion of |buffer|
// after |append_window_end| will be marked for post-decode discard.
// If |buffer| lies entirely before |append_window_start|, and thus would
// normally be discarded, |audio_preroll_buffer_| will be updated and the
// method will return false. In this case, the updated preroll will be
// |buffer| iff |buffer| is a keyframe, otherwise the preroll will be cleared.
bool HandlePartialAppendWindowTrimming(
base::TimeDelta append_window_start,
base::TimeDelta append_window_end,
scoped_refptr<StreamParserBuffer> buffer);
// Enables rejection of audio frame streams with nonkeyframe timestamps that
// do not monotonically increase since the last keyframe. Returns true if
// |frame| appears to be in order, false if |frame|'s order is not supported.
// |track_needs_random_access_point| should be the corresponding value for the
// frame's track buffer. This helper should only be called when
// |has_dependent_audio_frames_| is true, and only for an audio |frame|. This
// method also uses and updates
// |last_audio_pts_for_nonkeyframe_monotonicity_check_|.
bool CheckAudioPresentationOrder(const StreamParserBuffer& frame,
bool track_needs_random_access_point);
// Helper that processes one frame with the coded frame processing algorithm.
// Returns false on error or true on success.
bool ProcessFrame(scoped_refptr<StreamParserBuffer> frame,
base::TimeDelta append_window_start,
base::TimeDelta append_window_end,
base::TimeDelta* timestamp_offset);
// TrackId-indexed map of each track's stream.
using TrackBuffersMap =
std::map<StreamParser::TrackId, std::unique_ptr<MseTrackBuffer>>;
TrackBuffersMap track_buffers_;
// The last audio buffer seen by the frame processor that was removed because
// it was entirely before the start of the append window.
scoped_refptr<StreamParserBuffer> audio_preroll_buffer_;
// The AudioDecoderConfig associated with buffers handed to ProcessFrames().
// TODO(wolenetz): Associate current audio config and the derived
// |has_dependent_audio_frames_|, |sample_duration_| and
// |last_audio_pts_for_nonkeyframe_monotonicity_check_| with MseTrackBuffer
// instead to enable handling more than 1 audio track in a SourceBuffer
// simultaneously. See
AudioDecoderConfig current_audio_config_;
bool has_dependent_audio_frames_ = false;
base::TimeDelta sample_duration_;
// When |has_dependent_audio_frames_| is true, holds the PTS of the last
// successfully processed audio frame. If the next audio frame is not a
// keyframe and has lower PTS, the stream is invalid. Currently, the only
// supported audio streams that could contain nonkeyframes are in-order (PTS
// increases monotonically since last keyframe), e.g. xHE-AAC.
base::TimeDelta last_audio_pts_for_nonkeyframe_monotonicity_check_ =
// The AppendMode of the associated SourceBuffer.
// See SetSequenceMode() for interpretation of |sequence_mode_|.
// Per
// Controls how a sequence of media segments are handled. This is initially
// set to false ("segments").
bool sequence_mode_ = false;
// Tracks whether or not we need to notify all track buffers of a new coded
// frame group (see
// upon the next successfully processed frame. Set true initially and upon
// detection of DTS discontinuity, parser reset during 'segments' mode, or
// switching from 'sequence' to 'segments' mode. Individual track buffers can
// also be notified of an updated coded frame group start in edge cases. See
// further comments in ProcessFrame().
bool pending_notify_all_group_start_ = true;
// Tracks the MSE coded frame processing variable of same name.
// Initially kNoTimestamp, meaning "unset".
base::TimeDelta group_start_timestamp_;
// Tracks the MSE coded frame processing variable of same name. It stores the
// highest coded frame end timestamp across all coded frames in the current
// coded frame group. It is set to 0 when the SourceBuffer object is created
// and gets updated by ProcessFrames().
base::TimeDelta group_end_timestamp_;
const UpdateDurationCB update_duration_cb_;
// MediaLog for reporting messages and properties to debug content and engine.
MediaLog* media_log_;
// Callback for reporting problematic conditions that are not necessarily
// errors.
SourceBufferParseWarningCB parse_warning_cb_;
// Counters that limit spam to |media_log_| for frame processor warnings.
int num_dropped_preroll_warnings_ = 0;
int num_audio_non_keyframe_warnings_ = 0;
int num_muxed_sequence_mode_warnings_ = 0;
int num_skipped_empty_frame_warnings_ = 0;
int num_partial_discard_warnings_ = 0;
int num_dropped_frame_warnings_ = 0;
} // namespace media