| // Copyright 2017 The Cobalt Authors. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #ifndef STARBOARD_ANDROID_SHARED_PLAYER_COMPONENTS_FACTORY_H_ |
| #define STARBOARD_ANDROID_SHARED_PLAYER_COMPONENTS_FACTORY_H_ |
| |
| #include <string> |
| #include <vector> |
| |
| #include "starboard/android/shared/audio_decoder.h" |
| #include "starboard/android/shared/audio_track_audio_sink_type.h" |
| #include "starboard/android/shared/drm_system.h" |
| #include "starboard/android/shared/jni_env_ext.h" |
| #include "starboard/android/shared/jni_utils.h" |
| #include "starboard/android/shared/media_common.h" |
| #include "starboard/android/shared/video_decoder.h" |
| #include "starboard/atomic.h" |
| #include "starboard/common/log.h" |
| #include "starboard/common/media.h" |
| #include "starboard/common/ref_counted.h" |
| #include "starboard/common/scoped_ptr.h" |
| #include "starboard/media.h" |
| #include "starboard/shared/opus/opus_audio_decoder.h" |
| #include "starboard/shared/starboard/media/mime_type.h" |
| #include "starboard/shared/starboard/player/filter/adaptive_audio_decoder_internal.h" |
| #include "starboard/shared/starboard/player/filter/audio_decoder_internal.h" |
| #include "starboard/shared/starboard/player/filter/audio_renderer_sink.h" |
| #include "starboard/shared/starboard/player/filter/audio_renderer_sink_impl.h" |
| #include "starboard/shared/starboard/player/filter/player_components.h" |
| #include "starboard/shared/starboard/player/filter/video_decoder_internal.h" |
| #include "starboard/shared/starboard/player/filter/video_render_algorithm.h" |
| #include "starboard/shared/starboard/player/filter/video_render_algorithm_impl.h" |
| #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" |
| |
| namespace starboard { |
| namespace android { |
| namespace shared { |
| |
| using starboard::shared::starboard::media::MimeType; |
| |
| // On some platforms tunnel mode is only supported in the secure pipeline. Set |
| // the following variable to true to force creating a secure pipeline in tunnel |
| // mode, even for clear content. |
| // TODO: Allow this to be configured per playback at run time from the web app. |
| constexpr bool kForceSecurePipelineInTunnelModeWhenRequired = true; |
| |
| // This class allows us to force int16 sample type when tunnel mode is enabled. |
| class AudioRendererSinkAndroid : public ::starboard::shared::starboard::player:: |
| filter::AudioRendererSinkImpl { |
| public: |
| explicit AudioRendererSinkAndroid(bool enable_audio_routing, |
| int tunnel_mode_audio_session_id = -1) |
| : AudioRendererSinkImpl( |
| [=](SbTime start_media_time, |
| int channels, |
| int sampling_frequency_hz, |
| SbMediaAudioSampleType audio_sample_type, |
| SbMediaAudioFrameStorageType audio_frame_storage_type, |
| SbAudioSinkFrameBuffers frame_buffers, |
| int frame_buffers_size_in_frames, |
| SbAudioSinkUpdateSourceStatusFunc update_source_status_func, |
| SbAudioSinkPrivate::ConsumeFramesFunc consume_frames_func, |
| SbAudioSinkPrivate::ErrorFunc error_func, |
| void* context) { |
| |
| auto type = static_cast<AudioTrackAudioSinkType*>( |
| SbAudioSinkPrivate::GetPreferredType()); |
| |
| return type->Create( |
| channels, sampling_frequency_hz, audio_sample_type, |
| audio_frame_storage_type, frame_buffers, |
| frame_buffers_size_in_frames, update_source_status_func, |
| consume_frames_func, error_func, start_media_time, |
| tunnel_mode_audio_session_id, enable_audio_routing, context); |
| }) {} |
| |
| private: |
| bool IsAudioSampleTypeSupported( |
| SbMediaAudioSampleType audio_sample_type) const override { |
| // Currently the implementation only supports tunnel mode with int16 audio |
| // samples. |
| return audio_sample_type == kSbMediaAudioSampleTypeInt16Deprecated; |
| } |
| }; |
| |
| class AudioRendererSinkCallbackStub |
| : public starboard::shared::starboard::player::filter::AudioRendererSink:: |
| RenderCallback { |
| public: |
| bool error_occurred() const { return error_occurred_.load(); } |
| |
| private: |
| void GetSourceStatus(int* frames_in_buffer, |
| int* offset_in_frames, |
| bool* is_playing, |
| bool* is_eos_reached) override { |
| *frames_in_buffer = *offset_in_frames = 0; |
| *is_playing = true; |
| *is_eos_reached = false; |
| } |
| void ConsumeFrames(int frames_consumed, SbTime frames_consumed_at) override { |
| SB_DCHECK(frames_consumed == 0); |
| } |
| |
| void OnError(bool capability_changed) override { |
| error_occurred_.store(true); |
| } |
| |
| atomic_bool error_occurred_; |
| }; |
| |
| class PlayerComponentsFactory : public starboard::shared::starboard::player:: |
| filter::PlayerComponents::Factory { |
| typedef starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; |
| typedef starboard::shared::starboard::player::filter::AdaptiveAudioDecoder |
| AdaptiveAudioDecoder; |
| typedef starboard::shared::starboard::player::filter::AudioDecoder |
| AudioDecoderBase; |
| typedef starboard::shared::starboard::player::filter::AudioRendererSink |
| AudioRendererSink; |
| typedef starboard::shared::starboard::player::filter::AudioRendererSinkImpl |
| AudioRendererSinkImpl; |
| typedef starboard::shared::starboard::player::filter::VideoDecoder |
| VideoDecoderBase; |
| typedef starboard::shared::starboard::player::filter::VideoRenderAlgorithm |
| VideoRenderAlgorithmBase; |
| typedef starboard::shared::starboard::player::filter::VideoRendererSink |
| VideoRendererSink; |
| |
| const int kAudioSinkFramesAlignment = 256; |
| const int kDefaultAudioSinkMinFramesPerAppend = 1024; |
| const int kDefaultAudioSinkMaxCachedFrames = |
| 8 * kDefaultAudioSinkMinFramesPerAppend; |
| |
| virtual SbDrmSystem GetExtendedDrmSystem(SbDrmSystem drm_system) { |
| return drm_system; |
| } |
| |
| static int AlignUp(int value, int alignment) { |
| return (value + alignment - 1) / alignment * alignment; |
| } |
| |
| bool CreateSubComponents( |
| const CreationParameters& creation_parameters, |
| scoped_ptr<AudioDecoderBase>* audio_decoder, |
| scoped_ptr<AudioRendererSink>* audio_renderer_sink, |
| scoped_ptr<VideoDecoderBase>* video_decoder, |
| scoped_ptr<VideoRenderAlgorithmBase>* video_render_algorithm, |
| scoped_refptr<VideoRendererSink>* video_renderer_sink, |
| std::string* error_message) override { |
| SB_DCHECK(error_message); |
| |
| int tunnel_mode_audio_session_id = -1; |
| bool enable_tunnel_mode = false; |
| if (creation_parameters.audio_codec() != kSbMediaAudioCodecNone && |
| creation_parameters.video_codec() != kSbMediaVideoCodecNone) { |
| MimeType audio_mime_type(creation_parameters.audio_mime()); |
| MimeType video_mime_type(creation_parameters.video_mime()); |
| auto enable_tunnel_mode_audio_parameter_value = |
| audio_mime_type.GetParamStringValue("tunnelmode", ""); |
| auto enable_tunnel_mode_video_parameter_value = |
| video_mime_type.GetParamStringValue("tunnelmode", ""); |
| if (enable_tunnel_mode_audio_parameter_value == "true" && |
| enable_tunnel_mode_video_parameter_value == "true") { |
| enable_tunnel_mode = true; |
| } else { |
| if (enable_tunnel_mode_audio_parameter_value.empty()) { |
| enable_tunnel_mode_audio_parameter_value = "not provided"; |
| } |
| if (enable_tunnel_mode_video_parameter_value.empty()) { |
| enable_tunnel_mode_video_parameter_value = "not provided"; |
| } |
| SB_LOG(INFO) << "Tunnel mode is disabled. Audio mime parameter " |
| "\"tunnelmode\" value: " |
| << enable_tunnel_mode_audio_parameter_value |
| << ", video mime parameter \"tunnelmode\" value: " |
| << enable_tunnel_mode_video_parameter_value << "."; |
| } |
| } else { |
| SB_LOG(INFO) << "Tunnel mode requires both an audio and video stream. " |
| << "Audio codec: " |
| << GetMediaAudioCodecName(creation_parameters.audio_codec()) |
| << ", Video codec: " |
| << GetMediaVideoCodecName(creation_parameters.video_codec()) |
| << ". Tunnel mode is disabled."; |
| } |
| |
| bool force_secure_pipeline_under_tunnel_mode = false; |
| if (enable_tunnel_mode && |
| IsTunnelModeSupported(creation_parameters, |
| &force_secure_pipeline_under_tunnel_mode)) { |
| tunnel_mode_audio_session_id = |
| GenerateAudioSessionId(creation_parameters); |
| } |
| |
| if (tunnel_mode_audio_session_id == -1) { |
| SB_LOG(INFO) << "Create non-tunnel mode pipeline."; |
| } else { |
| SB_LOG(INFO) << "Create tunnel mode pipeline with audio session id " |
| << tunnel_mode_audio_session_id << '.'; |
| } |
| |
| if (creation_parameters.audio_codec() != kSbMediaAudioCodecNone) { |
| SB_DCHECK(audio_decoder); |
| SB_DCHECK(audio_renderer_sink); |
| |
| auto decoder_creator = [](const SbMediaAudioSampleInfo& audio_sample_info, |
| SbDrmSystem drm_system) { |
| if (audio_sample_info.codec == kSbMediaAudioCodecAac) { |
| scoped_ptr<AudioDecoder> audio_decoder_impl(new AudioDecoder( |
| audio_sample_info.codec, audio_sample_info, drm_system)); |
| if (audio_decoder_impl->is_valid()) { |
| return audio_decoder_impl.PassAs<AudioDecoderBase>(); |
| } |
| } else if (audio_sample_info.codec == kSbMediaAudioCodecOpus) { |
| scoped_ptr<OpusAudioDecoder> audio_decoder_impl( |
| new OpusAudioDecoder(audio_sample_info)); |
| if (audio_decoder_impl->is_valid()) { |
| return audio_decoder_impl.PassAs<AudioDecoderBase>(); |
| } |
| } else { |
| SB_NOTREACHED(); |
| } |
| return scoped_ptr<AudioDecoderBase>(); |
| }; |
| |
| audio_decoder->reset(new AdaptiveAudioDecoder( |
| creation_parameters.audio_sample_info(), |
| GetExtendedDrmSystem(creation_parameters.drm_system()), |
| decoder_creator)); |
| bool enable_audio_routing = true; |
| MimeType audio_mime_type(creation_parameters.audio_mime()); |
| auto enable_audio_routing_parameter_value = |
| audio_mime_type.GetParamStringValue("enableaudiorouting", ""); |
| if (enable_audio_routing_parameter_value.empty() || |
| enable_audio_routing_parameter_value == "true") { |
| SB_LOG(INFO) << "AudioRouting is enabled."; |
| } else { |
| enable_audio_routing = false; |
| SB_LOG(INFO) << "Mime attribute \"enableaudiorouting\" is set to: " |
| << enable_audio_routing_parameter_value |
| << ". AudioRouting is disabled."; |
| } |
| if (tunnel_mode_audio_session_id != -1) { |
| *audio_renderer_sink = TryToCreateTunnelModeAudioRendererSink( |
| tunnel_mode_audio_session_id, creation_parameters, |
| enable_audio_routing); |
| if (!*audio_renderer_sink) { |
| tunnel_mode_audio_session_id = -1; |
| } |
| } |
| if (!*audio_renderer_sink) { |
| audio_renderer_sink->reset( |
| new AudioRendererSinkAndroid(enable_audio_routing)); |
| } |
| } |
| |
| if (creation_parameters.video_codec() != kSbMediaVideoCodecNone) { |
| SB_DCHECK(video_decoder); |
| SB_DCHECK(video_render_algorithm); |
| SB_DCHECK(video_renderer_sink); |
| SB_DCHECK(error_message); |
| |
| if (tunnel_mode_audio_session_id == -1) { |
| force_secure_pipeline_under_tunnel_mode = false; |
| } |
| |
| scoped_ptr<VideoDecoder> video_decoder_impl(new VideoDecoder( |
| creation_parameters.video_codec(), |
| GetExtendedDrmSystem(creation_parameters.drm_system()), |
| creation_parameters.output_mode(), |
| creation_parameters.decode_target_graphics_context_provider(), |
| creation_parameters.max_video_capabilities(), |
| tunnel_mode_audio_session_id, force_secure_pipeline_under_tunnel_mode, |
| error_message)); |
| if (creation_parameters.video_codec() == kSbMediaVideoCodecAv1 || |
| video_decoder_impl->is_decoder_created()) { |
| *video_render_algorithm = video_decoder_impl->GetRenderAlgorithm(); |
| *video_renderer_sink = video_decoder_impl->GetSink(); |
| video_decoder->reset(video_decoder_impl.release()); |
| } else { |
| video_decoder->reset(); |
| *video_renderer_sink = NULL; |
| *error_message = |
| "Failed to create video decoder with error: " + *error_message; |
| return false; |
| } |
| } |
| |
| return true; |
| } |
| |
| void GetAudioRendererParams(const CreationParameters& creation_parameters, |
| int* max_cached_frames, |
| int* min_frames_per_append) const override { |
| SB_DCHECK(max_cached_frames); |
| SB_DCHECK(min_frames_per_append); |
| SB_DCHECK(kDefaultAudioSinkMinFramesPerAppend % kAudioSinkFramesAlignment == |
| 0); |
| *min_frames_per_append = kDefaultAudioSinkMinFramesPerAppend; |
| |
| // AudioRenderer prefers to use kSbMediaAudioSampleTypeFloat32 and only uses |
| // kSbMediaAudioSampleTypeInt16Deprecated when float32 is not supported. |
| int min_frames_required = SbAudioSinkGetMinBufferSizeInFrames( |
| creation_parameters.audio_sample_info().number_of_channels, |
| SbAudioSinkIsAudioSampleTypeSupported(kSbMediaAudioSampleTypeFloat32) |
| ? kSbMediaAudioSampleTypeFloat32 |
| : kSbMediaAudioSampleTypeInt16Deprecated, |
| creation_parameters.audio_sample_info().samples_per_second); |
| // On Android 5.0, the size of audio renderer sink buffer need to be two |
| // times larger than AudioTrack minBufferSize. Otherwise, AudioTrack may |
| // stop working after pause. |
| *max_cached_frames = |
| min_frames_required * 2 + kDefaultAudioSinkMinFramesPerAppend; |
| *max_cached_frames = AlignUp(*max_cached_frames, kAudioSinkFramesAlignment); |
| } |
| |
| bool IsTunnelModeSupported(const CreationParameters& creation_parameters, |
| bool* force_secure_pipeline_under_tunnel_mode) { |
| SB_DCHECK(force_secure_pipeline_under_tunnel_mode); |
| *force_secure_pipeline_under_tunnel_mode = false; |
| |
| if (!SbAudioSinkIsAudioSampleTypeSupported( |
| kSbMediaAudioSampleTypeInt16Deprecated)) { |
| SB_LOG(INFO) << "Disable tunnel mode because int16 sample is required " |
| "but not supported."; |
| return false; |
| } |
| |
| if (creation_parameters.output_mode() != kSbPlayerOutputModePunchOut) { |
| SB_LOG(INFO) |
| << "Disable tunnel mode because output mode is not punchout."; |
| return false; |
| } |
| |
| if (creation_parameters.audio_codec() == kSbMediaAudioCodecNone) { |
| SB_LOG(INFO) << "Disable tunnel mode because audio codec is none."; |
| return false; |
| } |
| |
| if (creation_parameters.video_codec() == kSbMediaVideoCodecNone) { |
| SB_LOG(INFO) << "Disable tunnel mode because video codec is none."; |
| return false; |
| } |
| |
| const char* mime = |
| SupportedVideoCodecToMimeType(creation_parameters.video_codec()); |
| if (!mime) { |
| SB_LOG(INFO) << "Disable tunnel mode because " |
| << creation_parameters.video_codec() << " is not supported."; |
| return false; |
| } |
| JniEnvExt* env = JniEnvExt::Get(); |
| ScopedLocalJavaRef<jstring> j_mime(env->NewStringStandardUTFOrAbort(mime)); |
| DrmSystem* drm_system_ptr = |
| static_cast<DrmSystem*>(creation_parameters.drm_system()); |
| jobject j_media_crypto = |
| drm_system_ptr ? drm_system_ptr->GetMediaCrypto() : NULL; |
| |
| bool is_encrypted = !!j_media_crypto; |
| if (env->CallStaticBooleanMethodOrAbort( |
| "dev/cobalt/media/MediaCodecUtil", "hasVideoDecoderFor", |
| "(Ljava/lang/String;ZIIIIZZ)Z", j_mime.Get(), is_encrypted, 0, 0, 0, |
| 0, false, true) == JNI_TRUE) { |
| return true; |
| } |
| |
| if (kForceSecurePipelineInTunnelModeWhenRequired && !is_encrypted) { |
| const bool kIsEncrypted = true; |
| auto support_tunnel_mode_under_secure_pipeline = |
| env->CallStaticBooleanMethodOrAbort( |
| "dev/cobalt/media/MediaCodecUtil", "hasVideoDecoderFor", |
| "(Ljava/lang/String;ZIIIIZZ)Z", j_mime.Get(), kIsEncrypted, 0, 0, |
| 0, 0, false, true) == JNI_TRUE; |
| if (support_tunnel_mode_under_secure_pipeline) { |
| *force_secure_pipeline_under_tunnel_mode = true; |
| return true; |
| } |
| } |
| |
| SB_LOG(INFO) << "Disable tunnel mode because no tunneled decoder for " |
| << mime << '.'; |
| return false; |
| } |
| |
| int GenerateAudioSessionId(const CreationParameters& creation_parameters) { |
| bool force_secure_pipeline_under_tunnel_mode = false; |
| SB_DCHECK(IsTunnelModeSupported(creation_parameters, |
| &force_secure_pipeline_under_tunnel_mode)); |
| |
| JniEnvExt* env = JniEnvExt::Get(); |
| ScopedLocalJavaRef<jobject> j_audio_output_manager( |
| env->CallStarboardObjectMethodOrAbort( |
| "getAudioOutputManager", |
| "()Ldev/cobalt/media/AudioOutputManager;")); |
| int tunnel_mode_audio_session_id = env->CallIntMethodOrAbort( |
| j_audio_output_manager.Get(), "generateTunnelModeAudioSessionId", |
| "(I)I", creation_parameters.audio_sample_info().number_of_channels); |
| |
| // AudioManager.generateAudioSessionId() return ERROR (-1) to indicate a |
| // failure, please see the following url for more details: |
| // https://developer.android.com/reference/android/media/AudioManager#generateAudioSessionId() |
| SB_LOG_IF(WARNING, tunnel_mode_audio_session_id == -1) |
| << "Failed to generate audio session id for tunnel mode."; |
| |
| return tunnel_mode_audio_session_id; |
| } |
| |
| scoped_ptr<AudioRendererSink> TryToCreateTunnelModeAudioRendererSink( |
| int tunnel_mode_audio_session_id, |
| const CreationParameters& creation_parameters, |
| bool enable_audio_routing) { |
| scoped_ptr<AudioRendererSink> audio_sink(new AudioRendererSinkAndroid( |
| enable_audio_routing, tunnel_mode_audio_session_id)); |
| // We need to double check if the audio sink can actually be created. |
| int max_cached_frames, min_frames_per_append; |
| GetAudioRendererParams(creation_parameters, &max_cached_frames, |
| &min_frames_per_append); |
| AudioRendererSinkCallbackStub callback_stub; |
| std::vector<uint16_t> frame_buffer( |
| max_cached_frames * |
| creation_parameters.audio_sample_info().number_of_channels); |
| uint16_t* frame_buffers[] = {frame_buffer.data()}; |
| audio_sink->Start( |
| 0, creation_parameters.audio_sample_info().number_of_channels, |
| creation_parameters.audio_sample_info().samples_per_second, |
| kSbMediaAudioSampleTypeInt16Deprecated, |
| kSbMediaAudioFrameStorageTypeInterleaved, |
| reinterpret_cast<SbAudioSinkFrameBuffers>(frame_buffers), |
| max_cached_frames, &callback_stub); |
| if (audio_sink->HasStarted() && !callback_stub.error_occurred()) { |
| audio_sink->Stop(); |
| return audio_sink.Pass(); |
| } |
| SB_LOG(WARNING) |
| << "AudioTrack does not support tunnel mode with sample rate:" |
| << creation_parameters.audio_sample_info().samples_per_second |
| << ", channels:" |
| << creation_parameters.audio_sample_info().number_of_channels |
| << ", audio format:" << creation_parameters.audio_codec() |
| << ", and audio buffer frames:" << max_cached_frames; |
| return scoped_ptr<AudioRendererSink>(); |
| } |
| }; |
| |
| } // namespace shared |
| } // namespace android |
| } // namespace starboard |
| |
| #endif // STARBOARD_ANDROID_SHARED_PLAYER_COMPONENTS_FACTORY_H_ |
