media/gpu/video_decode_accelerator_perf_tests.cc - cobalt - Git at Google

 // Copyright 2019 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include <algorithm>
 #include <numeric>
 #include <vector>

 #include "base/command_line.h"
 #include "base/files/file_util.h"
 #include "base/json/json_writer.h"
 #include "build/build_config.h"
 #include "media/base/test_data_util.h"
 #include "media/gpu/test/video.h"
 #include "media/gpu/test/video_player/frame_renderer_dummy.h"
 #include "media/gpu/test/video_player/video_decoder_client.h"
 #include "media/gpu/test/video_player/video_player.h"
 #include "media/gpu/test/video_player/video_player_test_environment.h"
 #include "sandbox/linux/services/resource_limits.h"
 #include "testing/gtest/include/gtest/gtest.h"

 namespace media {
 namespace test {

 namespace {

 // Video decoder perf tests usage message. Make sure to also update the
 // documentation under docs/media/gpu/video_decoder_perf_test_usage.md when
 // making changes here.
 constexpr const char* usage_msg =
     "usage: video_decode_accelerator_perf_tests\n"
     "           [-v=<level>] [--vmodule=<config>] [--output_folder]\n"
     "           ([--use-legacy]|[--use_vd]|[--use_vd_vda]) [--gtest_help]\n"
     "           [--help] [<video path>] [<video metadata path>]\n";

 // Video decoder perf tests help message.
 constexpr const char* help_msg =
     "Run the video decode accelerator performance tests on the video\n"
     "specified by <video path>. If no <video path> is given the default\n"
     "\"test-25fps.h264\" video will be used.\n"
     "\nThe <video metadata path> should specify the location of a json file\n"
     "containing the video's metadata, such as frame checksums. By default\n"
     "<video path>.json will be used.\n"
     "\nThe following arguments are supported:\n"
     "   -v                  enable verbose mode, e.g. -v=2.\n"
     "  --vmodule            enable verbose mode for the specified module,\n"
     "                       e.g. --vmodule=*media/gpu*=2.\n"
     "  --output_folder      overwrite the output folder used to store\n"
     "                       performance metrics, if not specified results\n"
     "                       will be stored in the current working directory.\n"
     "  --use-legacy         use the legacy VDA-based video decoders.\n"
     "  --use_vd             use the new VD-based video decoders.\n"
     "                       (enabled by default)\n"
     "  --use_vd_vda         use the new VD-based video decoders with a\n"
     "                       wrapper that translates to the VDA interface,\n"
     "                       used to test interaction with older components\n"
     "                       expecting the VDA interface.\n"
     "  --gtest_help         display the gtest help and exit.\n"
     "  --help               display this help and exit.\n";

 media::test::VideoPlayerTestEnvironment* g_env;

 // Default output folder used to store performance metrics.
 constexpr const base::FilePath::CharType* kDefaultOutputFolder =
     FILE_PATH_LITERAL("perf_metrics");

 // Struct storing various time-related statistics.
 struct PerformanceTimeStats {
   PerformanceTimeStats() {}
   explicit PerformanceTimeStats(const std::vector<double>& times);
   double avg_ms_ = 0.0;
   double percentile_25_ms_ = 0.0;
   double percentile_50_ms_ = 0.0;
   double percentile_75_ms_ = 0.0;
 };

 PerformanceTimeStats::PerformanceTimeStats(const std::vector<double>& times) {
   avg_ms_ = std::accumulate(times.begin(), times.end(), 0.0) / times.size();
   std::vector<double> sorted_times = times;
   std::sort(sorted_times.begin(), sorted_times.end());
   percentile_25_ms_ = sorted_times[sorted_times.size() / 4];
   percentile_50_ms_ = sorted_times[sorted_times.size() / 2];
   percentile_75_ms_ = sorted_times[(sorted_times.size() * 3) / 4];
 }

 struct PerformanceMetrics {
   // Total measurement duration.
   base::TimeDelta total_duration_;
   // The number of frames decoded.
   size_t frames_decoded_ = 0;
   // The overall number of frames decoded per second.
   double frames_per_second_ = 0.0;
   // The number of frames dropped because of the decoder running behind, only
   // relevant for capped performance tests.
   size_t frames_dropped_ = 0;
   // The percentage of frames dropped because of the decoder running behind,
   // only relevant for capped performance tests.
   double dropped_frame_percentage_ = 0.0;
   // Statistics about the time between subsequent frame deliveries.
   PerformanceTimeStats delivery_time_stats_;
   // Statistics about the time between decode start and frame deliveries.
   PerformanceTimeStats decode_time_stats_;
 };

 // The performance evaluator can be plugged into the video player to collect
 // various performance metrics.
 // TODO(dstaessens@) Check and post warning when CPU frequency scaling is
 // enabled as this affects test results.
 class PerformanceEvaluator : public VideoFrameProcessor {
  public:
   // Create a new performance evaluator. The caller should makes sure
   // |frame_renderer| outlives the performance evaluator.
   explicit PerformanceEvaluator(const FrameRendererDummy* const frame_renderer)
       : frame_renderer_(frame_renderer) {}

   // Interface VideoFrameProcessor
   void ProcessVideoFrame(scoped_refptr<const VideoFrame> video_frame,
                          size_t frame_index) override;
   bool WaitUntilDone() override { return true; }

   // Start/Stop collecting performance metrics.
   void StartMeasuring();
   void StopMeasuring();

   // Write the collected performance metrics to file.
   void WriteMetricsToFile() const;

  private:
   // Start/end time of the measurement period.
   base::TimeTicks start_time_;
   base::TimeTicks end_time_;

   // Time at which the previous frame was delivered.
   base::TimeTicks prev_frame_delivery_time_;
   // List of times between subsequent frame deliveries.
   std::vector<double> frame_delivery_times_;
   // List of times between decode start and frame delivery.
   std::vector<double> frame_decode_times_;

   // Collection of various performance metrics.
   PerformanceMetrics perf_metrics_;

   // Frame renderer used to get the dropped frame rate, owned by the creator of
   // the performance evaluator.
   const FrameRendererDummy* const frame_renderer_;
 };

 void PerformanceEvaluator::ProcessVideoFrame(
     scoped_refptr<const VideoFrame> video_frame,
     size_t frame_index) {
   base::TimeTicks now = base::TimeTicks::Now();

   base::TimeDelta delivery_time = (now - prev_frame_delivery_time_);
   frame_delivery_times_.push_back(delivery_time.InMillisecondsF());
   prev_frame_delivery_time_ = now;

   base::TimeDelta decode_time = now.since_origin() - video_frame->timestamp();
   frame_decode_times_.push_back(decode_time.InMillisecondsF());

   perf_metrics_.frames_decoded_++;
 }

 void PerformanceEvaluator::StartMeasuring() {
   start_time_ = base::TimeTicks::Now();
   prev_frame_delivery_time_ = start_time_;
 }

 void PerformanceEvaluator::StopMeasuring() {
   end_time_ = base::TimeTicks::Now();
   perf_metrics_.total_duration_ = end_time_ - start_time_;
   perf_metrics_.frames_per_second_ = perf_metrics_.frames_decoded_ /
                                      perf_metrics_.total_duration_.InSecondsF();
   perf_metrics_.frames_dropped_ = frame_renderer_->FramesDropped();

   // Calculate the dropped frame percentage.
   perf_metrics_.dropped_frame_percentage_ =
       static_cast<double>(perf_metrics_.frames_dropped_) /
       static_cast<double>(
           std::max<size_t>(perf_metrics_.frames_decoded_, 1ul)) *
       100.0;

   // Calculate delivery and decode time metrics.
   perf_metrics_.delivery_time_stats_ =
       PerformanceTimeStats(frame_delivery_times_);
   perf_metrics_.decode_time_stats_ = PerformanceTimeStats(frame_decode_times_);

   std::cout << "Frames decoded:     " << perf_metrics_.frames_decoded_
             << std::endl;
   std::cout << "Total duration:     "
             << perf_metrics_.total_duration_.InMillisecondsF() << "ms"
             << std::endl;
   std::cout << "FPS:                " << perf_metrics_.frames_per_second_
             << std::endl;
   std::cout << "Frames Dropped:     " << perf_metrics_.frames_dropped_
             << std::endl;
   std::cout << "Dropped frame percentage: "
             << perf_metrics_.dropped_frame_percentage_ << "%" << std::endl;
   std::cout << "Frame delivery time - average:       "
             << perf_metrics_.delivery_time_stats_.avg_ms_ << "ms" << std::endl;
   std::cout << "Frame delivery time - percentile 25: "
             << perf_metrics_.delivery_time_stats_.percentile_25_ms_ << "ms"
             << std::endl;
   std::cout << "Frame delivery time - percentile 50: "
             << perf_metrics_.delivery_time_stats_.percentile_50_ms_ << "ms"
             << std::endl;
   std::cout << "Frame delivery time - percentile 75: "
             << perf_metrics_.delivery_time_stats_.percentile_75_ms_ << "ms"
             << std::endl;
   std::cout << "Frame decode time - average:       "
             << perf_metrics_.decode_time_stats_.avg_ms_ << "ms" << std::endl;
   std::cout << "Frame decode time - percentile 25: "
             << perf_metrics_.decode_time_stats_.percentile_25_ms_ << "ms"
             << std::endl;
   std::cout << "Frame decode time - percentile 50: "
             << perf_metrics_.decode_time_stats_.percentile_50_ms_ << "ms"
             << std::endl;
   std::cout << "Frame decode time - percentile 75: "
             << perf_metrics_.decode_time_stats_.percentile_75_ms_ << "ms"
             << std::endl;
 }

 void PerformanceEvaluator::WriteMetricsToFile() const {
   base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
   if (!DirectoryExists(output_folder_path))
     base::CreateDirectory(output_folder_path);
   output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);

   // Write performance metrics to json.
   base::Value metrics(base::Value::Type::DICTIONARY);
   metrics.SetKey(
       "FramesDecoded",
       base::Value(base::checked_cast<int>(perf_metrics_.frames_decoded_)));
   metrics.SetKey("TotalDurationMs",
                  base::Value(perf_metrics_.total_duration_.InMillisecondsF()));
   metrics.SetKey("FPS", base::Value(perf_metrics_.frames_per_second_));
   metrics.SetKey(
       "FramesDropped",
       base::Value(base::checked_cast<int>(perf_metrics_.frames_dropped_)));
   metrics.SetKey("DroppedFramePercentage",
                  base::Value(perf_metrics_.dropped_frame_percentage_));
   metrics.SetKey("FrameDeliveryTimeAverage",
                  base::Value(perf_metrics_.delivery_time_stats_.avg_ms_));
   metrics.SetKey(
       "FrameDeliveryTimePercentile25",
       base::Value(perf_metrics_.delivery_time_stats_.percentile_25_ms_));
   metrics.SetKey(
       "FrameDeliveryTimePercentile50",
       base::Value(perf_metrics_.delivery_time_stats_.percentile_50_ms_));
   metrics.SetKey(
       "FrameDeliveryTimePercentile75",
       base::Value(perf_metrics_.delivery_time_stats_.percentile_75_ms_));
   metrics.SetKey("FrameDecodeTimeAverage",
                  base::Value(perf_metrics_.decode_time_stats_.avg_ms_));
   metrics.SetKey(
       "FrameDecodeTimePercentile25",
       base::Value(perf_metrics_.decode_time_stats_.percentile_25_ms_));
   metrics.SetKey(
       "FrameDecodeTimePercentile50",
       base::Value(perf_metrics_.decode_time_stats_.percentile_50_ms_));
   metrics.SetKey(
       "FrameDecodeTimePercentile75",
       base::Value(perf_metrics_.decode_time_stats_.percentile_75_ms_));

   // Write frame delivery times to json.
   base::Value delivery_times(base::Value::Type::LIST);
   for (double frame_delivery_time : frame_delivery_times_) {
     delivery_times.Append(frame_delivery_time);
   }
   metrics.SetKey("FrameDeliveryTimes", std::move(delivery_times));

   // Write frame decodes times to json.
   base::Value decode_times(base::Value::Type::LIST);
   for (double frame_decode_time : frame_decode_times_) {
     decode_times.Append(frame_decode_time);
   }
   metrics.SetKey("FrameDecodeTimes", std::move(decode_times));

   // Write json to file.
   std::string metrics_str;
   ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
       metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
   base::FilePath metrics_file_path = output_folder_path.Append(
       g_env->GetTestOutputFilePath().AddExtension(FILE_PATH_LITERAL(".json")));
   // Make sure that the directory into which json is saved is created.
   LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
   base::File metrics_output_file(
       base::FilePath(metrics_file_path),
       base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
   int bytes_written = metrics_output_file.WriteAtCurrentPos(
       metrics_str.data(), metrics_str.length());
   ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
   VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
 }

 // Video decode test class. Performs setup and teardown for each single test.
 class VideoDecoderTest : public ::testing::Test {
  public:
   // Create a new video player instance. |render_frame_rate| is the rate at
   // which the video player will simulate rendering frames, if 0 no rendering is
   // simulated. The |vsync_rate| is used during simulated rendering, if 0 Vsync
   // is disabled.
   std::unique_ptr<VideoPlayer> CreateVideoPlayer(const Video* video,
                                                  uint32_t render_frame_rate = 0,
                                                  uint32_t vsync_rate = 0) {
     LOG_ASSERT(video);

     // Create dummy frame renderer, simulates rendering at specified frame rate.
     base::TimeDelta frame_duration;
     base::TimeDelta vsync_interval_duration;
     if (render_frame_rate > 0) {
       frame_duration = base::Seconds(1) / render_frame_rate;
       vsync_interval_duration = base::Seconds(1) / vsync_rate;
     }
     auto frame_renderer =
         FrameRendererDummy::Create(frame_duration, vsync_interval_duration);

     std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors;
     auto performance_evaluator =
         std::make_unique<PerformanceEvaluator>(frame_renderer.get());
     performance_evaluator_ = performance_evaluator.get();
     frame_processors.push_back(std::move(performance_evaluator));

     // Use the new VD-based video decoders if requested.
     VideoDecoderClientConfig config;
     config.implementation = g_env->GetDecoderImplementation();

     auto video_player = VideoPlayer::Create(
         config, g_env->GetGpuMemoryBufferFactory(), std::move(frame_renderer),
         std::move(frame_processors));
     LOG_ASSERT(video_player);
     LOG_ASSERT(video_player->Initialize(video));

     // Make sure the event timeout is at least as long as the video's duration.
     video_player->SetEventWaitTimeout(
         std::max(kDefaultEventWaitTimeout, g_env->Video()->GetDuration()));
     return video_player;
   }

   PerformanceEvaluator* performance_evaluator_;
 };

 }  // namespace

 // Play video from start to end while measuring uncapped performance. This test
 // will decode a video as fast as possible, and gives an idea about the maximum
 // output of the decoder.
 TEST_F(VideoDecoderTest, MeasureUncappedPerformance) {
   auto tvp = CreateVideoPlayer(g_env->Video());

   performance_evaluator_->StartMeasuring();
   tvp->Play();
   EXPECT_TRUE(tvp->WaitForFlushDone());
   performance_evaluator_->StopMeasuring();
   performance_evaluator_->WriteMetricsToFile();

   EXPECT_EQ(tvp->GetFlushDoneCount(), 1u);
   EXPECT_EQ(tvp->GetFrameDecodedCount(), g_env->Video()->NumFrames());
 }

 // Play video from start to end while measuring capped performance. This test
 // will simulate rendering the video at its actual frame rate, and will
 // calculate the number of frames that were dropped. Vsync is enabled at 60 FPS.
 TEST_F(VideoDecoderTest, MeasureCappedPerformance) {
   auto tvp = CreateVideoPlayer(g_env->Video(), g_env->Video()->FrameRate(), 60);

   performance_evaluator_->StartMeasuring();
   tvp->Play();
   EXPECT_TRUE(tvp->WaitForFlushDone());
   tvp->WaitForRenderer();
   performance_evaluator_->StopMeasuring();
   performance_evaluator_->WriteMetricsToFile();

   EXPECT_EQ(tvp->GetFlushDoneCount(), 1u);
   EXPECT_EQ(tvp->GetFrameDecodedCount(), g_env->Video()->NumFrames());
 }

 // Play multiple videos simultaneously from start to finish.
 TEST_F(VideoDecoderTest,
        MeasureUncappedPerformance_MultipleConcurrentDecoders) {
   // Set RLIMIT_NOFILE soft limit to its hard limit value.
   if (sandbox::ResourceLimits::AdjustCurrent(
           RLIMIT_NOFILE, std::numeric_limits<long long int>::max())) {
     DPLOG(ERROR) << "Unable to increase soft limit of RLIMIT_NOFILE";
   }

 // The minimal number of concurrent decoders we expect to be supported on
 // platforms.
 #if defined(ARCH_CPU_X86_FAMILY)
   constexpr size_t kMinSupportedConcurrentDecoders = 25;
 #elif defined(ARCH_CPU_ARM_FAMILY)
   constexpr size_t kMinSupportedConcurrentDecoders = 10;
 #endif

   std::vector<std::unique_ptr<VideoPlayer>> players(
       kMinSupportedConcurrentDecoders);
   for (auto&& player : players)
     player = CreateVideoPlayer(g_env->Video());

   performance_evaluator_->StartMeasuring();

   for (auto&& player : players)
     player->Play();

   for (auto&& player : players) {
     EXPECT_TRUE(player->WaitForFlushDone());
     EXPECT_EQ(player->GetFlushDoneCount(), 1u);
     EXPECT_EQ(player->GetFrameDecodedCount(), g_env->Video()->NumFrames());
   }
   performance_evaluator_->StopMeasuring();
   performance_evaluator_->WriteMetricsToFile();
 }

 }  // namespace test
 }  // namespace media

 int main(int argc, char** argv) {
   // Set the default test data path.
   media::test::Video::SetTestDataPath(media::GetTestDataPath());

   // Print the help message if requested. This needs to be done before
   // initializing gtest, to overwrite the default gtest help message.
   base::CommandLine::Init(argc, argv);
   base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
   LOG_ASSERT(cmd_line);
   if (cmd_line->HasSwitch("help")) {
     std::cout << media::test::usage_msg << "\n" << media::test::help_msg;
     return 0;
   }

   // Check if a video was specified on the command line.
   base::CommandLine::StringVector args = cmd_line->GetArgs();
   base::FilePath video_path =
       (args.size() >= 1) ? base::FilePath(args[0]) : base::FilePath();
   base::FilePath video_metadata_path =
       (args.size() >= 2) ? base::FilePath(args[1]) : base::FilePath();

   // Parse command line arguments.
   base::FilePath::StringType output_folder = media::test::kDefaultOutputFolder;
   bool use_legacy = false;
   bool use_vd = false;
   bool use_vd_vda = false;
   media::test::DecoderImplementation implementation =
       media::test::DecoderImplementation::kVD;
   base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
   for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
        it != switches.end(); ++it) {
     if (it->first.find("gtest_") == 0 ||               // Handled by GoogleTest
         it->first == "v" || it->first == "vmodule") {  // Handled by Chrome
       continue;
     }

     if (it->first == "output_folder") {
       output_folder = it->second;
     } else if (it->first == "use-legacy") {
       use_legacy = true;
       implementation = media::test::DecoderImplementation::kVDA;
     } else if (it->first == "use_vd") {
       use_vd = true;
       implementation = media::test::DecoderImplementation::kVD;
     } else if (it->first == "use_vd_vda") {
       use_vd_vda = true;
       implementation = media::test::DecoderImplementation::kVDVDA;
     } else {
       std::cout << "unknown option: --" << it->first << "\n"
                 << media::test::usage_msg;
       return EXIT_FAILURE;
     }
   }

   if (use_legacy && use_vd) {
     std::cout << "--use-legacy and --use_vd cannot be enabled together.\n"
               << media::test::usage_msg;
     return EXIT_FAILURE;
   }
   if (use_legacy && use_vd_vda) {
     std::cout << "--use-legacy and --use_vd_vda cannot be enabled together.\n"
               << media::test::usage_msg;
     return EXIT_FAILURE;
   }
   if (use_vd && use_vd_vda) {
     std::cout << "--use_vd and --use_vd_vda cannot be enabled together.\n"
               << media::test::usage_msg;
     return EXIT_FAILURE;
   }

   testing::InitGoogleTest(&argc, argv);

   // Add the command line flag for HEVC testing which will be checked by the
   // video decoder to allow clear HEVC decoding.
   cmd_line->AppendSwitch("enable-clear-hevc-for-testing");

   // Set up our test environment.
   media::test::VideoPlayerTestEnvironment* test_environment =
       media::test::VideoPlayerTestEnvironment::Create(
           video_path, video_metadata_path, /*validator_type=*/
           media::test::VideoPlayerTestEnvironment::ValidatorType::kNone,
           implementation,
           base::FilePath(output_folder));
   if (!test_environment)
     return EXIT_FAILURE;

   media::test::g_env = static_cast<media::test::VideoPlayerTestEnvironment*>(
       testing::AddGlobalTestEnvironment(test_environment));

   return RUN_ALL_TESTS();
 }
	// Copyright 2019 The Chromium Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include <algorithm>
	#include <numeric>
	#include <vector>

	#include "base/command_line.h"
	#include "base/files/file_util.h"
	#include "base/json/json_writer.h"
	#include "build/build_config.h"
	#include "media/base/test_data_util.h"
	#include "media/gpu/test/video.h"
	#include "media/gpu/test/video_player/frame_renderer_dummy.h"
	#include "media/gpu/test/video_player/video_decoder_client.h"
	#include "media/gpu/test/video_player/video_player.h"
	#include "media/gpu/test/video_player/video_player_test_environment.h"
	#include "sandbox/linux/services/resource_limits.h"
	#include "testing/gtest/include/gtest/gtest.h"

	namespace media {
	namespace test {

	namespace {

	// Video decoder perf tests usage message. Make sure to also update the
	// documentation under docs/media/gpu/video_decoder_perf_test_usage.md when
	// making changes here.
	constexpr const char* usage_msg =
	"usage: video_decode_accelerator_perf_tests\n"
	" [-v=<level>] [--vmodule=<config>] [--output_folder]\n"
	" ([--use-legacy]\|[--use_vd]\|[--use_vd_vda]) [--gtest_help]\n"
	" [--help] [<video path>] [<video metadata path>]\n";

	// Video decoder perf tests help message.
	constexpr const char* help_msg =
	"Run the video decode accelerator performance tests on the video\n"
	"specified by <video path>. If no <video path> is given the default\n"
	"\"test-25fps.h264\" video will be used.\n"
	"\nThe <video metadata path> should specify the location of a json file\n"
	"containing the video's metadata, such as frame checksums. By default\n"
	"<video path>.json will be used.\n"
	"\nThe following arguments are supported:\n"
	" -v enable verbose mode, e.g. -v=2.\n"
	" --vmodule enable verbose mode for the specified module,\n"
	" e.g. --vmodule=media/gpu=2.\n"
	" --output_folder overwrite the output folder used to store\n"
	" performance metrics, if not specified results\n"
	" will be stored in the current working directory.\n"
	" --use-legacy use the legacy VDA-based video decoders.\n"
	" --use_vd use the new VD-based video decoders.\n"
	" (enabled by default)\n"
	" --use_vd_vda use the new VD-based video decoders with a\n"
	" wrapper that translates to the VDA interface,\n"
	" used to test interaction with older components\n"
	" expecting the VDA interface.\n"
	" --gtest_help display the gtest help and exit.\n"
	" --help display this help and exit.\n";

	media::test::VideoPlayerTestEnvironment* g_env;

	// Default output folder used to store performance metrics.
	constexpr const base::FilePath::CharType* kDefaultOutputFolder =
	FILE_PATH_LITERAL("perf_metrics");

	// Struct storing various time-related statistics.
	struct PerformanceTimeStats {
	PerformanceTimeStats() {}
	explicit PerformanceTimeStats(const std::vector<double>& times);
	double avg_ms_ = 0.0;
	double percentile_25_ms_ = 0.0;
	double percentile_50_ms_ = 0.0;
	double percentile_75_ms_ = 0.0;
	};

	PerformanceTimeStats::PerformanceTimeStats(const std::vector<double>& times) {
	avg_ms_ = std::accumulate(times.begin(), times.end(), 0.0) / times.size();
	std::vector<double> sorted_times = times;
	std::sort(sorted_times.begin(), sorted_times.end());
	percentile_25_ms_ = sorted_times[sorted_times.size() / 4];
	percentile_50_ms_ = sorted_times[sorted_times.size() / 2];
	percentile_75_ms_ = sorted_times[(sorted_times.size() * 3) / 4];
	}

	struct PerformanceMetrics {
	// Total measurement duration.
	base::TimeDelta total_duration_;
	// The number of frames decoded.
	size_t frames_decoded_ = 0;
	// The overall number of frames decoded per second.
	double frames_per_second_ = 0.0;
	// The number of frames dropped because of the decoder running behind, only
	// relevant for capped performance tests.
	size_t frames_dropped_ = 0;
	// The percentage of frames dropped because of the decoder running behind,
	// only relevant for capped performance tests.
	double dropped_frame_percentage_ = 0.0;
	// Statistics about the time between subsequent frame deliveries.
	PerformanceTimeStats delivery_time_stats_;
	// Statistics about the time between decode start and frame deliveries.
	PerformanceTimeStats decode_time_stats_;
	};

	// The performance evaluator can be plugged into the video player to collect
	// various performance metrics.
	// TODO(dstaessens@) Check and post warning when CPU frequency scaling is
	// enabled as this affects test results.
	class PerformanceEvaluator : public VideoFrameProcessor {
	public:
	// Create a new performance evaluator. The caller should makes sure
	// \|frame_renderer\| outlives the performance evaluator.
	explicit PerformanceEvaluator(const FrameRendererDummy* const frame_renderer)
	: frame_renderer_(frame_renderer) {}

	// Interface VideoFrameProcessor
	void ProcessVideoFrame(scoped_refptr<const VideoFrame> video_frame,
	size_t frame_index) override;
	bool WaitUntilDone() override { return true; }

	// Start/Stop collecting performance metrics.
	void StartMeasuring();
	void StopMeasuring();

	// Write the collected performance metrics to file.
	void WriteMetricsToFile() const;

	private:
	// Start/end time of the measurement period.
	base::TimeTicks start_time_;
	base::TimeTicks end_time_;

	// Time at which the previous frame was delivered.
	base::TimeTicks prev_frame_delivery_time_;
	// List of times between subsequent frame deliveries.
	std::vector<double> frame_delivery_times_;
	// List of times between decode start and frame delivery.
	std::vector<double> frame_decode_times_;

	// Collection of various performance metrics.
	PerformanceMetrics perf_metrics_;

	// Frame renderer used to get the dropped frame rate, owned by the creator of
	// the performance evaluator.
	const FrameRendererDummy* const frame_renderer_;
	};

	void PerformanceEvaluator::ProcessVideoFrame(
	scoped_refptr<const VideoFrame> video_frame,
	size_t frame_index) {
	base::TimeTicks now = base::TimeTicks::Now();

	base::TimeDelta delivery_time = (now - prev_frame_delivery_time_);
	frame_delivery_times_.push_back(delivery_time.InMillisecondsF());
	prev_frame_delivery_time_ = now;

	base::TimeDelta decode_time = now.since_origin() - video_frame->timestamp();
	frame_decode_times_.push_back(decode_time.InMillisecondsF());

	perf_metrics_.frames_decoded_++;
	}

	void PerformanceEvaluator::StartMeasuring() {
	start_time_ = base::TimeTicks::Now();
	prev_frame_delivery_time_ = start_time_;
	}

	void PerformanceEvaluator::StopMeasuring() {
	end_time_ = base::TimeTicks::Now();
	perf_metrics_.total_duration_ = end_time_ - start_time_;
	perf_metrics_.frames_per_second_ = perf_metrics_.frames_decoded_ /
	perf_metrics_.total_duration_.InSecondsF();
	perf_metrics_.frames_dropped_ = frame_renderer_->FramesDropped();

	// Calculate the dropped frame percentage.
	perf_metrics_.dropped_frame_percentage_ =
	static_cast<double>(perf_metrics_.frames_dropped_) /
	static_cast<double>(
	std::max<size_t>(perf_metrics_.frames_decoded_, 1ul)) *
	100.0;

	// Calculate delivery and decode time metrics.
	perf_metrics_.delivery_time_stats_ =
	PerformanceTimeStats(frame_delivery_times_);
	perf_metrics_.decode_time_stats_ = PerformanceTimeStats(frame_decode_times_);

	std::cout << "Frames decoded: " << perf_metrics_.frames_decoded_
	<< std::endl;
	std::cout << "Total duration: "
	<< perf_metrics_.total_duration_.InMillisecondsF() << "ms"
	<< std::endl;
	std::cout << "FPS: " << perf_metrics_.frames_per_second_
	<< std::endl;
	std::cout << "Frames Dropped: " << perf_metrics_.frames_dropped_
	<< std::endl;
	std::cout << "Dropped frame percentage: "
	<< perf_metrics_.dropped_frame_percentage_ << "%" << std::endl;
	std::cout << "Frame delivery time - average: "
	<< perf_metrics_.delivery_time_stats_.avg_ms_ << "ms" << std::endl;
	std::cout << "Frame delivery time - percentile 25: "
	<< perf_metrics_.delivery_time_stats_.percentile_25_ms_ << "ms"
	<< std::endl;
	std::cout << "Frame delivery time - percentile 50: "
	<< perf_metrics_.delivery_time_stats_.percentile_50_ms_ << "ms"
	<< std::endl;
	std::cout << "Frame delivery time - percentile 75: "
	<< perf_metrics_.delivery_time_stats_.percentile_75_ms_ << "ms"
	<< std::endl;
	std::cout << "Frame decode time - average: "
	<< perf_metrics_.decode_time_stats_.avg_ms_ << "ms" << std::endl;
	std::cout << "Frame decode time - percentile 25: "
	<< perf_metrics_.decode_time_stats_.percentile_25_ms_ << "ms"
	<< std::endl;
	std::cout << "Frame decode time - percentile 50: "
	<< perf_metrics_.decode_time_stats_.percentile_50_ms_ << "ms"
	<< std::endl;
	std::cout << "Frame decode time - percentile 75: "
	<< perf_metrics_.decode_time_stats_.percentile_75_ms_ << "ms"
	<< std::endl;
	}

	void PerformanceEvaluator::WriteMetricsToFile() const {
	base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
	if (!DirectoryExists(output_folder_path))
	base::CreateDirectory(output_folder_path);
	output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);

	// Write performance metrics to json.
	base::Value metrics(base::Value::Type::DICTIONARY);
	metrics.SetKey(
	"FramesDecoded",
	base::Value(base::checked_cast<int>(perf_metrics_.frames_decoded_)));
	metrics.SetKey("TotalDurationMs",
	base::Value(perf_metrics_.total_duration_.InMillisecondsF()));
	metrics.SetKey("FPS", base::Value(perf_metrics_.frames_per_second_));
	metrics.SetKey(
	"FramesDropped",
	base::Value(base::checked_cast<int>(perf_metrics_.frames_dropped_)));
	metrics.SetKey("DroppedFramePercentage",
	base::Value(perf_metrics_.dropped_frame_percentage_));
	metrics.SetKey("FrameDeliveryTimeAverage",
	base::Value(perf_metrics_.delivery_time_stats_.avg_ms_));
	metrics.SetKey(
	"FrameDeliveryTimePercentile25",
	base::Value(perf_metrics_.delivery_time_stats_.percentile_25_ms_));
	metrics.SetKey(
	"FrameDeliveryTimePercentile50",
	base::Value(perf_metrics_.delivery_time_stats_.percentile_50_ms_));
	metrics.SetKey(
	"FrameDeliveryTimePercentile75",
	base::Value(perf_metrics_.delivery_time_stats_.percentile_75_ms_));
	metrics.SetKey("FrameDecodeTimeAverage",
	base::Value(perf_metrics_.decode_time_stats_.avg_ms_));
	metrics.SetKey(
	"FrameDecodeTimePercentile25",
	base::Value(perf_metrics_.decode_time_stats_.percentile_25_ms_));
	metrics.SetKey(
	"FrameDecodeTimePercentile50",
	base::Value(perf_metrics_.decode_time_stats_.percentile_50_ms_));
	metrics.SetKey(
	"FrameDecodeTimePercentile75",
	base::Value(perf_metrics_.decode_time_stats_.percentile_75_ms_));

	// Write frame delivery times to json.
	base::Value delivery_times(base::Value::Type::LIST);
	for (double frame_delivery_time : frame_delivery_times_) {
	delivery_times.Append(frame_delivery_time);
	}
	metrics.SetKey("FrameDeliveryTimes", std::move(delivery_times));

	// Write frame decodes times to json.
	base::Value decode_times(base::Value::Type::LIST);
	for (double frame_decode_time : frame_decode_times_) {
	decode_times.Append(frame_decode_time);
	}
	metrics.SetKey("FrameDecodeTimes", std::move(decode_times));

	// Write json to file.
	std::string metrics_str;
	ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
	metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
	base::FilePath metrics_file_path = output_folder_path.Append(
	g_env->GetTestOutputFilePath().AddExtension(FILE_PATH_LITERAL(".json")));
	// Make sure that the directory into which json is saved is created.
	LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
	base::File metrics_output_file(
	base::FilePath(metrics_file_path),
	base::File::FLAG_CREATE_ALWAYS \| base::File::FLAG_WRITE);
	int bytes_written = metrics_output_file.WriteAtCurrentPos(
	metrics_str.data(), metrics_str.length());
	ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
	VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
	}

	// Video decode test class. Performs setup and teardown for each single test.
	class VideoDecoderTest : public ::testing::Test {
	public:
	// Create a new video player instance. \|render_frame_rate\| is the rate at
	// which the video player will simulate rendering frames, if 0 no rendering is
	// simulated. The \|vsync_rate\| is used during simulated rendering, if 0 Vsync
	// is disabled.
	std::unique_ptr<VideoPlayer> CreateVideoPlayer(const Video* video,
	uint32_t render_frame_rate = 0,
	uint32_t vsync_rate = 0) {
	LOG_ASSERT(video);

	// Create dummy frame renderer, simulates rendering at specified frame rate.
	base::TimeDelta frame_duration;
	base::TimeDelta vsync_interval_duration;
	if (render_frame_rate > 0) {
	frame_duration = base::Seconds(1) / render_frame_rate;
	vsync_interval_duration = base::Seconds(1) / vsync_rate;
	}
	auto frame_renderer =
	FrameRendererDummy::Create(frame_duration, vsync_interval_duration);

	std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors;
	auto performance_evaluator =
	std::make_unique<PerformanceEvaluator>(frame_renderer.get());
	performance_evaluator_ = performance_evaluator.get();
	frame_processors.push_back(std::move(performance_evaluator));

	// Use the new VD-based video decoders if requested.
	VideoDecoderClientConfig config;
	config.implementation = g_env->GetDecoderImplementation();

	auto video_player = VideoPlayer::Create(
	config, g_env->GetGpuMemoryBufferFactory(), std::move(frame_renderer),
	std::move(frame_processors));
	LOG_ASSERT(video_player);
	LOG_ASSERT(video_player->Initialize(video));

	// Make sure the event timeout is at least as long as the video's duration.
	video_player->SetEventWaitTimeout(
	std::max(kDefaultEventWaitTimeout, g_env->Video()->GetDuration()));
	return video_player;
	}

	PerformanceEvaluator* performance_evaluator_;
	};

	} // namespace

	// Play video from start to end while measuring uncapped performance. This test
	// will decode a video as fast as possible, and gives an idea about the maximum
	// output of the decoder.
	TEST_F(VideoDecoderTest, MeasureUncappedPerformance) {
	auto tvp = CreateVideoPlayer(g_env->Video());

	performance_evaluator_->StartMeasuring();
	tvp->Play();
	EXPECT_TRUE(tvp->WaitForFlushDone());
	performance_evaluator_->StopMeasuring();
	performance_evaluator_->WriteMetricsToFile();

	EXPECT_EQ(tvp->GetFlushDoneCount(), 1u);
	EXPECT_EQ(tvp->GetFrameDecodedCount(), g_env->Video()->NumFrames());
	}

	// Play video from start to end while measuring capped performance. This test
	// will simulate rendering the video at its actual frame rate, and will
	// calculate the number of frames that were dropped. Vsync is enabled at 60 FPS.
	TEST_F(VideoDecoderTest, MeasureCappedPerformance) {
	auto tvp = CreateVideoPlayer(g_env->Video(), g_env->Video()->FrameRate(), 60);

	performance_evaluator_->StartMeasuring();
	tvp->Play();
	EXPECT_TRUE(tvp->WaitForFlushDone());
	tvp->WaitForRenderer();
	performance_evaluator_->StopMeasuring();
	performance_evaluator_->WriteMetricsToFile();

	EXPECT_EQ(tvp->GetFlushDoneCount(), 1u);
	EXPECT_EQ(tvp->GetFrameDecodedCount(), g_env->Video()->NumFrames());
	}

	// Play multiple videos simultaneously from start to finish.
	TEST_F(VideoDecoderTest,
	MeasureUncappedPerformance_MultipleConcurrentDecoders) {
	// Set RLIMIT_NOFILE soft limit to its hard limit value.
	if (sandbox::ResourceLimits::AdjustCurrent(
	RLIMIT_NOFILE, std::numeric_limits<long long int>::max())) {
	DPLOG(ERROR) << "Unable to increase soft limit of RLIMIT_NOFILE";
	}

	// The minimal number of concurrent decoders we expect to be supported on
	// platforms.
	#if defined(ARCH_CPU_X86_FAMILY)
	constexpr size_t kMinSupportedConcurrentDecoders = 25;
	#elif defined(ARCH_CPU_ARM_FAMILY)
	constexpr size_t kMinSupportedConcurrentDecoders = 10;
	#endif

	std::vector<std::unique_ptr<VideoPlayer>> players(
	kMinSupportedConcurrentDecoders);
	for (auto&& player : players)
	player = CreateVideoPlayer(g_env->Video());

	performance_evaluator_->StartMeasuring();

	for (auto&& player : players)
	player->Play();

	for (auto&& player : players) {
	EXPECT_TRUE(player->WaitForFlushDone());
	EXPECT_EQ(player->GetFlushDoneCount(), 1u);
	EXPECT_EQ(player->GetFrameDecodedCount(), g_env->Video()->NumFrames());
	}
	performance_evaluator_->StopMeasuring();
	performance_evaluator_->WriteMetricsToFile();
	}

	} // namespace test
	} // namespace media

	int main(int argc, char** argv) {
	// Set the default test data path.
	media::test::Video::SetTestDataPath(media::GetTestDataPath());

	// Print the help message if requested. This needs to be done before
	// initializing gtest, to overwrite the default gtest help message.
	base::CommandLine::Init(argc, argv);
	base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
	LOG_ASSERT(cmd_line);
	if (cmd_line->HasSwitch("help")) {
	std::cout << media::test::usage_msg << "\n" << media::test::help_msg;
	return 0;
	}

	// Check if a video was specified on the command line.
	base::CommandLine::StringVector args = cmd_line->GetArgs();
	base::FilePath video_path =
	(args.size() >= 1) ? base::FilePath(args[0]) : base::FilePath();
	base::FilePath video_metadata_path =
	(args.size() >= 2) ? base::FilePath(args[1]) : base::FilePath();

	// Parse command line arguments.
	base::FilePath::StringType output_folder = media::test::kDefaultOutputFolder;
	bool use_legacy = false;
	bool use_vd = false;
	bool use_vd_vda = false;
	media::test::DecoderImplementation implementation =
	media::test::DecoderImplementation::kVD;
	base::CommandLine::SwitchMap switches = cmd_line->GetSwitches();
	for (base::CommandLine::SwitchMap::const_iterator it = switches.begin();
	it != switches.end(); ++it) {
	if (it->first.find("gtest_") == 0 \|\| // Handled by GoogleTest
	it->first == "v" \|\| it->first == "vmodule") { // Handled by Chrome
	continue;
	}

	if (it->first == "output_folder") {
	output_folder = it->second;
	} else if (it->first == "use-legacy") {
	use_legacy = true;
	implementation = media::test::DecoderImplementation::kVDA;
	} else if (it->first == "use_vd") {
	use_vd = true;
	implementation = media::test::DecoderImplementation::kVD;
	} else if (it->first == "use_vd_vda") {
	use_vd_vda = true;
	implementation = media::test::DecoderImplementation::kVDVDA;
	} else {
	std::cout << "unknown option: --" << it->first << "\n"
	<< media::test::usage_msg;
	return EXIT_FAILURE;
	}
	}

	if (use_legacy && use_vd) {
	std::cout << "--use-legacy and --use_vd cannot be enabled together.\n"
	<< media::test::usage_msg;
	return EXIT_FAILURE;
	}
	if (use_legacy && use_vd_vda) {
	std::cout << "--use-legacy and --use_vd_vda cannot be enabled together.\n"
	<< media::test::usage_msg;
	return EXIT_FAILURE;
	}
	if (use_vd && use_vd_vda) {
	std::cout << "--use_vd and --use_vd_vda cannot be enabled together.\n"
	<< media::test::usage_msg;
	return EXIT_FAILURE;
	}

	testing::InitGoogleTest(&argc, argv);

	// Add the command line flag for HEVC testing which will be checked by the
	// video decoder to allow clear HEVC decoding.
	cmd_line->AppendSwitch("enable-clear-hevc-for-testing");

	// Set up our test environment.
	media::test::VideoPlayerTestEnvironment* test_environment =
	media::test::VideoPlayerTestEnvironment::Create(
	video_path, video_metadata_path, /validator_type=/
	media::test::VideoPlayerTestEnvironment::ValidatorType::kNone,
	implementation,
	base::FilePath(output_folder));
	if (!test_environment)
	return EXIT_FAILURE;

	media::test::g_env = static_cast<media::test::VideoPlayerTestEnvironment*>(
	testing::AddGlobalTestEnvironment(test_environment));

	return RUN_ALL_TESTS();
	}