third_party/chromium/media/capture/video/chromeos/camera_app_device_impl.cc

// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/capture/video/chromeos/camera_app_device_impl.h"

#include <cmath>

#include "base/bind_post_task.h"
#include "base/time/time.h"
#include "gpu/ipc/common/gpu_memory_buffer_impl.h"
#include "media/base/bind_to_current_loop.h"
#include "media/capture/video/chromeos/camera_app_device_bridge_impl.h"
#include "media/capture/video/chromeos/camera_device_context.h"
#include "media/capture/video/chromeos/camera_metadata_utils.h"
#include "third_party/libyuv/include/libyuv.h"

namespace media {

namespace {

constexpr int kDetectionWidth = 256;
constexpr int kDetectionHeight = 256;

void OnStillCaptureDone(media::mojom::ImageCapture::TakePhotoCallback callback,
                        int status,
                        mojom::BlobPtr blob) {
  DCHECK_EQ(status, kReprocessSuccess);
  std::move(callback).Run(std::move(blob));
}

}  // namespace

ReprocessTask::ReprocessTask() = default;

ReprocessTask::ReprocessTask(ReprocessTask&& other)
    : effect(other.effect),
      callback(std::move(other.callback)),
      extra_metadata(std::move(other.extra_metadata)) {}

ReprocessTask::~ReprocessTask() = default;

// static
int CameraAppDeviceImpl::GetReprocessReturnCode(
    cros::mojom::Effect effect,
    const cros::mojom::CameraMetadataPtr* metadata) {
  if (effect == cros::mojom::Effect::PORTRAIT_MODE) {
    auto portrait_mode_segmentation_result = GetMetadataEntryAsSpan<uint8_t>(
        *metadata, static_cast<cros::mojom::CameraMetadataTag>(
                       kPortraitModeSegmentationResultVendorKey));
    DCHECK(!portrait_mode_segmentation_result.empty());
    return static_cast<int>(portrait_mode_segmentation_result[0]);
  }
  return kReprocessSuccess;
}

// static
ReprocessTaskQueue CameraAppDeviceImpl::GetSingleShotReprocessOptions(
    media::mojom::ImageCapture::TakePhotoCallback take_photo_callback) {
  ReprocessTaskQueue result_task_queue;
  ReprocessTask still_capture_task;
  still_capture_task.effect = cros::mojom::Effect::NO_EFFECT;
  still_capture_task.callback =
      base::BindOnce(&OnStillCaptureDone, std::move(take_photo_callback));
  // Explicitly disable edge enhancement and noise reduction for YUV -> JPG
  // conversion.
  DisableEeNr(&still_capture_task);
  result_task_queue.push(std::move(still_capture_task));
  return result_task_queue;
}

CameraAppDeviceImpl::CameraAppDeviceImpl(const std::string& device_id,
                                         cros::mojom::CameraInfoPtr camera_info)
    : device_id_(device_id),
      allow_new_ipc_weak_ptrs_(true),
      camera_info_(std::move(camera_info)),
      capture_intent_(cros::mojom::CaptureIntent::DEFAULT),
      camera_device_context_(nullptr) {}

CameraAppDeviceImpl::~CameraAppDeviceImpl() {
  // If the instance is bound, then this instance should only be destroyed when
  // the mojo connection is dropped, which also happens on the mojo thread.
  DCHECK(!mojo_task_runner_ || mojo_task_runner_->BelongsToCurrentThread());

  // All the weak pointers of |weak_ptr_factory_| should be invalidated on
  // camera device IPC thread before destroying CameraAppDeviceImpl.
  DCHECK(!weak_ptr_factory_.HasWeakPtrs());
}

void CameraAppDeviceImpl::BindReceiver(
    mojo::PendingReceiver<cros::mojom::CameraAppDevice> receiver) {
  receivers_.Add(this, std::move(receiver));
  receivers_.set_disconnect_handler(
      base::BindRepeating(&CameraAppDeviceImpl::OnMojoConnectionError,
                          weak_ptr_factory_for_mojo_.GetWeakPtr()));
  mojo_task_runner_ = base::ThreadTaskRunnerHandle::Get();

  document_scanner_service_ = chromeos::DocumentScannerServiceClient::Create();
}

base::WeakPtr<CameraAppDeviceImpl> CameraAppDeviceImpl::GetWeakPtr() {
  return allow_new_ipc_weak_ptrs_ ? weak_ptr_factory_.GetWeakPtr() : nullptr;
}

void CameraAppDeviceImpl::ResetOnDeviceIpcThread(base::OnceClosure callback,
                                                 bool should_disable_new_ptrs) {
  if (should_disable_new_ptrs) {
    allow_new_ipc_weak_ptrs_ = false;
  }
  weak_ptr_factory_.InvalidateWeakPtrs();
  std::move(callback).Run();
}

void CameraAppDeviceImpl::ConsumeReprocessOptions(
    media::mojom::ImageCapture::TakePhotoCallback take_photo_callback,
    base::OnceCallback<void(ReprocessTaskQueue)> consumption_callback) {
  ReprocessTaskQueue result_task_queue;

  ReprocessTask still_capture_task;
  still_capture_task.effect = cros::mojom::Effect::NO_EFFECT;
  still_capture_task.callback =
      base::BindOnce(&OnStillCaptureDone, std::move(take_photo_callback));
  // Explicitly disable edge enhancement and noise reduction for YUV -> JPG
  // conversion.
  DisableEeNr(&still_capture_task);
  result_task_queue.push(std::move(still_capture_task));

  base::AutoLock lock(reprocess_tasks_lock_);
  while (!reprocess_task_queue_.empty()) {
    result_task_queue.push(std::move(reprocess_task_queue_.front()));
    reprocess_task_queue_.pop();
  }
  std::move(consumption_callback).Run(std::move(result_task_queue));
}

absl::optional<gfx::Range> CameraAppDeviceImpl::GetFpsRange() {
  base::AutoLock lock(fps_ranges_lock_);

  return specified_fps_range_;
}

gfx::Size CameraAppDeviceImpl::GetStillCaptureResolution() {
  base::AutoLock lock(still_capture_resolution_lock_);

  return still_capture_resolution_;
}

cros::mojom::CaptureIntent CameraAppDeviceImpl::GetCaptureIntent() {
  base::AutoLock lock(capture_intent_lock_);
  return capture_intent_;
}

void CameraAppDeviceImpl::OnResultMetadataAvailable(
    const cros::mojom::CameraMetadataPtr& metadata,
    cros::mojom::StreamType streamType) {
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::NotifyResultMetadataOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr(), metadata.Clone(),
                     streamType));
}

void CameraAppDeviceImpl::OnShutterDone() {
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::NotifyShutterDoneOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr()));
}

void CameraAppDeviceImpl::SetCameraDeviceContext(
    CameraDeviceContext* camera_device_context) {
  base::AutoLock lock(camera_device_context_lock_);
  camera_device_context_ = camera_device_context;
}

void CameraAppDeviceImpl::MaybeDetectDocumentCorners(
    std::unique_ptr<gpu::GpuMemoryBufferImpl> gmb,
    VideoRotation rotation) {
  {
    base::AutoLock lock(capture_intent_lock_);
    if (capture_intent_ != cros::mojom::CaptureIntent::DOCUMENT) {
      return;
    }
  }
  if (!chromeos::DocumentScannerServiceClient::IsSupported()) {
    return;
  }
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::DetectDocumentCornersOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr(), std::move(gmb),
                     rotation));
}

void CameraAppDeviceImpl::GetCameraInfo(GetCameraInfoCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());
  DCHECK(camera_info_);

  std::move(callback).Run(camera_info_.Clone());
}

void CameraAppDeviceImpl::SetReprocessOptions(
    const std::vector<cros::mojom::Effect>& effects,
    mojo::PendingRemote<cros::mojom::ReprocessResultListener> listener,
    SetReprocessOptionsCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(reprocess_tasks_lock_);
  reprocess_listener_.reset();
  reprocess_listener_.Bind(std::move(listener));
  reprocess_task_queue_ = {};
  for (const auto& effect : effects) {
    ReprocessTask task;
    task.effect = effect;
    task.callback = media::BindToCurrentLoop(
        base::BindOnce(&CameraAppDeviceImpl::SetReprocessResultOnMojoThread,
                       weak_ptr_factory_for_mojo_.GetWeakPtr(), effect));

    if (effect == cros::mojom::Effect::PORTRAIT_MODE) {
      auto e = BuildMetadataEntry(
          static_cast<cros::mojom::CameraMetadataTag>(kPortraitModeVendorKey),
          1);
      task.extra_metadata.push_back(std::move(e));
    }
    reprocess_task_queue_.push(std::move(task));
  }
  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetFpsRange(const gfx::Range& fps_range,
                                      SetFpsRangeCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  const int entry_length = 2;

  auto& static_metadata = camera_info_->static_camera_characteristics;
  auto available_fps_range_entries = GetMetadataEntryAsSpan<int32_t>(
      static_metadata, cros::mojom::CameraMetadataTag::
                           ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES);
  DCHECK(available_fps_range_entries.size() % entry_length == 0);

  bool is_valid = false;
  int min_fps = static_cast<int>(fps_range.GetMin());
  int max_fps = static_cast<int>(fps_range.GetMax());
  for (size_t i = 0; i < available_fps_range_entries.size();
       i += entry_length) {
    if (available_fps_range_entries[i] == min_fps &&
        available_fps_range_entries[i + 1] == max_fps) {
      is_valid = true;
      break;
    }
  }

  base::AutoLock lock(fps_ranges_lock_);

  if (is_valid) {
    specified_fps_range_ = fps_range;
  } else {
    specified_fps_range_ = {};
  }
  std::move(callback).Run(is_valid);
}

void CameraAppDeviceImpl::SetStillCaptureResolution(
    const gfx::Size& resolution,
    SetStillCaptureResolutionCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(still_capture_resolution_lock_);
  still_capture_resolution_ = resolution;
  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetCaptureIntent(
    cros::mojom::CaptureIntent capture_intent,
    SetCaptureIntentCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  {
    base::AutoLock lock(capture_intent_lock_);
    capture_intent_ = capture_intent;
  }
  // Reset fps range for VCD to determine it if not explicitly set by app.
  {
    base::AutoLock lock(fps_ranges_lock_);
    specified_fps_range_ = {};
  }
  std::move(callback).Run();
}

void CameraAppDeviceImpl::AddResultMetadataObserver(
    mojo::PendingRemote<cros::mojom::ResultMetadataObserver> observer,
    cros::mojom::StreamType stream_type,
    AddResultMetadataObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  stream_to_metadata_observers_map_[stream_type].Add(std::move(observer));
  std::move(callback).Run();
}

void CameraAppDeviceImpl::AddCameraEventObserver(
    mojo::PendingRemote<cros::mojom::CameraEventObserver> observer,
    AddCameraEventObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  camera_event_observers_.Add(std::move(observer));
  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetCameraFrameRotationEnabledAtSource(
    bool is_enabled,
    SetCameraFrameRotationEnabledAtSourceCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  bool is_success = false;
  {
    base::AutoLock lock(camera_device_context_lock_);
    if (camera_device_context_) {
      camera_device_context_->SetCameraFrameRotationEnabledAtSource(is_enabled);
      is_success = true;
    }
  }
  std::move(callback).Run(is_success);
}

void CameraAppDeviceImpl::GetCameraFrameRotation(
    GetCameraFrameRotationCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  uint32_t rotation = 0;
  {
    base::AutoLock lock(camera_device_context_lock_);
    if (camera_device_context_ &&
        !camera_device_context_->IsCameraFrameRotationEnabledAtSource()) {
      // The camera rotation value can only be [0, 90, 180, 270].
      rotation = static_cast<uint32_t>(
          camera_device_context_->GetCameraFrameRotation());
    }
  }
  std::move(callback).Run(rotation);
}

void CameraAppDeviceImpl::RegisterDocumentCornersObserver(
    mojo::PendingRemote<cros::mojom::DocumentCornersObserver> observer,
    RegisterDocumentCornersObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  document_corners_observers_.Add(std::move(observer));
  std::move(callback).Run();
}

// static
void CameraAppDeviceImpl::DisableEeNr(ReprocessTask* task) {
  auto ee_entry =
      BuildMetadataEntry(cros::mojom::CameraMetadataTag::ANDROID_EDGE_MODE,
                         cros::mojom::AndroidEdgeMode::ANDROID_EDGE_MODE_OFF);
  auto nr_entry = BuildMetadataEntry(
      cros::mojom::CameraMetadataTag::ANDROID_NOISE_REDUCTION_MODE,
      cros::mojom::AndroidNoiseReductionMode::ANDROID_NOISE_REDUCTION_MODE_OFF);
  task->extra_metadata.push_back(std::move(ee_entry));
  task->extra_metadata.push_back(std::move(nr_entry));
}

void CameraAppDeviceImpl::OnMojoConnectionError() {
  CameraAppDeviceBridgeImpl::GetInstance()->OnDeviceMojoDisconnected(
      device_id_);
}

bool CameraAppDeviceImpl::IsCloseToPreviousDetectionRequest() {
  return document_detection_timer_ &&
         document_detection_timer_->Elapsed().InMilliseconds() < 300;
}

void CameraAppDeviceImpl::DetectDocumentCornersOnMojoThread(
    std::unique_ptr<gpu::GpuMemoryBufferImpl> image,
    VideoRotation rotation) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());
  DCHECK(document_scanner_service_);

  if (!document_scanner_service_->IsLoaded() ||
      IsCloseToPreviousDetectionRequest() ||
      has_ongoing_document_detection_task_) {
    return;
  }

  DCHECK(image);
  if (!image->Map()) {
    LOG(ERROR) << "Failed to map frame buffer";
    return;
  }
  auto frame_size = image->GetSize();
  int width = frame_size.width();
  int height = frame_size.height();

  base::MappedReadOnlyRegion memory = base::ReadOnlySharedMemoryRegion::Create(
      kDetectionWidth * kDetectionHeight * 3 / 2);

  auto* y_data = memory.mapping.GetMemoryAs<uint8_t>();
  auto* uv_data = y_data + kDetectionWidth * kDetectionHeight;

  int status = libyuv::NV12Scale(
      static_cast<uint8_t*>(image->memory(0)), image->stride(0),
      static_cast<uint8_t*>(image->memory(1)), image->stride(1), width, height,
      y_data, kDetectionWidth, uv_data, kDetectionWidth, kDetectionWidth,
      kDetectionHeight, libyuv::FilterMode::kFilterNone);
  image->Unmap();
  if (status != 0) {
    LOG(ERROR) << "Failed to scale buffer";
    return;
  }

  has_ongoing_document_detection_task_ = true;
  document_detection_timer_ = std::make_unique<base::ElapsedTimer>();
  // Since we destroy |document_scanner_service_| on mojo thread and this
  // callback is also called on mojo thread, it should be safe to just use
  // base::Unretained(this) here.
  document_scanner_service_->DetectCornersFromNV12Image(
      std::move(memory.region),
      base::BindOnce(
          &CameraAppDeviceImpl::OnDetectedDocumentCornersOnMojoThread,
          base::Unretained(this), rotation));
}

void CameraAppDeviceImpl::OnDetectedDocumentCornersOnMojoThread(
    VideoRotation rotation,
    bool success,
    const std::vector<gfx::PointF>& corners) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  has_ongoing_document_detection_task_ = false;
  if (!success) {
    LOG(ERROR) << "Failed to detect document corners";
    return;
  }

  // Rotate a point in coordination space {x: [0.0, 1.0], y: [0.0, 1.0]} with
  // anchor point {x: 0.5, y: 0.5}.
  auto rotate_corner = [&](const gfx::PointF& corner) -> gfx::PointF {
    float x = base::clamp(corner.x(), 0.0f, 1.0f);
    float y = base::clamp(corner.y(), 0.0f, 1.0f);

    switch (rotation) {
      case VIDEO_ROTATION_0:
        return {x, y};
      case VIDEO_ROTATION_90:
        return {1.0f - y, x};
      case VIDEO_ROTATION_180:
        return {1.0f - x, 1.0f - y};
      case VIDEO_ROTATION_270:
        return {y, 1.0f - x};
      default:
        NOTREACHED();
    }
  };

  std::vector<gfx::PointF> rotated_corners;
  for (auto& corner : corners) {
    rotated_corners.push_back(rotate_corner(corner));
  }

  for (auto& observer : document_corners_observers_) {
    observer->OnDocumentCornersUpdated(rotated_corners);
  }
}

void CameraAppDeviceImpl::SetReprocessResultOnMojoThread(
    cros::mojom::Effect effect,
    const int32_t status,
    media::mojom::BlobPtr blob) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(reprocess_tasks_lock_);
  reprocess_listener_->OnReprocessDone(effect, status, std::move(blob));
}

void CameraAppDeviceImpl::NotifyShutterDoneOnMojoThread() {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  for (auto& observer : camera_event_observers_) {
    observer->OnShutterDone();
  }
}

void CameraAppDeviceImpl::NotifyResultMetadataOnMojoThread(
    cros::mojom::CameraMetadataPtr metadata,
    cros::mojom::StreamType streamType) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  auto& metadata_observers = stream_to_metadata_observers_map_[streamType];
  for (auto& observer : metadata_observers) {
    observer->OnMetadataAvailable(metadata.Clone());
  }
}

}  // namespace media