src/cobalt/speech/speech_recognition_manager.cc

// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "cobalt/speech/speech_recognition_manager.h"

#include "base/bind.h"
#include "cobalt/base/tokens.h"
#include "cobalt/dom/dom_exception.h"
#if defined(ENABLE_FAKE_MICROPHONE)
#include "cobalt/speech/microphone_fake.h"
#include "cobalt/speech/url_fetcher_fake.h"
#endif  // defined(ENABLE_FAKE_MICROPHONE)
#include "cobalt/speech/microphone_manager.h"
#if defined(SB_USE_SB_MICROPHONE)
#include "cobalt/speech/microphone_starboard.h"
#endif  // defined(SB_USE_SB_MICROPHONE)
#include "net/url_request/url_fetcher.h"

namespace cobalt {
namespace speech {

namespace {
const int kSampleRate = 16000;
const float kAudioPacketDurationInSeconds = 0.1f;

scoped_ptr<net::URLFetcher> CreateURLFetcher(
    const GURL& url, net::URLFetcher::RequestType request_type,
    net::URLFetcherDelegate* delegate) {
  return make_scoped_ptr<net::URLFetcher>(
      net::URLFetcher::Create(url, request_type, delegate));
}

scoped_ptr<Microphone> CreateMicrophone(int buffer_size_bytes) {
#if defined(SB_USE_SB_MICROPHONE)
  return make_scoped_ptr<Microphone>(
      new MicrophoneStarboard(kSampleRate, buffer_size_bytes));
#else
  UNREFERENCED_PARAMETER(buffer_size_bytes);
  return scoped_ptr<Microphone>();
#endif  // defined(SB_USE_SB_MICROPHONE)
}

#if defined(SB_USE_SB_MICROPHONE)
#if defined(ENABLE_FAKE_MICROPHONE)
scoped_ptr<net::URLFetcher> CreateFakeURLFetcher(
    const GURL& url, net::URLFetcher::RequestType request_type,
    net::URLFetcherDelegate* delegate) {
  return make_scoped_ptr<net::URLFetcher>(
      new URLFetcherFake(url, request_type, delegate));
}

scoped_ptr<Microphone> CreateFakeMicrophone(const Microphone::Options& options,
                                            int /*buffer_size_bytes*/) {
  return make_scoped_ptr<Microphone>(new MicrophoneFake(options));
}
#endif  // defined(ENABLE_FAKE_MICROPHONE)
#endif  // defined(SB_USE_SB_MICROPHONE)

}  // namespace

SpeechRecognitionManager::SpeechRecognitionManager(
    network::NetworkModule* network_module, const EventCallback& event_callback,
    const Microphone::Options& microphone_options)
    : ALLOW_THIS_IN_INITIALIZER_LIST(weak_ptr_factory_(this)),
      weak_this_(weak_ptr_factory_.GetWeakPtr()),
      main_message_loop_(base::MessageLoopProxy::current()),
      event_callback_(event_callback),
      endpointer_delegate_(kSampleRate),
      state_(kStopped) {
  UNREFERENCED_PARAMETER(microphone_options);

  SpeechRecognizer::URLFetcherCreator url_fetcher_creator =
      base::Bind(&CreateURLFetcher);
  MicrophoneManager::MicrophoneCreator microphone_creator =
      base::Bind(&CreateMicrophone);

#if defined(SB_USE_SB_MICROPHONE)
#if defined(ENABLE_FAKE_MICROPHONE)
  if (microphone_options.enable_fake_microphone) {
    // If fake microphone is enabled, fake URL fetchers should be enabled as
    // well.
    url_fetcher_creator = base::Bind(&CreateFakeURLFetcher);
    microphone_creator = base::Bind(&CreateFakeMicrophone, microphone_options);
  }
#endif  // defined(ENABLE_FAKE_MICROPHONE)
#endif  // defined(SB_USE_SB_MICROPHONE)

  recognizer_.reset(new SpeechRecognizer(
      network_module, base::Bind(&SpeechRecognitionManager::OnRecognizerEvent,
                                 base::Unretained(this)),
      url_fetcher_creator));
  microphone_manager_.reset(new MicrophoneManager(
      base::Bind(&SpeechRecognitionManager::OnDataReceived,
                 base::Unretained(this)),
      base::Bind(&SpeechRecognitionManager::OnDataCompletion,
                 base::Unretained(this)),
      base::Bind(&SpeechRecognitionManager::OnMicError, base::Unretained(this)),
      microphone_creator));
}

SpeechRecognitionManager::~SpeechRecognitionManager() { Stop(false); }

void SpeechRecognitionManager::Start(const SpeechRecognitionConfig& config,
                                     script::ExceptionState* exception_state) {
  DCHECK(main_message_loop_->BelongsToCurrentThread());

  // If the start method is called on an already started object, the user agent
  // MUST throw an InvalidStateError exception and ignore the call.
  if (state_ == kStarted) {
    dom::DOMException::Raise(dom::DOMException::kInvalidStateErr,
                             exception_state);
    return;
  }

  recognizer_->Start(config, kSampleRate);
  microphone_manager_->Open();
  endpointer_delegate_.Start();
  state_ = kStarted;
}

void SpeechRecognitionManager::Stop(bool run_callback) {
  DCHECK(main_message_loop_->BelongsToCurrentThread());

  // If the stop method is called on an object which is already stopped or being
  // stopped, the user agent MUST ignore the call.
  if (state_ != kStarted) {
    return;
  }

  endpointer_delegate_.Stop();
  microphone_manager_->Close();
  recognizer_->Stop();
  state_ = kStopped;
  if (run_callback) {
    event_callback_.Run(new dom::Event(base::Tokens::soundend()));
  }
}

void SpeechRecognitionManager::Abort() {
  DCHECK(main_message_loop_->BelongsToCurrentThread());

  // If the abort method is called on an object which is already stopped or
  // aborting, the user agent MUST ignore the call.
  if (state_ != kStarted) {
    return;
  }

  endpointer_delegate_.Stop();
  microphone_manager_->Close();
  recognizer_->Stop();
  state_ = kAborted;
  event_callback_.Run(new dom::Event(base::Tokens::soundend()));
}

void SpeechRecognitionManager::OnDataReceived(
    scoped_ptr<ShellAudioBus> audio_bus) {
  if (!main_message_loop_->BelongsToCurrentThread()) {
    // Called from mic thread.
    main_message_loop_->PostTask(
        FROM_HERE, base::Bind(&SpeechRecognitionManager::OnDataReceived,
                              weak_this_, base::Passed(&audio_bus)));
    return;
  }

  // Stop recognizing if in the abort state.
  if (state_ != kAborted) {
    if (endpointer_delegate_.IsFirstTimeSoundStarted(*audio_bus)) {
      event_callback_.Run(new dom::Event(base::Tokens::soundstart()));
    }
    recognizer_->RecognizeAudio(audio_bus.Pass(), false);
  }
}

void SpeechRecognitionManager::OnDataCompletion() {
  if (!main_message_loop_->BelongsToCurrentThread()) {
    // Called from mic thread.
    main_message_loop_->PostTask(
        FROM_HERE,
        base::Bind(&SpeechRecognitionManager::OnDataCompletion, weak_this_));
    return;
  }

  // Stop recognizing if in the abort state.
  if (state_ != kAborted) {
    // The encoder requires a non-empty final buffer, so encoding a packet of
    // silence at the end in case encoder had no data already.
    size_t dummy_frames =
        static_cast<size_t>(kSampleRate * kAudioPacketDurationInSeconds);
    scoped_ptr<ShellAudioBus> dummy_audio_bus(new ShellAudioBus(
        1, dummy_frames, ShellAudioBus::kInt16, ShellAudioBus::kInterleaved));
    dummy_audio_bus->ZeroAllFrames();
    recognizer_->RecognizeAudio(dummy_audio_bus.Pass(), true);
  }
}

void SpeechRecognitionManager::OnRecognizerEvent(
    const scoped_refptr<dom::Event>& event) {
  if (!main_message_loop_->BelongsToCurrentThread()) {
    // Called from recognizer thread.
    main_message_loop_->PostTask(
        FROM_HERE, base::Bind(&SpeechRecognitionManager::OnRecognizerEvent,
                              weak_this_, event));
    return;
  }

  // Do not return any information if in the abort state.
  if (state_ != kAborted) {
    event_callback_.Run(event);
  }
}

void SpeechRecognitionManager::OnMicError(
    const scoped_refptr<dom::Event>& event) {
  if (!main_message_loop_->BelongsToCurrentThread()) {
    // Called from mic thread.
    main_message_loop_->PostTask(
        FROM_HERE,
        base::Bind(&SpeechRecognitionManager::OnMicError, weak_this_, event));
    return;
  }

  event_callback_.Run(event);

  // An error is occured in Mic, so stop the energy endpointer and recognizer.
  endpointer_delegate_.Stop();
  recognizer_->Stop();
  state_ = kAborted;
  event_callback_.Run(new dom::Event(base::Tokens::soundend()));
}

}  // namespace speech
}  // namespace cobalt