This source file includes following definitions.
- DumpResponse
- state_
- SetConfig
- StartRecognition
- EndRecognition
- TakeAudioChunk
- AudioChunksEnded
- OnURLFetchComplete
- OnURLFetchDownloadProgress
- DispatchHTTPResponse
- IsRecognitionPending
- GetDesiredAudioChunkDurationMs
- DispatchEvent
- ExecuteTransitionAndGetNextState
- ConnectBothStreams
- TransmitAudioUpstream
- ProcessDownstreamResponse
- RaiseNoMatchErrorIfGotNoResults
- CloseUpstreamAndWaitForResults
- CloseDownstream
- AbortSilently
- AbortWithError
- Abort
- DoNothing
- NotFeasible
- GetAcceptedLanguages
- GenerateRequestKey
#include "content/browser/speech/google_streaming_remote_engine.h"
#include <vector>
#include "base/bind.h"
#include "base/rand_util.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_util.h"
#include "base/strings/utf_string_conversions.h"
#include "base/time/time.h"
#include "content/browser/speech/audio_buffer.h"
#include "content/browser/speech/proto/google_streaming_api.pb.h"
#include "content/public/common/content_switches.h"
#include "content/public/common/speech_recognition_error.h"
#include "content/public/common/speech_recognition_result.h"
#include "google_apis/google_api_keys.h"
#include "net/base/escape.h"
#include "net/base/load_flags.h"
#include "net/url_request/http_user_agent_settings.h"
#include "net/url_request/url_fetcher.h"
#include "net/url_request/url_request_context.h"
#include "net/url_request/url_request_context_getter.h"
#include "net/url_request/url_request_status.h"
using net::URLFetcher;
namespace content {
namespace {
const char kWebServiceBaseUrl[] =
"https://www.google.com/speech-api/full-duplex/v1";
const char kDownstreamUrl[] = "/down?";
const char kUpstreamUrl[] = "/up?";
const AudioEncoder::Codec kDefaultAudioCodec = AudioEncoder::CODEC_FLAC;
const uint32 kMaxMaxAlternatives = 30;
void DumpResponse(const std::string& response) {
DVLOG(1) << "------------";
proto::SpeechRecognitionEvent event;
if (!event.ParseFromString(response)) {
DVLOG(1) << "Parse failed!";
return;
}
if (event.has_status())
DVLOG(1) << "STATUS\t" << event.status();
for (int i = 0; i < event.result_size(); ++i) {
DVLOG(1) << "RESULT #" << i << ":";
const proto::SpeechRecognitionResult& res = event.result(i);
if (res.has_final())
DVLOG(1) << " FINAL:\t" << res.final();
if (res.has_stability())
DVLOG(1) << " STABILITY:\t" << res.stability();
for (int j = 0; j < res.alternative_size(); ++j) {
const proto::SpeechRecognitionAlternative& alt =
res.alternative(j);
if (alt.has_confidence())
DVLOG(1) << " CONFIDENCE:\t" << alt.confidence();
if (alt.has_transcript())
DVLOG(1) << " TRANSCRIPT:\t" << alt.transcript();
}
}
}
}
const int GoogleStreamingRemoteEngine::kAudioPacketIntervalMs = 100;
const int GoogleStreamingRemoteEngine::kUpstreamUrlFetcherIdForTesting = 0;
const int GoogleStreamingRemoteEngine::kDownstreamUrlFetcherIdForTesting = 1;
const int GoogleStreamingRemoteEngine::kWebserviceStatusNoError = 0;
const int GoogleStreamingRemoteEngine::kWebserviceStatusErrorNoMatch = 5;
GoogleStreamingRemoteEngine::GoogleStreamingRemoteEngine(
net::URLRequestContextGetter* context)
: url_context_(context),
previous_response_length_(0),
got_last_definitive_result_(false),
is_dispatching_event_(false),
state_(STATE_IDLE) {}
GoogleStreamingRemoteEngine::~GoogleStreamingRemoteEngine() {}
void GoogleStreamingRemoteEngine::SetConfig(
const SpeechRecognitionEngineConfig& config) {
config_ = config;
}
void GoogleStreamingRemoteEngine::StartRecognition() {
FSMEventArgs event_args(EVENT_START_RECOGNITION);
DispatchEvent(event_args);
}
void GoogleStreamingRemoteEngine::EndRecognition() {
FSMEventArgs event_args(EVENT_END_RECOGNITION);
DispatchEvent(event_args);
}
void GoogleStreamingRemoteEngine::TakeAudioChunk(const AudioChunk& data) {
FSMEventArgs event_args(EVENT_AUDIO_CHUNK);
event_args.audio_data = &data;
DispatchEvent(event_args);
}
void GoogleStreamingRemoteEngine::AudioChunksEnded() {
FSMEventArgs event_args(EVENT_AUDIO_CHUNKS_ENDED);
DispatchEvent(event_args);
}
void GoogleStreamingRemoteEngine::OnURLFetchComplete(const URLFetcher* source) {
const bool kResponseComplete = true;
DispatchHTTPResponse(source, kResponseComplete);
}
void GoogleStreamingRemoteEngine::OnURLFetchDownloadProgress(
const URLFetcher* source, int64 current, int64 total) {
const bool kPartialResponse = false;
DispatchHTTPResponse(source, kPartialResponse);
}
void GoogleStreamingRemoteEngine::DispatchHTTPResponse(const URLFetcher* source,
bool end_of_response) {
DCHECK(CalledOnValidThread());
DCHECK(source);
const bool response_is_good = source->GetStatus().is_success() &&
source->GetResponseCode() == 200;
std::string response;
if (response_is_good)
source->GetResponseAsString(&response);
const size_t current_response_length = response.size();
DVLOG(1) << (source == downstream_fetcher_.get() ? "Downstream" : "Upstream")
<< "HTTP, code: " << source->GetResponseCode()
<< " length: " << current_response_length
<< " eor: " << end_of_response;
if (current_response_length != 0) {
DCHECK_GE(current_response_length, previous_response_length_);
response.erase(0, previous_response_length_);
previous_response_length_ = current_response_length;
}
if (!response_is_good && source == downstream_fetcher_.get()) {
DVLOG(1) << "Downstream error " << source->GetResponseCode();
FSMEventArgs event_args(EVENT_DOWNSTREAM_ERROR);
DispatchEvent(event_args);
return;
}
if (!response_is_good && source == upstream_fetcher_.get()) {
DVLOG(1) << "Upstream error " << source->GetResponseCode()
<< " EOR " << end_of_response;
FSMEventArgs event_args(EVENT_UPSTREAM_ERROR);
DispatchEvent(event_args);
return;
}
if (source == upstream_fetcher_.get())
return;
DCHECK(response_is_good && source == downstream_fetcher_.get());
chunked_byte_buffer_.Append(response);
while (chunked_byte_buffer_.HasChunks()) {
FSMEventArgs event_args(EVENT_DOWNSTREAM_RESPONSE);
event_args.response = chunked_byte_buffer_.PopChunk();
DCHECK(event_args.response.get());
DumpResponse(std::string(event_args.response->begin(),
event_args.response->end()));
DispatchEvent(event_args);
}
if (end_of_response) {
FSMEventArgs event_args(EVENT_DOWNSTREAM_CLOSED);
DispatchEvent(event_args);
}
}
bool GoogleStreamingRemoteEngine::IsRecognitionPending() const {
DCHECK(CalledOnValidThread());
return state_ != STATE_IDLE;
}
int GoogleStreamingRemoteEngine::GetDesiredAudioChunkDurationMs() const {
return kAudioPacketIntervalMs;
}
void GoogleStreamingRemoteEngine::DispatchEvent(
const FSMEventArgs& event_args) {
DCHECK(CalledOnValidThread());
DCHECK_LE(event_args.event, EVENT_MAX_VALUE);
DCHECK_LE(state_, STATE_MAX_VALUE);
DCHECK(!is_dispatching_event_);
is_dispatching_event_ = true;
state_ = ExecuteTransitionAndGetNextState(event_args);
is_dispatching_event_ = false;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::ExecuteTransitionAndGetNextState(
const FSMEventArgs& event_args) {
const FSMEvent event = event_args.event;
switch (state_) {
case STATE_IDLE:
switch (event) {
case EVENT_START_RECOGNITION:
return ConnectBothStreams(event_args);
case EVENT_END_RECOGNITION:
case EVENT_AUDIO_CHUNK:
case EVENT_AUDIO_CHUNKS_ENDED:
case EVENT_DOWNSTREAM_CLOSED:
return DoNothing(event_args);
case EVENT_UPSTREAM_ERROR:
case EVENT_DOWNSTREAM_ERROR:
case EVENT_DOWNSTREAM_RESPONSE:
return NotFeasible(event_args);
}
break;
case STATE_BOTH_STREAMS_CONNECTED:
switch (event) {
case EVENT_AUDIO_CHUNK:
return TransmitAudioUpstream(event_args);
case EVENT_DOWNSTREAM_RESPONSE:
return ProcessDownstreamResponse(event_args);
case EVENT_AUDIO_CHUNKS_ENDED:
return CloseUpstreamAndWaitForResults(event_args);
case EVENT_END_RECOGNITION:
return AbortSilently(event_args);
case EVENT_UPSTREAM_ERROR:
case EVENT_DOWNSTREAM_ERROR:
case EVENT_DOWNSTREAM_CLOSED:
return AbortWithError(event_args);
case EVENT_START_RECOGNITION:
return NotFeasible(event_args);
}
break;
case STATE_WAITING_DOWNSTREAM_RESULTS:
switch (event) {
case EVENT_DOWNSTREAM_RESPONSE:
return ProcessDownstreamResponse(event_args);
case EVENT_DOWNSTREAM_CLOSED:
return RaiseNoMatchErrorIfGotNoResults(event_args);
case EVENT_END_RECOGNITION:
return AbortSilently(event_args);
case EVENT_UPSTREAM_ERROR:
case EVENT_DOWNSTREAM_ERROR:
return AbortWithError(event_args);
case EVENT_START_RECOGNITION:
case EVENT_AUDIO_CHUNK:
case EVENT_AUDIO_CHUNKS_ENDED:
return NotFeasible(event_args);
}
break;
}
return NotFeasible(event_args);
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::ConnectBothStreams(const FSMEventArgs&) {
DCHECK(!upstream_fetcher_.get());
DCHECK(!downstream_fetcher_.get());
encoder_.reset(AudioEncoder::Create(kDefaultAudioCodec,
config_.audio_sample_rate,
config_.audio_num_bits_per_sample));
DCHECK(encoder_.get());
const std::string request_key = GenerateRequestKey();
std::vector<std::string> downstream_args;
downstream_args.push_back(
"key=" + net::EscapeQueryParamValue(google_apis::GetAPIKey(), true));
downstream_args.push_back("pair=" + request_key);
downstream_args.push_back("output=pb");
GURL downstream_url(std::string(kWebServiceBaseUrl) +
std::string(kDownstreamUrl) +
JoinString(downstream_args, '&'));
downstream_fetcher_.reset(URLFetcher::Create(
kDownstreamUrlFetcherIdForTesting, downstream_url, URLFetcher::GET,
this));
downstream_fetcher_->SetRequestContext(url_context_.get());
downstream_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
net::LOAD_DO_NOT_SEND_COOKIES |
net::LOAD_DO_NOT_SEND_AUTH_DATA);
downstream_fetcher_->Start();
std::vector<std::string> upstream_args;
upstream_args.push_back("key=" +
net::EscapeQueryParamValue(google_apis::GetAPIKey(), true));
upstream_args.push_back("pair=" + request_key);
upstream_args.push_back("output=pb");
upstream_args.push_back(
"lang=" + net::EscapeQueryParamValue(GetAcceptedLanguages(), true));
upstream_args.push_back(
config_.filter_profanities ? "pFilter=2" : "pFilter=0");
if (config_.max_hypotheses > 0U) {
int max_alternatives = std::min(kMaxMaxAlternatives,
config_.max_hypotheses);
upstream_args.push_back("maxAlternatives=" +
base::UintToString(max_alternatives));
}
upstream_args.push_back("client=chromium");
if (!config_.hardware_info.empty()) {
upstream_args.push_back(
"xhw=" + net::EscapeQueryParamValue(config_.hardware_info, true));
}
if (config_.continuous)
upstream_args.push_back("continuous");
if (config_.interim_results)
upstream_args.push_back("interim");
GURL upstream_url(std::string(kWebServiceBaseUrl) +
std::string(kUpstreamUrl) +
JoinString(upstream_args, '&'));
upstream_fetcher_.reset(URLFetcher::Create(
kUpstreamUrlFetcherIdForTesting, upstream_url, URLFetcher::POST, this));
upstream_fetcher_->SetChunkedUpload(encoder_->mime_type());
upstream_fetcher_->SetRequestContext(url_context_.get());
upstream_fetcher_->SetReferrer(config_.origin_url);
upstream_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
net::LOAD_DO_NOT_SEND_COOKIES |
net::LOAD_DO_NOT_SEND_AUTH_DATA);
upstream_fetcher_->Start();
previous_response_length_ = 0;
return STATE_BOTH_STREAMS_CONNECTED;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::TransmitAudioUpstream(
const FSMEventArgs& event_args) {
DCHECK(upstream_fetcher_.get());
DCHECK(event_args.audio_data.get());
const AudioChunk& audio = *(event_args.audio_data.get());
DCHECK_EQ(audio.bytes_per_sample(), config_.audio_num_bits_per_sample / 8);
encoder_->Encode(audio);
scoped_refptr<AudioChunk> encoded_data(encoder_->GetEncodedDataAndClear());
upstream_fetcher_->AppendChunkToUpload(encoded_data->AsString(), false);
return state_;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::ProcessDownstreamResponse(
const FSMEventArgs& event_args) {
DCHECK(event_args.response.get());
proto::SpeechRecognitionEvent ws_event;
if (!ws_event.ParseFromString(std::string(event_args.response->begin(),
event_args.response->end())))
return AbortWithError(event_args);
if (!ws_event.result_size() && (!ws_event.has_status() ||
ws_event.status() == proto::SpeechRecognitionEvent::STATUS_SUCCESS)) {
DVLOG(1) << "Received empty response";
return state_;
}
if (ws_event.has_status()) {
switch (ws_event.status()) {
case proto::SpeechRecognitionEvent::STATUS_SUCCESS:
break;
case proto::SpeechRecognitionEvent::STATUS_NO_SPEECH:
return Abort(SPEECH_RECOGNITION_ERROR_NO_SPEECH);
case proto::SpeechRecognitionEvent::STATUS_ABORTED:
return Abort(SPEECH_RECOGNITION_ERROR_ABORTED);
case proto::SpeechRecognitionEvent::STATUS_AUDIO_CAPTURE:
return Abort(SPEECH_RECOGNITION_ERROR_AUDIO);
case proto::SpeechRecognitionEvent::STATUS_NETWORK:
return Abort(SPEECH_RECOGNITION_ERROR_NETWORK);
case proto::SpeechRecognitionEvent::STATUS_NOT_ALLOWED:
return Abort(SPEECH_RECOGNITION_ERROR_ABORTED);
case proto::SpeechRecognitionEvent::STATUS_SERVICE_NOT_ALLOWED:
return Abort(SPEECH_RECOGNITION_ERROR_ABORTED);
case proto::SpeechRecognitionEvent::STATUS_BAD_GRAMMAR:
return Abort(SPEECH_RECOGNITION_ERROR_BAD_GRAMMAR);
case proto::SpeechRecognitionEvent::STATUS_LANGUAGE_NOT_SUPPORTED:
return Abort(SPEECH_RECOGNITION_ERROR_ABORTED);
}
}
SpeechRecognitionResults results;
for (int i = 0; i < ws_event.result_size(); ++i) {
const proto::SpeechRecognitionResult& ws_result = ws_event.result(i);
results.push_back(SpeechRecognitionResult());
SpeechRecognitionResult& result = results.back();
result.is_provisional = !(ws_result.has_final() && ws_result.final());
if (!result.is_provisional)
got_last_definitive_result_ = true;
for (int j = 0; j < ws_result.alternative_size(); ++j) {
const proto::SpeechRecognitionAlternative& ws_alternative =
ws_result.alternative(j);
SpeechRecognitionHypothesis hypothesis;
if (ws_alternative.has_confidence())
hypothesis.confidence = ws_alternative.confidence();
else if (ws_result.has_stability())
hypothesis.confidence = ws_result.stability();
DCHECK(ws_alternative.has_transcript());
if (ws_alternative.has_transcript())
hypothesis.utterance = base::UTF8ToUTF16(ws_alternative.transcript());
result.hypotheses.push_back(hypothesis);
}
}
delegate()->OnSpeechRecognitionEngineResults(results);
return state_;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::RaiseNoMatchErrorIfGotNoResults(
const FSMEventArgs& event_args) {
if (!got_last_definitive_result_) {
delegate()->OnSpeechRecognitionEngineResults(SpeechRecognitionResults());
}
return AbortSilently(event_args);
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::CloseUpstreamAndWaitForResults(
const FSMEventArgs&) {
DCHECK(upstream_fetcher_.get());
DCHECK(encoder_.get());
DVLOG(1) << "Closing upstream.";
std::vector<short> samples(
config_.audio_sample_rate * kAudioPacketIntervalMs / 1000);
scoped_refptr<AudioChunk> dummy_chunk =
new AudioChunk(reinterpret_cast<uint8*>(&samples[0]),
samples.size() * sizeof(short),
encoder_->bits_per_sample() / 8);
encoder_->Encode(*dummy_chunk.get());
encoder_->Flush();
scoped_refptr<AudioChunk> encoded_dummy_data =
encoder_->GetEncodedDataAndClear();
DCHECK(!encoded_dummy_data->IsEmpty());
encoder_.reset();
upstream_fetcher_->AppendChunkToUpload(encoded_dummy_data->AsString(), true);
got_last_definitive_result_ = false;
return STATE_WAITING_DOWNSTREAM_RESULTS;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::CloseDownstream(const FSMEventArgs&) {
DCHECK(!upstream_fetcher_.get());
DCHECK(downstream_fetcher_.get());
DVLOG(1) << "Closing downstream.";
downstream_fetcher_.reset();
return STATE_IDLE;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::AbortSilently(const FSMEventArgs&) {
return Abort(SPEECH_RECOGNITION_ERROR_NONE);
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::AbortWithError(const FSMEventArgs&) {
return Abort(SPEECH_RECOGNITION_ERROR_NETWORK);
}
GoogleStreamingRemoteEngine::FSMState GoogleStreamingRemoteEngine::Abort(
SpeechRecognitionErrorCode error_code) {
DVLOG(1) << "Aborting with error " << error_code;
if (error_code != SPEECH_RECOGNITION_ERROR_NONE) {
delegate()->OnSpeechRecognitionEngineError(
SpeechRecognitionError(error_code));
}
downstream_fetcher_.reset();
upstream_fetcher_.reset();
encoder_.reset();
return STATE_IDLE;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::DoNothing(const FSMEventArgs&) {
return state_;
}
GoogleStreamingRemoteEngine::FSMState
GoogleStreamingRemoteEngine::NotFeasible(const FSMEventArgs& event_args) {
NOTREACHED() << "Unfeasible event " << event_args.event
<< " in state " << state_;
return state_;
}
std::string GoogleStreamingRemoteEngine::GetAcceptedLanguages() const {
std::string langs = config_.language;
if (langs.empty() && url_context_.get()) {
net::URLRequestContext* request_context =
url_context_->GetURLRequestContext();
DCHECK(request_context);
if (request_context->http_user_agent_settings()) {
std::string accepted_language_list =
request_context->http_user_agent_settings()->GetAcceptLanguage();
size_t separator = accepted_language_list.find_first_of(",;");
if (separator != std::string::npos)
langs = accepted_language_list.substr(0, separator);
}
}
if (langs.empty())
langs = "en-US";
return langs;
}
std::string GoogleStreamingRemoteEngine::GenerateRequestKey() const {
const int64 kKeepLowBytes = GG_LONGLONG(0x00000000FFFFFFFF);
const int64 kKeepHighBytes = GG_LONGLONG(0xFFFFFFFF00000000);
int64 key = (base::Time::Now().ToInternalValue() & kKeepLowBytes) |
(base::RandUint64() & kKeepHighBytes);
return base::HexEncode(reinterpret_cast<void*>(&key), sizeof(key));
}
GoogleStreamingRemoteEngine::FSMEventArgs::FSMEventArgs(FSMEvent event_value)
: event(event_value) {
}
GoogleStreamingRemoteEngine::FSMEventArgs::~FSMEventArgs() {
}
}