From 0c67cb3c5f279ff5724b0bba64867d9da3c1c73c Mon Sep 17 00:00:00 2001 From: Paul Adenot Date: Wed, 2 Sep 2026 12:26:05 +0000 Subject: [PATCH] Bug 1940906 - Remove code related to online speech recognition, or generally the previous incarnation of Web Speech Recognition. r=pehrsons,frontend-codestyle-reviewers This has never been shipped and the endpoints aren't functional anymore, we can just remove everything. I don't think we'll want to ship non-local speech recognition at this point. This is mostly to ease review, we'll fill in the blanks in a later patch. Differential Revision: https://phabricator.services.mozilla.com/D268404 --- .clang-format-ignore | 6 - .prettierignore | 6 - .../OnlineSpeechRecognitionService.cpp | 451 --------- .../OnlineSpeechRecognitionService.h | 130 --- .../recognition/SpeechRecognition.cpp | 890 +----------------- .../webspeech/recognition/SpeechRecognition.h | 148 +-- .../recognition/SpeechTrackListener.cpp | 59 +- .../recognition/SpeechTrackListener.h | 4 - dom/media/webspeech/recognition/endpointer.cc | 193 ---- dom/media/webspeech/recognition/endpointer.h | 180 ---- .../recognition/energy_endpointer.cc | 393 -------- .../webspeech/recognition/energy_endpointer.h | 180 ---- .../recognition/energy_endpointer_params.cc | 77 -- .../recognition/energy_endpointer_params.h | 159 ---- dom/media/webspeech/recognition/moz.build | 30 +- .../nsISpeechRecognitionService.idl | 42 - .../test/FakeSpeechRecognitionService.cpp | 115 --- .../test/FakeSpeechRecognitionService.h | 38 - .../recognition/test/http_requesthandler.sjs | 87 -- .../webspeech/recognition/test/mochitest.toml | 34 - .../recognition/test/test_abort.html | 73 -- .../test/test_audio_capture_error.html | 42 - .../test_call_start_from_end_handler.html | 102 -- .../test/test_nested_eventloop.html | 82 -- .../test/test_online_400_response.html | 46 - .../test_online_empty_result_handling.html | 47 - .../recognition/test/test_online_hangup.html | 46 - .../recognition/test/test_online_http.html | 86 -- .../test/test_online_http_webkit.html | 87 -- ...test_online_malformed_result_handling.html | 47 - .../test/test_recognition_service_error.html | 45 - ...t_success_without_recognition_service.html | 45 - .../recognition/test/test_timeout.html | 42 - dom/media/webspeech/synth/moz.build | 9 +- layout/build/components.conf | 19 +- layout/build/nsLayoutModule.cpp | 8 +- modules/libpref/Preferences.cpp | 1 - modules/libpref/init/StaticPrefList.yaml | 12 +- toolkit/moz.configure | 17 - tools/rewriting/ThirdPartyPaths.txt | 6 - 40 files changed, 34 insertions(+), 4050 deletions(-) delete mode 100644 dom/media/webspeech/recognition/OnlineSpeechRecognitionService.cpp delete mode 100644 dom/media/webspeech/recognition/OnlineSpeechRecognitionService.h delete mode 100644 dom/media/webspeech/recognition/endpointer.cc delete mode 100644 dom/media/webspeech/recognition/endpointer.h delete mode 100644 dom/media/webspeech/recognition/energy_endpointer.cc delete mode 100644 dom/media/webspeech/recognition/energy_endpointer.h delete mode 100644 dom/media/webspeech/recognition/energy_endpointer_params.cc delete mode 100644 dom/media/webspeech/recognition/energy_endpointer_params.h delete mode 100644 dom/media/webspeech/recognition/nsISpeechRecognitionService.idl delete mode 100644 dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp delete mode 100644 dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.h delete mode 100644 dom/media/webspeech/recognition/test/http_requesthandler.sjs delete mode 100644 dom/media/webspeech/recognition/test/test_abort.html delete mode 100644 dom/media/webspeech/recognition/test/test_audio_capture_error.html delete mode 100644 dom/media/webspeech/recognition/test/test_call_start_from_end_handler.html delete mode 100644 dom/media/webspeech/recognition/test/test_nested_eventloop.html delete mode 100644 dom/media/webspeech/recognition/test/test_online_400_response.html delete mode 100644 dom/media/webspeech/recognition/test/test_online_empty_result_handling.html delete mode 100644 dom/media/webspeech/recognition/test/test_online_hangup.html delete mode 100644 dom/media/webspeech/recognition/test/test_online_http.html delete mode 100644 dom/media/webspeech/recognition/test/test_online_http_webkit.html delete mode 100644 dom/media/webspeech/recognition/test/test_online_malformed_result_handling.html delete mode 100644 dom/media/webspeech/recognition/test/test_recognition_service_error.html delete mode 100644 dom/media/webspeech/recognition/test/test_success_without_recognition_service.html delete mode 100644 dom/media/webspeech/recognition/test/test_timeout.html diff --git a/.clang-format-ignore b/.clang-format-ignore index d9dc7e8ce074..5d594891083f 100644 --- a/.clang-format-ignore +++ b/.clang-format-ignore @@ -88,12 +88,6 @@ dom/media/platforms/ffmpeg/libav53/.* dom/media/platforms/ffmpeg/libav54/.* dom/media/platforms/ffmpeg/libav55/.* dom/media/webrtc/transport/third_party/.* -dom/media/webspeech/recognition/endpointer.cc -dom/media/webspeech/recognition/endpointer.h -dom/media/webspeech/recognition/energy_endpointer.cc -dom/media/webspeech/recognition/energy_endpointer.h -dom/media/webspeech/recognition/energy_endpointer_params.cc -dom/media/webspeech/recognition/energy_endpointer_params.h dom/webauthn/winwebauthn/webauthn.h editor/libeditor/tests/browserscope/lib/richtext/.* editor/libeditor/tests/browserscope/lib/richtext2/.* diff --git a/.prettierignore b/.prettierignore index 2151f1298698..0546a8105a0c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1388,12 +1388,6 @@ dom/media/platforms/ffmpeg/libav55/ dom/media/webaudio/test/blink/ dom/media/webrtc/tests/mochitests/helpers_from_wpt/sdp.js dom/media/webrtc/transport/third_party/ -dom/media/webspeech/recognition/endpointer.cc -dom/media/webspeech/recognition/endpointer.h -dom/media/webspeech/recognition/energy_endpointer.cc -dom/media/webspeech/recognition/energy_endpointer.h -dom/media/webspeech/recognition/energy_endpointer_params.cc -dom/media/webspeech/recognition/energy_endpointer_params.h dom/media/webvtt/vtt.sys.mjs dom/security/sanitizer/safe-default-configuration.json dom/tests/mochitest/ajax/ diff --git a/dom/media/webspeech/recognition/OnlineSpeechRecognitionService.cpp b/dom/media/webspeech/recognition/OnlineSpeechRecognitionService.cpp deleted file mode 100644 index 9b68e04b97f7..000000000000 --- a/dom/media/webspeech/recognition/OnlineSpeechRecognitionService.cpp +++ /dev/null @@ -1,451 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "OnlineSpeechRecognitionService.h" - -#include - -#include "OggWriter.h" -#include "OpusTrackEncoder.h" -#include "SpeechGrammar.h" -#include "SpeechRecognition.h" -#include "SpeechRecognitionAlternative.h" -#include "SpeechRecognitionResult.h" -#include "SpeechRecognitionResultList.h" -#include "mozilla/Preferences.h" -#include "mozilla/ScopeExit.h" -#include "mozilla/dom/Document.h" -#include "nsContentUtils.h" -#include "nsGlobalWindowInner.h" -#include "nsIChannel.h" -#include "nsIClassOfService.h" -#include "nsIHttpChannel.h" -#include "nsIOutputStream.h" -#include "nsIPrincipal.h" -#include "nsIStreamListener.h" -#include "nsIUploadChannel2.h" -#include "nsNetUtil.h" -#include "nsStringStream.h" -#include "nsThreadUtils.h" - -namespace mozilla { - -using namespace dom; - -#define PREFERENCE_DEFAULT_RECOGNITION_ENDPOINT \ - "media.webspeech.service.endpoint" -#define DEFAULT_RECOGNITION_ENDPOINT "https://speaktome-2.services.mozilla.com/" -#define MAX_LISTENING_TIME_MS 10000 - -NS_IMPL_ISUPPORTS(OnlineSpeechRecognitionService, nsISpeechRecognitionService, - nsIStreamListener) - -NS_IMETHODIMP -OnlineSpeechRecognitionService::OnStartRequest(nsIRequest* aRequest) { - MOZ_ASSERT(NS_IsMainThread()); - return NS_OK; -} - -static nsresult AssignResponseToBuffer(nsIInputStream* aIn, void* aClosure, - const char* aFromRawSegment, - uint32_t aToOffset, uint32_t aCount, - uint32_t* aWriteCount) { - nsCString* buf = static_cast(aClosure); - buf->Append(aFromRawSegment, aCount); - *aWriteCount = aCount; - return NS_OK; -} - -NS_IMETHODIMP -OnlineSpeechRecognitionService::OnDataAvailable(nsIRequest* aRequest, - nsIInputStream* aInputStream, - uint64_t aOffset, - uint32_t aCount) { - MOZ_ASSERT(NS_IsMainThread()); - nsresult rv; - uint32_t readCount; - rv = aInputStream->ReadSegments(AssignResponseToBuffer, &mBuf, aCount, - &readCount); - NS_ENSURE_SUCCESS(rv, rv); - return NS_OK; -} - -NS_IMETHODIMP -OnlineSpeechRecognitionService::OnStopRequest(nsIRequest* aRequest, - nsresult aStatusCode) { - MOZ_ASSERT(NS_IsMainThread()); - - auto clearBuf = MakeScopeExit([&] { mBuf.Truncate(); }); - - if (mAborted) { - return NS_OK; - } - - bool success; - float confidence = 0; - Json::Value root; - Json::CharReaderBuilder builder; - bool parsingSuccessful; - nsAutoCString result; - nsAutoCString hypoValue; - nsAutoCString errorMsg; - SpeechRecognitionErrorCode errorCode; - - SR_LOG("STT Result: {}", mBuf.get()); - - if (NS_FAILED(aStatusCode)) { - success = false; - errorMsg.AssignLiteral("Error connecting to the service."); - errorCode = SpeechRecognitionErrorCode::Network; - } else { - success = true; - UniquePtr const reader(builder.newCharReader()); - parsingSuccessful = - reader->parse(mBuf.BeginReading(), mBuf.EndReading(), &root, nullptr); - if (!parsingSuccessful) { - // there's an internal server error - success = false; - errorMsg.AssignLiteral("Internal server error"); - errorCode = SpeechRecognitionErrorCode::Network; - } else { - result.Assign(root.get("status", "error").asString().c_str()); - if (result.EqualsLiteral("ok")) { - // ok, we have a result - if (!root["data"].empty()) { - hypoValue.Assign(root["data"][0].get("text", "").asString().c_str()); - confidence = root["data"][0].get("confidence", "0").asFloat(); - } else { - success = false; - errorMsg.AssignLiteral("Error reading result data."); - errorCode = SpeechRecognitionErrorCode::Network; - } - } else { - success = false; - errorMsg.Assign(root.get("message", "").asString().c_str()); - errorCode = SpeechRecognitionErrorCode::No_speech; - } - } - } - - if (!success) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, errorCode, errorMsg); - } else { - // Declare javascript result events - RefPtr event = new SpeechEvent( - mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT); - SpeechRecognitionResultList* resultList = - new SpeechRecognitionResultList(mRecognition); - SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); - - if (mRecognition->MaxAlternatives() > 0) { - SpeechRecognitionAlternative* alternative = - new SpeechRecognitionAlternative(mRecognition); - - alternative->mTranscript = NS_ConvertUTF8toUTF16(hypoValue); - alternative->mConfidence = confidence; - - result->mItems.AppendElement(alternative); - } - resultList->mItems.AppendElement(result); - - event->mRecognitionResultList = resultList; - NS_DispatchToMainThread(event); - } - - return NS_OK; -} - -OnlineSpeechRecognitionService::OnlineSpeechRecognitionService() = default; -OnlineSpeechRecognitionService::~OnlineSpeechRecognitionService() = default; - -NS_IMETHODIMP -OnlineSpeechRecognitionService::Initialize( - WeakPtr aSpeechRecognition) { - MOZ_ASSERT(NS_IsMainThread()); - mWriter = MakeUnique(); - mRecognition = new nsMainThreadPtrHolder( - "OnlineSpeechRecognitionService::mRecognition", aSpeechRecognition); - mEncodeTaskQueue = mRecognition->GetTaskQueueForEncoding(); - MOZ_ASSERT(mEncodeTaskQueue); - return NS_OK; -} - -void OnlineSpeechRecognitionService::EncoderFinished() { - MOZ_ASSERT(!NS_IsMainThread()); - MOZ_ASSERT(mEncodedAudioQueue.IsFinished()); - - while (RefPtr frame = mEncodedAudioQueue.PopFront()) { - AutoTArray, 1> frames({frame}); - DebugOnly rv = - mWriter->WriteEncodedTrack(frames, mEncodedAudioQueue.AtEndOfStream() - ? ContainerWriter::END_OF_STREAM - : 0); - MOZ_ASSERT(NS_SUCCEEDED(rv)); - } - - mWriter->GetContainerData(&mEncodedData, ContainerWriter::FLUSH_NEEDED); - MOZ_ASSERT(mWriter->IsWritingComplete()); - - NS_DispatchToMainThread( - NewRunnableMethod("OnlineSpeechRecognitionService::DoSTT", this, - &OnlineSpeechRecognitionService::DoSTT)); -} - -void OnlineSpeechRecognitionService::EncoderInitialized() { - MOZ_ASSERT(!NS_IsMainThread()); - AutoTArray, 1> metadata; - metadata.AppendElement(mAudioEncoder->GetMetadata()); - if (metadata[0]->GetKind() != TrackMetadataBase::METADATA_OPUS) { - SR_LOG("wrong meta data type!"); - MOZ_ASSERT_UNREACHABLE(); - } - - nsresult rv = mWriter->SetMetadata(metadata); - MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); - - rv = mWriter->GetContainerData(&mEncodedData, ContainerWriter::GET_HEADER); - MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); - - (void)rv; -} - -void OnlineSpeechRecognitionService::EncoderError() { - MOZ_ASSERT(!NS_IsMainThread()); - SR_LOG("Error encoding frames."); - mEncodedData.Clear(); - NS_DispatchToMainThread(NS_NewRunnableFunction( - "SpeechRecognition::DispatchError", - [this, self = RefPtr(this)]() { - if (!mRecognition) { - return; - } - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Audio_capture, "Encoder error"); - })); -} - -NS_IMETHODIMP -OnlineSpeechRecognitionService::ProcessAudioSegment(AudioSegment* aAudioSegment, - int32_t aSampleRate) { - MOZ_ASSERT(!NS_IsMainThread()); - int64_t duration = aAudioSegment->GetDuration(); - if (duration <= 0) { - return NS_OK; - } - - if (!mAudioEncoder) { - mSpeechEncoderListener = new SpeechEncoderListener(this); - mAudioEncoder = - MakeUnique(aSampleRate, mEncodedAudioQueue); - RefPtr mEncoderThread = AbstractThread::GetCurrent(); - mAudioEncoder->SetWorkerThread(mEncoderThread); - mAudioEncoder->RegisterListener(mSpeechEncoderListener); - } - - mAudioEncoder->AppendAudioSegment(std::move(*aAudioSegment)); - - TimeStamp now = TimeStamp::Now(); - if (mFirstIteration.IsNull()) { - mFirstIteration = now; - } - - if ((now - mFirstIteration).ToMilliseconds() >= MAX_LISTENING_TIME_MS) { - NS_DispatchToMainThread(NS_NewRunnableFunction( - "SpeechRecognition::Stop", - [this, self = RefPtr(this)]() { - if (!mRecognition) { - return; - } - mRecognition->Stop(); - })); - - return NS_OK; - } - - return NS_OK; -} - -void OnlineSpeechRecognitionService::DoSTT() { - MOZ_ASSERT(NS_IsMainThread()); - - if (mAborted) { - return; - } - - nsresult rv; - nsCOMPtr chan; - nsCOMPtr uri; - nsAutoCString speechRecognitionEndpoint; - nsAutoCString prefEndpoint; - nsAutoString language; - - Preferences::GetCString(PREFERENCE_DEFAULT_RECOGNITION_ENDPOINT, - prefEndpoint); - - if (!prefEndpoint.IsEmpty()) { - speechRecognitionEndpoint = std::move(prefEndpoint); - } else { - speechRecognitionEndpoint = DEFAULT_RECOGNITION_ENDPOINT; - } - - rv = NS_NewURI(getter_AddRefs(uri), speechRecognitionEndpoint, nullptr, - nullptr); - if (NS_WARN_IF(NS_FAILED(rv))) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Network, "Unknown URI"); - return; - } - - nsSecurityFlags secFlags = nsILoadInfo::SEC_REQUIRE_CORS_INHERITS_SEC_CONTEXT; - nsLoadFlags loadFlags = - nsIRequest::LOAD_NORMAL | nsIChannel::LOAD_BYPASS_SERVICE_WORKER; - nsContentPolicyType contentPolicy = nsIContentPolicy::TYPE_OTHER; - - nsGlobalWindowInner* window = mRecognition->GetOwnerWindow(); - if (NS_WARN_IF(!window)) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Aborted, "No window"); - return; - } - - Document* doc = window->GetExtantDoc(); - if (NS_WARN_IF(!doc)) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Aborted, "No document"); - } - rv = NS_NewChannel(getter_AddRefs(chan), uri, doc->NodePrincipal(), secFlags, - contentPolicy, nullptr, nullptr, nullptr, nullptr, - loadFlags); - if (NS_WARN_IF(NS_FAILED(rv))) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Network, "Failed to open channel"); - return; - } - - nsCOMPtr httpChan = do_QueryInterface(chan); - if (httpChan) { - rv = httpChan->SetRequestMethod("POST"_ns); - MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); - } - - if (httpChan) { - mRecognition->GetLang(language); - // Accept-Language-STT is a custom header of our backend server used to set - // the language of the speech sample being submitted by the client - rv = httpChan->SetRequestHeader("Accept-Language-STT"_ns, - NS_ConvertUTF16toUTF8(language), false); - MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); - // Tell the server to not store the transcription by default - rv = httpChan->SetRequestHeader("Store-Transcription"_ns, "0"_ns, false); - MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); - // Tell the server to not store the sample by default - rv = httpChan->SetRequestHeader("Store-Sample"_ns, "0"_ns, false); - MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); - // Set the product tag as the web speech api - rv = httpChan->SetRequestHeader("Product-Tag"_ns, "wsa"_ns, false); - MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); - } - - nsCOMPtr cos(do_QueryInterface(chan)); - if (cos) { - cos->AddClassFlags(nsIClassOfService::UrgentStart); - } - - nsCOMPtr uploadChan = do_QueryInterface(chan); - if (uploadChan) { - nsCOMPtr bodyStream; - uint32_t length = 0; - for (const nsTArray& chunk : mEncodedData) { - length += chunk.Length(); - } - - nsTArray audio; - if (!audio.SetCapacity(length, fallible)) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Audio_capture, "Allocation error"); - return; - } - - for (const nsTArray& chunk : mEncodedData) { - audio.AppendElements(chunk); - } - - mEncodedData.Clear(); - - rv = NS_NewByteInputStream(getter_AddRefs(bodyStream), std::move(audio)); - if (NS_WARN_IF(NS_FAILED(rv))) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Network, "Failed to open stream"); - return; - } - if (bodyStream) { - rv = uploadChan->ExplicitSetUploadStream(bodyStream, "audio/ogg"_ns, - length, "POST"_ns); - MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); - } - } - - rv = chan->AsyncOpen(this); - if (NS_WARN_IF(NS_FAILED(rv))) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Network, "Internal server error"); - } -} - -NS_IMETHODIMP -OnlineSpeechRecognitionService::SoundEnd() { - MOZ_ASSERT(NS_IsMainThread()); - - if (!mEncodeTaskQueue) { - // Not initialized - return NS_OK; - } - - nsresult rv = mEncodeTaskQueue->Dispatch(NS_NewRunnableFunction( - "OnlineSpeechRecognitionService::SoundEnd", - [this, self = RefPtr(this)]() { - if (mAudioEncoder) { - mAudioEncoder->NotifyEndOfStream(); - mAudioEncoder->UnregisterListener(mSpeechEncoderListener); - mSpeechEncoderListener = nullptr; - mAudioEncoder = nullptr; - EncoderFinished(); - } - })); - MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); - (void)rv; - - mEncodeTaskQueue = nullptr; - - return NS_OK; -} - -NS_IMETHODIMP -OnlineSpeechRecognitionService::ValidateAndSetGrammarList( - SpeechGrammar* aSpeechGrammar, - nsISpeechGrammarCompilationCallback* aCallback) { - // This is an online LVCSR (STT) service, - // so we don't need to set a grammar - return NS_OK; -} - -NS_IMETHODIMP -OnlineSpeechRecognitionService::Abort() { - MOZ_ASSERT(NS_IsMainThread()); - if (mAborted) { - return NS_OK; - } - mAborted = true; - return SoundEnd(); -} -} // namespace mozilla diff --git a/dom/media/webspeech/recognition/OnlineSpeechRecognitionService.h b/dom/media/webspeech/recognition/OnlineSpeechRecognitionService.h deleted file mode 100644 index 8079292b1662..000000000000 --- a/dom/media/webspeech/recognition/OnlineSpeechRecognitionService.h +++ /dev/null @@ -1,130 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef mozilla_dom_OnlineRecognitionService_h -#define mozilla_dom_OnlineRecognitionService_h - -#include "ContainerWriter.h" -#include "OpusTrackEncoder.h" -#include "nsCOMPtr.h" -#include "nsISpeechRecognitionService.h" -#include "nsIStreamListener.h" -#include "nsTArray.h" -#include "speex/speex_resampler.h" - -#define NS_ONLINE_SPEECH_RECOGNITION_SERVICE_CID \ - {0x0ff5ce56, \ - 0x5b09, \ - 0x4db8, \ - {0xad, 0xc6, 0x82, 0x66, 0xaf, 0x95, 0xf8, 0x64}}; - -namespace mozilla { - -namespace ipc { -class PrincipalInfo; -} // namespace ipc - -/** - * Online implementation of the nsISpeechRecognitionService interface - */ -class OnlineSpeechRecognitionService : public nsISpeechRecognitionService, - public nsIStreamListener { - public: - // Add XPCOM glue code - NS_DECL_THREADSAFE_ISUPPORTS - NS_DECL_NSISPEECHRECOGNITIONSERVICE - NS_DECL_NSIREQUESTOBSERVER - NS_DECL_NSISTREAMLISTENER - - /** - * Listener responsible for handling the events raised by the TrackEncoder - */ - class SpeechEncoderListener : public TrackEncoderListener { - public: - explicit SpeechEncoderListener(OnlineSpeechRecognitionService* aService) - : mService(aService), mOwningThread(AbstractThread::GetCurrent()) {} - - void Started(TrackEncoder* aEncoder) override {} - - void Initialized(TrackEncoder* aEncoder) override { - MOZ_ASSERT(mOwningThread->IsCurrentThreadIn()); - mService->EncoderInitialized(); - } - - void Error(TrackEncoder* aEncoder) override { - MOZ_ASSERT(mOwningThread->IsCurrentThreadIn()); - mService->EncoderError(); - } - - private: - const RefPtr mService; - const RefPtr mOwningThread; - }; - - /** - * Default constructs a OnlineSpeechRecognitionService - */ - OnlineSpeechRecognitionService(); - - /** - * Called by SpeechEncoderListener when the AudioTrackEncoder has been - * initialized. - */ - void EncoderInitialized(); - - /** - * Called after the AudioTrackEncoder has encoded all data for us to wrap in a - * container and pass along. - */ - void EncoderFinished(); - - /** - * Called by SpeechEncoderListener when the AudioTrackEncoder has - * encountered an error. - */ - void EncoderError(); - - private: - /** - * Private destructor to prevent bypassing of reference counting - */ - virtual ~OnlineSpeechRecognitionService(); - - /** The associated SpeechRecognition */ - nsMainThreadPtrHandle mRecognition; - - /** - * Builds a mock SpeechRecognitionResultList - */ - dom::SpeechRecognitionResultList* BuildMockResultList(); - - /** - * Method responsible for uploading the audio to the remote endpoint - */ - void DoSTT(); - - // Encoded and packaged ogg audio data - nsTArray> mEncodedData; - // Member responsible for holding a reference to the TrackEncoderListener - RefPtr mSpeechEncoderListener; - // MediaQueue fed encoded data by mAudioEncoder - MediaQueue mEncodedAudioQueue; - // Encoder responsible for encoding the frames from pcm to opus which is the - // format supported by our backend - UniquePtr mAudioEncoder; - // Object responsible for wrapping the opus frames into an ogg container - UniquePtr mWriter; - // Member responsible for storing the json string returned by the endpoint - nsCString mBuf; - // Used to calculate a ceiling on the time spent listening. - TimeStamp mFirstIteration; - // flag responsible to control if the user choose to abort - bool mAborted = false; - // reference to the audio encoder queue - RefPtr mEncodeTaskQueue; -}; - -} // namespace mozilla - -#endif diff --git a/dom/media/webspeech/recognition/SpeechRecognition.cpp b/dom/media/webspeech/recognition/SpeechRecognition.cpp index df163401e59d..68f316a62420 100644 --- a/dom/media/webspeech/recognition/SpeechRecognition.cpp +++ b/dom/media/webspeech/recognition/SpeechRecognition.cpp @@ -8,13 +8,15 @@ #include "AudioSegment.h" #include "MediaEnginePrefs.h" +#include "SpeechRecognitionAlternative.h" +#include "SpeechRecognitionBackend.h" +#include "SpeechRecognitionResult.h" +#include "SpeechRecognitionResultList.h" #include "SpeechTrackListener.h" #include "VideoUtils.h" -#include "endpointer.h" #include "mozilla/AbstractThread.h" #include "mozilla/MediaManager.h" #include "mozilla/Preferences.h" -#include "mozilla/ResultVariant.h" #include "mozilla/Services.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/dom/AudioStreamTrack.h" @@ -46,97 +48,12 @@ namespace mozilla::dom { -#define PREFERENCE_DEFAULT_RECOGNITION_SERVICE "media.webspeech.service.default" -#define DEFAULT_RECOGNITION_SERVICE "online" - -#define PREFERENCE_ENDPOINTER_SILENCE_LENGTH "media.webspeech.silence_length" -#define PREFERENCE_ENDPOINTER_LONG_SILENCE_LENGTH \ - "media.webspeech.long_silence_length" -#define PREFERENCE_ENDPOINTER_LONG_SPEECH_LENGTH \ - "media.webspeech.long_speech_length" -#define PREFERENCE_SPEECH_DETECTION_TIMEOUT_MS \ - "media.webspeech.recognition.timeout" - -static const uint32_t kSAMPLE_RATE = 16000; - -// number of frames corresponding to 300ms of audio to send to endpointer while -// it's in environment estimation mode -// kSAMPLE_RATE frames = 1s, kESTIMATION_FRAMES frames = 300ms -static const uint32_t kESTIMATION_SAMPLES = 300 * kSAMPLE_RATE / 1000; - -LogModule* GetSpeechRecognitionLog() { - static LazyLogModule sLog("SpeechRecognition"); - return sLog; -} -#define SR_LOG(...) \ - MOZ_LOG_FMT(GetSpeechRecognitionLog(), mozilla::LogLevel::Debug, __VA_ARGS__) - -namespace { -class SpeechRecognitionShutdownBlocker : public media::ShutdownBlocker { - public: - SpeechRecognitionShutdownBlocker(SpeechRecognition* aRecognition, - const nsString& aName) - : media::ShutdownBlocker(aName), mRecognition(aRecognition) {} - - NS_IMETHOD BlockShutdown(nsIAsyncShutdownClient*) override { - MOZ_ASSERT(NS_IsMainThread()); - // AbortSilently will eventually clear the blocker. - mRecognition->Abort(); - return NS_OK; - } - - private: - const RefPtr mRecognition; -}; - -enum class ServiceCreationError { - ServiceNotFound, -}; - -Result, ServiceCreationError> -CreateSpeechRecognitionService(nsPIDOMWindowInner* aWindow, - SpeechRecognition* aRecognition, - const nsAString& aLang) { - nsAutoCString speechRecognitionServiceCID; - - nsAutoCString prefValue; - Preferences::GetCString(PREFERENCE_DEFAULT_RECOGNITION_SERVICE, prefValue); - nsAutoCString speechRecognitionService; - - if (!prefValue.IsEmpty()) { - speechRecognitionService = std::move(prefValue); - } else { - speechRecognitionService = DEFAULT_RECOGNITION_SERVICE; - } - - if (StaticPrefs::media_webspeech_test_fake_recognition_service()) { - speechRecognitionServiceCID = - NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "fake"; - } else { - speechRecognitionServiceCID = - nsLiteralCString(NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX) + - speechRecognitionService; - } - - nsresult rv; - nsCOMPtr recognitionService; - recognitionService = - do_CreateInstance(speechRecognitionServiceCID.get(), &rv); - if (!recognitionService) { - return Err(ServiceCreationError::ServiceNotFound); - } - - return recognitionService; -} -} // namespace - NS_IMPL_CYCLE_COLLECTION_WEAK_PTR_INHERITED(SpeechRecognition, DOMEventTargetHelper, mStream, - mTrack, mRecognitionService, - mSpeechGrammarList, mListener) + mTrack, mSpeechGrammarList, + mListener) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(SpeechRecognition) - NS_INTERFACE_MAP_ENTRY(nsIObserver) NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) NS_IMPL_ADDREF_INHERITED(SpeechRecognition, DOMEventTargetHelper) @@ -154,45 +71,15 @@ NS_INTERFACE_MAP_END_INHERITING(DOMMediaStream::TrackListener) SpeechRecognition::SpeechRecognition(nsPIDOMWindowInner* aOwnerWindow) : DOMEventTargetHelper(aOwnerWindow), - mEndpointer(kSAMPLE_RATE), - mAudioSamplesPerChunk(mEndpointer.FrameSize()), - mSpeechDetectionTimer(NS_NewTimer()), - mSpeechGrammarList(new SpeechGrammarList(GetRelevantGlobal())), + mSpeechGrammarList(new SpeechGrammarList(aOwnerWindow)), mContinuous(false), mInterimResults(false), mMaxAlternatives(1) { - SR_LOG("created SpeechRecognition"); - - if (StaticPrefs::media_webspeech_test_enable()) { - nsCOMPtr obs = services::GetObserverService(); - obs->AddObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC, false); - obs->AddObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC, false); - } - - mEndpointer.set_speech_input_complete_silence_length( - Preferences::GetInt(PREFERENCE_ENDPOINTER_SILENCE_LENGTH, 1250000)); - mEndpointer.set_long_speech_input_complete_silence_length( - Preferences::GetInt(PREFERENCE_ENDPOINTER_LONG_SILENCE_LENGTH, 2500000)); - mEndpointer.set_long_speech_length( - Preferences::GetInt(PREFERENCE_ENDPOINTER_SILENCE_LENGTH, 3 * 1000000)); - - mSpeechDetectionTimeoutMs = - Preferences::GetInt(PREFERENCE_SPEECH_DETECTION_TIMEOUT_MS, 10000); - Reset(); } SpeechRecognition::~SpeechRecognition() = default; -bool SpeechRecognition::StateBetween(FSMState begin, FSMState end) { - return mCurrentState >= begin && mCurrentState <= end; -} - -void SpeechRecognition::SetState(FSMState state) { - mCurrentState = state; - SR_LOG("Transitioned to state {}", GetName(mCurrentState)); -} - JSObject* SpeechRecognition::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { return SpeechRecognition_Binding::Wrap(aCx, this, aGivenProto); @@ -210,249 +97,7 @@ already_AddRefed SpeechRecognition::Constructor( return object.forget(); } -void SpeechRecognition::ProcessEvent(SpeechEvent* aEvent) { - SR_LOG("Processing {}, current state is {}", GetName(aEvent), - GetName(mCurrentState)); - - if (mAborted && aEvent->mType != EVENT_ABORT) { - // ignore all events while aborting - return; - } - - Transition(aEvent); -} - -void SpeechRecognition::Transition(SpeechEvent* aEvent) { - switch (mCurrentState) { - case STATE_IDLE: - switch (aEvent->mType) { - case EVENT_START: - // TODO: may want to time out if we wait too long - // for user to approve - WaitForAudioData(aEvent); - break; - case EVENT_STOP: - case EVENT_ABORT: - case EVENT_AUDIO_DATA: - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - DoNothing(aEvent); - break; - case EVENT_AUDIO_ERROR: - case EVENT_RECOGNITIONSERVICE_ERROR: - AbortError(aEvent); - break; - default: - MOZ_CRASH("Invalid event"); - } - break; - case STATE_STARTING: - switch (aEvent->mType) { - case EVENT_AUDIO_DATA: - StartedAudioCapture(aEvent); - break; - case EVENT_AUDIO_ERROR: - case EVENT_RECOGNITIONSERVICE_ERROR: - AbortError(aEvent); - break; - case EVENT_ABORT: - AbortSilently(aEvent); - break; - case EVENT_STOP: - ResetAndEnd(); - break; - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - DoNothing(aEvent); - break; - case EVENT_START: - SR_LOG("STATE_STARTING: Unhandled event {}", GetName(aEvent)); - MOZ_CRASH(); - default: - MOZ_CRASH("Invalid event"); - } - break; - case STATE_ESTIMATING: - switch (aEvent->mType) { - case EVENT_AUDIO_DATA: - WaitForEstimation(aEvent); - break; - case EVENT_STOP: - StopRecordingAndRecognize(aEvent); - break; - case EVENT_ABORT: - AbortSilently(aEvent); - break; - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - case EVENT_RECOGNITIONSERVICE_ERROR: - DoNothing(aEvent); - break; - case EVENT_AUDIO_ERROR: - AbortError(aEvent); - break; - case EVENT_START: - SR_LOG("STATE_ESTIMATING: Unhandled event {}", - static_cast(aEvent->mType)); - MOZ_CRASH(); - default: - MOZ_CRASH("Invalid event"); - } - break; - case STATE_WAITING_FOR_SPEECH: - switch (aEvent->mType) { - case EVENT_AUDIO_DATA: - DetectSpeech(aEvent); - break; - case EVENT_STOP: - StopRecordingAndRecognize(aEvent); - break; - case EVENT_ABORT: - AbortSilently(aEvent); - break; - case EVENT_AUDIO_ERROR: - AbortError(aEvent); - break; - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - case EVENT_RECOGNITIONSERVICE_ERROR: - DoNothing(aEvent); - break; - case EVENT_START: - SR_LOG("STATE_STARTING: Unhandled event {}", GetName(aEvent)); - MOZ_CRASH(); - default: - MOZ_CRASH("Invalid event"); - } - break; - case STATE_RECOGNIZING: - switch (aEvent->mType) { - case EVENT_AUDIO_DATA: - WaitForSpeechEnd(aEvent); - break; - case EVENT_STOP: - StopRecordingAndRecognize(aEvent); - break; - case EVENT_AUDIO_ERROR: - case EVENT_RECOGNITIONSERVICE_ERROR: - AbortError(aEvent); - break; - case EVENT_ABORT: - AbortSilently(aEvent); - break; - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - DoNothing(aEvent); - break; - case EVENT_START: - SR_LOG("STATE_RECOGNIZING: Unhandled aEvent {}", GetName(aEvent)); - MOZ_CRASH(); - default: - MOZ_CRASH("Invalid event"); - } - break; - case STATE_WAITING_FOR_RESULT: - switch (aEvent->mType) { - case EVENT_STOP: - DoNothing(aEvent); - break; - case EVENT_AUDIO_ERROR: - case EVENT_RECOGNITIONSERVICE_ERROR: - AbortError(aEvent); - break; - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - NotifyFinalResult(aEvent); - break; - case EVENT_AUDIO_DATA: - DoNothing(aEvent); - break; - case EVENT_ABORT: - AbortSilently(aEvent); - break; - case EVENT_START: - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - SR_LOG("STATE_WAITING_FOR_RESULT: Unhandled aEvent {}", - GetName(aEvent)); - MOZ_CRASH(); - default: - MOZ_CRASH("Invalid event"); - } - break; - case STATE_ABORTING: - switch (aEvent->mType) { - case EVENT_STOP: - case EVENT_ABORT: - case EVENT_AUDIO_DATA: - case EVENT_AUDIO_ERROR: - case EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT: - case EVENT_RECOGNITIONSERVICE_FINAL_RESULT: - case EVENT_RECOGNITIONSERVICE_ERROR: - DoNothing(aEvent); - break; - case EVENT_START: - SR_LOG("STATE_ABORTING: Unhandled aEvent {}", GetName(aEvent)); - MOZ_CRASH(); - default: - MOZ_CRASH("Invalid event"); - } - break; - default: - MOZ_CRASH("Invalid state"); - } -} - -/* - * Handle a segment of recorded audio data. - * Returns the number of samples that were processed. - */ -uint32_t SpeechRecognition::ProcessAudioSegment(AudioSegment* aSegment, - TrackRate aTrackRate) { - AudioSegment::ChunkIterator iterator(*aSegment); - uint32_t samples = 0; - while (!iterator.IsEnded()) { - float out; - mEndpointer.ProcessAudio(*iterator, &out); - samples += iterator->GetDuration(); - iterator.Next(); - } - - // we need to call the nsISpeechRecognitionService::ProcessAudioSegment - // in a separate thread so that any eventual encoding or pre-processing - // of the audio does not block the main thread - nsresult rv = mEncodeTaskQueue->Dispatch(NS_NewRunnableFunction( - "nsISpeechRecognitionService::ProcessAudioSegment", - [=, service = mRecognitionService, - segment = std::move(*aSegment)]() mutable { - service->ProcessAudioSegment(&segment, aTrackRate); - })); - - MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); - (void)rv; - return samples; -} - -/**************************************************************************** - * FSM Transition functions - * - * If a transition function may cause a DOM event to be fired, - * it may also be re-entered, since the event handler may cause the - * event loop to spin and new SpeechEvents to be processed. - * - * Rules: - * 1) These methods should call SetState as soon as possible. - * 2) If these methods dispatch DOM events, or call methods that dispatch - * DOM events, that should be done as late as possible. - * 3) If anything must happen after dispatching a DOM event, make sure - * the state is still what the method expected it to be. - ****************************************************************************/ - void SpeechRecognition::Reset() { - SetState(STATE_IDLE); - - // This breaks potential ref-cycles. - mRecognitionService = nullptr; - - ++mStreamGeneration; if (mStream) { mStream->UnregisterTrackListener(mListener); mStream = nullptr; @@ -461,10 +106,6 @@ void SpeechRecognition::Reset() { mTrack = nullptr; mTrackIsOwned = false; mStopRecordingPromise = nullptr; - mEncodeTaskQueue = nullptr; - mEstimationSamples = 0; - mBufferedSamples = 0; - mSpeechDetectionTimer->Cancel(); mAborted = false; } @@ -473,129 +114,6 @@ void SpeechRecognition::ResetAndEnd() { DispatchTrustedEvent(u"end"_ns); } -void SpeechRecognition::WaitForAudioData(SpeechEvent* aEvent) { - SetState(STATE_STARTING); -} - -void SpeechRecognition::StartedAudioCapture(SpeechEvent* aEvent) { - SetState(STATE_ESTIMATING); - - mEndpointer.SetEnvironmentEstimationMode(); - mEstimationSamples += - ProcessAudioSegment(aEvent->mAudioSegment, aEvent->mTrackRate); - - DispatchTrustedEvent(u"audiostart"_ns); - if (mCurrentState == STATE_ESTIMATING) { - DispatchTrustedEvent(u"start"_ns); - } -} - -void SpeechRecognition::StopRecordingAndRecognize(SpeechEvent* aEvent) { - SetState(STATE_WAITING_FOR_RESULT); - - MOZ_ASSERT(mRecognitionService, "Service deleted before recording done"); - - // This will run SoundEnd on the service just before StopRecording begins - // shutting the encode thread down. - mSpeechListener->mRemovedPromise->Then( - GetCurrentSerialEventTarget(), __func__, - [service = mRecognitionService] { service->SoundEnd(); }); - - StopRecording(); -} - -void SpeechRecognition::WaitForEstimation(SpeechEvent* aEvent) { - SetState(STATE_ESTIMATING); - - mEstimationSamples += - ProcessAudioSegment(aEvent->mAudioSegment, aEvent->mTrackRate); - if (mEstimationSamples > kESTIMATION_SAMPLES) { - mEndpointer.SetUserInputMode(); - SetState(STATE_WAITING_FOR_SPEECH); - } -} - -void SpeechRecognition::DetectSpeech(SpeechEvent* aEvent) { - SetState(STATE_WAITING_FOR_SPEECH); - - ProcessAudioSegment(aEvent->mAudioSegment, aEvent->mTrackRate); - if (mEndpointer.DidStartReceivingSpeech()) { - mSpeechDetectionTimer->Cancel(); - SetState(STATE_RECOGNIZING); - DispatchTrustedEvent(u"speechstart"_ns); - } -} - -void SpeechRecognition::WaitForSpeechEnd(SpeechEvent* aEvent) { - SetState(STATE_RECOGNIZING); - - ProcessAudioSegment(aEvent->mAudioSegment, aEvent->mTrackRate); - if (mEndpointer.speech_input_complete()) { - DispatchTrustedEvent(u"speechend"_ns); - - if (mCurrentState == STATE_RECOGNIZING) { - // FIXME: StopRecordingAndRecognize should only be called for single - // shot services for continuous we should just inform the service - StopRecordingAndRecognize(aEvent); - } - } -} - -void SpeechRecognition::NotifyFinalResult(SpeechEvent* aEvent) { - ResetAndEnd(); - - RootedDictionary init(RootingCx()); - init.mBubbles = true; - init.mCancelable = false; - // init.mResultIndex = 0; - init.mResults = aEvent->mRecognitionResultList; - init.mInterpretation = JS::NullValue(); - // init.mEmma = nullptr; - - RefPtr event = - SpeechRecognitionEvent::Constructor(this, u"result"_ns, init); - event->SetTrusted(true); - - DispatchEvent(*event); -} - -void SpeechRecognition::DoNothing(SpeechEvent* aEvent) {} - -void SpeechRecognition::AbortSilently(SpeechEvent* aEvent) { - if (mRecognitionService) { - if (mTrack) { - // This will run Abort on the service just before StopRecording begins - // shutting the encode thread down. - mSpeechListener->mRemovedPromise->Then( - GetCurrentSerialEventTarget(), __func__, - [service = mRecognitionService] { service->Abort(); }); - } else { - // Recording hasn't started yet. We can just call Abort(). - mRecognitionService->Abort(); - } - } - - StopRecording()->Then( - GetCurrentSerialEventTarget(), __func__, - [self = RefPtr(this), this] { ResetAndEnd(); }); - - SetState(STATE_ABORTING); -} - -void SpeechRecognition::AbortError(SpeechEvent* aEvent) { - AbortSilently(aEvent); - NotifyError(aEvent); -} - -void SpeechRecognition::NotifyError(SpeechEvent* aEvent) { - aEvent->mError->SetTrusted(true); - - DispatchEvent(*aEvent->mError); -} - -/************************************** - * Event triggers and other functions * - **************************************/ NS_IMETHODIMP SpeechRecognition::StartRecording(RefPtr& aTrack) { // hold a reference so that the underlying track doesn't get collected. @@ -605,18 +123,7 @@ SpeechRecognition::StartRecording(RefPtr& aTrack) { mSpeechListener = SpeechTrackListener::Create(this); mTrack->AddListener(mSpeechListener); - nsString blockerName; - blockerName.AppendPrintf("SpeechRecognition %p shutdown", this); - mShutdownBlocker = - MakeAndAddRef(this, blockerName); - media::MustGetShutdownBarrier()->AddBlocker( - mShutdownBlocker, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__, - u"SpeechRecognition shutdown"_ns); - - mEndpointer.StartSession(); - - return mSpeechDetectionTimer->Init(this, mSpeechDetectionTimeoutMs, - nsITimer::TYPE_ONE_SHOT); + return NS_OK; } RefPtr SpeechRecognition::StopRecording() { @@ -640,81 +147,9 @@ RefPtr SpeechRecognition::StopRecording() { mTrack->Stop(); } - mEndpointer.EndSession(); DispatchTrustedEvent(u"audioend"_ns); - // Block shutdown until the speech track listener has been removed from the - // MSG, as it holds a reference to us, and we reference the world, which we - // don't want to leak. - mStopRecordingPromise = - mSpeechListener->mRemovedPromise - ->Then( - GetCurrentSerialEventTarget(), __func__, - [self = RefPtr(this), this] { - SR_LOG("Shutting down encoding thread"); - return mEncodeTaskQueue->BeginShutdown(); - }, - [] { - MOZ_CRASH("Unexpected rejection"); - return ShutdownPromise::CreateAndResolve(false, __func__); - }) - ->Then( - GetCurrentSerialEventTarget(), __func__, - [self = RefPtr(this), this] { - media::MustGetShutdownBarrier()->RemoveBlocker( - mShutdownBlocker); - mShutdownBlocker = nullptr; - - MOZ_DIAGNOSTIC_ASSERT(mCurrentState != STATE_IDLE); - return GenericNonExclusivePromise::CreateAndResolve(true, - __func__); - }, - [] { - MOZ_CRASH("Unexpected rejection"); - return GenericNonExclusivePromise::CreateAndResolve(false, - __func__); - }); - return mStopRecordingPromise; -} - -NS_IMETHODIMP -SpeechRecognition::Observe(nsISupports* aSubject, const char* aTopic, - const char16_t* aData) { - MOZ_ASSERT(NS_IsMainThread(), "Observer invoked off the main thread"); - - if (!strcmp(aTopic, NS_TIMER_CALLBACK_TOPIC) && - StateBetween(STATE_IDLE, STATE_WAITING_FOR_SPEECH)) { - DispatchError(SpeechRecognition::EVENT_AUDIO_ERROR, - SpeechRecognitionErrorCode::No_speech, - "No speech detected (timeout)"); - } else if (!strcmp(aTopic, SPEECH_RECOGNITION_TEST_END_TOPIC)) { - nsCOMPtr obs = services::GetObserverService(); - obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC); - obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC); - } else if (StaticPrefs::media_webspeech_test_fake_fsm_events() && - !strcmp(aTopic, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC)) { - ProcessTestEventRequest(aSubject, nsDependentString(aData)); - } - - return NS_OK; -} - -void SpeechRecognition::ProcessTestEventRequest(nsISupports* aSubject, - const nsAString& aEventName) { - if (aEventName.EqualsLiteral("EVENT_ABORT")) { - Abort(); - } else if (aEventName.EqualsLiteral("EVENT_AUDIO_ERROR")) { - DispatchError( - SpeechRecognition::EVENT_AUDIO_ERROR, - SpeechRecognitionErrorCode::Audio_capture, // TODO different codes? - "AUDIO_ERROR test event"); - } else { - NS_ASSERTION(StaticPrefs::media_webspeech_test_fake_recognition_service(), - "Got request for fake recognition service event, but " - "media.webspeech.test.fake_recognition_service is unset"); - - // let the fake recognition service handle the request - } + return nullptr; } already_AddRefed SpeechRecognition::Grammars() const { @@ -759,29 +194,6 @@ void SpeechRecognition::SetServiceURI(const nsAString& aArg, ErrorResult& aRv) { void SpeechRecognition::Start(const Optional>& aStream, CallerType aCallerType, ErrorResult& aRv) { - if (mCurrentState != STATE_IDLE) { - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - return; - } - - if (!SetRecognitionService(aRv)) { - return; - } - - if (!ValidateAndSetGrammarList(aRv)) { - return; - } - - mEncodeTaskQueue = - TaskQueue::Create(GetMediaThreadPool(MediaThreadType::WEBRTC_WORKER), - "WebSpeechEncoderThread"); - - nsresult rv; - rv = mRecognitionService->Initialize(this); - if (NS_WARN_IF(NS_FAILED(rv))) { - return; - } - MediaStreamConstraints constraints; constraints.mAudio.SetAsBoolean() = true; @@ -813,12 +225,10 @@ void SpeechRecognition::Start(const Optional>& aStream, ->GetUserMedia(win, constraints, aCallerType) ->Then( GetCurrentSerialEventTarget(), __func__, - [this, self, - generation = mStreamGeneration](RefPtr&& aStream) { + [this, self](RefPtr&& aStream) { nsTArray> tracks; aStream->GetAudioTracks(tracks); - if (mAborted || mCurrentState != STATE_STARTING || - mStreamGeneration != generation) { + if (mAborted) { // We were probably aborted. Exit early. for (const RefPtr& track : tracks) { track->Stop(); @@ -833,10 +243,8 @@ void SpeechRecognition::Start(const Optional>& aStream, } } }, - [this, self, - generation = mStreamGeneration](RefPtr&& error) { - if (mAborted || mCurrentState != STATE_STARTING || - mStreamGeneration != generation) { + [this, self](RefPtr&& error) { + if (mAborted) { // We were probably aborted. Exit early. return; } @@ -847,86 +255,12 @@ void SpeechRecognition::Start(const Optional>& aStream, } else { errorCode = SpeechRecognitionErrorCode::Audio_capture; } - DispatchError(SpeechRecognition::EVENT_AUDIO_ERROR, errorCode, - error->mMessage); + DispatchError(errorCode, error->mMessage); }); } - - RefPtr event = new SpeechEvent(this, EVENT_START); - NS_DispatchToMainThread(event); } -bool SpeechRecognition::SetRecognitionService(ErrorResult& aRv) { - if (!GetOwnerWindow()) { - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - return false; - } - - // See: - // https://dvcs.w3.org/hg/speech-api/raw-file/tip/webspeechapi.html#dfn-lang - nsAutoString lang; - if (!mLang.IsEmpty()) { - lang = mLang; - } else { - nsCOMPtr document = GetOwnerWindow()->GetExtantDoc(); - if (!document) { - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - return false; - } - nsCOMPtr element = document->GetRootElement(); - if (!element) { - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - return false; - } - - nsAutoString lang; - element->GetLang(lang); - } - - auto result = CreateSpeechRecognitionService(GetOwnerWindow(), this, lang); - - if (result.isErr()) { - switch (result.unwrapErr()) { - case ServiceCreationError::ServiceNotFound: - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - break; - default: - MOZ_CRASH("Unknown error"); - } - return false; - } - - mRecognitionService = result.unwrap(); - MOZ_DIAGNOSTIC_ASSERT(mRecognitionService); - return true; -} - -bool SpeechRecognition::ValidateAndSetGrammarList(ErrorResult& aRv) { - if (!mSpeechGrammarList) { - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - return false; - } - - uint32_t grammarListLength = mSpeechGrammarList->Length(); - for (uint32_t count = 0; count < grammarListLength; ++count) { - RefPtr speechGrammar = mSpeechGrammarList->Item(count, aRv); - if (aRv.Failed()) { - return false; - } - if (NS_FAILED(mRecognitionService->ValidateAndSetGrammarList( - speechGrammar.get(), nullptr))) { - aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); - return false; - } - } - - return true; -} - -void SpeechRecognition::Stop() { - RefPtr event = new SpeechEvent(this, EVENT_STOP); - NS_DispatchToMainThread(event); -} +void SpeechRecognition::Stop() { ResetAndEnd(); } void SpeechRecognition::Abort() { if (mAborted) { @@ -934,9 +268,7 @@ void SpeechRecognition::Abort() { } mAborted = true; - - RefPtr event = new SpeechEvent(this, EVENT_ABORT); - NS_DispatchToMainThread(event); + ResetAndEnd(); } void SpeechRecognition::NotifyTrackAdded( @@ -957,202 +289,18 @@ void SpeechRecognition::NotifyTrackAdded( StartRecording(audioTrack); } -void SpeechRecognition::DispatchError(EventType aErrorType, - SpeechRecognitionErrorCode aErrorCode, +void SpeechRecognition::DispatchError(SpeechRecognitionErrorCode aErrorCode, const nsACString& aMessage) { MOZ_ASSERT(NS_IsMainThread()); - MOZ_ASSERT(aErrorType == EVENT_RECOGNITIONSERVICE_ERROR || - aErrorType == EVENT_AUDIO_ERROR, - "Invalid error type!"); RefPtr srError = new SpeechRecognitionError(nullptr, nullptr, nullptr); srError->InitSpeechRecognitionError(u"error"_ns, true, false, aErrorCode, aMessage); + srError->SetTrusted(true); - RefPtr event = new SpeechEvent(this, aErrorType); - event->mError = srError; - NS_DispatchToMainThread(event); -} - -/* - * Buffer audio samples into mAudioSamplesBuffer until aBufferSize. - * Updates mBufferedSamples and returns the number of samples that were - * buffered. - */ -uint32_t SpeechRecognition::FillSamplesBuffer(const int16_t* aSamples, - uint32_t aSampleCount) { - MOZ_ASSERT(mBufferedSamples < mAudioSamplesPerChunk); - MOZ_ASSERT(mAudioSamplesBuffer); - - int16_t* samplesBuffer = static_cast(mAudioSamplesBuffer->Data()); - size_t samplesToCopy = - std::min(aSampleCount, mAudioSamplesPerChunk - mBufferedSamples); - - PodCopy(samplesBuffer + mBufferedSamples, aSamples, samplesToCopy); - - mBufferedSamples += samplesToCopy; - return samplesToCopy; -} - -/* - * Split a samples buffer starting of a given size into - * chunks of equal size. The chunks are stored in the array - * received as argument. - * Returns the offset of the end of the last chunk that was - * created. - */ -uint32_t SpeechRecognition::SplitSamplesBuffer( - const int16_t* aSamplesBuffer, uint32_t aSampleCount, - nsTArray>& aResult) { - uint32_t chunkStart = 0; - - while (chunkStart + mAudioSamplesPerChunk <= aSampleCount) { - CheckedInt bufferSize(sizeof(int16_t)); - bufferSize *= mAudioSamplesPerChunk; - RefPtr chunk = SharedBuffer::Create(bufferSize); - - PodCopy(static_cast(chunk->Data()), aSamplesBuffer + chunkStart, - mAudioSamplesPerChunk); - - aResult.AppendElement(chunk.forget()); - chunkStart += mAudioSamplesPerChunk; - } - - return chunkStart; -} - -AudioSegment* SpeechRecognition::CreateAudioSegment( - nsTArray>& aChunks) { - AudioSegment* segment = new AudioSegment(); - for (uint32_t i = 0; i < aChunks.Length(); ++i) { - RefPtr buffer = aChunks[i]; - const int16_t* chunkData = static_cast(buffer->Data()); - - AutoTArray channels; - channels.AppendElement(chunkData); - segment->AppendFrames(buffer.forget(), channels, mAudioSamplesPerChunk, - PRINCIPAL_HANDLE_NONE); - } - - return segment; -} - -void SpeechRecognition::FeedAudioData( - nsMainThreadPtrHandle& aRecognition, - already_AddRefed aSamples, uint32_t aDuration, - MediaTrackListener* aProvider, TrackRate aTrackRate) { - NS_ASSERTION(!NS_IsMainThread(), - "FeedAudioData should not be called in the main thread"); - - // Endpointer expects to receive samples in chunks whose size is a - // multiple of its frame size. - // Since we can't assume we will receive the frames in appropriate-sized - // chunks, we must buffer and split them in chunks of mAudioSamplesPerChunk - // (a multiple of Endpointer's frame size) before feeding to Endpointer. - - // ensure aSamples is deleted - RefPtr refSamples = aSamples; - - uint32_t samplesIndex = 0; - const int16_t* samples = static_cast(refSamples->Data()); - AutoTArray, 5> chunksToSend; - - // fill up our buffer and make a chunk out of it, if possible - if (mBufferedSamples > 0) { - samplesIndex += FillSamplesBuffer(samples, aDuration); - - if (mBufferedSamples == mAudioSamplesPerChunk) { - chunksToSend.AppendElement(mAudioSamplesBuffer.forget()); - mBufferedSamples = 0; - } - } - - // create sample chunks of correct size - if (samplesIndex < aDuration) { - samplesIndex += SplitSamplesBuffer(samples + samplesIndex, - aDuration - samplesIndex, chunksToSend); - } - - // buffer remaining samples - if (samplesIndex < aDuration) { - mBufferedSamples = 0; - CheckedInt bufferSize(sizeof(int16_t)); - bufferSize *= mAudioSamplesPerChunk; - mAudioSamplesBuffer = SharedBuffer::Create(bufferSize); - - FillSamplesBuffer(samples + samplesIndex, aDuration - samplesIndex); - } - - AudioSegment* segment = CreateAudioSegment(chunksToSend); - RefPtr event = new SpeechEvent(aRecognition, EVENT_AUDIO_DATA); - event->mAudioSegment = segment; - event->mProvider = aProvider; - event->mTrackRate = aTrackRate; - NS_DispatchToMainThread(event); -} - -const char* SpeechRecognition::GetName(FSMState aId) { - static const char* names[] = { - "STATE_IDLE", "STATE_STARTING", - "STATE_ESTIMATING", "STATE_WAITING_FOR_SPEECH", - "STATE_RECOGNIZING", "STATE_WAITING_FOR_RESULT", - "STATE_ABORTING", - }; - - MOZ_ASSERT(aId < STATE_COUNT); - MOZ_ASSERT(std::size(names) == STATE_COUNT); - return names[aId]; -} - -const char* SpeechRecognition::GetName(SpeechEvent* aEvent) { - static const char* names[] = {"EVENT_START", - "EVENT_STOP", - "EVENT_ABORT", - "EVENT_AUDIO_DATA", - "EVENT_AUDIO_ERROR", - "EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT", - "EVENT_RECOGNITIONSERVICE_FINAL_RESULT", - "EVENT_RECOGNITIONSERVICE_ERROR"}; - - MOZ_ASSERT(aEvent->mType < EVENT_COUNT); - MOZ_ASSERT(std::size(names) == EVENT_COUNT); - return names[aEvent->mType]; -} - -TaskQueue* SpeechRecognition::GetTaskQueueForEncoding() const { - MOZ_ASSERT(NS_IsMainThread()); - return mEncodeTaskQueue; -} - -SpeechEvent::SpeechEvent(SpeechRecognition* aRecognition, - SpeechRecognition::EventType aType) - : Runnable("dom::SpeechEvent"), - mAudioSegment(nullptr), - mRecognitionResultList(nullptr), - mError(nullptr), - mRecognition(new nsMainThreadPtrHolder( - "SpeechEvent::SpeechEvent", aRecognition)), - mType(aType), - mTrackRate(0) {} - -SpeechEvent::SpeechEvent(nsMainThreadPtrHandle& aRecognition, - SpeechRecognition::EventType aType) - : Runnable("dom::SpeechEvent"), - mAudioSegment(nullptr), - mRecognitionResultList(nullptr), - mError(nullptr), - mRecognition(aRecognition), - mType(aType), - mTrackRate(0) {} - -SpeechEvent::~SpeechEvent() { delete mAudioSegment; } - -NS_IMETHODIMP -SpeechEvent::Run() { - mRecognition->ProcessEvent(this); - return NS_OK; + DispatchEvent(*srError); } } // namespace mozilla::dom diff --git a/dom/media/webspeech/recognition/SpeechRecognition.h b/dom/media/webspeech/recognition/SpeechRecognition.h index c0bfb4725047..3fae22734d99 100644 --- a/dom/media/webspeech/recognition/SpeechRecognition.h +++ b/dom/media/webspeech/recognition/SpeechRecognition.h @@ -10,15 +10,12 @@ #include "MediaTrackGraph.h" #include "SpeechGrammarList.h" #include "SpeechRecognitionResultList.h" -#include "endpointer.h" #include "js/TypeDecls.h" #include "mozilla/DOMEventTargetHelper.h" #include "mozilla/WeakPtr.h" #include "mozilla/dom/BindingDeclarations.h" #include "mozilla/dom/SpeechRecognitionError.h" #include "nsCOMPtr.h" -#include "nsISpeechRecognitionService.h" -#include "nsITimer.h" #include "nsProxyRelease.h" #include "nsString.h" #include "nsTArray.h" @@ -38,15 +35,10 @@ namespace dom { class GlobalObject; class AudioStreamTrack; -class SpeechEvent; +class MediaStreamTrack; class SpeechTrackListener; -LogModule* GetSpeechRecognitionLog(); -#define SR_LOG(...) \ - MOZ_LOG_FMT(GetSpeechRecognitionLog(), mozilla::LogLevel::Debug, __VA_ARGS__) - class SpeechRecognition final : public DOMEventTargetHelper, - public nsIObserver, public SupportsWeakPtr { public: explicit SpeechRecognition(nsPIDOMWindowInner* aOwnerWindow); @@ -55,8 +47,6 @@ class SpeechRecognition final : public DOMEventTargetHelper, NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(SpeechRecognition, DOMEventTargetHelper) - NS_DECL_NSIOBSERVER - JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override; @@ -113,18 +103,6 @@ class SpeechRecognition final : public DOMEventTargetHelper, IMPL_EVENT_HANDLER(start) IMPL_EVENT_HANDLER(end) - enum EventType { - EVENT_START, - EVENT_STOP, - EVENT_ABORT, - EVENT_AUDIO_DATA, - EVENT_AUDIO_ERROR, - EVENT_RECOGNITIONSERVICE_INTERMEDIATE_RESULT, - EVENT_RECOGNITIONSERVICE_FINAL_RESULT, - EVENT_RECOGNITIONSERVICE_ERROR, - EVENT_COUNT - }; - void NotifyTrackAdded(const RefPtr& aTrack); class TrackListener final : public DOMMediaStream::TrackListener { @@ -145,121 +123,39 @@ class SpeechRecognition final : public DOMEventTargetHelper, // aMessage should be valid UTF-8, but invalid UTF-8 byte sequences are // replaced with the REPLACEMENT CHARACTER on conversion to UTF-16. - void DispatchError(EventType aErrorType, - SpeechRecognitionErrorCode aErrorCode, + void DispatchError(SpeechRecognitionErrorCode aErrorCode, const nsACString& aMessage); template - void DispatchError(EventType aErrorType, - SpeechRecognitionErrorCode aErrorCode, + void DispatchError(SpeechRecognitionErrorCode aErrorCode, const char (&aMessage)[N]) { - DispatchError(aErrorType, aErrorCode, nsLiteralCString(aMessage)); + DispatchError(aErrorCode, nsLiteralCString(aMessage)); } - uint32_t FillSamplesBuffer(const int16_t* aSamples, uint32_t aSampleCount); - uint32_t SplitSamplesBuffer(const int16_t* aSamplesBuffer, - uint32_t aSampleCount, - nsTArray>& aResult); - AudioSegment* CreateAudioSegment(nsTArray>& aChunks); - void FeedAudioData(nsMainThreadPtrHandle& aRecognition, - already_AddRefed aSamples, - uint32_t aDuration, MediaTrackListener* aProvider, - TrackRate aTrackRate); - - friend class SpeechEvent; private: virtual ~SpeechRecognition(); - enum FSMState { - STATE_IDLE, - STATE_STARTING, - STATE_ESTIMATING, - STATE_WAITING_FOR_SPEECH, - STATE_RECOGNIZING, - STATE_WAITING_FOR_RESULT, - STATE_ABORTING, - STATE_COUNT - }; - - void SetState(FSMState state); - bool StateBetween(FSMState begin, FSMState end); - - bool SetRecognitionService(ErrorResult& aRv); - bool ValidateAndSetGrammarList(ErrorResult& aRv); - NS_IMETHOD StartRecording(RefPtr& aDOMStream); RefPtr StopRecording(); uint32_t ProcessAudioSegment(AudioSegment* aSegment, TrackRate aTrackRate); - void NotifyError(SpeechEvent* aEvent); - - void ProcessEvent(SpeechEvent* aEvent); - void Transition(SpeechEvent* aEvent); void Reset(); void ResetAndEnd(); - void WaitForAudioData(SpeechEvent* aEvent); - void StartedAudioCapture(SpeechEvent* aEvent); - void StopRecordingAndRecognize(SpeechEvent* aEvent); - void WaitForEstimation(SpeechEvent* aEvent); - void DetectSpeech(SpeechEvent* aEvent); - void WaitForSpeechEnd(SpeechEvent* aEvent); - void NotifyFinalResult(SpeechEvent* aEvent); - void DoNothing(SpeechEvent* aEvent); - void AbortSilently(SpeechEvent* aEvent); - void AbortError(SpeechEvent* aEvent); RefPtr mStream; RefPtr mTrack; bool mTrackIsOwned = false; RefPtr mStopRecordingPromise; RefPtr mSpeechListener; - nsCOMPtr mRecognitionService; RefPtr mShutdownBlocker; - // TaskQueue responsible for pre-processing the samples by the service - // it runs in a separate thread from the main thread - RefPtr mEncodeTaskQueue; - // A generation ID of the MediaStream a started session is for, so that - // a gUM request that resolves after the session has stopped, and a new - // one has started, can exit early. Main thread only. Can wrap. - uint8_t mStreamGeneration = 0; - - FSMState mCurrentState; - - Endpointer mEndpointer; - uint32_t mEstimationSamples; - - uint32_t mAudioSamplesPerChunk; - - // maximum amount of seconds the engine will wait for voice - // until returning a 'no speech detected' error - uint32_t mSpeechDetectionTimeoutMs; - - // buffer holds one chunk of mAudioSamplesPerChunk - // samples before feeding it to mEndpointer - RefPtr mAudioSamplesBuffer; - uint32_t mBufferedSamples; - - nsCOMPtr mSpeechDetectionTimer; bool mAborted; nsString mLang; RefPtr mSpeechGrammarList; - // private flag used to hold if the user called the setContinuous() method - // of the API bool mContinuous; - - // WebSpeechAPI (http://bit.ly/1gIl7DC) states: - // - // 1. Default value MUST be false - // 2. If true, interim results SHOULD be returned - // 3. If false, interim results MUST NOT be returned - // - // Pocketsphinx does not return interm results; so, defaulting - // mInterimResults to false, then ignoring its subsequent value - // is a conforming implementation. bool mInterimResults; // WebSpeechAPI (http://bit.ly/1JAiqeo) states: @@ -276,42 +172,6 @@ class SpeechRecognition final : public DOMEventTargetHelper, uint32_t mMaxAlternatives; RefPtr mListener; - - void ProcessTestEventRequest(nsISupports* aSubject, - const nsAString& aEventName); - - const char* GetName(FSMState aId); - const char* GetName(SpeechEvent* aEvent); -}; - -class SpeechEvent : public Runnable { - public: - SpeechEvent(SpeechRecognition* aRecognition, - SpeechRecognition::EventType aType); - SpeechEvent(nsMainThreadPtrHandle& aRecognition, - SpeechRecognition::EventType aType); - - ~SpeechEvent(); - - NS_IMETHOD Run() override; - AudioSegment* mAudioSegment; - RefPtr - mRecognitionResultList; // TODO: make this a session being passed which - // also has index and stuff - RefPtr mError; - - friend class SpeechRecognition; - - private: - nsMainThreadPtrHandle mRecognition; - - // for AUDIO_DATA events, keep a reference to the provider - // of the data (i.e., the SpeechTrackListener) to ensure it - // is kept alive (and keeps SpeechRecognition alive) until this - // event gets processed. - RefPtr mProvider; - SpeechRecognition::EventType mType; - TrackRate mTrackRate; }; } // namespace dom diff --git a/dom/media/webspeech/recognition/SpeechTrackListener.cpp b/dom/media/webspeech/recognition/SpeechTrackListener.cpp index 782d5ba63340..4f62e9585678 100644 --- a/dom/media/webspeech/recognition/SpeechTrackListener.cpp +++ b/dom/media/webspeech/recognition/SpeechTrackListener.cpp @@ -2,9 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "SpeechTrackListener.h" - #include "SpeechRecognition.h" +#include "SpeechTrackListener.h" #include "nsProxyRelease.h" namespace mozilla::dom { @@ -31,61 +30,7 @@ already_AddRefed SpeechTrackListener::Create( void SpeechTrackListener::NotifyQueuedChanges( MediaTrackGraph* aGraph, TrackTime aTrackOffset, - const MediaSegment& aQueuedMedia) { - AudioSegment* audio = const_cast( - static_cast(&aQueuedMedia)); - - AudioSegment::ChunkIterator iterator(*audio); - while (!iterator.IsEnded()) { - // Skip over-large chunks so we don't crash! - if (iterator->GetDuration() > INT_MAX) { - continue; - } - int duration = int(iterator->GetDuration()); - - if (iterator->IsNull()) { - nsTArray nullData; - PodZero(nullData.AppendElements(duration), duration); - ConvertAndDispatchAudioChunk(duration, iterator->mVolume, - nullData.Elements(), aGraph->GraphRate()); - } else { - AudioSampleFormat format = iterator->mBufferFormat; - - MOZ_ASSERT(format == AUDIO_FORMAT_S16 || format == AUDIO_FORMAT_FLOAT32); - - if (format == AUDIO_FORMAT_S16) { - ConvertAndDispatchAudioChunk( - duration, iterator->mVolume, - static_cast(iterator->mChannelData[0]), - aGraph->GraphRate()); - } else if (format == AUDIO_FORMAT_FLOAT32) { - ConvertAndDispatchAudioChunk( - duration, iterator->mVolume, - static_cast(iterator->mChannelData[0]), - aGraph->GraphRate()); - } - } - - iterator.Next(); - } -} - -template -void SpeechTrackListener::ConvertAndDispatchAudioChunk(int aDuration, - float aVolume, - SampleFormatType* aData, - TrackRate aTrackRate) { - CheckedInt bufferSize(sizeof(int16_t)); - bufferSize *= aDuration; - bufferSize *= 1; // channel - RefPtr samples(SharedBuffer::Create(bufferSize)); - - int16_t* to = static_cast(samples->Data()); - ConvertAudioSamplesWithScale(aData, to, aDuration, aVolume); - - mRecognition->FeedAudioData(mRecognition, samples.forget(), aDuration, this, - aTrackRate); -} + const MediaSegment& aQueuedMedia) {} void SpeechTrackListener::NotifyEnded(MediaTrackGraph* aGraph) { // TODO dispatch SpeechEnd event so services can be informed diff --git a/dom/media/webspeech/recognition/SpeechTrackListener.h b/dom/media/webspeech/recognition/SpeechTrackListener.h index 880b444494ae..37add634ba4f 100644 --- a/dom/media/webspeech/recognition/SpeechTrackListener.h +++ b/dom/media/webspeech/recognition/SpeechTrackListener.h @@ -36,10 +36,6 @@ class SpeechTrackListener : public MediaTrackListener { void NotifyRemoved(MediaTrackGraph* aGraph) override; private: - template - void ConvertAndDispatchAudioChunk(int aDuration, float aVolume, - SampleFormatType* aData, - TrackRate aTrackRate); nsMainThreadPtrHandle mRecognition; MozPromiseHolder mRemovedHolder; diff --git a/dom/media/webspeech/recognition/endpointer.cc b/dom/media/webspeech/recognition/endpointer.cc deleted file mode 100644 index 2347043d4bb3..000000000000 --- a/dom/media/webspeech/recognition/endpointer.cc +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "endpointer.h" - -#include "AudioSegment.h" - -namespace { -const int kFrameRate = 200; // 1 frame = 5ms of audio. -} - -namespace mozilla { - -Endpointer::Endpointer(int sample_rate) - : speech_input_possibly_complete_silence_length_us_(-1), - speech_input_complete_silence_length_us_(-1), - audio_frame_time_us_(0), - sample_rate_(sample_rate), - frame_size_(0) { - Reset(); - - frame_size_ = static_cast(sample_rate / static_cast(kFrameRate)); - - speech_input_minimum_length_us_ = - static_cast(1.7 * 1000000); - speech_input_complete_silence_length_us_ = - static_cast(0.5 * 1000000); - long_speech_input_complete_silence_length_us_ = -1; - long_speech_length_us_ = -1; - speech_input_possibly_complete_silence_length_us_ = - 1 * 1000000; - - // Set the default configuration for Push To Talk mode. - EnergyEndpointerParams ep_config; - ep_config.set_frame_period(1.0f / static_cast(kFrameRate)); - ep_config.set_frame_duration(1.0f / static_cast(kFrameRate)); - ep_config.set_endpoint_margin(0.2f); - ep_config.set_onset_window(0.15f); - ep_config.set_speech_on_window(0.4f); - ep_config.set_offset_window(0.15f); - ep_config.set_onset_detect_dur(0.09f); - ep_config.set_onset_confirm_dur(0.075f); - ep_config.set_on_maintain_dur(0.10f); - ep_config.set_offset_confirm_dur(0.12f); - ep_config.set_decision_threshold(1000.0f); - ep_config.set_min_decision_threshold(50.0f); - ep_config.set_fast_update_dur(0.2f); - ep_config.set_sample_rate(static_cast(sample_rate)); - ep_config.set_min_fundamental_frequency(57.143f); - ep_config.set_max_fundamental_frequency(400.0f); - ep_config.set_contamination_rejection_period(0.25f); - energy_endpointer_.Init(ep_config); -} - -void Endpointer::Reset() { - old_ep_status_ = EP_PRE_SPEECH; - waiting_for_speech_possibly_complete_timeout_ = false; - waiting_for_speech_complete_timeout_ = false; - speech_previously_detected_ = false; - speech_input_complete_ = false; - audio_frame_time_us_ = 0; // Reset time for packets sent to endpointer. - speech_end_time_us_ = -1; - speech_start_time_us_ = -1; -} - -void Endpointer::StartSession() { - Reset(); - energy_endpointer_.StartSession(); -} - -void Endpointer::EndSession() { - energy_endpointer_.EndSession(); -} - -void Endpointer::SetEnvironmentEstimationMode() { - Reset(); - energy_endpointer_.SetEnvironmentEstimationMode(); -} - -void Endpointer::SetUserInputMode() { - energy_endpointer_.SetUserInputMode(); -} - -EpStatus Endpointer::Status(int64_t *time) { - return energy_endpointer_.Status(time); -} - -EpStatus Endpointer::ProcessAudio(const AudioChunk& raw_audio, float* rms_out) { - MOZ_ASSERT(raw_audio.mBufferFormat == AUDIO_FORMAT_S16, "Audio is not in 16 bit format"); - const int16_t* audio_data = static_cast(raw_audio.mChannelData[0]); - const int num_samples = raw_audio.mDuration; - EpStatus ep_status = EP_PRE_SPEECH; - - // Process the input data in blocks of frame_size_, dropping any incomplete - // frames at the end (which is ok since typically the caller will be recording - // audio in multiples of our frame size). - int sample_index = 0; - while (sample_index + frame_size_ <= num_samples) { - // Have the endpointer process the frame. - energy_endpointer_.ProcessAudioFrame(audio_frame_time_us_, - audio_data + sample_index, - frame_size_, - rms_out); - sample_index += frame_size_; - audio_frame_time_us_ += (frame_size_ * 1000000) / - sample_rate_; - - // Get the status of the endpointer. - int64_t ep_time; - ep_status = energy_endpointer_.Status(&ep_time); - if (old_ep_status_ != ep_status) - fprintf(stderr, "Status changed old= %d, new= %d\n", old_ep_status_, ep_status); - - // Handle state changes. - if ((EP_SPEECH_PRESENT == ep_status) && - (EP_POSSIBLE_ONSET == old_ep_status_)) { - speech_end_time_us_ = -1; - waiting_for_speech_possibly_complete_timeout_ = false; - waiting_for_speech_complete_timeout_ = false; - // Trigger SpeechInputDidStart event on first detection. - if (false == speech_previously_detected_) { - speech_previously_detected_ = true; - speech_start_time_us_ = ep_time; - } - } - if ((EP_PRE_SPEECH == ep_status) && - (EP_POSSIBLE_OFFSET == old_ep_status_)) { - speech_end_time_us_ = ep_time; - waiting_for_speech_possibly_complete_timeout_ = true; - waiting_for_speech_complete_timeout_ = true; - } - if (ep_time > speech_input_minimum_length_us_) { - // Speech possibly complete timeout. - if ((waiting_for_speech_possibly_complete_timeout_) && - (ep_time - speech_end_time_us_ > - speech_input_possibly_complete_silence_length_us_)) { - waiting_for_speech_possibly_complete_timeout_ = false; - } - if (waiting_for_speech_complete_timeout_) { - // The length of the silence timeout period can be held constant, or it - // can be changed after a fixed amount of time from the beginning of - // speech. - bool has_stepped_silence = - (long_speech_length_us_ > 0) && - (long_speech_input_complete_silence_length_us_ > 0); - int64_t requested_silence_length; - if (has_stepped_silence && - (ep_time - speech_start_time_us_) > long_speech_length_us_) { - requested_silence_length = - long_speech_input_complete_silence_length_us_; - } else { - requested_silence_length = - speech_input_complete_silence_length_us_; - } - - // Speech complete timeout. - if ((ep_time - speech_end_time_us_) > requested_silence_length) { - waiting_for_speech_complete_timeout_ = false; - speech_input_complete_ = true; - } - } - } - old_ep_status_ = ep_status; - } - return ep_status; -} - -} // namespace mozilla diff --git a/dom/media/webspeech/recognition/endpointer.h b/dom/media/webspeech/recognition/endpointer.h deleted file mode 100644 index 7879d6b9f3b7..000000000000 --- a/dom/media/webspeech/recognition/endpointer.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef CONTENT_BROWSER_SPEECH_ENDPOINTER_ENDPOINTER_H_ -#define CONTENT_BROWSER_SPEECH_ENDPOINTER_ENDPOINTER_H_ - -#include "energy_endpointer.h" - -namespace mozilla { - -struct AudioChunk; - -// A simple interface to the underlying energy-endpointer implementation, this -// class lets callers provide audio as being recorded and let them poll to find -// when the user has stopped speaking. -// -// There are two events that may trigger the end of speech: -// -// speechInputPossiblyComplete event: -// -// Signals that silence/noise has been detected for a *short* amount of -// time after some speech has been detected. It can be used for low latency -// UI feedback. To disable it, set it to a large amount. -// -// speechInputComplete event: -// -// This event is intended to signal end of input and to stop recording. -// The amount of time to wait after speech is set by -// speech_input_complete_silence_length_ and optionally two other -// parameters (see below). -// This time can be held constant, or can change as more speech is detected. -// In the latter case, the time changes after a set amount of time from the -// *beginning* of speech. This is motivated by the expectation that there -// will be two distinct types of inputs: short search queries and longer -// dictation style input. -// -// Three parameters are used to define the piecewise constant timeout function. -// The timeout length is speech_input_complete_silence_length until -// long_speech_length, when it changes to -// long_speech_input_complete_silence_length. -class Endpointer { - public: - explicit Endpointer(int sample_rate); - - // Start the endpointer. This should be called at the beginning of a session. - void StartSession(); - - // Stop the endpointer. - void EndSession(); - - // Start environment estimation. Audio will be used for environment estimation - // i.e. noise level estimation. - void SetEnvironmentEstimationMode(); - - // Start user input. This should be called when the user indicates start of - // input, e.g. by pressing a button. - void SetUserInputMode(); - - // Process a segment of audio, which may be more than one frame. - // The status of the last frame will be returned. - EpStatus ProcessAudio(const AudioChunk& raw_audio, float* rms_out); - - // Get the status of the endpointer. - EpStatus Status(int64_t *time_us); - - // Get the expected frame size for audio chunks. Audio chunks are expected - // to contain a number of samples that is a multiple of this number, and extra - // samples will be dropped. - int32_t FrameSize() const { - return frame_size_; - } - - // Returns true if the endpointer detected reasonable audio levels above - // background noise which could be user speech, false if not. - bool DidStartReceivingSpeech() const { - return speech_previously_detected_; - } - - bool IsEstimatingEnvironment() const { - return energy_endpointer_.estimating_environment(); - } - - void set_speech_input_complete_silence_length(int64_t time_us) { - speech_input_complete_silence_length_us_ = time_us; - } - - void set_long_speech_input_complete_silence_length(int64_t time_us) { - long_speech_input_complete_silence_length_us_ = time_us; - } - - void set_speech_input_possibly_complete_silence_length(int64_t time_us) { - speech_input_possibly_complete_silence_length_us_ = time_us; - } - - void set_long_speech_length(int64_t time_us) { - long_speech_length_us_ = time_us; - } - - bool speech_input_complete() const { - return speech_input_complete_; - } - - // RMS background noise level in dB. - float NoiseLevelDb() const { return energy_endpointer_.GetNoiseLevelDb(); } - - private: - // Reset internal states. Helper method common to initial input utterance - // and following input utternaces. - void Reset(); - - // Minimum allowable length of speech input. - int64_t speech_input_minimum_length_us_; - - // The speechInputPossiblyComplete event signals that silence/noise has been - // detected for a *short* amount of time after some speech has been detected. - // This proporty specifies the time period. - int64_t speech_input_possibly_complete_silence_length_us_; - - // The speechInputComplete event signals that silence/noise has been - // detected for a *long* amount of time after some speech has been detected. - // This property specifies the time period. - int64_t speech_input_complete_silence_length_us_; - - // Same as above, this specifies the required silence period after speech - // detection. This period is used instead of - // speech_input_complete_silence_length_ when the utterance is longer than - // long_speech_length_. This parameter is optional. - int64_t long_speech_input_complete_silence_length_us_; - - // The period of time after which the endpointer should consider - // long_speech_input_complete_silence_length_ as a valid silence period - // instead of speech_input_complete_silence_length_. This parameter is - // optional. - int64_t long_speech_length_us_; - - // First speech onset time, used in determination of speech complete timeout. - int64_t speech_start_time_us_; - - // Most recent end time, used in determination of speech complete timeout. - int64_t speech_end_time_us_; - - int64_t audio_frame_time_us_; - EpStatus old_ep_status_; - bool waiting_for_speech_possibly_complete_timeout_; - bool waiting_for_speech_complete_timeout_; - bool speech_previously_detected_; - bool speech_input_complete_; - EnergyEndpointer energy_endpointer_; - int sample_rate_; - int32_t frame_size_; -}; - -} // namespace mozilla - -#endif // CONTENT_BROWSER_SPEECH_ENDPOINTER_ENDPOINTER_H_ diff --git a/dom/media/webspeech/recognition/energy_endpointer.cc b/dom/media/webspeech/recognition/energy_endpointer.cc deleted file mode 100644 index b1c1ee0bcf1b..000000000000 --- a/dom/media/webspeech/recognition/energy_endpointer.cc +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "energy_endpointer.h" - -#include - -namespace { - -// Returns the RMS (quadratic mean) of the input signal. -float RMS(const int16_t* samples, int num_samples) { - int64_t ssq_int64_t = 0; - int64_t sum_int64_t = 0; - for (int i = 0; i < num_samples; ++i) { - sum_int64_t += samples[i]; - ssq_int64_t += samples[i] * samples[i]; - } - // now convert to floats. - double sum = static_cast(sum_int64_t); - sum /= num_samples; - double ssq = static_cast(ssq_int64_t); - return static_cast(sqrt((ssq / num_samples) - (sum * sum))); -} - -int64_t Secs2Usecs(float seconds) { - return static_cast(0.5 + (1.0e6 * seconds)); -} - -float GetDecibel(float value) { - if (value > 1.0e-100) - return 20 * log10(value); - return -2000.0; -} - -} // namespace - -namespace mozilla { - -// Stores threshold-crossing histories for making decisions about the speech -// state. -class EnergyEndpointer::HistoryRing { - public: - HistoryRing() : insertion_index_(0) {} - - // Resets the ring to |size| elements each with state |initial_state| - void SetRing(int size, bool initial_state); - - // Inserts a new entry into the ring and drops the oldest entry. - void Insert(int64_t time_us, bool decision); - - // Returns the time in microseconds of the most recently added entry. - int64_t EndTime() const; - - // Returns the sum of all intervals during which 'decision' is true within - // the time in seconds specified by 'duration'. The returned interval is - // in seconds. - float RingSum(float duration_sec); - - private: - struct DecisionPoint { - int64_t time_us; - bool decision; - }; - - std::vector decision_points_; - int insertion_index_; // Index at which the next item gets added/inserted. - - HistoryRing(const HistoryRing&); - void operator=(const HistoryRing&); -}; - -void EnergyEndpointer::HistoryRing::SetRing(int size, bool initial_state) { - insertion_index_ = 0; - decision_points_.clear(); - DecisionPoint init = { -1, initial_state }; - decision_points_.resize(size, init); -} - -void EnergyEndpointer::HistoryRing::Insert(int64_t time_us, bool decision) { - decision_points_[insertion_index_].time_us = time_us; - decision_points_[insertion_index_].decision = decision; - insertion_index_ = (insertion_index_ + 1) % decision_points_.size(); -} - -int64_t EnergyEndpointer::HistoryRing::EndTime() const { - int ind = insertion_index_ - 1; - if (ind < 0) - ind = decision_points_.size() - 1; - return decision_points_[ind].time_us; -} - -float EnergyEndpointer::HistoryRing::RingSum(float duration_sec) { - if (decision_points_.empty()) - return 0.0; - - int64_t sum_us = 0; - int ind = insertion_index_ - 1; - if (ind < 0) - ind = decision_points_.size() - 1; - int64_t end_us = decision_points_[ind].time_us; - bool is_on = decision_points_[ind].decision; - int64_t start_us = end_us - static_cast(0.5 + (1.0e6 * duration_sec)); - if (start_us < 0) - start_us = 0; - size_t n_summed = 1; // n points ==> (n-1) intervals - while ((decision_points_[ind].time_us > start_us) && - (n_summed < decision_points_.size())) { - --ind; - if (ind < 0) - ind = decision_points_.size() - 1; - if (is_on) - sum_us += end_us - decision_points_[ind].time_us; - is_on = decision_points_[ind].decision; - end_us = decision_points_[ind].time_us; - n_summed++; - } - - return 1.0e-6f * sum_us; // Returns total time that was super threshold. -} - -EnergyEndpointer::EnergyEndpointer() - : status_(EP_PRE_SPEECH), - offset_confirm_dur_sec_(0), - endpointer_time_us_(0), - fast_update_frames_(0), - frame_counter_(0), - max_window_dur_(4.0), - sample_rate_(0), - history_(new HistoryRing()), - decision_threshold_(0), - estimating_environment_(false), - noise_level_(0), - rms_adapt_(0), - start_lag_(0), - end_lag_(0), - user_input_start_time_us_(0) { -} - -EnergyEndpointer::~EnergyEndpointer() { -} - -int EnergyEndpointer::TimeToFrame(float time) const { - return static_cast(0.5 + (time / params_.frame_period())); -} - -void EnergyEndpointer::Restart(bool reset_threshold) { - status_ = EP_PRE_SPEECH; - user_input_start_time_us_ = 0; - - if (reset_threshold) { - decision_threshold_ = params_.decision_threshold(); - rms_adapt_ = decision_threshold_; - noise_level_ = params_.decision_threshold() / 2.0f; - frame_counter_ = 0; // Used for rapid initial update of levels. - } - - // Set up the memories to hold the history windows. - history_->SetRing(TimeToFrame(max_window_dur_), false); - - // Flag that indicates that current input should be used for - // estimating the environment. The user has not yet started input - // by e.g. pressed the push-to-talk button. By default, this is - // false for backward compatibility. - estimating_environment_ = false; -} - -void EnergyEndpointer::Init(const EnergyEndpointerParams& params) { - params_ = params; - - // Find the longest history interval to be used, and make the ring - // large enough to accommodate that number of frames. NOTE: This - // depends upon ep_frame_period being set correctly in the factory - // that did this instantiation. - max_window_dur_ = params_.onset_window(); - if (params_.speech_on_window() > max_window_dur_) - max_window_dur_ = params_.speech_on_window(); - if (params_.offset_window() > max_window_dur_) - max_window_dur_ = params_.offset_window(); - Restart(true); - - offset_confirm_dur_sec_ = params_.offset_window() - - params_.offset_confirm_dur(); - if (offset_confirm_dur_sec_ < 0.0) - offset_confirm_dur_sec_ = 0.0; - - user_input_start_time_us_ = 0; - - // Flag that indicates that current input should be used for - // estimating the environment. The user has not yet started input - // by e.g. pressed the push-to-talk button. By default, this is - // false for backward compatibility. - estimating_environment_ = false; - // The initial value of the noise and speech levels is inconsequential. - // The level of the first frame will overwrite these values. - noise_level_ = params_.decision_threshold() / 2.0f; - fast_update_frames_ = - static_cast(params_.fast_update_dur() / params_.frame_period()); - - frame_counter_ = 0; // Used for rapid initial update of levels. - - sample_rate_ = params_.sample_rate(); - start_lag_ = static_cast(sample_rate_ / - params_.max_fundamental_frequency()); - end_lag_ = static_cast(sample_rate_ / - params_.min_fundamental_frequency()); -} - -void EnergyEndpointer::StartSession() { - Restart(true); -} - -void EnergyEndpointer::EndSession() { - status_ = EP_POST_SPEECH; -} - -void EnergyEndpointer::SetEnvironmentEstimationMode() { - Restart(true); - estimating_environment_ = true; -} - -void EnergyEndpointer::SetUserInputMode() { - estimating_environment_ = false; - user_input_start_time_us_ = endpointer_time_us_; -} - -void EnergyEndpointer::ProcessAudioFrame(int64_t time_us, - const int16_t* samples, - int num_samples, - float* rms_out) { - endpointer_time_us_ = time_us; - float rms = RMS(samples, num_samples); - - // Check that this is user input audio vs. pre-input adaptation audio. - // Input audio starts when the user indicates start of input, by e.g. - // pressing push-to-talk. Audio recieved prior to that is used to update - // noise and speech level estimates. - if (!estimating_environment_) { - bool decision = false; - if ((endpointer_time_us_ - user_input_start_time_us_) < - Secs2Usecs(params_.contamination_rejection_period())) { - decision = false; - //PR_LOG(GetSpeechRecognitionLog(), PR_LOG_DEBUG, ("decision: forced to false, time: %d", endpointer_time_us_)); - } else { - decision = (rms > decision_threshold_); - } - - history_->Insert(endpointer_time_us_, decision); - - switch (status_) { - case EP_PRE_SPEECH: - if (history_->RingSum(params_.onset_window()) > - params_.onset_detect_dur()) { - status_ = EP_POSSIBLE_ONSET; - } - break; - - case EP_POSSIBLE_ONSET: { - float tsum = history_->RingSum(params_.onset_window()); - if (tsum > params_.onset_confirm_dur()) { - status_ = EP_SPEECH_PRESENT; - } else { // If signal is not maintained, drop back to pre-speech. - if (tsum <= params_.onset_detect_dur()) - status_ = EP_PRE_SPEECH; - } - break; - } - - case EP_SPEECH_PRESENT: { - // To induce hysteresis in the state residency, we allow a - // smaller residency time in the on_ring, than was required to - // enter the SPEECH_PERSENT state. - float on_time = history_->RingSum(params_.speech_on_window()); - if (on_time < params_.on_maintain_dur()) - status_ = EP_POSSIBLE_OFFSET; - break; - } - - case EP_POSSIBLE_OFFSET: - if (history_->RingSum(params_.offset_window()) <= - offset_confirm_dur_sec_) { - // Note that this offset time may be beyond the end - // of the input buffer in a real-time system. It will be up - // to the RecognizerSession to decide what to do. - status_ = EP_PRE_SPEECH; // Automatically reset for next utterance. - } else { // If speech picks up again we allow return to SPEECH_PRESENT. - if (history_->RingSum(params_.speech_on_window()) >= - params_.on_maintain_dur()) - status_ = EP_SPEECH_PRESENT; - } - break; - - default: - break; - } - - // If this is a quiet, non-speech region, slowly adapt the detection - // threshold to be about 6dB above the average RMS. - if ((!decision) && (status_ == EP_PRE_SPEECH)) { - decision_threshold_ = (0.98f * decision_threshold_) + (0.02f * 2 * rms); - rms_adapt_ = decision_threshold_; - } else { - // If this is in a speech region, adapt the decision threshold to - // be about 10dB below the average RMS. If the noise level is high, - // the threshold is pushed up. - // Adaptation up to a higher level is 5 times faster than decay to - // a lower level. - if ((status_ == EP_SPEECH_PRESENT) && decision) { - if (rms_adapt_ > rms) { - rms_adapt_ = (0.99f * rms_adapt_) + (0.01f * rms); - } else { - rms_adapt_ = (0.95f * rms_adapt_) + (0.05f * rms); - } - float target_threshold = 0.3f * rms_adapt_ + noise_level_; - decision_threshold_ = (.90f * decision_threshold_) + - (0.10f * target_threshold); - } - } - - // Set a floor - if (decision_threshold_ < params_.min_decision_threshold()) - decision_threshold_ = params_.min_decision_threshold(); - } - - // Update speech and noise levels. - UpdateLevels(rms); - ++frame_counter_; - - if (rms_out) - *rms_out = GetDecibel(rms); -} - -float EnergyEndpointer::GetNoiseLevelDb() const { - return GetDecibel(noise_level_); -} - -void EnergyEndpointer::UpdateLevels(float rms) { - // Update quickly initially. We assume this is noise and that - // speech is 6dB above the noise. - if (frame_counter_ < fast_update_frames_) { - // Alpha increases from 0 to (k-1)/k where k is the number of time - // steps in the initial adaptation period. - float alpha = static_cast(frame_counter_) / - static_cast(fast_update_frames_); - noise_level_ = (alpha * noise_level_) + ((1 - alpha) * rms); - //PR_LOG(GetSpeechRecognitionLog(), PR_LOG_DEBUG, ("FAST UPDATE, frame_counter_ %d, fast_update_frames_ %d", frame_counter_, fast_update_frames_)); - } else { - // Update Noise level. The noise level adapts quickly downward, but - // slowly upward. The noise_level_ parameter is not currently used - // for threshold adaptation. It is used for UI feedback. - if (noise_level_ < rms) - noise_level_ = (0.999f * noise_level_) + (0.001f * rms); - else - noise_level_ = (0.95f * noise_level_) + (0.05f * rms); - } - if (estimating_environment_ || (frame_counter_ < fast_update_frames_)) { - decision_threshold_ = noise_level_ * 2; // 6dB above noise level. - // Set a floor - if (decision_threshold_ < params_.min_decision_threshold()) - decision_threshold_ = params_.min_decision_threshold(); - } -} - -EpStatus EnergyEndpointer::Status(int64_t* status_time) const { - *status_time = history_->EndTime(); - return status_; -} - -} // namespace mozilla diff --git a/dom/media/webspeech/recognition/energy_endpointer.h b/dom/media/webspeech/recognition/energy_endpointer.h deleted file mode 100644 index 12d3c736e338..000000000000 --- a/dom/media/webspeech/recognition/energy_endpointer.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The EnergyEndpointer class finds likely speech onset and offset points. -// -// The implementation described here is about the simplest possible. -// It is based on timings of threshold crossings for overall signal -// RMS. It is suitable for light weight applications. -// -// As written, the basic idea is that one specifies intervals that -// must be occupied by super- and sub-threshold energy levels, and -// defers decisions re onset and offset times until these -// specifications have been met. Three basic intervals are tested: an -// onset window, a speech-on window, and an offset window. We require -// super-threshold to exceed some mimimum total durations in the onset -// and speech-on windows before declaring the speech onset time, and -// we specify a required sub-threshold residency in the offset window -// before declaring speech offset. As the various residency requirements are -// met, the EnergyEndpointer instance assumes various states, and can return the -// ID of these states to the client (see EpStatus below). -// -// The levels of the speech and background noise are continuously updated. It is -// important that the background noise level be estimated initially for -// robustness in noisy conditions. The first frames are assumed to be background -// noise and a fast update rate is used for the noise level. The duration for -// fast update is controlled by the fast_update_dur_ paramter. -// -// If used in noisy conditions, the endpointer should be started and run in the -// EnvironmentEstimation mode, for at least 200ms, before switching to -// UserInputMode. -// Audio feedback contamination can appear in the input audio, if not cut -// out or handled by echo cancellation. Audio feedback can trigger a false -// accept. The false accepts can be ignored by setting -// ep_contamination_rejection_period. - -#ifndef CONTENT_BROWSER_SPEECH_ENDPOINTER_ENERGY_ENDPOINTER_H_ -#define CONTENT_BROWSER_SPEECH_ENDPOINTER_ENERGY_ENDPOINTER_H_ - -#include - -#include "mozilla/UniquePtr.h" - -#include "energy_endpointer_params.h" - -namespace mozilla { - -// Endpointer status codes -enum EpStatus { - EP_PRE_SPEECH = 10, - EP_POSSIBLE_ONSET, - EP_SPEECH_PRESENT, - EP_POSSIBLE_OFFSET, - EP_POST_SPEECH, -}; - -class EnergyEndpointer { - public: - // The default construction MUST be followed by Init(), before any - // other use can be made of the instance. - EnergyEndpointer(); - virtual ~EnergyEndpointer(); - - void Init(const EnergyEndpointerParams& params); - - // Start the endpointer. This should be called at the beginning of a session. - void StartSession(); - - // Stop the endpointer. - void EndSession(); - - // Start environment estimation. Audio will be used for environment estimation - // i.e. noise level estimation. - void SetEnvironmentEstimationMode(); - - // Start user input. This should be called when the user indicates start of - // input, e.g. by pressing a button. - void SetUserInputMode(); - - // Computes the next input frame and modifies EnergyEndpointer status as - // appropriate based on the computation. - void ProcessAudioFrame(int64_t time_us, - const int16_t* samples, int num_samples, - float* rms_out); - - // Returns the current state of the EnergyEndpointer and the time - // corresponding to the most recently computed frame. - EpStatus Status(int64_t* status_time_us) const; - - bool estimating_environment() const { - return estimating_environment_; - } - - // Returns estimated noise level in dB. - float GetNoiseLevelDb() const; - - private: - class HistoryRing; - - // Resets the endpointer internal state. If reset_threshold is true, the - // state will be reset completely, including adaptive thresholds and the - // removal of all history information. - void Restart(bool reset_threshold); - - // Update internal speech and noise levels. - void UpdateLevels(float rms); - - // Returns the number of frames (or frame number) corresponding to - // the 'time' (in seconds). - int TimeToFrame(float time) const; - - EpStatus status_; // The current state of this instance. - float offset_confirm_dur_sec_; // max on time allowed to confirm POST_SPEECH - int64_t endpointer_time_us_; // Time of the most recently received audio frame. - int64_t fast_update_frames_; // Number of frames for initial level adaptation. - int64_t frame_counter_; // Number of frames seen. Used for initial adaptation. - float max_window_dur_; // Largest search window size (seconds) - float sample_rate_; // Sampling rate. - - // Ring buffers to hold the speech activity history. - UniquePtr history_; - - // Configuration parameters. - EnergyEndpointerParams params_; - - // RMS which must be exceeded to conclude frame is speech. - float decision_threshold_; - - // Flag to indicate that audio should be used to estimate environment, prior - // to receiving user input. - bool estimating_environment_; - - // Estimate of the background noise level. Used externally for UI feedback. - float noise_level_; - - // An adaptive threshold used to update decision_threshold_ when appropriate. - float rms_adapt_; - - // Start lag corresponds to the highest fundamental frequency. - int start_lag_; - - // End lag corresponds to the lowest fundamental frequency. - int end_lag_; - - // Time when mode switched from environment estimation to user input. This - // is used to time forced rejection of audio feedback contamination. - int64_t user_input_start_time_us_; - - // prevent copy constructor and assignment - EnergyEndpointer(const EnergyEndpointer&); - void operator=(const EnergyEndpointer&); -}; - -} // namespace mozilla - -#endif // CONTENT_BROWSER_SPEECH_ENDPOINTER_ENERGY_ENDPOINTER_H_ diff --git a/dom/media/webspeech/recognition/energy_endpointer_params.cc b/dom/media/webspeech/recognition/energy_endpointer_params.cc deleted file mode 100644 index cac4f1b238b8..000000000000 --- a/dom/media/webspeech/recognition/energy_endpointer_params.cc +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "energy_endpointer_params.h" - -namespace mozilla { - -EnergyEndpointerParams::EnergyEndpointerParams() { - SetDefaults(); -} - -void EnergyEndpointerParams::SetDefaults() { - frame_period_ = 0.01f; - frame_duration_ = 0.01f; - endpoint_margin_ = 0.2f; - onset_window_ = 0.15f; - speech_on_window_ = 0.4f; - offset_window_ = 0.15f; - onset_detect_dur_ = 0.09f; - onset_confirm_dur_ = 0.075f; - on_maintain_dur_ = 0.10f; - offset_confirm_dur_ = 0.12f; - decision_threshold_ = 150.0f; - min_decision_threshold_ = 50.0f; - fast_update_dur_ = 0.2f; - sample_rate_ = 8000.0f; - min_fundamental_frequency_ = 57.143f; - max_fundamental_frequency_ = 400.0f; - contamination_rejection_period_ = 0.25f; -} - -void EnergyEndpointerParams::operator=(const EnergyEndpointerParams& source) { - frame_period_ = source.frame_period(); - frame_duration_ = source.frame_duration(); - endpoint_margin_ = source.endpoint_margin(); - onset_window_ = source.onset_window(); - speech_on_window_ = source.speech_on_window(); - offset_window_ = source.offset_window(); - onset_detect_dur_ = source.onset_detect_dur(); - onset_confirm_dur_ = source.onset_confirm_dur(); - on_maintain_dur_ = source.on_maintain_dur(); - offset_confirm_dur_ = source.offset_confirm_dur(); - decision_threshold_ = source.decision_threshold(); - min_decision_threshold_ = source.min_decision_threshold(); - fast_update_dur_ = source.fast_update_dur(); - sample_rate_ = source.sample_rate(); - min_fundamental_frequency_ = source.min_fundamental_frequency(); - max_fundamental_frequency_ = source.max_fundamental_frequency(); - contamination_rejection_period_ = source.contamination_rejection_period(); -} - -} // namespace mozilla diff --git a/dom/media/webspeech/recognition/energy_endpointer_params.h b/dom/media/webspeech/recognition/energy_endpointer_params.h deleted file mode 100644 index 6437c6dc0f07..000000000000 --- a/dom/media/webspeech/recognition/energy_endpointer_params.h +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef CONTENT_BROWSER_SPEECH_ENDPOINTER_ENERGY_ENDPOINTER_PARAMS_H_ -#define CONTENT_BROWSER_SPEECH_ENDPOINTER_ENERGY_ENDPOINTER_PARAMS_H_ - -namespace mozilla { - -// Input parameters for the EnergyEndpointer class. -class EnergyEndpointerParams { - public: - EnergyEndpointerParams(); - - void SetDefaults(); - - void operator=(const EnergyEndpointerParams& source); - - // Accessors and mutators - float frame_period() const { return frame_period_; } - void set_frame_period(float frame_period) { - frame_period_ = frame_period; - } - - float frame_duration() const { return frame_duration_; } - void set_frame_duration(float frame_duration) { - frame_duration_ = frame_duration; - } - - float endpoint_margin() const { return endpoint_margin_; } - void set_endpoint_margin(float endpoint_margin) { - endpoint_margin_ = endpoint_margin; - } - - float onset_window() const { return onset_window_; } - void set_onset_window(float onset_window) { onset_window_ = onset_window; } - - float speech_on_window() const { return speech_on_window_; } - void set_speech_on_window(float speech_on_window) { - speech_on_window_ = speech_on_window; - } - - float offset_window() const { return offset_window_; } - void set_offset_window(float offset_window) { - offset_window_ = offset_window; - } - - float onset_detect_dur() const { return onset_detect_dur_; } - void set_onset_detect_dur(float onset_detect_dur) { - onset_detect_dur_ = onset_detect_dur; - } - - float onset_confirm_dur() const { return onset_confirm_dur_; } - void set_onset_confirm_dur(float onset_confirm_dur) { - onset_confirm_dur_ = onset_confirm_dur; - } - - float on_maintain_dur() const { return on_maintain_dur_; } - void set_on_maintain_dur(float on_maintain_dur) { - on_maintain_dur_ = on_maintain_dur; - } - - float offset_confirm_dur() const { return offset_confirm_dur_; } - void set_offset_confirm_dur(float offset_confirm_dur) { - offset_confirm_dur_ = offset_confirm_dur; - } - - float decision_threshold() const { return decision_threshold_; } - void set_decision_threshold(float decision_threshold) { - decision_threshold_ = decision_threshold; - } - - float min_decision_threshold() const { return min_decision_threshold_; } - void set_min_decision_threshold(float min_decision_threshold) { - min_decision_threshold_ = min_decision_threshold; - } - - float fast_update_dur() const { return fast_update_dur_; } - void set_fast_update_dur(float fast_update_dur) { - fast_update_dur_ = fast_update_dur; - } - - float sample_rate() const { return sample_rate_; } - void set_sample_rate(float sample_rate) { sample_rate_ = sample_rate; } - - float min_fundamental_frequency() const { return min_fundamental_frequency_; } - void set_min_fundamental_frequency(float min_fundamental_frequency) { - min_fundamental_frequency_ = min_fundamental_frequency; - } - - float max_fundamental_frequency() const { return max_fundamental_frequency_; } - void set_max_fundamental_frequency(float max_fundamental_frequency) { - max_fundamental_frequency_ = max_fundamental_frequency; - } - - float contamination_rejection_period() const { - return contamination_rejection_period_; - } - void set_contamination_rejection_period( - float contamination_rejection_period) { - contamination_rejection_period_ = contamination_rejection_period; - } - - private: - float frame_period_; // Frame period - float frame_duration_; // Window size - float onset_window_; // Interval scanned for onset activity - float speech_on_window_; // Inverval scanned for ongoing speech - float offset_window_; // Interval scanned for offset evidence - float offset_confirm_dur_; // Silence duration required to confirm offset - float decision_threshold_; // Initial rms detection threshold - float min_decision_threshold_; // Minimum rms detection threshold - float fast_update_dur_; // Period for initial estimation of levels. - float sample_rate_; // Expected sample rate. - - // Time to add on either side of endpoint threshold crossings - float endpoint_margin_; - // Total dur within onset_window required to enter ONSET state - float onset_detect_dur_; - // Total on time within onset_window required to enter SPEECH_ON state - float onset_confirm_dur_; - // Minimum dur in SPEECH_ON state required to maintain ON state - float on_maintain_dur_; - // Minimum fundamental frequency for autocorrelation. - float min_fundamental_frequency_; - // Maximum fundamental frequency for autocorrelation. - float max_fundamental_frequency_; - // Period after start of user input that above threshold values are ignored. - // This is to reject audio feedback contamination. - float contamination_rejection_period_; -}; - -} // namespace mozilla - -#endif // CONTENT_BROWSER_SPEECH_ENDPOINTER_ENERGY_ENDPOINTER_PARAMS_H_ diff --git a/dom/media/webspeech/recognition/moz.build b/dom/media/webspeech/recognition/moz.build index cb221cb02a81..47972d046379 100644 --- a/dom/media/webspeech/recognition/moz.build +++ b/dom/media/webspeech/recognition/moz.build @@ -4,12 +4,7 @@ MOCHITEST_MANIFESTS += ["test/mochitest.toml"] -XPIDL_MODULE = "dom_webspeechrecognition" - -XPIDL_SOURCES = ["nsISpeechRecognitionService.idl"] - EXPORTS.mozilla.dom += [ - "OnlineSpeechRecognitionService.h", "SpeechGrammar.h", "SpeechGrammarList.h", "SpeechRecognition.h", @@ -19,22 +14,9 @@ EXPORTS.mozilla.dom += [ "SpeechTrackListener.h", ] -EXPORTS += [ - "endpointer.h", - "energy_endpointer.h", - "energy_endpointer_params.h", -] - -if CONFIG["MOZ_WEBSPEECH_TEST_BACKEND"]: - EXPORTS.mozilla.dom += [ - "test/FakeSpeechRecognitionService.h", - ] +EXPORTS += [] UNIFIED_SOURCES += [ - "endpointer.cc", - "energy_endpointer.cc", - "energy_endpointer_params.cc", - "OnlineSpeechRecognitionService.cpp", "SpeechGrammar.cpp", "SpeechGrammarList.cpp", "SpeechRecognition.cpp", @@ -44,18 +26,10 @@ UNIFIED_SOURCES += [ "SpeechTrackListener.cpp", ] -if CONFIG["MOZ_WEBSPEECH_TEST_BACKEND"]: - UNIFIED_SOURCES += [ - "test/FakeSpeechRecognitionService.cpp", - ] - -USE_LIBS += [ - "jsoncpp", -] +USE_LIBS += [] LOCAL_INCLUDES += [ "/dom/base", - "/toolkit/components/jsoncpp/include", ] # Bug 1244242 is about enabling on Android diff --git a/dom/media/webspeech/recognition/nsISpeechRecognitionService.idl b/dom/media/webspeech/recognition/nsISpeechRecognitionService.idl deleted file mode 100644 index 93aa6c9f877b..000000000000 --- a/dom/media/webspeech/recognition/nsISpeechRecognitionService.idl +++ /dev/null @@ -1,42 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "nsISupports.idl" - -%{C++ -#include "mozilla/WeakPtr.h" - -namespace mozilla { -class AudioSegment; -namespace dom { -class SpeechRecognition; -class SpeechRecognitionResultList; -class SpeechGrammarList; -class SpeechGrammar; -} -} -%} - -native SpeechRecognitionWeakPtr(mozilla::WeakPtr); -[ptr] native AudioSegmentPtr(mozilla::AudioSegment); -[ptr] native SpeechGrammarPtr(mozilla::dom::SpeechGrammar); -[ptr] native SpeechGrammarListPtr(mozilla::dom::SpeechGrammarList); - -[uuid(6fcb6ee8-a6db-49ba-9f06-355d7ee18ea7)] -interface nsISpeechGrammarCompilationCallback : nsISupports { - void grammarCompilationEnd(in SpeechGrammarPtr grammarObject, in boolean success); -}; - -[uuid(8e97f287-f322-44e8-8888-8344fa408ef8)] -interface nsISpeechRecognitionService : nsISupports { - void initialize(in SpeechRecognitionWeakPtr aSpeechRecognition); - void processAudioSegment(in AudioSegmentPtr aAudioSegment, in long aSampleRate); - void validateAndSetGrammarList(in SpeechGrammarPtr aSpeechGrammar, in nsISpeechGrammarCompilationCallback aCallback); - void soundEnd(); - void abort(); -}; - -%{C++ -#define NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "@mozilla.org/webspeech/service;1?name=" -%} diff --git a/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp b/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp deleted file mode 100644 index 85a5be9fa606..000000000000 --- a/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "FakeSpeechRecognitionService.h" - -#include "SpeechRecognition.h" -#include "SpeechRecognitionAlternative.h" -#include "SpeechRecognitionResult.h" -#include "SpeechRecognitionResultList.h" -#include "mozilla/Services.h" -#include "mozilla/StaticPrefs_media.h" -#include "nsIObserverService.h" -#include "nsThreadUtils.h" - -namespace mozilla { - -using namespace dom; - -NS_IMPL_ISUPPORTS(FakeSpeechRecognitionService, nsISpeechRecognitionService, - nsIObserver) - -FakeSpeechRecognitionService::FakeSpeechRecognitionService() = default; - -FakeSpeechRecognitionService::~FakeSpeechRecognitionService() = default; - -NS_IMETHODIMP -FakeSpeechRecognitionService::Initialize( - WeakPtr aSpeechRecognition) { - MOZ_ASSERT(NS_IsMainThread()); - mRecognition = aSpeechRecognition; - nsCOMPtr obs = services::GetObserverService(); - obs->AddObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC, false); - obs->AddObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC, false); - return NS_OK; -} - -NS_IMETHODIMP -FakeSpeechRecognitionService::ProcessAudioSegment(AudioSegment* aAudioSegment, - int32_t aSampleRate) { - MOZ_ASSERT(!NS_IsMainThread()); - return NS_OK; -} - -NS_IMETHODIMP -FakeSpeechRecognitionService::SoundEnd() { - MOZ_ASSERT(NS_IsMainThread()); - return NS_OK; -} - -NS_IMETHODIMP -FakeSpeechRecognitionService::ValidateAndSetGrammarList( - mozilla::dom::SpeechGrammar*, nsISpeechGrammarCompilationCallback*) { - return NS_OK; -} - -NS_IMETHODIMP -FakeSpeechRecognitionService::Abort() { - MOZ_ASSERT(NS_IsMainThread()); - return NS_OK; -} - -NS_IMETHODIMP -FakeSpeechRecognitionService::Observe(nsISupports* aSubject, const char* aTopic, - const char16_t* aData) { - MOZ_ASSERT(StaticPrefs::media_webspeech_test_fake_recognition_service(), - "Got request to fake recognition service event, but " - "media.webspeech.test.fake_recognition_service is not set"); - - if (!strcmp(aTopic, SPEECH_RECOGNITION_TEST_END_TOPIC)) { - nsCOMPtr obs = services::GetObserverService(); - obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC); - obs->RemoveObserver(this, SPEECH_RECOGNITION_TEST_END_TOPIC); - - return NS_OK; - } - - const nsDependentString eventName = nsDependentString(aData); - - if (eventName.EqualsLiteral("EVENT_RECOGNITIONSERVICE_ERROR")) { - mRecognition->DispatchError( - SpeechRecognition::EVENT_RECOGNITIONSERVICE_ERROR, - SpeechRecognitionErrorCode::Network, // TODO different codes? - "RECOGNITIONSERVICE_ERROR test event"); - - } else if (eventName.EqualsLiteral("EVENT_RECOGNITIONSERVICE_FINAL_RESULT")) { - RefPtr event = new SpeechEvent( - mRecognition, SpeechRecognition::EVENT_RECOGNITIONSERVICE_FINAL_RESULT); - - event->mRecognitionResultList = BuildMockResultList(); - NS_DispatchToMainThread(event); - } - return NS_OK; -} - -SpeechRecognitionResultList* -FakeSpeechRecognitionService::BuildMockResultList() { - SpeechRecognitionResultList* resultList = - new SpeechRecognitionResultList(mRecognition); - SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); - if (0 < mRecognition->MaxAlternatives()) { - SpeechRecognitionAlternative* alternative = - new SpeechRecognitionAlternative(mRecognition); - - alternative->mTranscript = u"Mock final result"_ns; - alternative->mConfidence = 0.0f; - - result->mItems.AppendElement(alternative); - } - resultList->mItems.AppendElement(result); - - return resultList; -} - -} // namespace mozilla diff --git a/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.h b/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.h deleted file mode 100644 index 8edd803081a8..000000000000 --- a/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.h +++ /dev/null @@ -1,38 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef mozilla_dom_FakeSpeechRecognitionService_h -#define mozilla_dom_FakeSpeechRecognitionService_h - -#include "nsCOMPtr.h" -#include "nsIObserver.h" -#include "nsISpeechRecognitionService.h" - -#define NS_FAKE_SPEECH_RECOGNITION_SERVICE_CID \ - {0x48c345e7, \ - 0x9929, \ - 0x4f9a, \ - {0xa5, 0x63, 0xf4, 0x78, 0x22, 0x2d, 0xab, 0xcd}}; - -namespace mozilla { - -class FakeSpeechRecognitionService : public nsISpeechRecognitionService, - public nsIObserver { - public: - NS_DECL_THREADSAFE_ISUPPORTS - NS_DECL_NSISPEECHRECOGNITIONSERVICE - NS_DECL_NSIOBSERVER - - FakeSpeechRecognitionService(); - - private: - virtual ~FakeSpeechRecognitionService(); - - WeakPtr mRecognition; - dom::SpeechRecognitionResultList* BuildMockResultList(); -}; - -} // namespace mozilla - -#endif diff --git a/dom/media/webspeech/recognition/test/http_requesthandler.sjs b/dom/media/webspeech/recognition/test/http_requesthandler.sjs deleted file mode 100644 index e79662b850b4..000000000000 --- a/dom/media/webspeech/recognition/test/http_requesthandler.sjs +++ /dev/null @@ -1,87 +0,0 @@ -const CC = Components.Constructor; - -// Context structure - we need to set this up properly to pass to setObjectState -const ctx = { - QueryInterface(iid) { - if (iid.equals(Ci.nsISupports)) { - return this; - } - throw Components.Exception("", Cr.NS_ERROR_NO_INTERFACE); - }, -}; - -function setRequest(request) { - setObjectState(key, request); -} -function getRequest() { - let request; - getObjectState(v => { - request = v; - }); - return request; -} - -function handleRequest(request, response) { - response.processAsync(); - if (request.queryString == "save") { - // Get the context structure and finish the old request - getObjectState("context", function (obj) { - savedCtx = obj.wrappedJSObject; - request = savedCtx.request; - - response.setHeader("Content-Type", "application/octet-stream", false); - response.setHeader("Access-Control-Allow-Origin", "*", false); - response.setHeader("Cache-Control", "no-cache", false); - response.setStatusLine(request.httpVersion, 200, "OK"); - - const input = request.bodyInputStream; - const output = response.bodyOutputStream; - let bodyAvail; - while ((bodyAvail = input.available()) > 0) { - output.writeFrom(input, bodyAvail); - } - response.finish(); - }); - return; - } - - if ( - request.queryString == "malformedresult=1" || - request.queryString == "emptyresult=1" - ) { - jsonOK = - request.queryString == "malformedresult=1" - ? '{"status":"ok","dat' - : '{"status":"ok","data":[]}'; - response.setHeader("Content-Length", String(jsonOK.length), false); - response.setHeader("Content-Type", "application/json", false); - response.setHeader("Access-Control-Allow-Origin", "*", false); - response.setHeader("Cache-Control", "no-cache", false); - response.setStatusLine(request.httpVersion, 200, "OK"); - response.write(jsonOK, jsonOK.length); - response.finish(); - } else if (request.queryString == "hangup=1") { - response.finish(); - } else if (request.queryString == "return400=1") { - jsonOK = "{'message':'Bad header:accept-language-stt'}"; - response.setHeader("Content-Length", String(jsonOK.length), false); - response.setHeader("Content-Type", "application/json", false); - response.setHeader("Access-Control-Allow-Origin", "*", false); - response.setHeader("Cache-Control", "no-cache", false); - response.setStatusLine(request.httpVersion, 400, "Bad Request"); - response.write(jsonOK, jsonOK.length); - response.finish(); - } else { - ctx.wrappedJSObject = ctx; - ctx.request = request; - setObjectState("context", ctx); - jsonOK = '{"status":"ok","data":[{"confidence":0.9085610,"text":"hello"}]}'; - response.setHeader("Content-Length", String(jsonOK.length), false); - response.setHeader("Content-Type", "application/json", false); - response.setHeader("Access-Control-Allow-Origin", "*", false); - response.setHeader("Cache-Control", "no-cache", false); - response.setStatusLine(request.httpVersion, 200, "OK"); - response.write(jsonOK, jsonOK.length); - response.finish(); - } -} diff --git a/dom/media/webspeech/recognition/test/mochitest.toml b/dom/media/webspeech/recognition/test/mochitest.toml index 1021cfd3d258..7a71731e52f7 100644 --- a/dom/media/webspeech/recognition/test/mochitest.toml +++ b/dom/media/webspeech/recognition/test/mochitest.toml @@ -5,44 +5,10 @@ support-files = [ "head.js", "hello.ogg", "hello.ogg^headers^", - "http_requesthandler.sjs", "sinoid+hello.ogg", "sinoid+hello.ogg^headers^", "silence.ogg", "silence.ogg^headers^", ] -["test_abort.html"] - -["test_audio_capture_error.html"] - -["test_call_start_from_end_handler.html"] -tags = "capturestream" - -["test_nested_eventloop.html"] -skip-if = [ - "os == 'android'", -] - -["test_online_400_response.html"] - -["test_online_empty_result_handling.html"] - -["test_online_hangup.html"] - -["test_online_http.html"] - -["test_online_http_webkit.html"] - -["test_online_malformed_result_handling.html"] - ["test_preference_enable.html"] - -["test_recognition_service_error.html"] - -["test_success_without_recognition_service.html"] - -["test_timeout.html"] -skip-if = [ - "os == 'linux' && os_version == '24.04' && arch == 'x86_64' && display == 'x11'", # Bug 1307991 - low frequency on try pushes -] diff --git a/dom/media/webspeech/recognition/test/test_abort.html b/dom/media/webspeech/recognition/test/test_abort.html deleted file mode 100644 index 0f22770cc7c8..000000000000 --- a/dom/media/webspeech/recognition/test/test_abort.html +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - Test for Bug 650295 -- Call abort from inside handlers - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_audio_capture_error.html b/dom/media/webspeech/recognition/test/test_audio_capture_error.html deleted file mode 100644 index 0c054dbf0b53..000000000000 --- a/dom/media/webspeech/recognition/test/test_audio_capture_error.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - Test for Bug 650295 -- Behavior on audio error - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_call_start_from_end_handler.html b/dom/media/webspeech/recognition/test/test_call_start_from_end_handler.html deleted file mode 100644 index 895648ad9e2e..000000000000 --- a/dom/media/webspeech/recognition/test/test_call_start_from_end_handler.html +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - Test for Bug 650295 -- Restart recognition from end handler - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_nested_eventloop.html b/dom/media/webspeech/recognition/test/test_nested_eventloop.html deleted file mode 100644 index 4924766b447d..000000000000 --- a/dom/media/webspeech/recognition/test/test_nested_eventloop.html +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - Test for Bug 650295 -- Spin the event loop from inside a callback - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_online_400_response.html b/dom/media/webspeech/recognition/test/test_online_400_response.html deleted file mode 100644 index d3f615e480c4..000000000000 --- a/dom/media/webspeech/recognition/test/test_online_400_response.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - Test for Bug 1248897 -- Online speech service - - - - - -Mozilla Bug 1248897 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_online_empty_result_handling.html b/dom/media/webspeech/recognition/test/test_online_empty_result_handling.html deleted file mode 100644 index 32a44dda32e2..000000000000 --- a/dom/media/webspeech/recognition/test/test_online_empty_result_handling.html +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - Test for Bug 1248897 -- Online speech service - - - - - -Mozilla Bug 1248897 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_online_hangup.html b/dom/media/webspeech/recognition/test/test_online_hangup.html deleted file mode 100644 index 3d0cd97c0158..000000000000 --- a/dom/media/webspeech/recognition/test/test_online_hangup.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - Test for Bug 1248897 -- Online speech service - - - - - -Mozilla Bug 1248897 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_online_http.html b/dom/media/webspeech/recognition/test/test_online_http.html deleted file mode 100644 index 3d0987933d3b..000000000000 --- a/dom/media/webspeech/recognition/test/test_online_http.html +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - Test for Bug 1248897 -- Online speech service - - - - - -Mozilla Bug 1248897 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_online_http_webkit.html b/dom/media/webspeech/recognition/test/test_online_http_webkit.html deleted file mode 100644 index a6a2de5d3bea..000000000000 --- a/dom/media/webspeech/recognition/test/test_online_http_webkit.html +++ /dev/null @@ -1,87 +0,0 @@ - - - - - - Test for Bug 1248897 -- Online speech service - - - - - -Mozilla Bug 1248897 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_online_malformed_result_handling.html b/dom/media/webspeech/recognition/test/test_online_malformed_result_handling.html deleted file mode 100644 index 23f256019e7d..000000000000 --- a/dom/media/webspeech/recognition/test/test_online_malformed_result_handling.html +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - Test for Bug 1248897 -- Online speech service - - - - - -Mozilla Bug 1248897 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_recognition_service_error.html b/dom/media/webspeech/recognition/test/test_recognition_service_error.html deleted file mode 100644 index e8e59e2afc53..000000000000 --- a/dom/media/webspeech/recognition/test/test_recognition_service_error.html +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - Test for Bug 650295 -- Behavior on recognition service error - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_success_without_recognition_service.html b/dom/media/webspeech/recognition/test/test_success_without_recognition_service.html deleted file mode 100644 index 38748ed5cbc5..000000000000 --- a/dom/media/webspeech/recognition/test/test_success_without_recognition_service.html +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - Test for Bug 650295 -- Success with fake recognition service - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/recognition/test/test_timeout.html b/dom/media/webspeech/recognition/test/test_timeout.html deleted file mode 100644 index 8334c9e7799a..000000000000 --- a/dom/media/webspeech/recognition/test/test_timeout.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - Test for Bug 650295 -- Timeout for user speech - - - - - -Mozilla Bug 650295 -

- -
-
-
- - diff --git a/dom/media/webspeech/synth/moz.build b/dom/media/webspeech/synth/moz.build index e1bf34515970..3f29c33c797c 100644 --- a/dom/media/webspeech/synth/moz.build +++ b/dom/media/webspeech/synth/moz.build @@ -31,12 +31,11 @@ UNIFIED_SOURCES += [ "SpeechSynthesisVoice.cpp", ] -if CONFIG["MOZ_WEBSPEECH_TEST_BACKEND"]: - UNIFIED_SOURCES += ["test/nsFakeSynthServices.cpp"] +UNIFIED_SOURCES += ["test/nsFakeSynthServices.cpp"] - XPCOM_MANIFESTS += [ - "test/components.conf", - ] +XPCOM_MANIFESTS += [ + "test/components.conf", +] if CONFIG["MOZ_WIDGET_TOOLKIT"] == "windows": DIRS += ["windows"] diff --git a/layout/build/components.conf b/layout/build/components.conf index 6b5ee1d8a25f..9c292e67b33a 100644 --- a/layout/build/components.conf +++ b/layout/build/components.conf @@ -470,25 +470,8 @@ if defined('MOZ_WEBSPEECH'): 'type': 'mozilla::dom::nsSynthVoiceRegistry', 'headers': ['mozilla/dom/nsSynthVoiceRegistry.h'], 'constructor': 'mozilla::dom::nsSynthVoiceRegistry::GetInstanceForService', - }, - { - 'cid': '{0ff5ce56-5b09-4db8-adc6-8266af95f864}', - 'contract_ids': ['@mozilla.org/webspeech/service;1?name=online'], - 'type': 'mozilla::OnlineSpeechRecognitionService', - 'headers': ['mozilla/dom/OnlineSpeechRecognitionService.h'], - }, + } ] - -if defined('MOZ_WEBSPEECH_TEST_BACKEND'): - Classes += [ - { - 'cid': '{48c345e7-9929-4f9a-a563-f478222dabcd}', - 'contract_ids': ['@mozilla.org/webspeech/service;1?name=fake'], - 'type': 'mozilla::FakeSpeechRecognitionService', - 'headers': ['mozilla/dom/FakeSpeechRecognitionService.h'], - }, - ] - if defined('ACCESSIBILITY'): Classes += [ { diff --git a/layout/build/nsLayoutModule.cpp b/layout/build/nsLayoutModule.cpp index e3e936c72066..da10dfbea2b5 100644 --- a/layout/build/nsLayoutModule.cpp +++ b/layout/build/nsLayoutModule.cpp @@ -35,15 +35,9 @@ #include "mozilla/dom/LocalStorageCommon.h" #include "mozilla/dom/LocalStorageManager.h" #include "mozilla/dom/LocalStorageManager2.h" +#include "mozilla/dom/PushNotifier.h" #include "mozilla/dom/SessionStorageManager.h" #include "nsContentCreatorFunctions.h" - -#ifdef MOZ_WEBSPEECH -# include "mozilla/dom/OnlineSpeechRecognitionService.h" -# include "mozilla/dom/nsSynthVoiceRegistry.h" -#endif - -#include "mozilla/dom/PushNotifier.h" using mozilla::dom::PushNotifier; #define PUSHNOTIFIER_CID \ {0x2fc2d3e3, 0x020f, 0x404e, {0xb0, 0x6a, 0x6e, 0xcf, 0x3e, 0xa2, 0x33, 0x4a}} diff --git a/modules/libpref/Preferences.cpp b/modules/libpref/Preferences.cpp index cf7596ff78c4..7fe649219444 100644 --- a/modules/libpref/Preferences.cpp +++ b/modules/libpref/Preferences.cpp @@ -7290,7 +7290,6 @@ static const PrefListEntry sDynamicPrefOverrideList[]{ PREF_LIST_ENTRY("media.peerconnection.nat_simulator.redirect_targets"), PREF_LIST_ENTRY("media.peerconnection.nat_simulator.network_delay_ms"), PREF_LIST_ENTRY("media.video_loopback_dev"), - PREF_LIST_ENTRY("media.webspeech.service.endpoint"), PREF_LIST_ENTRY("network.protocol-handler.external."), PREF_LIST_ENTRY("network.security.ports.banned"), PREF_LIST_ENTRY("nimbus.syncdatastore."), diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml index 296312548b3b..c1cf2a4d65ea 100644 --- a/modules/libpref/init/StaticPrefList.yaml +++ b/modules/libpref/init/StaticPrefList.yaml @@ -13229,21 +13229,11 @@ value: false mirror: always -- name: media.webspeech.test.fake_fsm_events +- name: media.webspeech.recognition.enable type: bool value: false mirror: always -- name: media.webspeech.test.fake_recognition_service - type: bool - value: false - mirror: always - -- name: media.webspeech.recognition.enable - type: bool - value: false - mirror: always - - name: media.webspeech.synth.enabled type: bool value: true diff --git a/toolkit/moz.configure b/toolkit/moz.configure index 481ca540bdfb..65246b6cae69 100644 --- a/toolkit/moz.configure +++ b/toolkit/moz.configure @@ -1248,23 +1248,6 @@ def webspeech(value): set_config("MOZ_WEBSPEECH", webspeech) set_define("MOZ_WEBSPEECH", webspeech) -# Speech API test backend -# ============================================================== -option( - "--enable-webspeechtestbackend", - default=webspeech, - help="{Enable|Disable} support for HTML Speech API Test Backend", -) - - -@depends_if("--enable-webspeechtestbackend") -def webspeech_test_backend(value): - return True - - -set_config("MOZ_WEBSPEECH_TEST_BACKEND", webspeech_test_backend) -set_define("MOZ_WEBSPEECH_TEST_BACKEND", webspeech_test_backend) - # Graphics # ============================================================== diff --git a/tools/rewriting/ThirdPartyPaths.txt b/tools/rewriting/ThirdPartyPaths.txt index 81ed38c851a0..5484472d37bb 100644 --- a/tools/rewriting/ThirdPartyPaths.txt +++ b/tools/rewriting/ThirdPartyPaths.txt @@ -42,12 +42,6 @@ dom/media/platforms/ffmpeg/libav55/ dom/media/webaudio/test/blink/ dom/media/webrtc/tests/mochitests/helpers_from_wpt/sdp.js dom/media/webrtc/transport/third_party/ -dom/media/webspeech/recognition/endpointer.cc -dom/media/webspeech/recognition/endpointer.h -dom/media/webspeech/recognition/energy_endpointer.cc -dom/media/webspeech/recognition/energy_endpointer.h -dom/media/webspeech/recognition/energy_endpointer_params.cc -dom/media/webspeech/recognition/energy_endpointer_params.h dom/media/webvtt/vtt.sys.mjs dom/security/sanitizer/safe-default-configuration.json dom/tests/mochitest/ajax/