/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "EncoderTemplate.h" #include #include #include "EncoderTypes.h" #include "WebCodecsUtils.h" #include "mozilla/ScopeExit.h" #include "mozilla/StaticPrefs_dom.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/Try.h" #include "mozilla/dom/BindingDeclarations.h" #include "mozilla/dom/DOMException.h" #include "mozilla/dom/Event.h" #include "mozilla/dom/Promise.h" #include "mozilla/dom/VideoFrame.h" #include "mozilla/dom/WorkerCommon.h" #include "nsGkAtoms.h" #include "nsRFPService.h" #include "nsString.h" #include "nsThreadUtils.h" extern mozilla::LazyLogModule gWebCodecsLog; namespace mozilla::dom { #ifdef LOG_INTERNAL # undef LOG_INTERNAL #endif // LOG_INTERNAL #define LOG_INTERNAL(level, msg, ...) \ MOZ_LOG_FMT(gWebCodecsLog, LogLevel::level, msg, ##__VA_ARGS__) #ifdef LOG # undef LOG #endif // LOG #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) #ifdef LOGW # undef LOGW #endif // LOGW #define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) #ifdef LOGE # undef LOGE #endif // LOGE #define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) #ifdef LOGV # undef LOGV #endif // LOGV #define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) #define AUTO_ENCODER_MARKER(var, postfix) \ AutoWebCodecsMarker var(EncoderType::Name.get(), postfix); /* * Below are ControlMessage classes implementations */ template EncoderTemplate::ControlMessage::ControlMessage( WebCodecsId aConfigureId) : mConfigureId(aConfigureId), mMessageId(sNextId++) {} template EncoderTemplate::ConfigureMessage::ConfigureMessage( WebCodecsId aConfigureId, const RefPtr& aConfig) : ControlMessage(aConfigureId), mConfig(aConfig) {} template EncoderTemplate::EncodeMessage::EncodeMessage( WebCodecsId aConfigureId, already_AddRefed aData, Maybe&& aOptions) : ControlMessage(aConfigureId) { PushData(std::move(aData), std::move(aOptions)); } template EncoderTemplate::FlushMessage::FlushMessage( WebCodecsId aConfigureId) : ControlMessage(aConfigureId) {} /* * Below are EncoderTemplate implementation */ template EncoderTemplate::EncoderTemplate( nsIGlobalObject* aGlobalObject, RefPtr&& aErrorCallback, RefPtr&& aOutputCallback) : DOMEventTargetHelper(aGlobalObject), mErrorCallback(std::move(aErrorCallback)), mOutputCallback(std::move(aOutputCallback)), mState(CodecState::Unconfigured), mMessageQueueBlocked(false), mEncodeQueueSize(0), mDequeueEventScheduled(false), mLatestConfigureId(0), mEncodeCounter(0), mFlushCounter(0) {} template void EncoderTemplate::Configure(const ConfigType& aConfig, ErrorResult& aRv) { AssertIsOnOwningThread(); LOG("{}::Configure {} codec {}", EncoderType::Name.get(), fmt::ptr(this), NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); nsCString errorMessage; if (!EncoderType::Validate(aConfig, errorMessage)) { LOG("Configure: Validate error: {}", errorMessage.get()); aRv.ThrowTypeError(errorMessage); return; } if (mState == CodecState::Closed) { LOG("Configure: CodecState::Closed, rejecting with InvalidState"); aRv.ThrowInvalidStateError("The codec is no longer usable"); return; } // Clone a ConfigType as the active Encode config. RefPtr config = EncoderType::CreateConfigInternal(aConfig); if (!config) { CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } // Audio encoders are all software, no need to do anything. if constexpr (std::is_same_v) { ApplyResistFingerprintingIfNeeded(config, GetRelevantGlobal()); } mState = CodecState::Configured; mEncodeCounter = 0; mFlushCounter = 0; mControlMessageQueue.push(MakeRefPtr(sNextId++, config)); mLatestConfigureId = mControlMessageQueue.back()->mMessageId; LOG("{} {} enqueues {}", EncoderType::Name.get(), fmt::ptr(this), mControlMessageQueue.back()->ToString().get()); ProcessControlMessageQueue(); } template void EncoderTemplate::EncodeAudioData(InputType& aInput, ErrorResult& aRv) { AssertIsOnOwningThread(); LOG("{} {}, EncodeAudioData", EncoderType::Name.get(), fmt::ptr(this)); if (mState != CodecState::Configured) { aRv.ThrowInvalidStateError("Encoder must be configured first"); return; } if (aInput.IsClosed()) { aRv.ThrowTypeError("input AudioData has been closed"); return; } mAsyncDurationTracker.Start( aInput.Timestamp(), AutoWebCodecsMarker(EncoderType::Name.get(), ".encode-duration-a")); // Dummy options here as a shortcut PushEncodeRequest( mLatestConfigureId, EncoderType::CreateInputInternal(aInput, VideoEncoderEncodeOptions())); ProcessControlMessageQueue(); } template void EncoderTemplate::EncodeVideoFrame( InputType& aInput, const VideoEncoderEncodeOptions& aOptions, ErrorResult& aRv) { AssertIsOnOwningThread(); LOG("{}::Encode {} {}", EncoderType::Name.get(), fmt::ptr(this), aInput.ToString().get()); if (mState != CodecState::Configured) { aRv.ThrowInvalidStateError("Encoder must be configured first"); return; } if (aInput.IsClosed()) { aRv.ThrowTypeError("input VideoFrame has been closed"); return; } mAsyncDurationTracker.Start( aInput.Timestamp(), AutoWebCodecsMarker(EncoderType::Name.get(), ".encode-duration-v")); PushEncodeRequest(mLatestConfigureId, EncoderType::CreateInputInternal(aInput, aOptions), Some(aOptions)); ProcessControlMessageQueue(); } template already_AddRefed EncoderTemplate::Flush( ErrorResult& aRv) { AssertIsOnOwningThread(); LOG("{}::Flush {}", EncoderType::Name.get(), fmt::ptr(this)); if (mState != CodecState::Configured) { LOG("{} {}, wrong state!", EncoderType::Name.get(), fmt::ptr(this)); aRv.ThrowInvalidStateError("Encoder must be configured first"); return nullptr; } RefPtr p = Promise::Create(GetParentObject(), aRv); if (NS_WARN_IF(aRv.Failed())) { return p.forget(); } auto msg = MakeRefPtr(mLatestConfigureId); const auto flushPromiseId = static_cast(msg->mMessageId); MOZ_ASSERT(!mPendingFlushPromises.Contains(flushPromiseId)); mPendingFlushPromises.Insert(flushPromiseId, p); mControlMessageQueue.emplace(std::move(msg)); LOG("{} {} enqueues {}", EncoderType::Name.get(), fmt::ptr(this), mControlMessageQueue.back()->ToString().get()); ProcessControlMessageQueue(); return p.forget(); } template void EncoderTemplate::Reset(ErrorResult& aRv) { AssertIsOnOwningThread(); LOG("{} {}, Reset", EncoderType::Name.get(), fmt::ptr(this)); if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { aRv.Throw(r.unwrapErr()); } } template void EncoderTemplate::Close(ErrorResult& aRv) { AssertIsOnOwningThread(); LOG("{} {}, Close", EncoderType::Name.get(), fmt::ptr(this)); if (auto r = CloseInternalWithAbort(); r.isErr()) { aRv.Throw(r.unwrapErr()); } } template Result EncoderTemplate::ResetInternal( const nsresult& aResult) { AssertIsOnOwningThread(); LOG("{}::Reset {}", EncoderType::Name.get(), fmt::ptr(this)); if (mState == CodecState::Closed) { return Err(NS_ERROR_DOM_INVALID_STATE_ERR); } mState = CodecState::Unconfigured; mEncodeCounter = 0; mFlushCounter = 0; CancelPendingControlMessagesAndFlushPromises(aResult); DestroyEncoderAgentIfAny(); if (mEncodeQueueSize > 0) { mEncodeQueueSize = 0; ScheduleDequeueEvent(); } StopBlockingMessageQueue(); return Ok(); } template Result EncoderTemplate::CloseInternalWithAbort() { AssertIsOnOwningThread(); MOZ_TRY(ResetInternal(NS_ERROR_DOM_ABORT_ERR)); mState = CodecState::Closed; return Ok(); } template void EncoderTemplate::CloseInternal(const nsresult& aResult) { AssertIsOnOwningThread(); MOZ_ASSERT(aResult != NS_ERROR_DOM_ABORT_ERR, "Use CloseInternalWithAbort"); // Return early if already closed. This can happen when async error handling // tasks race with user-initiated close(). if (mState == CodecState::Closed) { return; } auto r = ResetInternal(aResult); if (r.isErr()) { nsCString name; GetErrorName(r.unwrapErr(), name); LOGE("Error during ResetInternal during CloseInternal: {}", name.get()); } mState = CodecState::Closed; nsCString error; GetErrorName(aResult, error); LOGE("{} {} Close on error: {}", EncoderType::Name.get(), fmt::ptr(this), error.get()); ReportError(aResult); } template void EncoderTemplate::ReportError(const nsresult& aResult) { AssertIsOnOwningThread(); RefPtr e = DOMException::Create(aResult); RefPtr cb(mErrorCallback); cb->Call(*e); } template <> void EncoderTemplate::OutputEncodedVideoData( const nsTArray>&& aData) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(mActiveConfig); // Get JSContext for RootedDictionary. // The EncoderType::MetadataType, VideoDecoderConfig, and VideoColorSpaceInit // below are rooted to work around the JS hazard issues. AutoJSAPI jsapi; if (!jsapi.Init(GetParentObject())) { LOGE("{} {} AutoJSAPI init failed", VideoEncoderTraits::Name.get(), fmt::ptr(this)); return; } JSContext* cx = jsapi.cx(); RefPtr cb(mOutputCallback); for (const auto& data : aData) { // It's possible to have reset() called in between this task having been // dispatched, and running -- no output callback should happen when that's // the case. // This is imprecise in the spec, but discussed in // https://github.com/w3c/webcodecs/issues/755 and agreed upon. if (!mActiveConfig) { return; } RefPtr encodedData = EncodedDataToOutputType(GetParentObject(), data); RootedDictionary metadata(cx); if (mOutputNewDecoderConfig) { RootedDictionary decoderConfig(cx); EncoderConfigToDecoderConfig(cx, data, *mActiveConfig, decoderConfig); metadata.mDecoderConfig.Construct(std::move(decoderConfig)); mOutputNewDecoderConfig = false; LOG("New config passed to output callback"); } nsAutoCString metadataInfo; if (data->mTemporalLayerId) { RootedDictionary svc(cx); svc.mTemporalLayerId.Construct(data->mTemporalLayerId.value()); metadata.mSvc.Construct(std::move(svc)); metadataInfo.Append( nsPrintfCString(", temporal layer id %d", metadata.mSvc.Value().mTemporalLayerId.Value())); } if (metadata.mDecoderConfig.WasPassed()) { metadataInfo.Append(", new decoder config"); } LOG("EncoderTemplate:: output callback (ts: {}){}", encodedData->Timestamp(), metadataInfo.get()); mAsyncDurationTracker.End(encodedData->Timestamp()); cb->Call((EncodedVideoChunk&)(*encodedData), metadata); } } template <> void EncoderTemplate::OutputEncodedAudioData( const nsTArray>&& aData) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(mActiveConfig); // Get JSContext for RootedDictionary. // The EncoderType::MetadataType, AudioDecoderConfig // below are rooted to work around the JS hazard issues. AutoJSAPI jsapi; if (!jsapi.Init(GetParentObject())) { LOGE("{} {} AutoJSAPI init failed", AudioEncoderTraits::Name.get(), fmt::ptr(this)); return; } JSContext* cx = jsapi.cx(); RefPtr cb(mOutputCallback); for (const auto& data : aData) { // It's possible to have reset() called in between this task having been // dispatched, and running -- no output callback should happen when that's // the case. // This is imprecise in the spec, but discussed in // https://github.com/w3c/webcodecs/issues/755 and agreed upon. if (!mActiveConfig) { return; } RefPtr encodedData = EncodedDataToOutputType(GetParentObject(), data); RootedDictionary metadata(cx); if (mOutputNewDecoderConfig) { RootedDictionary decoderConfig(cx); EncoderConfigToDecoderConfig(cx, data, *mActiveConfig, decoderConfig); metadata.mDecoderConfig.Construct(std::move(decoderConfig)); mOutputNewDecoderConfig = false; LOG("New config passed to output callback"); } nsAutoCString metadataInfo; if (metadata.mDecoderConfig.WasPassed()) { metadataInfo.Append(", new decoder config"); } LOG("EncoderTemplate:: output callback (ts: {}, duration: {}, {} bytes, " "{} so far)", encodedData->Timestamp(), !encodedData->GetDuration().IsNull() ? encodedData->GetDuration().Value() : 0, data->Size(), mPacketsOutput++); mAsyncDurationTracker.End(encodedData->Timestamp()); cb->Call((EncodedAudioChunk&)(*encodedData), metadata); } } template void EncoderTemplate::ScheduleDequeueEvent() { AssertIsOnOwningThread(); if (mDequeueEventScheduled) { return; } mDequeueEventScheduled = true; QueueATask("dequeue event task", [self = RefPtr{this}]() { self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns); self->mDequeueEventScheduled = false; }); } template nsresult EncoderTemplate::FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType) { if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) { return NS_ERROR_ABORT; } LOGV("Dispatching {} event to {} {}", NS_ConvertUTF16toUTF8(aEventType).get(), EncoderType::Name.get(), fmt::ptr(this)); RefPtr event = new Event(this, nullptr, nullptr); event->InitEvent(aEventType, true, true); event->SetTrusted(true); this->DispatchEvent(*event); return NS_OK; } template void EncoderTemplate::SchedulePromiseResolveOrReject( already_AddRefed aPromise, const nsresult& aResult) { AssertIsOnOwningThread(); RefPtr p = aPromise; auto resolver = [p, result = aResult] { if (NS_FAILED(result)) { p->MaybeReject(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); return; } p->MaybeResolveWithUndefined(); }; nsISerialEventTarget* target = GetCurrentSerialEventTarget(); if (NS_IsMainThread()) { MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( "SchedulePromiseResolveOrReject Runnable (main)", resolver))); return; } MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( "SchedulePromiseResolveOrReject Runnable (worker)", resolver))); } template void EncoderTemplate::ProcessControlMessageQueue() { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) { RefPtr& msg = mControlMessageQueue.front(); if (msg->AsConfigureMessage()) { if (ProcessConfigureMessage(msg->AsConfigureMessage()) == MessageProcessedResult::NotProcessed) { break; } } else if (msg->AsEncodeMessage()) { if (ProcessEncodeMessage(msg->AsEncodeMessage()) == MessageProcessedResult::NotProcessed) { break; } } else { MOZ_ASSERT(msg->AsFlushMessage()); if (ProcessFlushMessage(msg->AsFlushMessage()) == MessageProcessedResult::NotProcessed) { break; } } } } template void EncoderTemplate::CancelPendingControlMessagesAndFlushPromises( const nsresult& aResult) { AssertIsOnOwningThread(); mReconfigureRequest.DisconnectIfExists(); mDrainAfterReconfigureRequest.DisconnectIfExists(); // Cancel the message that is being processed. if (mProcessingMessage) { LOG("{} {} cancels current {}", EncoderType::Name.get(), fmt::ptr(this), mProcessingMessage->ToString().get()); mProcessingMessage->Cancel(); mProcessingMessage = nullptr; } // Clear the message queue. while (!mControlMessageQueue.empty()) { LOG("{} {} cancels pending {}", EncoderType::Name.get(), fmt::ptr(this), mControlMessageQueue.front()->ToString().get()); MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing()); mControlMessageQueue.pop(); } // If there are pending flush promises, reject them. mPendingFlushPromises.Clear([&](const int64_t& id, const RefPtr& p) { LOG("{} {}, reject the promise for flush {}", EncoderType::Name.get(), fmt::ptr(this), id); p->MaybeReject(aResult); }); } template template void EncoderTemplate::QueueATask(const char* aName, Func&& aSteps) { AssertIsOnOwningThread(); MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread( NS_NewRunnableFunction(aName, std::forward(aSteps)))); } template MessageProcessedResult EncoderTemplate::ProcessConfigureMessage( RefPtr aMessage) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(aMessage->AsConfigureMessage()); if (mProcessingMessage) { return MessageProcessedResult::NotProcessed; } mProcessingMessage = aMessage; mControlMessageQueue.pop(); LOG("{} {} Configuring, message queue processing blocked({})", EncoderType::Name.get(), fmt::ptr(this), aMessage->ToString().get()); StartBlockingMessageQueue(); bool supported = EncoderType::IsSupported(*aMessage->Config()); if (!supported) { LOGE("{} {} ProcessConfigureMessage error (sync): Not supported", EncoderType::Name.get(), fmt::ptr(this)); mProcessingMessage = nullptr; QueueATask( "Error while configuring encoder", [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { LOGE("{} {} ProcessConfigureMessage (async close): Not supported", EncoderType::Name.get(), fmt::ptr(self.get())); self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); }); return MessageProcessedResult::Processed; } if (mAgent) { Reconfigure(aMessage); } else { Configure(aMessage); } return MessageProcessedResult::Processed; } template void EncoderTemplate::StartBlockingMessageQueue() { LOG("=== Message queue blocked"); mMessageQueueBlocked = true; } template void EncoderTemplate::StopBlockingMessageQueue() { LOG("=== Message queue unblocked"); mMessageQueueBlocked = false; } template void EncoderTemplate::OutputEncodedData( const nsTArray>&& aData) { if constexpr (std::is_same_v) { OutputEncodedVideoData(std::move(aData)); } else { OutputEncodedAudioData(std::move(aData)); } } template void EncoderTemplate::DrainAndReconfigure( RefPtr aMessage) { MOZ_ASSERT(mAgent); mAgent->Drain() ->Then( GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = mAgent->mId, message = aMessage]( EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) { self->mDrainAfterReconfigureRequest.Complete(); if (aResult.IsReject()) { const MediaResult& error = aResult.RejectValue(); LOGE( "{} {}, EncoderAgent #{} failed to drain during " "reconfigure: {}", EncoderType::Name.get(), fmt::ptr(self.get()), id, error.Description().get()); self->QueueATask( "Error during drain during reconfigure", [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { self->CloseInternal( NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); }); return; } LOG("{} {} flush during reconfiguration succeeded.", EncoderType::Name.get(), fmt::ptr(self.get())); nsTArray> data = std::move(aResult.ResolveValue()); if (!data.IsEmpty()) { LOG("{} {} Outputing {} frames during flush " " for reconfiguration with encoder destruction", EncoderType::Name.get(), fmt::ptr(self.get()), data.Length()); self->QueueATask("Output encoded Data", [self = RefPtr{self}, data = std::move(data)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { self->OutputEncodedData(std::move(data)); }); } self->QueueATask( "Destroy + recreate encoder after failed reconfigure", [self = RefPtr(self), message]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { if (self->mState != CodecState::Configured) { return; } self->DestroyEncoderAgentIfAny(); self->Configure(message); }); }) ->Track(mDrainAfterReconfigureRequest); } template void EncoderTemplate::Reconfigure( RefPtr aMessage) { MOZ_ASSERT(mAgent); LOG("Reconfiguring encoder: {}", aMessage->Config()->ToString().get()); RefPtr config = aMessage->Config(); RefPtr configDiff = mActiveConfig->Diff(*config); // Nothing to do, return now, but per spec the config // must be output next time a packet is output. if (configDiff->Empty()) { mOutputNewDecoderConfig = true; LOG("Reconfigure with identical config, returning."); mProcessingMessage = nullptr; StopBlockingMessageQueue(); return; } LOG("Attempting to reconfigure encoder: old: {} new: {}, diff: {}", mActiveConfig->ToString().get(), config->ToString().get(), configDiff->ToString().get()); // Changes like codec or hardware acceleration cannot be done on the fly: // drain the encoder and create a fresh one with the new config. if (!configDiff->CanAttemptReconfigure()) { DrainAndReconfigure(aMessage); return; } RefPtr changeList = configDiff->ToPEMChangeList(); // Attempt to reconfigure the encoder on the fly. // If reconfiguring on the fly didn't work, flush the encoder and recreate. mAgent->Reconfigure(changeList) ->Then( GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = mAgent->mId, message = aMessage]( const EncoderAgent::ReconfigurationPromise::ResolveOrRejectValue& aResult) { self->mReconfigureRequest.Complete(); MOZ_ASSERT(self->mProcessingMessage); MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); MOZ_ASSERT(self->mState == CodecState::Configured); MOZ_ASSERT(self->mAgent); MOZ_ASSERT(id == self->mAgent->mId); MOZ_ASSERT(self->mActiveConfig); if (aResult.IsReject()) { LOGE( "Reconfiguring on the fly didn't succeed, flushing and " "configuring a new encoder"); self->DrainAndReconfigure(message); return; } LOG("{} {}, EncoderAgent #{} has been reconfigured on the fly to " "{}", EncoderType::Name.get(), fmt::ptr(self.get()), id, message->ToString().get()); self->mOutputNewDecoderConfig = true; self->mActiveConfig = message->Config(); self->mProcessingMessage = nullptr; self->StopBlockingMessageQueue(); self->ProcessControlMessageQueue(); }) ->Track(mReconfigureRequest); } template void EncoderTemplate::Configure( RefPtr aMessage) { MOZ_ASSERT(!mAgent); AUTO_ENCODER_MARKER(marker, ".configure"); LOG("Configuring encoder: {}", aMessage->Config()->ToString().get()); mOutputNewDecoderConfig = true; mActiveConfig = aMessage->Config(); bool encoderAgentCreated = CreateEncoderAgent(aMessage->mMessageId, aMessage->Config()); if (!encoderAgentCreated) { LOGE( "{} {} ProcessConfigureMessage error (sync): encoder agent " "creation " "failed", EncoderType::Name.get(), fmt::ptr(this)); mProcessingMessage = nullptr; QueueATask( "Error when configuring encoder (encoder agent creation failed)", [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { LOGE( "{} {} ProcessConfigureMessage (async close): encoder agent " "creation failed", EncoderType::Name.get(), fmt::ptr(self.get())); self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); }); return; } MOZ_ASSERT(mAgent); MOZ_ASSERT(mActiveConfig); LOG("Real configuration with fresh config: {}", mActiveConfig->ToString().get()); EncoderConfig config = mActiveConfig->ToEncoderConfig(); mAgent->Configure(config) ->Then(GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = mAgent->mId, aMessage, m = std::move(marker)]( const EncoderAgent::ConfigurePromise::ResolveOrRejectValue& aResult) mutable { MOZ_ASSERT(self->mProcessingMessage); MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); MOZ_ASSERT(self->mState == CodecState::Configured); MOZ_ASSERT(self->mAgent); MOZ_ASSERT(id == self->mAgent->mId); MOZ_ASSERT(self->mActiveConfig); LOG("{} {}, EncoderAgent #{} {} has been {}. now unblocks " "message-queue-processing", EncoderType::Name.get(), fmt::ptr(self.get()), id, aMessage->ToString().get(), aResult.IsResolve() ? "resolved" : "rejected"); aMessage->Complete(); self->mProcessingMessage = nullptr; if (aResult.IsReject()) { // The spec asks to close the decoder with an // NotSupportedError so we log the exact error here. const MediaResult& error = aResult.RejectValue(); LOGE("{} {}, EncoderAgent #{} failed to configure: {}", EncoderType::Name.get(), fmt::ptr(self.get()), id, error.Description().get()); self->QueueATask( "Error during configure", [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { self->CloseInternal( NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); }); return; } LOG("{} {}, EncoderAgent #{} configured successfully. {} " "encode requests are pending", EncoderType::Name.get(), fmt::ptr(self.get()), id, self->mEncodeQueueSize); self->StopBlockingMessageQueue(); self->ProcessControlMessageQueue(); }) ->Track(aMessage->Request()); } template MessageProcessedResult EncoderTemplate::ProcessEncodeMessage( RefPtr aMessage) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(aMessage->AsEncodeMessage()); MOZ_ASSERT(mEncodeQueueSize > 0); AUTO_ENCODER_MARKER(marker, ".encode-process"); if (mProcessingMessage) { return MessageProcessedResult::NotProcessed; } mProcessingMessage = aMessage; mControlMessageQueue.pop(); LOGV("{} {} processing {}", EncoderType::Name.get(), fmt::ptr(this), aMessage->ToString().get()); MOZ_ASSERT(AssertedCast(aMessage->BatchSize()) <= mEncodeQueueSize); mEncodeQueueSize -= AssertedCast(aMessage->BatchSize()); ScheduleDequeueEvent(); // Treat it like decode error if no EncoderAgent is available or the encoded // data is invalid. auto closeOnError = [&]() { mProcessingMessage = nullptr; QueueATask("Error during encode", [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { self->CloseInternal(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); }); return MessageProcessedResult::Processed; }; if (!mAgent) { LOGE("{} {} is not configured", EncoderType::Name.get(), fmt::ptr(this)); return closeOnError(); } MOZ_ASSERT(mActiveConfig); if (!aMessage->IsValid()) { LOGE("{} {}, {} has empty data", EncoderType::Name.get(), fmt::ptr(this), aMessage->ToString().get()); return closeOnError(); } mAgent->Encode(aMessage->TakeData()) ->Then(GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = mAgent->mId, m = std::move(marker)]( EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) mutable { MOZ_ASSERT(self->mProcessingMessage); MOZ_ASSERT(self->mProcessingMessage->AsEncodeMessage()); MOZ_ASSERT(self->mState == CodecState::Configured); MOZ_ASSERT(self->mAgent); MOZ_ASSERT(id == self->mAgent->mId); MOZ_ASSERT(self->mActiveConfig); RefPtr msg = self->mProcessingMessage->AsEncodeMessage(); nsCString msgStr = msg->ToString(); msg->Complete(); self->mProcessingMessage = nullptr; if (aResult.IsReject()) { // The spec asks to queue a task to run close the decoder // with an EncodingError so we log the exact error here. const MediaResult& error = aResult.RejectValue(); LOGE("{} {}, EncoderAgent #{} {} failed: {}", EncoderType::Name.get(), fmt::ptr(self.get()), id, msgStr.get(), error.Description().get()); self->QueueATask( "Error during encode runnable", [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { self->CloseInternal( NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); }); return; } MOZ_ASSERT(aResult.IsResolve()); nsTArray> data = std::move(aResult.ResolveValue()); if (data.IsEmpty()) { LOGV("{} {} got no data for {}", EncoderType::Name.get(), fmt::ptr(self.get()), msgStr.get()); } else { LOGV("{} {}, schedule {} encoded data output for {}", EncoderType::Name.get(), fmt::ptr(self.get()), data.Length(), msgStr.get()); m.End(); AUTO_ENCODER_MARKER(outMarker, ".encode-output"); self->QueueATask( "Output encoded Data", [self = RefPtr{self}, data2 = std::move(data), om = std::move(outMarker)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY mutable { self->OutputEncodedData(std::move(data2)); }); } self->ProcessControlMessageQueue(); }) ->Track(aMessage->Request()); return MessageProcessedResult::Processed; } template MessageProcessedResult EncoderTemplate::ProcessFlushMessage( RefPtr aMessage) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(aMessage->AsFlushMessage()); AUTO_ENCODER_MARKER(marker, ".flush"); if (mProcessingMessage) { return MessageProcessedResult::NotProcessed; } mProcessingMessage = aMessage; mControlMessageQueue.pop(); LOG("{} {} starts processing {}", EncoderType::Name.get(), fmt::ptr(this), aMessage->ToString().get()); // No agent, no thing to do. The promise has been rejected with the // appropriate error in ResetInternal already. if (!mAgent) { LOGE("{} {} no agent, nothing to do", EncoderType::Name.get(), fmt::ptr(this)); mProcessingMessage = nullptr; return MessageProcessedResult::Processed; } mAgent->Drain() ->Then(GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = mAgent->mId, aMessage, m = std::move(marker), this](EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) mutable { MOZ_ASSERT(self->mProcessingMessage); MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage()); MOZ_ASSERT(self->mState == CodecState::Configured); MOZ_ASSERT(self->mAgent); MOZ_ASSERT(id == self->mAgent->mId); MOZ_ASSERT(self->mActiveConfig); LOG("{} {}, EncoderAgent #{} {} has been {}", EncoderType::Name.get(), fmt::ptr(self.get()), id, aMessage->ToString().get(), aResult.IsResolve() ? "resolved" : "rejected"); nsCString msgStr = aMessage->ToString(); aMessage->Complete(); // If flush failed, it means encoder fails to encode the data // sent before, so we treat it like an encode error. We reject // the promise first and then queue a task to close VideoEncoder // with an EncodingError. if (aResult.IsReject()) { const MediaResult& error = aResult.RejectValue(); LOGE("{} {}, EncoderAgent #{} failed to flush: {}", EncoderType::Name.get(), fmt::ptr(self.get()), id, error.Description().get()); // Reject with an EncodingError instead of the error we got // above. self->QueueATask( "Error during flush runnable", [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { // If Reset() was invoked before this task executes, the // promise in mPendingFlushPromises is handled there. // Otherwise, the promise is going to be rejected by // CloseInternal() below. self->mProcessingMessage = nullptr; self->CloseInternal( NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); }); return; } // If flush succeeded, schedule to output encoded data first // and then resolve the promise, then keep processing the // control messages. MOZ_ASSERT(aResult.IsResolve()); nsTArray> data = std::move(aResult.ResolveValue()); if (data.IsEmpty()) { LOG("{} {} gets no data for {}", EncoderType::Name.get(), fmt::ptr(self.get()), msgStr.get()); } else { LOG("{} {}, schedule {} encoded data output for {}", EncoderType::Name.get(), fmt::ptr(self.get()), data.Length(), msgStr.get()); } const auto flushPromiseId = static_cast(aMessage->mMessageId); m.End(); AUTO_ENCODER_MARKER(outMarker, ".flush-output"); self->QueueATask( "Flush: output encoded data task", [self = RefPtr{self}, data = std::move(data), flushPromiseId, om = std::move( outMarker)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY mutable { self->OutputEncodedData(std::move(data)); // If Reset() was invoked before this task executes, or // during the output callback above in the execution of // this task, the promise in mPendingFlushPromises is // handled there. Otherwise, the promise is resolved here. if (Maybe> p = self->mPendingFlushPromises.Take(flushPromiseId)) { LOG("{} {}, resolving the promise for flush {}", EncoderType::Name.get(), fmt::ptr(self.get()), flushPromiseId); p.value()->MaybeResolveWithUndefined(); } }); self->mProcessingMessage = nullptr; self->ProcessControlMessageQueue(); }) ->Track(aMessage->Request()); return MessageProcessedResult::Processed; } // CreateEncoderAgent will create an EncoderAgent paired with a xpcom-shutdown // blocker and a worker-reference. Besides the needs mentioned in the header // file, the blocker and the worker-reference also provides an entry point for // us to clean up the resources. Other than the encoder dtor, Reset(), or // Close(), the resources should be cleaned up in the following situations: // 1. Encoder on window, closing document // 2. Encoder on worker, closing document // 3. Encoder on worker, terminating worker // // In case 1, the entry point to clean up is in the mShutdownBlocker's // ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's // shutting down callback. In case 3, the entry point is in mWorkerRef's // shutting down callback. template bool EncoderTemplate::CreateEncoderAgent( WebCodecsId aId, RefPtr aConfig) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(!mAgent); MOZ_ASSERT(!mShutdownBlocker); MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef); auto resetOnFailure = MakeScopeExit([&]() { mAgent = nullptr; mActiveConfig = nullptr; mShutdownBlocker = nullptr; mWorkerRef = nullptr; }); // If the encoder is on worker, get a worker reference. if (!NS_IsMainThread()) { WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); if (NS_WARN_IF(!workerPrivate)) { return false; } // Clean up all the resources when worker is going away. RefPtr workerRef = StrongWorkerRef::Create( workerPrivate, "EncoderTemplate::CreateEncoderAgent", [self = RefPtr{this}]() { LOG("{} {}, worker is going away", EncoderType::Name.get(), fmt::ptr(self.get())); (void)self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); }); if (NS_WARN_IF(!workerRef)) { return false; } mWorkerRef = new ThreadSafeWorkerRef(workerRef); } mAgent = MakeRefPtr(aId); // ShutdownBlockingTicket requires an unique name to register its own // nsIAsyncShutdownBlocker since each blocker needs a distinct name. // To do that, we use EncoderAgent's unique id to create a unique name. nsAutoString uniqueName; uniqueName.AppendPrintf( "Blocker for EncoderAgent #%zu (codec: %s) @ %p", mAgent->mId, NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get()); mShutdownBlocker = media::ShutdownBlockingTicket::Create( uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__); if (!mShutdownBlocker) { LOGE("{} {} failed to create {}", EncoderType::Name.get(), fmt::ptr(this), NS_ConvertUTF16toUTF8(uniqueName).get()); return false; } // Clean up all the resources when xpcom-will-shutdown arrives since the // page is going to be closed. mShutdownBlocker->ShutdownPromise()->Then( GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = mAgent->mId, ref = mWorkerRef](bool /* aUnUsed*/) { LOG("{} {} gets xpcom-will-shutdown notification for EncoderAgent " "#{}", EncoderType::Name.get(), fmt::ptr(self.get()), id); (void)self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); }, [self = RefPtr{this}, id = mAgent->mId, ref = mWorkerRef](bool /* aUnUsed*/) { LOG("{} {} removes shutdown-blocker #{} before getting any " "notification. EncoderAgent should have been dropped", EncoderType::Name.get(), fmt::ptr(self.get()), id); MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id); }); LOG("{} {} creates EncoderAgent #{} @ {} and its shutdown-blocker", EncoderType::Name.get(), fmt::ptr(this), mAgent->mId, fmt::ptr(mAgent.get())); resetOnFailure.release(); return true; } template void EncoderTemplate::DestroyEncoderAgentIfAny() { AssertIsOnOwningThread(); if (!mAgent) { LOG("{} {} has no EncoderAgent to destroy", EncoderType::Name.get(), fmt::ptr(this)); return; } MOZ_ASSERT(mActiveConfig); MOZ_ASSERT(mShutdownBlocker); MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); LOG("{} {} destroys EncoderAgent #{} @ {}", EncoderType::Name.get(), fmt::ptr(this), mAgent->mId, fmt::ptr(mAgent.get())); mActiveConfig = nullptr; RefPtr agent = std::move(mAgent); // mShutdownBlocker should be kept alive until the shutdown is done. // mWorkerRef is used to ensure this task won't be discarded in worker. agent->Shutdown()->Then( GetCurrentSerialEventTarget(), __func__, [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef), blocker = std::move(mShutdownBlocker)]( const ShutdownPromise::ResolveOrRejectValue& aResult) { LOG("{} {}, EncoderAgent #{}'s shutdown has been {}. Drop its " "shutdown-blocker now", EncoderType::Name.get(), fmt::ptr(self.get()), id, aResult.IsResolve() ? "resolved" : "rejected"); }); } template void EncoderTemplate::PushEncodeRequest( WebCodecsId aConfigureId, RefPtr&& aData, Maybe&& aOptions) { AssertIsOnOwningThread(); MOZ_ASSERT(mState == CodecState::Configured); // TODO(Bug 1984936): Enable batch encoding for selected encoders now. const size_t batchSize = (StaticPrefs::media_use_remote_encoder_video() && mActiveConfig && IsH264CodecString(mActiveConfig->mCodec)) ? std::max( StaticPrefs::dom_media_webcodecs_batch_encoding_size(), 1) : 1; RefPtr msg; if (!mControlMessageQueue.empty()) { msg = mControlMessageQueue.back()->AsEncodeMessage(); if (msg && (msg->mConfigureId != aConfigureId || msg->BatchSize() >= batchSize)) { msg = nullptr; } } const bool isNewMessage = !msg; if (isNewMessage) { msg = MakeRefPtr(aConfigureId, aData.forget(), std::move(aOptions)); mControlMessageQueue.push(msg); } else { msg->PushData(aData.forget(), std::move(aOptions)); } mEncodeQueueSize += 1; LOGV("{} {} {} {}, encode queue size: {}", EncoderType::Name.get(), fmt::ptr(this), isNewMessage ? "queued a new" : "appended data to", msg->ToString().get(), mEncodeQueueSize); } template class EncoderTemplate; template class EncoderTemplate; #undef LOG #undef LOGW #undef LOGE #undef LOGV #undef LOG_INTERNAL } // namespace mozilla::dom