/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "AndroidImageReader.h" #include #include #include #include "GLContext.h" #include "GLContextEGL.h" #include "GLContextProvider.h" #include "GLLibraryEGL.h" #include "GLReadTexImageHelper.h" #include "OGLShaderConfig.h" #include "ScopedGLHelpers.h" #include "mozilla/RemoteMediaManagerParent.h" #include "mozilla/ScopeExit.h" #include "mozilla/TimeStamp.h" #include "mozilla/gfx/2D.h" #include "mozilla/gfx/Logging.h" #include "mozilla/layers/AndroidImageConsumer.h" #include "mozilla/webrender/RenderThread.h" #include "nsProxyRelease.h" namespace mozilla { namespace layers { constinit static RefPtr sAImageSnapshotContext; AndroidImageReaderImage::AndroidImageReaderImage( const GpuProcessAndroidImageReaderId aImageReaderId, const gfx::IntSize& aSize, const bool aHasAlpha) : Image(nullptr, ImageFormat::ANDROID_IMAGE_READER), mImageReaderId(aImageReaderId), mFrameId(AndroidMediaCodecFrameId::GetNext()), mSize(aSize), mHasAlpha(aHasAlpha) { MOZ_ASSERT(XRE_IsGPUProcess()); } AndroidImageReaderImage::~AndroidImageReaderImage() { auto* imageReaderMap = layers::GpuProcessAndroidImageReaderMap::Get(); if (imageReaderMap) { imageReaderMap->MaybeReleaseFrameToCodec(mImageReaderId, mFrameId, /* aRender */ false); } } Maybe AndroidImageReaderImage::GetDesc() { return Nothing(); } void AndroidImageReaderImage::OnSetCurrent() {} bool AndroidImageReaderImage::MaybeReleaseFrameToCodec(bool aRender) { if (!mSetCurrentCallback) { return false; } bool ret = (*mSetCurrentCallback)(aRender); mSetCurrentCallback.reset(); return ret; } nsresult AndroidImageReaderImage::BuildSurfaceDescriptorBuffer( SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags, const std::function& aAllocate) { MOZ_ASSERT(RemoteMediaManagerParent::OnManagerThread()); if (!sAImageSnapshotContext) { nsCString discardFailureId; sAImageSnapshotContext = gl::GLContextProvider::CreateHeadless({}, &discardFailureId); if (!sAImageSnapshotContext) { NS_WARNING("Failed to create snapshot GLContext"); return NS_ERROR_FAILURE; } } auto* imageReaderMap = layers::GpuProcessAndroidImageReaderMap::Get(); if (!imageReaderMap) { return NS_ERROR_FAILURE; } RefPtr imageReader = imageReaderMap->GetImageReader(mImageReaderId); if (!imageReader) { return NS_ERROR_FAILURE; } gfx::IntSize size = GetSize(); // auto format = gfx::SurfaceFormat::R8G8B8X8 auto format = gfx::SurfaceFormat::B8G8R8A8; uint8_t* buffer = nullptr; int32_t stride = 0; nsresult rv = Image::AllocateSurfaceDescriptorBufferRgb( size, format, buffer, aSdBuffer, stride, aAllocate); if (NS_FAILED(rv)) { return rv; } RefPtr surface; surface = gfx::Factory::CreateWrappingDataSourceSurface(buffer, stride, size, format); if (!surface) { return NS_ERROR_FAILURE; } bool ret = imageReader->UpdateTexImageWithReadback( mFrameId, sAImageSnapshotContext, surface, size, format); if (!ret) { return NS_ERROR_FAILURE; } return NS_OK; } AndroidImageWrapper::AndroidImageWrapper(AndroidImageReader* aImageReader, AImage* aImage, AHardwareBuffer* aHardwareBuffer, const gfx::IntSize aSize, const gfx::SurfaceFormat aFormat, mozilla::UniqueFileHandle&& aFence) : mHardwareBuffer(aHardwareBuffer), mSize(aSize), mFormat(aFormat), mImageReader(aImageReader), mImage(aImage), mFence(std::move(aFence)) { MOZ_ASSERT(mImageReader); mImageReader->mAcquiredImageCount++; } AndroidImageWrapper::~AndroidImageWrapper() { AImage_delete(mImage); // XXX Add fence handling // AImage_deleteAsync(mImage fence); mImageReader->mAcquiredImageCount--; } mozilla::UniqueFileHandle AndroidImageWrapper::CloneFence() { auto fence = ipc::FileDescriptor(GetHandle()); return fence.TakePlatformHandle(); } /* static */ already_AddRefed AndroidImageReader::Create() { if (!XRE_IsGPUProcess()) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return nullptr; } auto* imageReaderMap = layers::GpuProcessAndroidImageReaderMap::Get(); if (!imageReaderMap) { return nullptr; } RefPtr imageReader = new AndroidImageReader(); bool ret = imageReader->Init(); if (!ret) { return nullptr; } imageReaderMap->Register(imageReader); return imageReader.forget(); } AndroidImageReader::AndroidImageReader() : mImageReaderId(GpuProcessAndroidImageReaderId::GetNext()), mMonitor("mozilla.layers.AndroidImageReader.mMonitor") {} AndroidImageReader::~AndroidImageReader() { auto* imageReaderMap = layers::GpuProcessAndroidImageReaderMap::Get(); if (imageReaderMap) { imageReaderMap->Unregister(mImageReaderId); } ReleaseResources(); } bool AndroidImageReader::Init() { MonitorAutoLock lock(mMonitor); MOZ_ASSERT(!mInited); // Set the width, height and format to some default value. This parameters // are/maybe overriden by the producer sending buffers to this imageReader's // Surface. const int32_t width = 1, height = 1; uint64_t usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE; // XXX set if video could be used for overlay. // usage |= AHARDWAREBUFFER_USAGE_COMPOSER_OVERLAY; media_status_t result; AImageReader* reader = nullptr; result = AImageReader_newWithUsage(width, height, AIMAGE_FORMAT_PRIVATE, usage, mMaxImageCount, &reader); if (result != AMEDIA_OK) { gfxCriticalNoteOnce << "AImageReader_newWithUsage failed" << static_cast(result); return false; } mAImageReader = reader; ANativeWindow* window = nullptr; result = AImageReader_getWindow(reader, &window); if (result != AMEDIA_OK) { gfxCriticalNoteOnce << "AImageReader_getWindow failed" << static_cast(result); return false; } mNativeWindow = window; auto listener = std::make_unique(); listener->context = reinterpret_cast(this); listener->onImageAvailable = &AndroidImageReader::OnFrameAvailable; result = AImageReader_setImageListener(reader, listener.get()); if (result != AMEDIA_OK) { gfxCriticalNoteOnce << "setImageListener failed" << static_cast(result); return false; } MOZ_ASSERT(mAImageReader); MOZ_ASSERT(mNativeWindow); mInited = true; return true; } void AndroidImageReader::ReleaseResources() { MonitorAutoLock lock(mMonitor); if (!mAImageReader) { return; } AImageReader_setImageListener(mAImageReader, nullptr); // Delete all images before closing the associated image reader. // Delete the image reader. AImageReader_delete(mAImageReader); mAImageReader = nullptr; // mNativeWindow is not owned by AndroidImageReader. // It is owned by AImageReader. mNativeWindow = nullptr; } /* static */ void AndroidImageReader::OnFrameAvailable(void* aContext, AImageReader* aReader) { auto* reader = static_cast(aContext); reader->NotifyFrameAvailable(); } void AndroidImageReader::NotifyFrameAvailable() { MonitorAutoLock lock(mMonitor); mWaitingFrameAvailable = false; mMonitor.NotifyAll(); } bool AndroidImageReader::UpdateTexImage(const AndroidMediaCodecFrameId aFrameId, gl::GLContext* aGL, GLuint aTexture, AndroidImageWrapper** aImage) { MOZ_ASSERT(aGL); MOZ_ASSERT(aImage); MonitorAutoLock lock(mMonitor); DebugOnly ret = DoUpdateTexImage(lock, aFrameId); if (!mCurrentImage || *aImage == mCurrentImage) { MOZ_ASSERT(!ret); return false; } MOZ_ASSERT(mCurrentImage); // XXX add fence handling const auto& gle = gl::GLContextEGL::Cast(aGL); const auto& egl = gle->mEgl; const EGLint attrs[] = { LOCAL_EGL_IMAGE_PRESERVED, LOCAL_EGL_TRUE, LOCAL_EGL_NONE, }; auto* nativeBuffer = mCurrentImage->mHardwareBuffer; EGLClientBuffer clientBuffer = egl->mLib->fGetNativeClientBufferANDROID(nativeBuffer); EGLImage eglImage = egl->fCreateImage( EGL_NO_CONTEXT, LOCAL_EGL_NATIVE_BUFFER_ANDROID, clientBuffer, attrs); MOZ_ASSERT(eglImage); if (!eglImage) { return false; } aGL->fBindTexture(LOCAL_GL_TEXTURE_EXTERNAL, aTexture); aGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE); aGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE); aGL->fEGLImageTargetTexture2D(LOCAL_GL_TEXTURE_EXTERNAL, eglImage); egl->fDestroyImage(eglImage); *aImage = mCurrentImage; NS_ADDREF(*aImage); return true; } bool AndroidImageReader::UpdateTexImageWithReadback( AndroidMediaCodecFrameId aFrameId, gl::GLContext* aGL, gfx::DataSourceSurface* aSurface, const gfx::IntSize aSize, const gfx::SurfaceFormat aFormat) { MOZ_ASSERT(aGL); MOZ_ASSERT(aSurface); MonitorAutoLock lock(mMonitor); DoUpdateTexImage(lock, aFrameId); if (!mCurrentImage) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return false; } aGL->MakeCurrent(); gl::ScopedTexture scopedTex(aGL); const auto& gle = gl::GLContextEGL::Cast(aGL); const auto& egl = gle->mEgl; const EGLint attrs[] = { LOCAL_EGL_IMAGE_PRESERVED, LOCAL_EGL_TRUE, LOCAL_EGL_NONE, }; auto* nativeBuffer = mCurrentImage->mHardwareBuffer; EGLClientBuffer clientBuffer = egl->mLib->fGetNativeClientBufferANDROID(nativeBuffer); EGLImage eglImage = egl->fCreateImage( EGL_NO_CONTEXT, LOCAL_EGL_NATIVE_BUFFER_ANDROID, clientBuffer, attrs); MOZ_ASSERT(eglImage); if (!eglImage) { return false; } aGL->fBindTexture(LOCAL_GL_TEXTURE_EXTERNAL, scopedTex.Texture()); aGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE); aGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE); aGL->fEGLImageTargetTexture2D(LOCAL_GL_TEXTURE_EXTERNAL, eglImage); egl->fDestroyImage(eglImage); ShaderConfigOGL config = ShaderConfigFromTargetAndFormat(LOCAL_GL_TEXTURE_EXTERNAL, aFormat); int shaderConfig = config.mFeatures; bool ret = aGL->ReadTexImageHelper()->ReadTexImage( aSurface, scopedTex.Texture(), LOCAL_GL_TEXTURE_EXTERNAL, aSize, gfx::Matrix4x4(), shaderConfig, /* aYInvert */ false); if (!ret) { return false; } return true; } bool AndroidImageReader::DoUpdateTexImage(const MonitorAutoLock& aProofOfLock, AndroidMediaCodecFrameId aFrameId) { MOZ_ASSERT(static_cast(mAcquiredImageCount) <= mMaxImageCount); if (mCurrentFrameId == aFrameId) { return false; } // Limit as only one thread could do DoUpdateTexImage() const TimeDuration pendingTimeout = TimeDuration::FromMilliseconds(20); while (mIsPendingNextImage) { CVStatus status = mMonitor.Wait(pendingTimeout); if (status == CVStatus::Timeout) { gfxCriticalNoteOnce << "Pending next image wait timeout"; return false; } } MOZ_ASSERT(!mIsPendingNextImage); auto scopeExit = MakeScopeExit([&]() MOZ_REQUIRES(mMonitor) { mIsPendingNextImage = false; mMonitor.NotifyAll(); }); mIsPendingNextImage = true; MOZ_ASSERT(!mWaitingFrameAvailable); mWaitingFrameAvailable = true; if (!MaybeReleaseFrameToCodec(aProofOfLock, aFrameId, /* aRender */ true)) { mWaitingFrameAvailable = false; return false; } const TimeDuration timeout = TimeDuration::FromMilliseconds(20); while (mWaitingFrameAvailable) { CVStatus status = mMonitor.Wait(timeout); if (status == CVStatus::Timeout) { gfxCriticalNoteOnce << "UpdateTexImage wait timeout"; return false; } } mWaitingFrameAvailable = false; AImage* image = nullptr; media_status_t ret = AMEDIA_OK; UniqueFileHandle fence; ret = AImageReader_acquireNextImageAsync(mAImageReader, &image, getter_Transfers(fence)); // XXX Add AImageReader_acquireLatestImageAsync() usage switch (ret) { case AMEDIA_ERROR_INVALID_PARAMETER: MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return false; case AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED: MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return false; case AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE: return false; case AMEDIA_ERROR_UNKNOWN: return false; case AMEDIA_OK: // Call succeeded. break; default: MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return false; } if (!image) { return false; } AHardwareBuffer* nativeBuffer = nullptr; media_status_t result = AImage_getHardwareBuffer(image, &nativeBuffer); if (!nativeBuffer) { gfxCriticalNoteOnce << "AImage_getHardwareBuffer failed" << static_cast(result); AImage_delete(image); // XXX Add fence handling // AImage_deleteAsync(image, fence); return false; } AHardwareBuffer_Desc bufferInfo = {}; AHardwareBuffer_describe(nativeBuffer, &bufferInfo); const gfx::IntSize size = gfx::IntSize(bufferInfo.width, bufferInfo.height); // XXX const gfx::SurfaceFormat format = gfx::SurfaceFormat::R8G8B8X8; // XXX add crop handling mCurrentImage = new AndroidImageWrapper(this, image, nativeBuffer, size, format, std::move(fence)); MOZ_ASSERT(static_cast(mAcquiredImageCount) <= mMaxImageCount); return true; } ANativeWindow* AndroidImageReader::GetANativeWindow() { MonitorAutoLock lock(mMonitor); return mNativeWindow; } RefPtr AndroidImageReader::GetCurrentImage() { MonitorAutoLock lock(mMonitor); return mCurrentImage; } void AndroidImageReader::RegisterReaderImage( AndroidImageReaderImage* aReaderImage) { if (!aReaderImage) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return; } MonitorAutoLock lock(mMonitor); MOZ_ASSERT(mPendingFrames.find(aReaderImage->mFrameId) == mPendingFrames.end()); mPendingFrames.emplace(aReaderImage->mFrameId, aReaderImage); } bool AndroidImageReader::MaybeReleaseFrameToCodec( const AndroidMediaCodecFrameId aFrameId, const bool aRender) { MonitorAutoLock lock(mMonitor); return MaybeReleaseFrameToCodec(lock, aFrameId, aRender); } bool AndroidImageReader::MaybeReleaseFrameToCodec( const MonitorAutoLock& aProofOfLock, const AndroidMediaCodecFrameId aFrameId, const bool aRender) { auto it = mPendingFrames.find(aFrameId); if (it == mPendingFrames.end()) { return false; } bool ret = it->second->MaybeReleaseFrameToCodec(aRender); mPendingFrames.erase(it); return ret; } StaticAutoPtr GpuProcessAndroidImageReaderMap::sInstance; /* static */ void GpuProcessAndroidImageReaderMap::Init() { MOZ_ASSERT(XRE_IsGPUProcess()); sInstance = new GpuProcessAndroidImageReaderMap(); } /* static */ void GpuProcessAndroidImageReaderMap::Shutdown() { sInstance = nullptr; } GpuProcessAndroidImageReaderMap::GpuProcessAndroidImageReaderMap() : mMonitor("GpuProcessAndroidImageReaderMap.mMonitor") {} void GpuProcessAndroidImageReaderMap::Register( AndroidImageReader* aImageReader) { MOZ_ASSERT(aImageReader); if (!aImageReader) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return; } MonitorAutoLock lock(mMonitor); auto it = mImageReaders.find(aImageReader->mImageReaderId); if (it != mImageReaders.end()) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return; } mImageReaders.emplace(aImageReader->mImageReaderId, MakeUnique(aImageReader)); } void GpuProcessAndroidImageReaderMap::Unregister( GpuProcessAndroidImageReaderId aImageReaderId) { MonitorAutoLock lock(mMonitor); const auto it = mImageReaders.find(aImageReaderId); MOZ_ASSERT(it != mImageReaders.end()); if (it == mImageReaders.end()) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return; } mImageReaders.erase(it); } RefPtr GpuProcessAndroidImageReaderMap::GetImageReader( GpuProcessAndroidImageReaderId aImageReaderId) { MonitorAutoLock lock(mMonitor); const auto it = mImageReaders.find(aImageReaderId); if (it == mImageReaders.end()) { return nullptr; } return it->second->mImageReader; } bool GpuProcessAndroidImageReaderMap::MaybeReleaseFrameToCodec( const GpuProcessAndroidImageReaderId aImageReaderId, const AndroidMediaCodecFrameId aFrameId, const bool aRender) { RefPtr reader = GetImageReader(aImageReaderId); if (!reader) { return false; } return reader->MaybeReleaseFrameToCodec(aFrameId, aRender); } RefPtr GpuProcessAndroidImageReaderMap::GetImageConsumer( const GpuProcessAndroidImageReaderId aImageReaderId, gl::GLContext* aGL) { MOZ_ASSERT(wr::RenderThread::IsInRenderThread()); MOZ_ASSERT(aGL); MonitorAutoLock lock(mMonitor); const auto it = mImageReaders.find(aImageReaderId); if (it == mImageReaders.end()) { return nullptr; } auto* holder = it->second.get(); if (holder->mImageConsumer) { if (holder->mImageConsumer->mGL == aGL) { return holder->mImageConsumer; } MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return nullptr; } RefPtr imageConsumer; imageConsumer = AndroidImageConsumer::Create(holder->mImageReader, aGL); if (!imageConsumer) { MOZ_ASSERT_UNREACHABLE("unexpected to be called"); return nullptr; } holder->mImageConsumer = imageConsumer; return imageConsumer; } void GpuProcessAndroidImageReaderMap::UnregisterImageConsumer( GpuProcessAndroidImageReaderId aImageReaderId) { MOZ_ASSERT(wr::RenderThread::IsInRenderThread()); MonitorAutoLock lock(mMonitor); const auto it = mImageReaders.find(aImageReaderId); if (it == mImageReaders.end()) { return; } auto* holder = it->second.get(); MOZ_ASSERT(holder->mImageConsumer); holder->mImageConsumer = nullptr; } GpuProcessAndroidImageReaderMap::ImageReaderHolder::ImageReaderHolder( AndroidImageReader* aImageReader) : mImageReader(aImageReader) {} GpuProcessAndroidImageReaderMap::ImageReaderHolder::~ImageReaderHolder() {} } // namespace layers } // namespace mozilla