media / gpu / android / frame_info_helper.cc [blame]

// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/gpu/android/frame_info_helper.h"

#include "base/memory/raw_ptr.h"
#include "base/metrics/histogram_functions.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/threading/sequence_bound.h"
#include "gpu/command_buffer/service/shared_image/android_video_image_backing.h"
#include "gpu/ipc/service/command_buffer_stub.h"
#include "gpu/ipc/service/gpu_channel.h"
#include "gpu/ipc/service/gpu_channel_manager.h"
#include "media/base/media_switches.h"
#include "media/gpu/android/codec_output_buffer_renderer.h"

namespace media {

FrameInfoHelper::FrameInfo::FrameInfo() = default;
FrameInfoHelper::FrameInfo::~FrameInfo() = default;
FrameInfoHelper::FrameInfo::FrameInfo(FrameInfo&&) = default;
FrameInfoHelper::FrameInfo::FrameInfo(const FrameInfoHelper::FrameInfo&) =
    default;
FrameInfoHelper::FrameInfo& FrameInfoHelper::FrameInfo::operator=(
    const FrameInfoHelper::FrameInfo&) = default;

// Concrete implementation of FrameInfoHelper that renders output buffers and
// gets the FrameInfo they need.
class FrameInfoHelperImpl : public FrameInfoHelper,
                            public gpu::RefCountedLockHelperDrDc {
 public:
  FrameInfoHelperImpl(scoped_refptr<base::SequencedTaskRunner> gpu_task_runner,
                      SharedImageVideoProvider::GetStubCB get_stub_cb,
                      scoped_refptr<gpu::RefCountedLock> drdc_lock)
      : gpu::RefCountedLockHelperDrDc(drdc_lock) {
    on_gpu_ = base::SequenceBound<OnGpu>(std::move(gpu_task_runner),
                                         std::move(get_stub_cb),
                                         std::move(drdc_lock));
  }

  ~FrameInfoHelperImpl() override = default;

  void GetFrameInfo(std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer,
                    FrameInfoReadyCB callback) override {
    Request request = {.buffer_renderer = std::move(buffer_renderer),
                       .callback = std::move(callback)};
    requests_.push(std::move(request));
    // If there were no pending requests start processing queue now.
    if (requests_.size() == 1)
      ProcessRequestsQueue();
  }

  bool IsStalled() const override { return waiting_for_real_frame_info_; }

 private:
  struct Request {
    std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer;
    FrameInfoReadyCB callback;
  };

  class OnGpu : public gpu::RefCountedLockHelperDrDc {
   public:
    OnGpu(SharedImageVideoProvider::GetStubCB get_stub_cb,
          scoped_refptr<gpu::RefCountedLock> drdc_lock)
        : gpu::RefCountedLockHelperDrDc(std::move(drdc_lock)),
          frame_info_helper_holder_(
              base::MakeRefCounted<FrameInfoHelperHolder>(this)) {
      auto* stub = get_stub_cb.Run();
      if (stub) {
        gpu::ContextResult result;
        shared_context_ =
            stub->channel()->gpu_channel_manager()->GetSharedContextState(
                &result);
        if (result == gpu::ContextResult::kSuccess) {
          DCHECK(shared_context_);
          if (shared_context_->GrContextIsVulkan()) {
            vulkan_context_provider_ = shared_context_->vk_context_provider();
          } else if (shared_context_->IsGraphiteDawnVulkan()) {
            dawn_context_provider_ = shared_context_->dawn_context_provider();
          }
        }
      }
    }

    ~OnGpu() {
      DCHECK(frame_info_helper_holder_);
      frame_info_helper_holder_->SetFrameInfoHelperOnGpuToNull();
    }

    void GetFrameInfoImpl(
        std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer,
        base::OnceCallback<void(std::unique_ptr<CodecOutputBufferRenderer>,
                                std::optional<FrameInfo>)> cb) {
      AssertAcquiredDrDcLock();
      DCHECK(buffer_renderer);

      auto texture_owner = buffer_renderer->texture_owner();
      DCHECK(texture_owner);

      std::optional<FrameInfo> info;

      if (buffer_renderer->RenderToTextureOwnerFrontBuffer()) {
        gfx::Size coded_size;
        gfx::Rect visible_rect;
        if (texture_owner->GetCodedSizeAndVisibleRect(
                buffer_renderer->size(), &coded_size, &visible_rect)) {
          info.emplace();
          info->coded_size = coded_size;
          info->visible_rect = visible_rect;
          info->ycbcr_info = gpu::AndroidVideoImageBacking::GetYcbcrInfo(
              texture_owner.get(), vulkan_context_provider_,
              dawn_context_provider_);
        }
      }
      std::move(cb).Run(std::move(buffer_renderer), info);
    }

    void GetFrameInfo(
        std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer,
        base::OnceCallback<void(std::unique_ptr<CodecOutputBufferRenderer>,
                                std::optional<FrameInfo>)> cb) {
      // Note that we need to ensure that no other thread renders another buffer
      // in between while we are getting frame info here. Otherwise we will get
      // wrong frame info. This is ensured by holding |drdc_lock| from all the
      // places from where GetFrameInfoImpl() call can be triggered. It can be
      // called from here via |texture_owner->RunWhenBufferIsAvailable()| below
      // or from |ImageReaderGLOwner::ReleaseRefOnImageLocked()| when the
      // buffer_vailable_cb is cached and triggered. So we lock here as well as
      // ensure lock is held during
      // |ImageReaderGLOwner::ReleaseRefOnImageLocked|.
      base::AutoLockMaybe auto_lock(GetDrDcLockPtr());
      DCHECK(buffer_renderer);

      auto texture_owner = buffer_renderer->texture_owner();
      DCHECK(texture_owner);

      auto buffer_available_cb =
          base::BindOnce(&FrameInfoHelperHolder::GetFrameInfoImpl,
                         base::RetainedRef(frame_info_helper_holder_),
                         std::move(buffer_renderer), std::move(cb));
      texture_owner->RunWhenBufferIsAvailable(std::move(buffer_available_cb));
    }

   private:
    // OnGpu::GetFrameInfoImpl can be called from any gpu thread (gpu main or
    // DrDc), hence we can not use WeakPtr to it in |buffer_available_cb|.
    // FrameInfoHelperHolder is used instead to mimic this weakPtr behavior of
    // OnGpu. FrameInfoHelperHolder is RefCountedThreadSafe, and has a pointer
    // to the OnGpu. OnGpu owns the FrameInfoHelperHolder and sets this pointer
    // to null in its destructor so that it can't be used once OnGpu is
    // destroyed. Note that since OnGpu::GetFrameInfoImpl needed to be called
    // from any gpu thread, we could not use WeakPtr to it.
    class FrameInfoHelperHolder
        : public base::RefCountedThreadSafe<FrameInfoHelperHolder> {
     public:
      explicit FrameInfoHelperHolder(raw_ptr<OnGpu> frame_info_helper_on_gpu)
          : frame_info_helper_on_gpu_(frame_info_helper_on_gpu) {
        DCHECK(frame_info_helper_on_gpu_);
      }

      void GetFrameInfoImpl(
          std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer,
          base::OnceCallback<void(std::unique_ptr<CodecOutputBufferRenderer>,
                                  std::optional<FrameInfo>)> cb) {
        base::AutoLock l(lock_);
        if (frame_info_helper_on_gpu_) {
          frame_info_helper_on_gpu_->GetFrameInfoImpl(
              std::move(buffer_renderer), std::move(cb));
        }
      }

      void SetFrameInfoHelperOnGpuToNull() {
        base::AutoLock l(lock_);
        frame_info_helper_on_gpu_ = nullptr;
      }

     private:
      friend class base::RefCountedThreadSafe<FrameInfoHelperHolder>;
      ~FrameInfoHelperHolder() = default;

      // |lock_| for thread safe access to |frame_info_helper_on_gpu_|.
      base::Lock lock_;
      raw_ptr<OnGpu> frame_info_helper_on_gpu_ GUARDED_BY(lock_) = nullptr;
    };

    // Note that |shared_context_| is to just keep ref on it until
    // context provider raw_ptrs are being used.
    scoped_refptr<gpu::SharedContextState> shared_context_;
    raw_ptr<viz::VulkanContextProvider> vulkan_context_provider_ = nullptr;
    raw_ptr<gpu::DawnContextProvider> dawn_context_provider_ = nullptr;
    scoped_refptr<FrameInfoHelperHolder> frame_info_helper_holder_;
  };

  bool CanGuessCodedSize(
      const CodecOutputBufferRenderer& buffer_renderer) const {
    // We never guess on the first frame since we can always render instead.
    if (!frame_info_) {
      return false;
    }
    // Coded size guessing will be disabled if the feature isn't enabled or we
    // didn't correctly guess the initial coded size or failed to guess a coded
    // size during a size change.
    if (disable_coded_size_guessing_) {
      return false;
    }
    // We can't guess non-origin visible rects.
    if (!frame_info_->visible_rect.origin().IsOrigin()) {
      return false;
    }
    return buffer_renderer.CanGuessCodedSize();
  }

  FrameInfo GuessFrameInfo(
      const CodecOutputBufferRenderer& buffer_renderer) const {
    FrameInfo info;
    info.coded_size = CanGuessCodedSize(buffer_renderer)
                          ? buffer_renderer.GuessCodedSize()
                          : buffer_renderer.size();
    info.visible_rect = gfx::Rect(buffer_renderer.size());
    return info;
  }

  void DisableCodedSizeGuessing(const gfx::Size& guessed_coded_size,
                                const gfx::Size& actual_coded_size) {
    disable_coded_size_guessing_ = true;
    LOG(ERROR) << "Guessed coded size incorrectly. Expected "
               << guessed_coded_size.ToString() << ", got "
               << actual_coded_size.ToString();
  }

  void OnRealFrameInfoAvailable(gfx::Size visible_size,
                                gfx::Size guessed_coded_size,
                                std::optional<gfx::Size> coded_size,
                                std::optional<gfx::Rect> visible_rect) {
    DVLOG(1) << __func__
             << ": coded_size=" << (coded_size ? coded_size->ToString() : "")
             << ", visible_rect="
             << (visible_rect ? visible_rect->ToString() : "");

    DCHECK(waiting_for_real_frame_info_);
    waiting_for_real_frame_info_ = false;

    if (coded_size && visible_rect) {
      const bool guessed_coded_size_correctly =
          guessed_coded_size == *coded_size;
      base::UmaHistogramBoolean("Media.FrameInfo.GuessedCodedSizeChangeSuccess",
                                guessed_coded_size_correctly);
      if (!guessed_coded_size_correctly) {
        DisableCodedSizeGuessing(guessed_coded_size, frame_info_->coded_size);
      }

      frame_info_->coded_size = *coded_size;
      frame_info_->visible_rect = *visible_rect;
      visible_size_ = visible_size;

      // There's no way to get ycbr_info at this point. The TextureOwner can't
      // provide it since it doesn't have a vulkan context and we can't get it
      // here since there may not be a TextureOwner anymore. We're not even on
      // the right thread anymore.
    } else {
      // This means the buffer was destroyed without being rendered to the front
      // buffer, so we must emit another frame to try and get real size info.
    }

    ProcessRequestsQueue();
  }

  void OnFrameInfoReady(
      std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer,
      std::optional<FrameInfo> frame_info) {
    DCHECK(buffer_renderer);
    DCHECK(!requests_.empty());

    auto& request = requests_.front();

    if (frame_info) {
      const bool is_first_frame = !frame_info_;

      visible_size_ = buffer_renderer->size();
      frame_info_ = *frame_info;

      // We always render the first frame, so we compare the real coded size
      // against what we would have guessed. We then turn off guessing for
      // future coded size changes if we guessed wrong.
      if (is_first_frame && CanGuessCodedSize(*buffer_renderer)) {
        const auto guessed_coded_size = buffer_renderer->GuessCodedSize();
        const bool guessed_coded_size_correctly =
            frame_info_->coded_size == guessed_coded_size;

        base::UmaHistogramBoolean(
            "Media.FrameInfo.GuessedInitialCodedSizeSuccess",
            guessed_coded_size_correctly);
        if (!guessed_coded_size_correctly) {
          DisableCodedSizeGuessing(guessed_coded_size, frame_info_->coded_size);
        }
      }

      std::move(request.callback).Run(std::move(buffer_renderer), *frame_info_);
    } else {
      // It's possible that we will fail to render frame and so weren't able to
      // obtain FrameInfo. In this case we don't cache new values and complete
      // current request with guessed values.
      auto info = GuessFrameInfo(*buffer_renderer);
      std::move(request.callback)
          .Run(std::move(buffer_renderer), std::move(info));
    }
    requests_.pop();
    ProcessRequestsQueue();
  }

  void ProcessRequestsQueue() {
    while (!requests_.empty()) {
      auto& request = requests_.front();
      if (!request.buffer_renderer) {
        // If we don't have buffer_renderer we can Run callback immediately.
        std::move(request.callback).Run(nullptr, FrameInfo());
      } else if (!request.buffer_renderer->texture_owner()) {
        // If there is no texture_owner (SurfaceView case), we can't render
        // frame and get proper size. But as Display Compositor won't render
        // this frame the actual size is not important, so just guess.
        auto info = GuessFrameInfo(*request.buffer_renderer);
        std::move(request.callback)
            .Run(std::move(request.buffer_renderer), std::move(info));
      } else if (waiting_for_real_frame_info_) {
        // Only allow emitting one frame at a time when we're waiting for frame
        // information so that any glitches caused by a wrong guess are limited
        // to a single visible frame.
        break;
      } else if (visible_size_ == request.buffer_renderer->size()) {
        // We have cached the results of last frame info request with the same
        // size. We assume that coded_size doesn't change if the visible_size
        // stays the same.
        std::move(request.callback)
            .Run(std::move(request.buffer_renderer), *frame_info_);
      } else if (CanGuessCodedSize(*request.buffer_renderer)) {
        DCHECK(frame_info_);  // We never guess on the first frame.

        // To avoid glitches during size changes, guess a likely coded size.
        auto info = GuessFrameInfo(*request.buffer_renderer);
        info.ycbcr_info = frame_info_->ycbcr_info;
        waiting_for_real_frame_info_ = true;

        // Ensure we get the real coded size for the next frame.
        request.buffer_renderer->set_frame_info_callback(
            base::BindPostTaskToCurrentDefault(base::BindOnce(
                &FrameInfoHelperImpl::OnRealFrameInfoAvailable,
                weak_factory_.GetWeakPtr(), request.buffer_renderer->size(),
                info.coded_size)));

        std::move(request.callback)
            .Run(std::move(request.buffer_renderer), info);
      } else {
        // We have texture_owner and we don't have cached value, so we need to
        // hop to GPU thread and render the frame to get proper size.
        auto cb = base::BindPostTaskToCurrentDefault(
            base::BindOnce(&FrameInfoHelperImpl::OnFrameInfoReady,
                           weak_factory_.GetWeakPtr()));

        on_gpu_.AsyncCall(&OnGpu::GetFrameInfo)
            .WithArgs(std::move(request.buffer_renderer), std::move(cb));
        // We didn't complete this request quite yet, so we can't process queue
        // any further.
        break;
      }
      requests_.pop();
    }
  }

  base::SequenceBound<OnGpu> on_gpu_;
  base::queue<Request> requests_;

  // Cached values.
  std::optional<FrameInfo> frame_info_;
  gfx::Size visible_size_;
  bool waiting_for_real_frame_info_ = false;
  bool disable_coded_size_guessing_ =
      !base::FeatureList::IsEnabled(kMediaCodecCodedSizeGuessing);

  base::WeakPtrFactory<FrameInfoHelperImpl> weak_factory_{this};
};

// static
std::unique_ptr<FrameInfoHelper> FrameInfoHelper::Create(
    scoped_refptr<base::SequencedTaskRunner> gpu_task_runner,
    SharedImageVideoProvider::GetStubCB get_stub_cb,
    scoped_refptr<gpu::RefCountedLock> drdc_lock) {
  return std::make_unique<FrameInfoHelperImpl>(
      std::move(gpu_task_runner), std::move(get_stub_cb), std::move(drdc_lock));
}

}  // namespace media