media / gpu / v4l2 / v4l2_video_decoder_delegate_av1.cc [blame]

// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/gpu/v4l2/v4l2_video_decoder_delegate_av1.h"

#include <linux/v4l2-controls.h>
#include <linux/videodev2.h>

#include "media/gpu/macros.h"
#include "media/gpu/v4l2/v4l2_decode_surface.h"
#include "media/gpu/v4l2/v4l2_decode_surface_handler.h"
#include "third_party/libgav1/src/src/obu_parser.h"
#include "third_party/libgav1/src/src/warp_prediction.h"

namespace media {

using DecodeStatus = AV1Decoder::AV1Accelerator::Status;

class V4L2AV1Picture : public AV1Picture {
 public:
  V4L2AV1Picture(scoped_refptr<V4L2DecodeSurface> dec_surface)
      : dec_surface_(std::move(dec_surface)) {}

  V4L2AV1Picture(const V4L2AV1Picture&) = delete;
  V4L2AV1Picture& operator=(const V4L2AV1Picture&) = delete;

  const scoped_refptr<V4L2DecodeSurface>& dec_surface() const {
    return dec_surface_;
  }

 private:
  ~V4L2AV1Picture() override = default;

  scoped_refptr<AV1Picture> CreateDuplicate() override {
    return new V4L2AV1Picture(dec_surface_);
  }

  scoped_refptr<V4L2DecodeSurface> dec_surface_;
};

namespace {
// TODO(stevecho): Remove this when AV1 uAPI RFC v3 change
// (crrev/c/3859126) lands.
#ifndef BIT
#define BIT(nr) (1U << (nr))
#endif

// Section 5.5. Sequence header OBU syntax in the AV1 spec.
// https://aomediacodec.github.io/av1-spec
struct v4l2_ctrl_av1_sequence FillSequenceParams(
    const libgav1::ObuSequenceHeader& seq_header) {
  struct v4l2_ctrl_av1_sequence v4l2_seq_params = {};

  if (seq_header.still_picture)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_STILL_PICTURE;

  if (seq_header.use_128x128_superblock)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK;

  if (seq_header.enable_filter_intra)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_FILTER_INTRA;

  if (seq_header.enable_intra_edge_filter)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTRA_EDGE_FILTER;

  if (seq_header.enable_interintra_compound)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTERINTRA_COMPOUND;

  if (seq_header.enable_masked_compound)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND;

  if (seq_header.enable_warped_motion)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_WARPED_MOTION;

  if (seq_header.enable_dual_filter)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_DUAL_FILTER;

  if (seq_header.enable_order_hint)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT;

  if (seq_header.enable_jnt_comp)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_JNT_COMP;

  if (seq_header.enable_ref_frame_mvs)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS;

  if (seq_header.enable_superres)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_SUPERRES;

  if (seq_header.enable_cdef)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF;

  if (seq_header.enable_restoration)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_ENABLE_RESTORATION;

  if (seq_header.color_config.is_monochrome)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_MONO_CHROME;

  if (seq_header.color_config.color_range)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_COLOR_RANGE;

  if (seq_header.color_config.subsampling_x)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X;

  if (seq_header.color_config.subsampling_y)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y;

  if (seq_header.film_grain_params_present)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_FILM_GRAIN_PARAMS_PRESENT;

  if (seq_header.color_config.separate_uv_delta_q)
    v4l2_seq_params.flags |= V4L2_AV1_SEQUENCE_FLAG_SEPARATE_UV_DELTA_Q;

  v4l2_seq_params.seq_profile = seq_header.profile;
  v4l2_seq_params.order_hint_bits = seq_header.order_hint_bits;
  v4l2_seq_params.bit_depth = seq_header.color_config.bitdepth;
  v4l2_seq_params.max_frame_width_minus_1 = seq_header.max_frame_width - 1;
  v4l2_seq_params.max_frame_height_minus_1 = seq_header.max_frame_height - 1;

  return v4l2_seq_params;
}

// Section 5.9.11. Loop filter params syntax.
// Note that |update_ref_delta| and |update_mode_delta| flags in the spec
// are not needed for V4L2 AV1 API.
void FillLoopFilterParams(v4l2_av1_loop_filter& v4l2_lf,
                          const libgav1::LoopFilter& lf) {
  if (lf.delta_enabled)
    v4l2_lf.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_ENABLED;

  if (lf.delta_update)
    v4l2_lf.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_UPDATE;

  static_assert(std::size(decltype(v4l2_lf.level){}) == libgav1::kFrameLfCount,
                "Invalid size of loop filter level (strength) array");
  for (size_t i = 0; i < libgav1::kFrameLfCount; i++)
    v4l2_lf.level[i] = base::checked_cast<__u8>(lf.level[i]);

  v4l2_lf.sharpness = lf.sharpness;

  static_assert(std::size(decltype(v4l2_lf.ref_deltas){}) ==
                    libgav1::kNumReferenceFrameTypes,
                "Invalid size of ref deltas array");
  for (size_t i = 0; i < libgav1::kNumReferenceFrameTypes; i++)
    v4l2_lf.ref_deltas[i] = lf.ref_deltas[i];

  static_assert(std::size(decltype(v4l2_lf.mode_deltas){}) ==
                    libgav1::kLoopFilterMaxModeDeltas,
                "Invalid size of mode deltas array");
  for (size_t i = 0; i < libgav1::kLoopFilterMaxModeDeltas; i++)
    v4l2_lf.mode_deltas[i] = lf.mode_deltas[i];
}

// Section 5.9.12. Quantization params syntax
void FillQuantizationParams(v4l2_av1_quantization& v4l2_quant,
                            const libgav1::QuantizerParameters& quant) {
  if (quant.use_matrix)
    v4l2_quant.flags |= V4L2_AV1_QUANTIZATION_FLAG_USING_QMATRIX;

  v4l2_quant.base_q_idx = quant.base_index;

  // Note that quant.delta_ac[0] is useless
  // because it is always 0 according to libgav1.
  v4l2_quant.delta_q_y_dc = quant.delta_dc[0];

  v4l2_quant.delta_q_u_dc = quant.delta_dc[1];
  v4l2_quant.delta_q_u_ac = quant.delta_ac[1];

  v4l2_quant.delta_q_v_dc = quant.delta_dc[2];
  v4l2_quant.delta_q_v_ac = quant.delta_ac[2];

  if (!quant.use_matrix)
    return;

  v4l2_quant.qm_y = base::checked_cast<uint8_t>(quant.matrix_level[0]);
  v4l2_quant.qm_u = base::checked_cast<uint8_t>(quant.matrix_level[1]);
  v4l2_quant.qm_v = base::checked_cast<uint8_t>(quant.matrix_level[2]);
}

// Section 5.9.14. Segmentation params syntax
struct v4l2_av1_segmentation FillSegmentationParams(
    const libgav1::Segmentation& seg) {
  struct v4l2_av1_segmentation v4l2_seg = {};

  if (seg.enabled)
    v4l2_seg.flags |= V4L2_AV1_SEGMENTATION_FLAG_ENABLED;

  if (seg.update_map)
    v4l2_seg.flags |= V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP;

  if (seg.temporal_update)
    v4l2_seg.flags |= V4L2_AV1_SEGMENTATION_FLAG_TEMPORAL_UPDATE;

  if (seg.update_data)
    v4l2_seg.flags |= V4L2_AV1_SEGMENTATION_FLAG_UPDATE_DATA;

  if (seg.segment_id_pre_skip)
    v4l2_seg.flags |= V4L2_AV1_SEGMENTATION_FLAG_SEG_ID_PRE_SKIP;

  static_assert(
      std::size(decltype(v4l2_seg.feature_enabled){}) == libgav1::kMaxSegments,
      "Invalid size of |feature_enabled| array in |v4l2_av1_segmentation| "
      "struct");

  static_assert(
      std::size(decltype(v4l2_seg.feature_data){}) == libgav1::kMaxSegments &&
          std::extent<decltype(v4l2_seg.feature_data), 0>::value ==
              libgav1::kSegmentFeatureMax,
      "Invalid size of |feature_data| array in |v4l2_av1_segmentation| struct");

  for (size_t i = 0; i < libgav1::kMaxSegments; ++i) {
    for (size_t j = 0; j < libgav1::kSegmentFeatureMax; ++j) {
      v4l2_seg.feature_enabled[i] |= (seg.feature_enabled[i][j] << j);
      v4l2_seg.feature_data[i][j] = seg.feature_data[i][j];
    }
  }

  v4l2_seg.last_active_seg_id = seg.last_active_segment_id;

  return v4l2_seg;
}

// Section 5.9.15. Tile info syntax
struct v4l2_av1_tile_info FillTileInfo(const libgav1::TileInfo& ti) {
  struct v4l2_av1_tile_info v4l2_ti = {};

  if (ti.uniform_spacing)
    v4l2_ti.flags |= V4L2_AV1_TILE_INFO_FLAG_UNIFORM_TILE_SPACING;

  static_assert(std::size(decltype(v4l2_ti.mi_col_starts){}) ==
                    (libgav1::kMaxTileColumns + 1),
                "Size of |mi_col_starts| array in |v4l2_av1_tile_info| struct "
                "does not match libgav1 expectation");

  for (size_t i = 0; i < libgav1::kMaxTileColumns + 1; i++) {
    v4l2_ti.mi_col_starts[i] =
        base::checked_cast<uint32_t>(ti.tile_column_start[i]);
  }
  static_assert(std::size(decltype(v4l2_ti.mi_row_starts){}) ==
                    (libgav1::kMaxTileRows + 1),
                "Size of |mi_row_starts| array in |v4l2_av1_tile_info| struct "
                "does not match libgav1 expectation");
  for (size_t i = 0; i < libgav1::kMaxTileRows + 1; i++) {
    v4l2_ti.mi_row_starts[i] =
        base::checked_cast<uint32_t>(ti.tile_row_start[i]);
  }

  if (!ti.uniform_spacing) {
    // Confirmed that |kMaxTileColumns| is enough size for
    // |width_in_sbs_minus_1| and |kMaxTileRows| is enough size for
    // |height_in_sbs_minus_1|
    // https://b.corp.google.com/issues/187828854#comment19
    static_assert(
        std::size(decltype(v4l2_ti.width_in_sbs_minus_1){}) ==
            libgav1::kMaxTileColumns,
        "Size of |width_in_sbs_minus_1| array in |v4l2_av1_tile_info| struct "
        "does not match libgav1 expectation");
    for (size_t i = 0; i < libgav1::kMaxTileColumns; i++) {
      if (ti.tile_column_width_in_superblocks[i] >= 1) {
        v4l2_ti.width_in_sbs_minus_1[i] = base::checked_cast<uint32_t>(
            ti.tile_column_width_in_superblocks[i] - 1);
      }
    }

    static_assert(
        std::size(decltype(v4l2_ti.height_in_sbs_minus_1){}) ==
            libgav1::kMaxTileRows,
        "Size of |height_in_sbs_minus_1| array in |v4l2_av1_tile_info| struct "
        "does not match libgav1 expectation");
    for (size_t i = 0; i < libgav1::kMaxTileRows; i++) {
      if (ti.tile_row_height_in_superblocks[i] >= 1) {
        v4l2_ti.height_in_sbs_minus_1[i] = base::checked_cast<uint32_t>(
            ti.tile_row_height_in_superblocks[i] - 1);
      }
    }
  }

  v4l2_ti.tile_size_bytes = ti.tile_size_bytes;
  v4l2_ti.context_update_tile_id = ti.context_update_id;
  v4l2_ti.tile_cols = ti.tile_columns;
  v4l2_ti.tile_rows = ti.tile_rows;

  return v4l2_ti;
}

// Section 5.9.17. Quantizer index delta parameters syntax
void FillQuantizerIndexDeltaParams(struct v4l2_av1_quantization& v4l2_quant,
                                   const libgav1::ObuSequenceHeader& seq_header,
                                   const libgav1::ObuFrameHeader& frm_header) {
  // |diff_uv_delta| in the spec doesn't exist in libgav1,
  // because libgav1 infers it using the following logic.
  const bool diff_uv_delta = (frm_header.quantizer.base_index != 0) &&
                             (!seq_header.color_config.is_monochrome) &&
                             (seq_header.color_config.separate_uv_delta_q);
  if (diff_uv_delta)
    v4l2_quant.flags |= V4L2_AV1_QUANTIZATION_FLAG_DIFF_UV_DELTA;

  if (frm_header.delta_q.present)
    v4l2_quant.flags |= V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT;

  // |scale| is used to store |delta_q_res| value. This is because libgav1 uses
  // the same struct |Delta| both for quantizer index delta parameters and loop
  // filter delta parameters.
  v4l2_quant.delta_q_res = frm_header.delta_q.scale;
}

// Section 5.9.18. Loop filter delta parameters syntax.
// Note that |delta_lf_res| in |v4l2_av1_loop_filter| corresponds to
// |delta_lf.scale| in the frame header defined in libgav1.
void FillLoopFilterDeltaParams(struct v4l2_av1_loop_filter& v4l2_lf,
                               const libgav1::Delta& delta_lf) {
  if (delta_lf.present)
    v4l2_lf.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_PRESENT;

  if (delta_lf.multi)
    v4l2_lf.flags |= V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_MULTI;

  v4l2_lf.delta_lf_res = delta_lf.scale;
}

// Section 5.9.19. CDEF params syntax
struct v4l2_av1_cdef FillCdefParams(const libgav1::Cdef& cdef,
                                    uint8_t color_bitdepth) {
  struct v4l2_av1_cdef v4l2_cdef = {};

  // Damping value parsed in libgav1 is from the spec + (|color_bitdepth| - 8).
  CHECK_GE(color_bitdepth, 8u);
  const uint8_t coeff_shift = color_bitdepth - 8u;

  v4l2_cdef.damping_minus_3 =
      base::checked_cast<uint8_t>(cdef.damping - coeff_shift - 3u);

  v4l2_cdef.bits = cdef.bits;

  static_assert(std::size(decltype(v4l2_cdef.y_pri_strength){}) ==
                    libgav1::kMaxCdefStrengths,
                "Invalid size of cdef y_pri_strength strength");

  static_assert(std::size(decltype(v4l2_cdef.y_sec_strength){}) ==
                    libgav1::kMaxCdefStrengths,
                "Invalid size of cdef y_sec_strength strength");

  static_assert(std::size(decltype(v4l2_cdef.uv_pri_strength){}) ==
                    libgav1::kMaxCdefStrengths,
                "Invalid size of cdef uv_pri_strength strength");

  static_assert(std::size(decltype(v4l2_cdef.uv_sec_strength){}) ==
                    libgav1::kMaxCdefStrengths,
                "Invalid size of cdef uv_sec_strength strength");

  SafeArrayMemcpy(v4l2_cdef.y_pri_strength, cdef.y_primary_strength);
  SafeArrayMemcpy(v4l2_cdef.y_sec_strength, cdef.y_secondary_strength);
  SafeArrayMemcpy(v4l2_cdef.uv_pri_strength, cdef.uv_primary_strength);
  SafeArrayMemcpy(v4l2_cdef.uv_sec_strength, cdef.uv_secondary_strength);

  // All the strength values parsed in libgav1 are from the AV1 spec and left
  // shifted by (|color_bitdepth| - 8). So these values need to be right shifted
  // by (|color_bitdepth| - 8) before passing to a driver.
  for (size_t i = 0; i < libgav1::kMaxCdefStrengths; i++) {
    v4l2_cdef.y_pri_strength[i] >>= coeff_shift;
    v4l2_cdef.y_sec_strength[i] >>= coeff_shift;
    v4l2_cdef.uv_pri_strength[i] >>= coeff_shift;
    v4l2_cdef.uv_sec_strength[i] >>= coeff_shift;
  }

  return v4l2_cdef;
}

// 5.9.20. Loop restoration params syntax
struct v4l2_av1_loop_restoration FillLoopRestorationParams(
    const libgav1::LoopRestoration& lr) {
  struct v4l2_av1_loop_restoration v4l2_lr = {};

  for (size_t i = 0; i < V4L2_AV1_NUM_PLANES_MAX; i++) {
    switch (lr.type[i]) {
      case libgav1::LoopRestorationType::kLoopRestorationTypeNone:
        v4l2_lr.frame_restoration_type[i] = V4L2_AV1_FRAME_RESTORE_NONE;
        break;
      case libgav1::LoopRestorationType::kLoopRestorationTypeWiener:
        v4l2_lr.frame_restoration_type[i] = V4L2_AV1_FRAME_RESTORE_WIENER;
        break;
      case libgav1::LoopRestorationType::kLoopRestorationTypeSgrProj:
        v4l2_lr.frame_restoration_type[i] = V4L2_AV1_FRAME_RESTORE_SGRPROJ;
        break;
      case libgav1::LoopRestorationType::kLoopRestorationTypeSwitchable:
        v4l2_lr.frame_restoration_type[i] = V4L2_AV1_FRAME_RESTORE_SWITCHABLE;
        break;
      default:
        NOTREACHED() << "Invalid loop restoration type";
    }

    if (v4l2_lr.frame_restoration_type[i] != V4L2_AV1_FRAME_RESTORE_NONE) {
      if (true)
        v4l2_lr.flags |= V4L2_AV1_LOOP_RESTORATION_FLAG_USES_LR;

      if (i > 0)
        v4l2_lr.flags |= V4L2_AV1_LOOP_RESTORATION_FLAG_USES_CHROMA_LR;
    }
  }

  const bool use_loop_restoration =
      std::find_if(std::begin(lr.type),
                   std::begin(lr.type) + libgav1::kMaxPlanes,
                   [](const auto type) {
                     return type != libgav1::kLoopRestorationTypeNone;
                   }) != (lr.type + libgav1::kMaxPlanes);

  if (use_loop_restoration) {
    DCHECK_GE(lr.unit_size_log2[0], lr.unit_size_log2[1]);
    DCHECK_LE(lr.unit_size_log2[0] - lr.unit_size_log2[1], 1);
    v4l2_lr.lr_unit_shift = lr.unit_size_log2[0] - 6;
    v4l2_lr.lr_uv_shift = lr.unit_size_log2[0] - lr.unit_size_log2[1];

    // AV1 spec (p.52) uses this formula with hard coded value 2.
    // https://aomediacodec.github.io/av1-spec/#loop-restoration-params-syntax
    v4l2_lr.loop_restoration_size[0] =
        V4L2_AV1_RESTORATION_TILESIZE_MAX >> (2 - v4l2_lr.lr_unit_shift);
    v4l2_lr.loop_restoration_size[1] =
        v4l2_lr.loop_restoration_size[0] >> v4l2_lr.lr_uv_shift;
    v4l2_lr.loop_restoration_size[2] =
        v4l2_lr.loop_restoration_size[0] >> v4l2_lr.lr_uv_shift;
  }

  return v4l2_lr;
}

// Section 5.9.24. Global motion params syntax
struct v4l2_av1_global_motion FillGlobalMotionParams(
    const std::array<libgav1::GlobalMotion, libgav1::kNumReferenceFrameTypes>&
        gm_array) {
  struct v4l2_av1_global_motion v4l2_gm = {};

  // gm_array[0] (for kReferenceFrameIntra) is not used because global motion is
  // not relevant for intra frames
  for (size_t i = 1; i < libgav1::kNumReferenceFrameTypes; ++i) {
    auto gm = gm_array[i];
    switch (gm.type) {
      case libgav1::kGlobalMotionTransformationTypeIdentity:
        v4l2_gm.type[i] = V4L2_AV1_WARP_MODEL_IDENTITY;
        break;
      case libgav1::kGlobalMotionTransformationTypeTranslation:
        v4l2_gm.type[i] = V4L2_AV1_WARP_MODEL_TRANSLATION;
        v4l2_gm.flags[i] |= V4L2_AV1_GLOBAL_MOTION_FLAG_IS_TRANSLATION;
        break;
      case libgav1::kGlobalMotionTransformationTypeRotZoom:
        v4l2_gm.type[i] = V4L2_AV1_WARP_MODEL_ROTZOOM;
        v4l2_gm.flags[i] |= V4L2_AV1_GLOBAL_MOTION_FLAG_IS_ROT_ZOOM;
        break;
      case libgav1::kGlobalMotionTransformationTypeAffine:
        v4l2_gm.type[i] = V4L2_AV1_WARP_MODEL_AFFINE;
        v4l2_gm.flags[i] |= V4L2_AV1_WARP_MODEL_AFFINE;
        break;
      default:
        NOTREACHED() << "Invalid global motion transformation type, "
                     << v4l2_gm.type[i];
    }

    if (gm.type != libgav1::kGlobalMotionTransformationTypeIdentity)
      v4l2_gm.flags[i] |= V4L2_AV1_GLOBAL_MOTION_FLAG_IS_GLOBAL;

    constexpr auto kNumGlobalMotionParams = std::size(decltype(gm.params){});

    for (size_t j = 0; j < kNumGlobalMotionParams; ++j) {
      static_assert(
          std::is_same<decltype(v4l2_gm.params[0][0]), int32_t&>::value,
          "|v4l2_av1_global_motion::params|'s data type must be int32_t "
          "starting from AV1 uAPI v4");
      v4l2_gm.params[i][j] = gm.params[j];
    }

    if (!libgav1::SetupShear(&gm))
      v4l2_gm.invalid |= V4L2_AV1_GLOBAL_MOTION_IS_INVALID(i);
  }

  return v4l2_gm;
}

// 5.9.2. Uncompressed header syntax
struct v4l2_ctrl_av1_frame SetupFrameParams(
    const libgav1::ObuSequenceHeader& sequence_header,
    const libgav1::ObuFrameHeader& frame_header,
    const AV1ReferenceFrameVector& ref_frames) {
  struct v4l2_ctrl_av1_frame v4l2_frame_params = {};

  FillLoopFilterParams(v4l2_frame_params.loop_filter, frame_header.loop_filter);
  FillLoopFilterDeltaParams(v4l2_frame_params.loop_filter,
                            frame_header.delta_lf);

  FillQuantizationParams(v4l2_frame_params.quantization,
                         frame_header.quantizer);
  FillQuantizerIndexDeltaParams(v4l2_frame_params.quantization, sequence_header,
                                frame_header);

  v4l2_frame_params.segmentation =
      FillSegmentationParams(frame_header.segmentation);

  const auto color_bitdepth = sequence_header.color_config.bitdepth;
  v4l2_frame_params.cdef = FillCdefParams(
      frame_header.cdef, base::strict_cast<int8_t>(color_bitdepth));

  v4l2_frame_params.loop_restoration =
      FillLoopRestorationParams(frame_header.loop_restoration);

  v4l2_frame_params.tile_info = FillTileInfo(frame_header.tile_info);

  v4l2_frame_params.global_motion =
      FillGlobalMotionParams(frame_header.global_motion);

  if (frame_header.show_frame)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_SHOW_FRAME;
  if (frame_header.showable_frame)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_SHOWABLE_FRAME;
  if (frame_header.error_resilient_mode)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE;
  if (frame_header.enable_cdf_update == false)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_DISABLE_CDF_UPDATE;
  if (frame_header.allow_screen_content_tools)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_ALLOW_SCREEN_CONTENT_TOOLS;
  if (frame_header.force_integer_mv)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_FORCE_INTEGER_MV;
  if (frame_header.allow_intrabc)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC;
  if (frame_header.use_superres)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_USE_SUPERRES;
  if (frame_header.allow_high_precision_mv)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV;
  if (frame_header.is_motion_mode_switchable)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE;
  if (frame_header.use_ref_frame_mvs)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;
  if (frame_header.enable_frame_end_update_cdf == false)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF;
  if (frame_header.allow_warped_motion)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_ALLOW_WARPED_MOTION;
  if (frame_header.reference_mode_select)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT;
  if (frame_header.reduced_tx_set)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_REDUCED_TX_SET;
  if (frame_header.skip_mode_frame[0] > 0)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_SKIP_MODE_ALLOWED;
  if (frame_header.skip_mode_present)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT;
  if (frame_header.frame_size_override_flag)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_FRAME_SIZE_OVERRIDE;
  // libgav1 header doesn't have |buffer_removal_time_present_flag|.
  if (frame_header.buffer_removal_time[0] > 0)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_BUFFER_REMOVAL_TIME_PRESENT;
  if (frame_header.frame_refs_short_signaling)
    v4l2_frame_params.flags |= V4L2_AV1_FRAME_FLAG_FRAME_REFS_SHORT_SIGNALING;

  switch (frame_header.frame_type) {
    case libgav1::kFrameKey:
      v4l2_frame_params.frame_type = V4L2_AV1_KEY_FRAME;
      break;
    case libgav1::kFrameInter:
      v4l2_frame_params.frame_type = V4L2_AV1_INTER_FRAME;
      break;
    case libgav1::kFrameIntraOnly:
      v4l2_frame_params.frame_type = V4L2_AV1_INTRA_ONLY_FRAME;
      break;
    case libgav1::kFrameSwitch:
      v4l2_frame_params.frame_type = V4L2_AV1_SWITCH_FRAME;
      break;
    default:
      NOTREACHED() << "Invalid frame type, " << frame_header.frame_type;
  }

  v4l2_frame_params.order_hint = frame_header.order_hint;
  v4l2_frame_params.superres_denom = frame_header.superres_scale_denominator;
  v4l2_frame_params.upscaled_width = frame_header.upscaled_width;

  switch (frame_header.interpolation_filter) {
    case libgav1::kInterpolationFilterEightTap:
      v4l2_frame_params.interpolation_filter =
          V4L2_AV1_INTERPOLATION_FILTER_EIGHTTAP;
      break;
    case libgav1::kInterpolationFilterEightTapSmooth:
      v4l2_frame_params.interpolation_filter =
          V4L2_AV1_INTERPOLATION_FILTER_EIGHTTAP_SMOOTH;
      break;
    case libgav1::kInterpolationFilterEightTapSharp:
      v4l2_frame_params.interpolation_filter =
          V4L2_AV1_INTERPOLATION_FILTER_EIGHTTAP_SHARP;
      break;
    case libgav1::kInterpolationFilterBilinear:
      v4l2_frame_params.interpolation_filter =
          V4L2_AV1_INTERPOLATION_FILTER_BILINEAR;
      break;
    case libgav1::kInterpolationFilterSwitchable:
      v4l2_frame_params.interpolation_filter =
          V4L2_AV1_INTERPOLATION_FILTER_SWITCHABLE;
      break;
    default:
      NOTREACHED() << "Invalid interpolation filter, "
                   << frame_header.interpolation_filter;
  }

  switch (frame_header.tx_mode) {
    case libgav1::kTxModeOnly4x4:
      v4l2_frame_params.tx_mode = V4L2_AV1_TX_MODE_ONLY_4X4;
      break;
    case libgav1::kTxModeLargest:
      v4l2_frame_params.tx_mode = V4L2_AV1_TX_MODE_LARGEST;
      break;
    case libgav1::kTxModeSelect:
      v4l2_frame_params.tx_mode = V4L2_AV1_TX_MODE_SELECT;
      break;
    default:
      NOTREACHED() << "Invalid tx mode, " << frame_header.tx_mode;
  }

  v4l2_frame_params.frame_width_minus_1 = frame_header.width - 1;
  v4l2_frame_params.frame_height_minus_1 = frame_header.height - 1;
  v4l2_frame_params.render_width_minus_1 = frame_header.render_width - 1;
  v4l2_frame_params.render_height_minus_1 = frame_header.render_height - 1;

  v4l2_frame_params.current_frame_id = frame_header.current_frame_id;
  v4l2_frame_params.primary_ref_frame = frame_header.primary_reference_frame;
  SafeArrayMemcpy(v4l2_frame_params.buffer_removal_time,
                  frame_header.buffer_removal_time);
  v4l2_frame_params.refresh_frame_flags = frame_header.refresh_frame_flags;

  // |reference_frame_index| indicates which reference frame slot is used for
  // different reference frame types: L(1), L2(2), L3(3), G(4), BWD(5), A2(6),
  // A(7). As |ref_frames[i]| is a |AV1Picture| with frame header info, we can
  // extract |order_hint| directly for each reference frame type instead of
  // maintaining |RefOrderHint| array in the AV1 spec.
  static_assert(std::size(decltype(v4l2_frame_params.order_hints){}) ==
                    libgav1::kNumInterReferenceFrameTypes + 1,
                "Invalid size of |order_hints| array");
  if (!libgav1::IsIntraFrame(frame_header.frame_type)) {
    for (size_t i = 0; i < libgav1::kNumInterReferenceFrameTypes; ++i) {
      const int8_t reference_frame_index =
          frame_header.reference_frame_index[i];

      // The DCHECK()s are guaranteed by
      // AV1Decoder::CheckAndCleanUpReferenceFrames().
      DCHECK_GE(reference_frame_index, 0);
      DCHECK_LT(reference_frame_index, libgav1::kNumReferenceFrameTypes);
      DCHECK(ref_frames[reference_frame_index]);

      const uint8_t order_hint =
          ref_frames[reference_frame_index]->frame_header.order_hint;
      v4l2_frame_params.order_hints[i + 1] =
          base::strict_cast<__u32>(order_hint);
    }
  }

  // TODO(b/230891887): use uint64_t when v4l2_timeval_to_ns() function is used.
  constexpr uint32_t kInvalidSurface = std::numeric_limits<uint32_t>::max();

  for (size_t i = 0; i < libgav1::kNumReferenceFrameTypes; ++i) {
    if (!ref_frames[i]) {
      v4l2_frame_params.reference_frame_ts[i] = kInvalidSurface;
      continue;
    }

    const auto* v4l2_ref_pic =
        static_cast<const V4L2AV1Picture*>(ref_frames[i].get());

    v4l2_frame_params.reference_frame_ts[i] =
        v4l2_ref_pic->dec_surface()->GetReferenceID();
  }

  static_assert(std::size(decltype(v4l2_frame_params.ref_frame_idx){}) ==
                    libgav1::kNumInterReferenceFrameTypes,
                "Invalid size of |ref_frame_idx| array");
  for (size_t i = 0; i < libgav1::kNumInterReferenceFrameTypes; i++) {
    LOG_IF(ERROR, (frame_header.frame_type == libgav1::kFrameKey) &&
                      (frame_header.reference_frame_index[i] != 0))
        << "|reference_frame_index| from the frame header is not 0 for the "
           "intra frame";

    static_assert(std::is_same<decltype(v4l2_frame_params.ref_frame_idx[0]),
                               int8_t&>::value,
                  "|v4l2_ctrl_av1_frame::ref_frame_idx|'s data type must be "
                  "int8_t starting from AV1 uAPI v4");
    v4l2_frame_params.ref_frame_idx[i] = frame_header.reference_frame_index[i];
  }

  v4l2_frame_params.skip_mode_frame[0] =
      base::checked_cast<__u8>(frame_header.skip_mode_frame[0]);
  v4l2_frame_params.skip_mode_frame[1] =
      base::checked_cast<__u8>(frame_header.skip_mode_frame[1]);

  return v4l2_frame_params;
}

// Section 5.11. Tile Group OBU syntax
std::vector<struct v4l2_ctrl_av1_tile_group_entry> FillTileGroupParams(
    const base::span<const uint8_t> frame_obu_data,
    const size_t tile_columns,
    const libgav1::Vector<libgav1::TileBuffer>& tile_buffers) {
  // This could happen in rare cases (for example, if there is a Metadata OBU
  // after the TileGroup OBU). We currently do not have a reason to handle those
  // cases. This is also the case in libgav1 at the moment.
  CHECK(!tile_buffers.empty());

  CHECK_GT(tile_columns, 0u);
  const uint32_t num_tiles = tile_buffers.size();

  std::vector<struct v4l2_ctrl_av1_tile_group_entry> tile_group_entry_vector(
      num_tiles);

  for (uint32_t tile_index = 0; tile_index < num_tiles; ++tile_index) {
    auto& tile_group_entry_params = tile_group_entry_vector[tile_index];

    CHECK(tile_buffers[tile_index].data >= frame_obu_data.data());
    tile_group_entry_params.tile_offset = base::checked_cast<uint32_t>(
        tile_buffers[tile_index].data - frame_obu_data.data());

    tile_group_entry_params.tile_size =
        base::checked_cast<uint32_t>(tile_buffers[tile_index].size);

    // The tiles are row-major. We use the number of columns |tile_columns|
    // to compute computation of the row and column for a given tile.
    tile_group_entry_params.tile_row =
        tile_index / base::checked_cast<uint32_t>(tile_columns);
    tile_group_entry_params.tile_col =
        tile_index % base::checked_cast<uint32_t>(tile_columns);

    base::CheckedNumeric<uint32_t> safe_tile_data_end(
        tile_group_entry_params.tile_offset);
    safe_tile_data_end += tile_group_entry_params.tile_size;
    size_t tile_data_end;
    if (!safe_tile_data_end.AssignIfValid(&tile_data_end) ||
        tile_data_end > frame_obu_data.size()) {
      DLOG(ERROR) << "Invalid tile offset and size"
                  << ", offset=" << tile_group_entry_params.tile_offset
                  << ", size=" << tile_group_entry_params.tile_size
                  << ", entire data size=" << frame_obu_data.size();

      return {};
    }
  }

  return tile_group_entry_vector;
}

}  // namespace

V4L2VideoDecoderDelegateAV1::V4L2VideoDecoderDelegateAV1(
    V4L2DecodeSurfaceHandler* surface_handler,
    V4L2Device* device)
    : surface_handler_(surface_handler), device_(device) {
  VLOGF(1);
  DCHECK(surface_handler_);
  DCHECK(device_);
}

V4L2VideoDecoderDelegateAV1::~V4L2VideoDecoderDelegateAV1() = default;

scoped_refptr<AV1Picture> V4L2VideoDecoderDelegateAV1::CreateAV1Picture(
    bool apply_grain) {
  scoped_refptr<V4L2DecodeSurface> dec_surface =
      surface_handler_->CreateSurface();
  if (!dec_surface)
    return nullptr;

  return new V4L2AV1Picture(std::move(dec_surface));
}

scoped_refptr<AV1Picture> V4L2VideoDecoderDelegateAV1::CreateAV1PictureSecure(
    bool apply_grain,
    uint64_t secure_handle) {
  scoped_refptr<V4L2DecodeSurface> dec_surface =
      surface_handler_->CreateSecureSurface(secure_handle);
  if (!dec_surface) {
    return nullptr;
  }

  return new V4L2AV1Picture(std::move(dec_surface));
}

DecodeStatus V4L2VideoDecoderDelegateAV1::SubmitDecode(
    const AV1Picture& pic,
    const libgav1::ObuSequenceHeader& sequence_header,
    const AV1ReferenceFrameVector& ref_frames,
    const libgav1::Vector<libgav1::TileBuffer>& tile_buffers,
    base::span<const uint8_t> stream) {
  struct v4l2_ctrl_av1_sequence v4l2_seq_params =
      FillSequenceParams(sequence_header);

  struct v4l2_ctrl_av1_frame v4l2_frame_params =
      SetupFrameParams(sequence_header, pic.frame_header, ref_frames);

  std::vector<struct v4l2_ctrl_av1_tile_group_entry> tile_group_entry_vectors =
      FillTileGroupParams(stream, pic.frame_header.tile_info.tile_columns,
                          tile_buffers);

  if (tile_group_entry_vectors.empty()) {
    VLOGF(1) << "Tile group entry setup failed";
    return DecodeStatus::kFail;
  }

  struct v4l2_ext_control ext_ctrl_array[] = {
      {.id = V4L2_CID_STATELESS_AV1_SEQUENCE,
       .size = sizeof(v4l2_seq_params),
       .ptr = &v4l2_seq_params},
      {.id = V4L2_CID_STATELESS_AV1_FRAME,
       .size = sizeof(v4l2_frame_params),
       .ptr = &v4l2_frame_params},
      {.id = V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,
       .size =
           base::checked_cast<__u32>(tile_group_entry_vectors.size() *
                                     sizeof(v4l2_ctrl_av1_tile_group_entry)),
       .ptr = tile_group_entry_vectors.data()}};

  struct v4l2_ext_controls ext_ctrls = {
      .count = base::checked_cast<__u32>(std::size(ext_ctrl_array)),
      .controls = ext_ctrl_array};

  const auto* v4l2_pic = static_cast<const V4L2AV1Picture*>(&pic);
  auto dec_surface = v4l2_pic->dec_surface();
  dec_surface->PrepareSetCtrls(&ext_ctrls);
  if (device_->Ioctl(VIDIOC_S_EXT_CTRLS, &ext_ctrls) != 0) {
    RecordVidiocIoctlErrorUMA(VidiocIoctlRequests::kVidiocSExtCtrls);
    VPLOGF(1) << "ioctl() failed: VIDIOC_S_EXT_CTRLS";
    return DecodeStatus::kFail;
  }

  std::vector<scoped_refptr<V4L2DecodeSurface>> ref_surfaces;
  for (size_t i = 0; i < libgav1::kNumReferenceFrameTypes; i++) {
    if (ref_frames[i]) {
      const auto* v4l2_ref_pic =
          static_cast<const V4L2AV1Picture*>(ref_frames[i].get());

      ref_surfaces.emplace_back(std::move(v4l2_ref_pic->dec_surface()));
    }
  }
  dec_surface->SetReferenceSurfaces(std::move(ref_surfaces));

  // Copies the frame data into the V4L2 buffer.
  if (!surface_handler_->SubmitSlice(
          dec_surface.get(),
          dec_surface->secure_handle() ? nullptr : stream.data(),
          stream.size())) {
    return DecodeStatus::kFail;
  }

  // Queues the buffers to the kernel driver.
  DVLOGF(4) << "Submitting decode for surface: "
            << v4l2_pic->dec_surface()->ToString();
  surface_handler_->DecodeSurface(v4l2_pic->dec_surface());

  return DecodeStatus::kOk;
}

bool V4L2VideoDecoderDelegateAV1::OutputPicture(const AV1Picture& pic) {
  VLOGF(3);
  const auto* v4l2_pic = static_cast<const V4L2AV1Picture*>(&pic);

  surface_handler_->SurfaceReady(
      v4l2_pic->dec_surface(), v4l2_pic->bitstream_id(),
      v4l2_pic->visible_rect(), v4l2_pic->get_colorspace());

  return true;
}

}  // namespace media