media / mojo / mojom / media_types.mojom [blame]

// Copyright 2014 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

module media.mojom;

import "gpu/ipc/common/exported_shared_image.mojom";
import "gpu/ipc/common/mailbox.mojom";
import "gpu/ipc/common/sync_token.mojom";
import "gpu/ipc/common/vulkan_ycbcr_info.mojom";
import "media/mojo/mojom/audio_data.mojom";
import "media/mojo/mojom/encryption_pattern.mojom";
import "mojo/public/mojom/base/byte_string.mojom";
import "mojo/public/mojom/base/shared_memory.mojom";
import "mojo/public/mojom/base/time.mojom";
import "mojo/public/mojom/base/values.mojom";
import "mojo/public/mojom/base/unguessable_token.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";
import "ui/gfx/mojom/buffer_types.mojom";
import "ui/gfx/mojom/color_space.mojom";
import "ui/gfx/mojom/hdr_metadata.mojom";

// See media/base/audio_codecs.h for descriptions.
[Native]
enum AudioCodec;
[Native]
enum AudioCodecProfile;

// See media/base/buffering_state.h for descriptions.
[Native]
enum BufferingState;
[Native]
enum BufferingStateChangeReason;

// See media/base/channel_layout.h for descriptions.
[Native]
enum ChannelLayout;

// See media/base/media_content_type.h for descriptions.
[Native]
enum MediaContentType;

// See media/base/media_log_record.h for description.
[Native]
struct MediaLogRecord;

// See media/base/output_device_info.h for descriptions.
[Native]
enum OutputDeviceStatus;

// See media/base/sample_format.h for descriptions.
[Native]
enum SampleFormat;

// See media/base/video_codecs.h for descriptions.
[Native]
enum VideoCodec;

// See media/base/video_codecs.h for descriptions.
[Native]
enum VideoCodecProfile;

// See media/base/video_types.h for descriptions.
[Native]
enum VideoPixelFormat;

// See media/base/decoder.h for descriptions.
[Native]
enum VideoDecoderType;

// See media/base/decoder.h for descriptions.
[Native]
enum AudioDecoderType;

// See media/base/video_transformation.h for descriptions.
enum VideoRotation {
  kVideoRotation0,
  kVideoRotation90,
  kVideoRotation180,
  kVideoRotation270,
};

// see third_party/blink/public/platform/web_fullscreen_video_status.h for
// descriptions.
[Native]
enum FullscreenVideoStatus;

// See media/base/video_transformation.h for descriptions.
struct VideoTransformation {
  VideoRotation rotation;
  bool mirrored;
};

// See media/base/waiting.h for descriptions.
[Native]
enum WaitingReason;

// See media/base/watch_time_keys.h for descriptions.
[Native]
enum WatchTimeKey;

// See media/base/container_names.h for descriptions.
[Native]
enum MediaContainerName;

// See media/base/media_status.h for description.
[Native]
enum MediaStatusState;

// See media/base/encryption_scheme.h for description.
[Native]
enum EncryptionScheme;

// See media::EncryptionType for descriptions.
enum EncryptionType {
  kNone,
  kClear,
  kEncrypted,
  kEncryptedWithClearLead,
};

// See media/base/svc_scalability_mode.h for description.
// This mojo enum only list hardware codec supported scalability mode.
enum SVCScalabilityMode {
  // kUnsupportedMode is used to handle the enum differ of C++ and Mojo
  // SVCScalabilityMode in ToMojom, should not be used in other place.
  kUnsupportedMode,
  kL1T1,
  kL1T2,
  kL1T3,
  kL2T1,
  kL2T2,
  kL2T3,
  kL3T1,
  kL3T2,
  kL3T3,
  kL2T1Key,
  kL2T2Key,
  kL2T3Key,
  kL3T1Key,
  kL3T2Key,
  kL3T3Key,
  kS2T1,
  kS2T2,
  kS2T3,
  kS3T1,
  kS3T2,
  kS3T3,
};

// This mojo enum only list hardware codec supported scalability mode.
// See media/base/svc_scalability_mode.h for description.
enum SVCInterLayerPredMode {
  kOff,
  kOn,
  kOnKeyPic
};

// This defines a mojo transport format for media::VideoAspectRatio.
// See media/base/video_aspect_ratio.h for description.
struct VideoAspectRatio {
  enum Type {
    kDisplay,
    kPixel,
  };

  Type type;
  double value;
};

// This defines a mojo transport format for media::VideoColorSpace.
// See media/base/video_color_space.h for description.
struct VideoColorSpace {
  [Native]
  enum PrimaryID;

  [Native]
  enum TransferID;

  [Native]
  enum MatrixID;

  [Native]
  enum RangeID;

  PrimaryID primaries;
  TransferID transfer;
  MatrixID matrix;
  RangeID range;
};

// This defines a mojo transport format for media::AudioDecoderConfig.
// See media/base/audio_decoder_config.h for descriptions.
// TODO(crbug.com/40191032): Support `channels_` set by
// SetChannelsForDiscrete().
struct AudioDecoderConfig {
  AudioCodec codec;
  SampleFormat sample_format;
  ChannelLayout channel_layout;
  int32 samples_per_second;
  array<uint8> extra_data;
  EncryptionScheme encryption_scheme;
  mojo_base.mojom.TimeDelta seek_preroll;
  int32 codec_delay;
  AudioCodecProfile profile;
  ChannelLayout target_output_channel_layout;
  SampleFormat target_output_sample_format;
  bool should_discard_decoder_delay;
  array<uint8> aac_extra_data;
};

// This defines a mojo transport format for media::VideoDecoderConfig.
// See media/base/video_decoder_config.h for descriptions.
struct VideoDecoderConfig {
  VideoCodec codec;
  VideoCodecProfile profile;
  uint32 level;
  bool has_alpha;
  VideoTransformation transformation;
  gfx.mojom.Size coded_size;
  gfx.mojom.Rect visible_rect;
  gfx.mojom.Size natural_size;
  VideoAspectRatio aspect_ratio;
  array<uint8> extra_data;
  EncryptionScheme encryption_scheme;
  VideoColorSpace color_space_info;
  gfx.mojom.HDRMetadata? hdr_metadata;
};

// Native struct media::SubsampleEntry;
[Native]
struct SubsampleEntry;

// Important events happened to the CDM. See media/cdm/cdm_document_service.h
// for descriptions.
[EnableIf=is_win]
enum CdmEvent {
  kSignificantPlayback,
  kPlaybackError,
  kCdmError,
  kHardwareContextReset,
};

// See media::CdmSessionClosedReason for descriptions.
enum CdmSessionClosedReason {
  kInternalError,
  kClose,
  kReleaseAcknowledged,
  kHardwareContextReset,
  kResourceEvicted,
};

// This defines a mojo transport format for media::DecryptConfig.
// See media/base/decrypt_config.h for descriptions.
struct DecryptConfig {
  EncryptionScheme encryption_scheme;
  mojo_base.mojom.ByteString key_id;
  mojo_base.mojom.ByteString iv;
  array<SubsampleEntry> subsamples;
  EncryptionPattern? encryption_pattern;
};

union DecoderBufferSideDataNextConfig {
  AudioDecoderConfig next_audio_config;
  VideoDecoderConfig next_video_config;
};

// This defines a mojo transport format for media::DecoderBufferSideData.
struct DecoderBufferSideData {
  // VP9 specific information.
  array<uint32> spatial_layers;
  array<uint8> alpha_data;

  // Secure buffer handle corresponding to the decrypted contents of the
  // associated DecoderBuffer. A non-zero value indicates this was set.
  //
  // Required by some hardware content protection mechanisms to ensure the
  // decrypted buffer never leaves secure memory. When set, this DecoderBuffer
  // generally carries the partially (headers in the clear) encrypted payload;
  uint64 secure_handle;

  // These fields indicate the amount of data to discard after decoding.
  mojo_base.mojom.TimeDelta front_discard;
  mojo_base.mojom.TimeDelta back_discard;
};

// This defines a mojo transport format for media::DecoderBuffer w/ data.
struct DataDecoderBuffer {
  mojo_base.mojom.TimeDelta timestamp;
  mojo_base.mojom.TimeDelta duration;

  // The number of bytes present in this buffer. The data is not serialized
  // along with this structure and must be read from a separate DataPipe.
  // Note that |data_size| could be zero even for a non-EOS buffer (e.g.
  // with non-empty |size_data|). See http://crbug.com/663438
  uint32 data_size;

  // Indicates whether or not this buffer is a random access point.
  bool is_key_frame;

  // DecryptConfig for a encrypted buffer. NULL if the buffer is not encrypted.
  DecryptConfig? decrypt_config;

  DecoderBufferSideData? side_data;
};

// This defines a mojo transport format for an EOS media::DecoderBuffer.
struct EosDecoderBuffer {
  // Indicates that the EOS is for a config change. Decoders may make more
  // optimal flushing decisions using this information.
  DecoderBufferSideDataNextConfig? next_config;
};

// Possible choices for DecoderBuffer types.
union DecoderBuffer {
  EosDecoderBuffer eos;
  DataDecoderBuffer data;
};

// This defines a mojo transport format for media::AudioBuffer.
struct AudioBuffer {
  // Format of the audio.
  SampleFormat sample_format;

  // How the channels are laid out.
  ChannelLayout channel_layout;

  // Number of channels.
  int32 channel_count;

  // Sample rate of the buffer.
  int32 sample_rate;

  // Number of frames in the buffer.
  int32 frame_count;

  // True if end of stream.
  bool end_of_stream;

  // Timestamp in microseconds of the first frame.
  mojo_base.mojom.TimeDelta timestamp;

  // Channel data. Will be empty for EOS buffers.
  array<uint8> data;
};

// See media/base/video_frame_metadata.h for a description of fields.
enum EffectState {
  kUnknown = 0,
  kDisabled = 1,
  kEnabled = 2
};

// See media/base/video_frame_metadata.h for a description of fields.
// TODO(crbug.com/40489779): Remove |has_*| values and use nullable types.
struct VideoFrameMetadata {
  bool allow_overlay;

  mojo_base.mojom.TimeTicks? capture_begin_time;
  mojo_base.mojom.TimeTicks? capture_end_time;

  bool has_capture_counter;
  int32 capture_counter;

  gfx.mojom.Rect? capture_update_rect;

  gfx.mojom.Size? source_size;

  gfx.mojom.Rect? region_capture_rect;
  uint32 sub_capture_target_version;

  bool copy_required;

  bool end_of_stream;

  mojo_base.mojom.TimeDelta? frame_duration;

  bool has_frame_rate;
  double frame_rate;

  bool interactive_content;

  mojo_base.mojom.TimeTicks? reference_time;

  bool read_lock_fences_enabled;

  VideoTransformation? transformation;

  bool texture_owner;

  bool wants_promotion_hint;

  bool protected_video;

  bool hw_protected;

  bool needs_detiling;

  bool is_webgpu_compatible;

  mojo_base.mojom.UnguessableToken? tracking_token;

  bool power_efficient;

  bool has_device_scale_factor;
  double device_scale_factor;

  bool has_page_scale_factor;
  double page_scale_factor;

  bool has_root_scroll_offset_x;
  double root_scroll_offset_x;

  bool has_root_scroll_offset_y;
  double root_scroll_offset_y;

  bool has_top_controls_visible_height;
  double top_controls_visible_height;

  mojo_base.mojom.TimeTicks? decode_begin_time;
  mojo_base.mojom.TimeTicks? decode_end_time;

  mojo_base.mojom.TimeDelta? processing_time;

  bool has_rtp_timestamp;
  double rtp_timestamp;

  mojo_base.mojom.TimeTicks? receive_time;

  mojo_base.mojom.TimeDelta? wallclock_frame_duration;

  uint64? frame_sequence;

  EffectState background_blur;
};

// This defines a mojo transport format for media::VideoFrame.
struct VideoFrame {
  // Format of the frame.
  VideoPixelFormat format;

  // Width and height of the video frame, in pixels.
  gfx.mojom.Size coded_size;

  // Visible size of the frame.
  gfx.mojom.Rect visible_rect;

  // Natural size of the frame.
  gfx.mojom.Size natural_size;

  // Timestamp in microseconds of the associated frame.
  mojo_base.mojom.TimeDelta timestamp;

  // Contents of the video frame (or EOS marker).
  VideoFrameData data;

  // Extra properties associated with the VideoFrame.
  VideoFrameMetadata metadata;

  gfx.mojom.ColorSpace color_space;
  gfx.mojom.HDRMetadata? hdr_metadata;
};

// Possible choices for storing VideoFrame data.
union VideoFrameData {
  EosVideoFrameData eos_data;
  SharedMemoryVideoFrameData shared_memory_data;
  GpuMemoryBufferSharedImageVideoFrameData gpu_memory_buffer_shared_image_data;
  SharedImageVideoFrameData shared_image_data;
  OpaqueVideoFrameData opaque_data;
};

// A marker for EOS frames.
struct EosVideoFrameData {
};

// This defines video frame data for STORAGE_SHMEM VideoFrame.
struct SharedMemoryVideoFrameData {
  // Shared memory region for the frame data.
  mojo_base.mojom.ReadOnlySharedMemoryRegion frame_data;

  // Stride and offsets for each plane. Offsets are relative to the start
  // of |frame_data|.
  array<int32> strides;
  array<uint32> offsets;
};

struct GpuMemoryBufferSharedImageVideoFrameData {
  gfx.mojom.GpuMemoryBufferHandle gpu_memory_buffer_handle;
  gpu.mojom.ExportedSharedImage? shared_image;
  gpu.mojom.SyncToken sync_token;
};

// This defines video frame data stored in texture shared images.
struct SharedImageVideoFrameData {
  gpu.mojom.ExportedSharedImage shared_image;
  gpu.mojom.SyncToken sync_token;
  // |ycbcr_data| is present on Android (and sometimes Fuchsia) when the
  // video is hardware decoded and the display compositor uses Vulkan.
  gpu.mojom.VulkanYCbCrInfo? ycbcr_data;
};

// An empty structure used for STORAGE_OPAQUE VideoFrame.
struct OpaqueVideoFrameData {
};

struct PipelineStatistics {
  uint64 audio_bytes_decoded;
  uint64 video_bytes_decoded;
  uint32 video_frames_decoded;
  uint32 video_frames_dropped;
  int64 audio_memory_usage;
  int64 video_memory_usage;
};

// Set of features for MediaCapabilities prediction.
// TODO(liberato): consider generalizing this.
struct PredictionFeatures {
    // It would be nice to initialize this to VIDEO_CODEC_PROFILE_UNKNOWN (-1),
    // but we can't do that with native enums.
    VideoCodecProfile profile;
    gfx.mojom.Size video_size;
    // Frames per second may ultimately be a bucketed as an integer, but we want
    // to do that as late as possible. Using a double avoids early truncation.
    double frames_per_sec = 0;
    // Name of the key system used for EME playbacks.
    string key_system;
    // Indicates when CDM will use HW secure decoding for EME playbacks.
    bool use_hw_secure_codecs = false;
};

// Target values for MediaCapabilities predictions.
// TODO(liberato): consider generalizing this.
struct PredictionTargets {
  uint32 frames_decoded = 0;
  uint32 frames_dropped = 0;
  uint32 frames_power_efficient = 0;
};

// See media/base/pipeline_status.h for descriptions.
struct AudioPipelineInfo {
  AudioDecoderType decoder_type;
  bool is_platform_decoder = false;
  bool has_decrypting_demuxer_stream = false;
  EncryptionType encryption_type;
};

// See media/base/pipeline_status.h for descriptions.
struct VideoPipelineInfo {
  VideoDecoderType decoder_type;
  bool is_platform_decoder = false;
  bool has_decrypting_demuxer_stream = false;
  EncryptionType encryption_type;
};

// See media/base/status.h for descriptions.
struct StatusData {
  string group;
  uint16 code;
  string message;
  mojo_base.mojom.ListValue frames;
  StatusData? cause;
  mojo_base.mojom.Value data;
  uint64 packed_root_cause;
};

struct EncoderStatus {
  StatusData? internal;
};

struct DecoderStatus {
  StatusData? internal;
};

struct PipelineStatus {
  StatusData? internal;
};

// Types of media stream, categorised by the media stream's source.
// The enum values are emitted to metrics. Do not reorder.
enum MediaStreamType {
  kLocalElementCapture = 0, // The source is a local capture from element.
  kLocalDeviceCapture = 1, // The source is a local device capture, e.g. webcam.
  kLocalTabCapture = 2, // The source is a local tab capture.
  kLocalDesktopCapture = 3, // The source is a local desktop capture.
  kLocalDisplayCapture = 4, // The source is a local display capture.
  kRemote = 5, // The source is a remote peer connection.
  kNone = 6, // Not a media stream.
};

// Error codes propagated by Input media streams OnError methods to indicate the
// reason for an error.
enum InputStreamErrorCode {
  kUnknown = 0,
  kSystemPermissions = 1,
  kDeviceInUse = 2,
};

// See media/base/renderer.h for description.
enum RendererType {
  kRendererImpl = 0,     // RendererImplFactory
  kMojo = 1,             // MojoRendererFactory
  kMediaPlayer = 2,      // MediaPlayerRendererClientFactory
  kCourier = 3,          // CourierRendererFactory
  kFlinging = 4,         // FlingingRendererClientFactory
  kCast = 5,             // CastRendererClientFactory
  kMediaFoundation = 6,  // MediaFoundationRendererClientFactory
  // kFuchsia = 7,       // Deprecated
  kRemoting = 8,         // RemotingRendererFactory for remoting::Receiver
  kCastStreaming = 9,    // CastStreamingRendererFactory
  kContentEmbedderDefined = 10,  // Defined by the content embedder
  kTest= 11,             // Renderer implementations used in tests
};

// See media/base/demuxer.h for description.
enum DemuxerType {
  kUnknownDemuxer = 0,
  kMockDemuxer = 1,
  kFFmpegDemuxer = 2,
  kChunkDemuxer = 3,
  kMediaUrlDemuxer = 4,
  kFrameInjectingDemuxer = 5,
  kStreamProviderDemuxer = 6,
  kManifestDemuxer = 7,
};

// See media::CreateCdmStatus
enum CreateCdmStatus {
  kSuccess = 0,
  kUnknownError = 1,
  kCdmCreationAborted = 2,
  //kLoadCdmFailed = 3,     // Deprecated
  kCreateCdmFuncNotAvailable = 4,
  kCdmHelperCreationFailed = 5,
  kGetCdmPrefDataFailed = 6,
  kGetCdmOriginIdFailed = 7,
  kInitCdmFailed = 8,
  kCdmFactoryCreationFailed = 9,
  kCdmNotSupported = 10,
  kInvalidCdmConfig = 11,
  kUnsupportedKeySystem = 12,
  kDisconnectionError = 13,
  kNotAllowedOnUniqueOrigin = 14,
  kMediaDrmBridgeCreationFailed = 15,
  kMediaCryptoNotAvailable = 16,
  kNoMoreInstances = 17,
  kInsufficientGpuResources = 18,
  kCrOsVerifiedAccessDisabled = 19,
  kCrOsRemoteFactoryCreationFailed = 20,
  kAndroidMediaDrmIllegalArgument = 21,
  kAndroidMediaDrmIllegalState = 22,
  kAndroidFailedL1SecurityLevel = 23,
  kAndroidFailedL3SecurityLevel = 24,
  kAndroidFailedSecurityOrigin = 25,
  kAndroidFailedMediaCryptoSession = 26,
  kAndroidFailedToStartProvisioning = 27,
  kAndroidFailedMediaCryptoCreate = 28,
  kAndroidUnsupportedMediaCryptoScheme = 29,
};