media / mojo / mojom / speech_recognition.mojom [blame]

// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

module media.mojom;

import "media/mojo/mojom/audio_data.mojom";
import "media/mojo/mojom/speech_recognition_audio_forwarder.mojom";
import "media/mojo/mojom/speech_recognition_recognition_context.mojom";
import "media/mojo/mojom/speech_recognizer.mojom";
import "mojo/public/mojom/base/time.mojom";
import "mojo/public/mojom/base/unguessable_token.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";

// Next MinVersion: 9

// Corresponds to the LangIdEvent.ConfidenceInterval defined in
// http://google3/speech/soda/public/soda_event.proto.
[Stable, Extensible]
enum ConfidenceLevel {
  [Default] kUnknown,
  kNotConfident,
  kConfident,
  kHighlyConfident,
};

// Corresponds to the LangIdEvent.AsrSwitchResult defined in
// http://google3/speech/soda/public/soda_event.proto.
[Stable, Extensible]
enum AsrSwitchResult {
  [Default] kDefaultNoSwitch,
  kSwitchSucceeded,
  kSwitchFailed,
  kSwitchSkipedNoLp,
};

// The main interface a client uses to interact with a speech
// recognition service process. For Live Caption, every renderer can own one
// or more Remote<SpeechRecognitionContext>, with the receiver bound through the
// BrowserInterfaceBroker. For the Web Speech API, the browser is the client and
// can own one or more Remote<SpeechRecognitionContext>. This is a stable
// interface that is used across the LaCrOS/Ash boundary.
[Stable]
interface SpeechRecognitionContext {
  // Bind the recognizers to the speech recognition service. Returns a flag
  // indicating whether multichannel audio is supported by the speech
  // recognition service.
  BindRecognizer@0(pending_receiver<SpeechRecognitionRecognizer> receiver,
                   pending_remote<SpeechRecognitionRecognizerClient> client,
                   SpeechRecognitionOptions options)
      => (bool is_multichannel_supported);

  // Bind the recognizer used by the Web Speech API.
  [MinVersion=6]
  BindWebSpeechRecognizer@1(
      pending_receiver<media.mojom.SpeechRecognitionSession> session_receiver,
      pending_remote<media.mojom.SpeechRecognitionSessionClient> session_client,
      pending_receiver<SpeechRecognitionAudioForwarder> audio_forwarder,
      int32 channel_count,
      int32 sample_rate,
      SpeechRecognitionOptions options,
      bool continuous);
};

// The interface used to pass raw audio from the renderer to the speech
// recognition service. The remote lives either in the renderer process (for web
// Live Caption) or the browser process (for CrOS features like system Live
// Caption and dictation) and the receiver lives in the speech recognition
// process.
[Stable]
interface SpeechRecognitionRecognizer {
  // Initialize the speech recognition instance. The speech recognition client
  // will return the recognition events containing the transcribed audio back
  // to the originating media.
  SendAudioToSpeechRecognitionService@0(AudioDataS16 buffer);

  // Mark audio stream done. This informs the speech recognition client to stop
  // speech recognition after it finishes processing the audio it has received
  // already. This will eventually trigger the
  // SpeechRecognitionRecognizerClient::OnSpeechRecognitionStopped callback.
  MarkDone@1();

  // Notify the speech recognition recognizer that the language changed. Takes
  // in the locale string (e.g. "en-US").
  OnLanguageChanged@2(string language);

  // Notify the speech recognition recognizer that the mask offensive words
  // setting has changed.
  [MinVersion=2]
  OnMaskOffensiveWordsChanged@3(bool mask_offensive_words);
};

// The interface used to return speech recognition events from the speech
// recognition service to the client that will display the results to the user.
// The remote lives in the speech recognition process and the receiver lives in
// the browser process.
[Stable]
interface SpeechRecognitionRecognizerClient {
  // Triggered by speech recognition process on a speech recognition event.
  //
  // Returns false if the client wants to halt speech recognition e.g. in
  // response to user input or in the case of an error.
  OnSpeechRecognitionRecognitionEvent@0(SpeechRecognitionResult result)
      => (bool continue_recognition);

  // Called when speech recognition stops.
  OnSpeechRecognitionStopped@1();

  // Triggered by an error within the speech recognition service.
  OnSpeechRecognitionError@2();

  // Triggered by speech recognition process on a language identification event.
  OnLanguageIdentificationEvent@3(LanguageIdentificationEvent event);
};

// The hypothesis parts that provides timing information for each word in
// recognized speech.
[Stable]
struct HypothesisParts {
  // A section of the final transcription text. Either an entire word or single
  // character (depending on the language) with adjacent punctuation. There will
  // usually only be one value here. If formatting is enabled in the speech
  // recognition, then the raw text will be included as the second element.
  array<string> text;

  // Time offset from this event's |audio_start_time| defined below. We enforce
  // the following invariant: 0 <= hypothesis_part_offset < |audio_end_time -
  // audio_start_time|.
  mojo_base.mojom.TimeDelta hypothesis_part_offset;
};

// The timing information for the transcript.
[Stable]
struct TimingInformation {
  // Start time in audio time from the start of the SODA session.
  // This time measures the amount of audio input into SODA.
  mojo_base.mojom.TimeDelta audio_start_time;

  // Elapsed processed audio from first frame after preamble.
  mojo_base.mojom.TimeDelta audio_end_time;

  // The timing information for each word/letter in the transription.
  // HypothesisPartsInResult was introduced in min version 1 in
  // chromeos/services/machine_learning/public/mojom/soda.mojom. Therefore, it
  // must be optional. Hypothesis parts maybe non-empty optional containing a
  // zero length vector if no words were spoken during the event's time span.
  array<HypothesisParts>? hypothesis_parts;
};

// A speech recognition result created by the speech service and passed to the
// browser.
[Stable]
struct SpeechRecognitionResult {
  string transcription;

  // A flag indicating whether the result is final. If true, the result is
  // locked in and the next result returned will not overlap with the previous
  // final result.
  bool is_final;

  // Timing information for the current transcription. |timing_information| is
  // expected to be valid if:
  //   1. speech recognition is provided by |CrosSodaClient| and
  //   2. |is_final| is true.
  TimingInformation? timing_information;
};

// A language identification event created by the speech recognition service
// and passed to the browser and renderer.
[Stable]
struct LanguageIdentificationEvent {
  // The locale of the language with the highest confidence.
  string language;

  // The confidence interval.
  ConfidenceLevel confidence_level;

  // If multilang is enabled, describes the actions Automatic Speech Recognition
  // took as a result of this event.
  [MinVersion=1]
  AsrSwitchResult? asr_switch_result;
};

// The interface used to notify the speech recognition client of events
// triggered by the browser. The remote lives in the browser process and the
// receiver lives either in the renderer process (for web Live Caption) or
// the browser process (for CrOS system Live Caption).
[Stable]
interface SpeechRecognitionBrowserObserver {
  // Notify the speech recognition client when speech recognition availability
  // changes.
  SpeechRecognitionAvailabilityChanged@0(
      bool is_speech_recognition_available);

  // Notify the speech recognition client when the speech recognition language
  // changes.
  SpeechRecognitionLanguageChanged@1(string language);

  // Notify the speech recognition client when the mask offensive words pref
  // changes.
  [MinVersion=2]
  SpeechRecognitionMaskOffensiveWordsChanged@2(bool mask_offensive_words);
};

// The user-facing source of recognized speech; typically a tab. The remote
// lives in the Ash browser process and is used to trigger behavior in lacros
// (like focusing the tab). The receiver lives in the lacros browser process.
[Stable]
interface SpeechRecognitionSurface {
  // "Activate" the surface - i.e. bring it to the front and focus it.
  Activate@0();

  // Fetch the bounds of the surface in screen coordinates. A nullopt is
  // returned if no bounds could be fetched.
  GetBounds@1() => (gfx.mojom.Rect? bounds);
};

// The OS-side observer of a lacros-side speech surface. Used to close or
// re-render a live caption bubble based on user interaction with the
// lacros-side surface. The remote lives in the lacros browser process, and the
// receiver lives in the Ash browser process.
[Stable]
interface SpeechRecognitionSurfaceClient {
  // Called when the user navigates away or refreshes the current tab. This
  // comprises the end of a live caption "session", after which the caption
  // bubble can be shown even if it was explicitly dismissed by the user.
  OnSessionEnded@0();

  // Called when the user fullscreens or un-fullscreens the speech surface.
  OnFullscreenToggled@1();
};

// This interface between the speech recognition client and the browser.
// The remote lives in the renderer process and the receiver lives in the
// browser process. Not necessary for browser-side features (e.g. CrOS system
// Live Caption), which can access browser functionality directly.
[Stable]
interface SpeechRecognitionClientBrowserInterface {
  // Bind the speech recognition availability observer.
  BindSpeechRecognitionBrowserObserver@0(
      pending_remote<SpeechRecognitionBrowserObserver> observer);

  [MinVersion=1]
  REMOVED_1@1();

  // Similar to BindSpeechRecognitionBrowserObserver, however binds Browser
  // observers that listen to events specific to BabelOrca rather than
  // the traditional LiveCaption preference.
  [MinVersion=7, EnableIf=is_chromeos_ash]
  BindBabelOrcaSpeechRecognitionBrowserObserver@2(
      pending_remote<SpeechRecognitionBrowserObserver> observer);
};

// Corresponds to ExtendedSodaConfigMsg.RecognitionMode in
// chrome/services/speech/soda/proto/soda_api.proto and
// SodaRecognitionMode in
// chromeos/services/machine_learning/public/mojom/soda.mojom.
[Stable, Extensible]
enum SpeechRecognitionMode {
  [Default] kUnknown,
  // Intended for voice input for keyboard usage.
  kIme,
  // Intended to caption a stream of audio.
  kCaption,
};

// Which Chrome/ChromeOS application that is triggering the
// speech recognition session to start.
[Stable, Extensible]
enum RecognizerClientType {
  [Default] kUnknown,
  // Dictation on ChromeOS.
  kDictation,
  // LiveCaption on Chrome/ChromeOS.
  kLiveCaption,
  // Projector on ChromeOS.
  kProjector,
  // CastModerator on ChromeOS.
  kCastModerator,
};

// Options for speech recognition.
[Stable]
struct SpeechRecognitionOptions {
  // What kind of recognition to use.
  // In the case of web fallback (not for launch, used for development only),
  // this option will be ignored.
  SpeechRecognitionMode recognition_mode;

  // Whether to enable formatting and punctuation in the recognition results.
  bool enable_formatting;

  // The BCP-47 localized language code to use (e.g. "en-US").
  // TODO(crbug.com/40162502): Language needs to be required when multiple
  // languages are supported by SODA, so that each SpeechRecognitionRecognizer
  // can use its own language.
  string? language;

  // Whether the recognition is happening on-device or remotely on a server.
  [MinVersion=1]
  bool is_server_based;

  // Which client is requesting the speech recognition session.
  [MinVersion=1]
  RecognizerClientType recognizer_client_type;

  // When true, if the incoming audio buffer is zero for an extended period
  // (e.g. 10 seconds), audio won't be fed to the captioning model until nonzero
  // audio is received.
  // When false, even empty audio is captioned indefinitely.
  // Set to false if accurate TimingInfo relative to the start of captioning is
  // needed.
  [MinVersion=4]
  bool skip_continuously_empty_audio = false;

  // The optional experiment recognizer routing key for current request.
  [MinVersion=5]
  string? experiment_recognizer_routing_key;

  // The channel count of the forwarded audio.
  [MinVersion=6]
  int32 channel_count;

  // The sample rate of the forwarded audio.
  [MinVersion=6]
  int32 sample_rate;

  // The optional recognition context for speech recognition biasing.
  [MinVersion=8]
  SpeechRecognitionRecognitionContext? recognition_context;
};