media / audio / cras / cras_input.cc [blame]

// Copyright 2012 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/audio/cras/cras_input.h"

#include <inttypes.h>
#include <math.h>

#include <algorithm>
#include <ctime>

#include "base/files/file_util.h"
#include "base/logging.h"
#include "base/metrics/histogram_functions.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/stringprintf.h"
#include "base/time/time.h"
#include "media/audio/audio_device_description.h"
#include "media/audio/cras/audio_manager_cras_base.h"
#include "media/base/audio_timestamp_helper.h"
#include "media/base/media_switches.h"

namespace media {

namespace {

// Used to log errors in `CrasInputStream::Open`.
enum class StreamOpenResult {
  kCallbackOpenSuccess = 0,
  kCallbackOpenClientAlreadyOpen = 1,
  kCallbackOpenUnsupportedAudioFrequency = 2,
  kCallbackOpenUnsupportedAudioFormat = 3,
  kCallbackOpenCrasClientCreationFailed = 4,
  kCallbackOpenCannotConnectToCrasClient = 5,
  kCallbackOpenCannotRunCrasClient = 6,
  kCallbackOpenCannotSynchronizeData = 7,
  kCallbackOpenCannotFindLoopbackDevice = 8,
  kMaxValue = kCallbackOpenCannotFindLoopbackDevice
};

// Used to log errors in `CrasInputStream::Start`.
enum class StreamStartResult {
  kCallbackStartSuccess = 0,
  kCallbackStartErrorCreatingStreamParameters = 1,
  kCallbackStartErrorSettingUpStreamParameters = 2,
  kCallbackStartErrorSettingUpChannelLayout = 3,
  kCallbackStartFailedAddingStream = 4,
  kMaxValue = kCallbackStartFailedAddingStream
};

void ReportStreamOpenResult(StreamOpenResult result) {
  base::UmaHistogramEnumeration("Media.Audio.CrasInputStreamOpenSuccess",
                                result);
}

void ReportStreamStartResult(StreamStartResult result) {
  base::UmaHistogramEnumeration("Media.Audio.CrasInputStreamStartSuccess",
                                result);
}

void ReportNotifyStreamErrors(int err) {
  base::UmaHistogramSparse("Media.Audio.CrasInputStreamNotifyStreamError", err);
}

static constexpr char kVoiceIsolationEffectStateHistogramName[] =
    "Cras.StreamEffectState.VoiceIsolation";

// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
// Used to log stream effects in `CrasInputStream::Start`.
enum class StreamEffectState {
  kForceDisable = 0,
  kForceEnable = 1,
  kPlatformDefault = 2,
  kMaxValue = kPlatformDefault
};

void RecordVoiceIsolationState(StreamEffectState state) {
  base::UmaHistogramEnumeration(kVoiceIsolationEffectStateHistogramName, state);
}

}  // namespace

CrasInputStream::CrasInputStream(const AudioParameters& params,
                                 AudioManagerCrasBase* manager,
                                 const std::string& device_id,
                                 const AudioManager::LogCallback& log_callback)
    : audio_manager_(manager),
      params_(params),
      is_loopback_(AudioDeviceDescription::IsLoopbackDevice(device_id)),
      is_loopback_without_chrome_(
          device_id == AudioDeviceDescription::kLoopbackWithoutChromeId),
      mute_system_audio_(device_id ==
                         AudioDeviceDescription::kLoopbackWithMuteDeviceId),
#if DCHECK_IS_ON()
      recording_enabled_(false),
#endif
      glitch_reporter_(SystemGlitchReporter::StreamType::kCapture),
      log_callback_(std::move(log_callback)),
      peak_detector_(base::BindRepeating(&AudioManager::TraceAmplitudePeak,
                                         base::Unretained(audio_manager_),
                                         /*trace_start=*/true)) {
  DCHECK(audio_manager_);
  audio_bus_ = AudioBus::Create(params_);
  if (!audio_manager_->IsDefault(device_id, true)) {
    uint64_t cras_node_id;
    base::StringToUint64(device_id, &cras_node_id);
    pin_device_ = dev_index_of(cras_node_id);
  }
}

CrasInputStream::~CrasInputStream() {
  DCHECK(!client_);
}

AudioInputStream::OpenOutcome CrasInputStream::Open() {
  if (client_) {
    NOTREACHED() << "CrasInputStream already open";
  }

  // Sanity check input values.
  if (params_.sample_rate() <= 0) {
    DLOG(WARNING) << "Unsupported audio frequency.";
    ReportStreamOpenResult(
        StreamOpenResult::kCallbackOpenUnsupportedAudioFrequency);
    return OpenOutcome::kFailed;
  }

  if (AudioParameters::AUDIO_PCM_LINEAR != params_.format() &&
      AudioParameters::AUDIO_PCM_LOW_LATENCY != params_.format()) {
    DLOG(WARNING) << "Unsupported audio format.";
    ReportStreamOpenResult(
        StreamOpenResult::kCallbackOpenUnsupportedAudioFormat);
    return OpenOutcome::kFailed;
  }

  // Create the client and connect to the CRAS server.
  client_ = libcras_client_create();
  if (!client_) {
    DLOG(WARNING) << "Couldn't create CRAS client.\n";
    ReportStreamOpenResult(
        StreamOpenResult::kCallbackOpenCrasClientCreationFailed);
    client_ = NULL;
    return OpenOutcome::kFailed;
  }

  if (libcras_client_connect(client_)) {
    DLOG(WARNING) << "Couldn't connect CRAS client.\n";
    ReportStreamOpenResult(
        StreamOpenResult::kCallbackOpenCannotConnectToCrasClient);
    libcras_client_destroy(client_);
    client_ = NULL;
    return OpenOutcome::kFailed;
  }

  // Then start running the client.
  if (libcras_client_run_thread(client_)) {
    DLOG(WARNING) << "Couldn't run CRAS client.\n";
    ReportStreamOpenResult(StreamOpenResult::kCallbackOpenCannotRunCrasClient);
    libcras_client_destroy(client_);
    client_ = NULL;
    return OpenOutcome::kFailed;
  }

  if (is_loopback_) {
    if (libcras_client_connected_wait(client_) < 0) {
      DLOG(WARNING) << "Couldn't synchronize data.";
      // TODO(chinyue): Add a DestroyClientOnError method to de-duplicate the
      // cleanup code.
      ReportStreamOpenResult(
          StreamOpenResult::kCallbackOpenCannotSynchronizeData);
      libcras_client_destroy(client_);
      client_ = NULL;
      return OpenOutcome::kFailed;
    }

    int rc;
    if (is_loopback_without_chrome_) {
      uint32_t client_types = 0;
      client_types |= 1 << CRAS_CLIENT_TYPE_CHROME;
      client_types |= 1 << CRAS_CLIENT_TYPE_LACROS;
      client_types = ~client_types;
      rc = pin_device_ = libcras_client_get_floop_dev_idx_by_client_types(
          client_, client_types);
    } else {
      if (base::FeatureList::IsEnabled(
              media::kAudioFlexibleLoopbackForSystemLoopback)) {
        rc = pin_device_ = libcras_client_get_floop_dev_idx_by_client_types(
            client_, ~(uint32_t)0);
      } else {
        rc = libcras_client_get_loopback_dev_idx(client_, &pin_device_);
      }
    }
    if (rc < 0) {
      DLOG(WARNING) << "Couldn't find CRAS loopback device "
                    << (is_loopback_without_chrome_
                            ? " for non-chrome loopback."
                            : " for full loopback.");
      ReportStreamOpenResult(
          StreamOpenResult::kCallbackOpenCannotFindLoopbackDevice);
      libcras_client_destroy(client_);
      client_ = NULL;
      return OpenOutcome::kFailed;
    }
  }

  ReportStreamOpenResult(StreamOpenResult::kCallbackOpenSuccess);
  return OpenOutcome::kSuccess;
}

void CrasInputStream::Close() {
  Stop();

  if (client_) {
    libcras_client_stop(client_);
    libcras_client_destroy(client_);
    client_ = NULL;
  }

  // Signal to the manager that we're closed and can be removed.
  // Should be last call in the method as it deletes "this".
  audio_manager_->ReleaseInputStream(this);
}

inline bool CrasInputStream::UseCrasAec() const {
  return params_.effects() & AudioParameters::ECHO_CANCELLER;
}

inline bool CrasInputStream::UseCrasNs() const {
  return params_.effects() & AudioParameters::NOISE_SUPPRESSION;
}

inline bool CrasInputStream::UseCrasAgc() const {
  return params_.effects() & AudioParameters::AUTOMATIC_GAIN_CONTROL;
}

inline bool CrasInputStream::UseClientControlledVoiceIsolation() const {
  return params_.effects() & AudioParameters::CLIENT_CONTROLLED_VOICE_ISOLATION;
}

inline bool CrasInputStream::UseCrasVoiceIsolation() const {
  return params_.effects() & AudioParameters::VOICE_ISOLATION;
}

inline bool CrasInputStream::DspBasedAecIsAllowed() const {
  return params_.effects() & AudioParameters::ALLOW_DSP_ECHO_CANCELLER;
}

inline bool CrasInputStream::DspBasedNsIsAllowed() const {
  return params_.effects() & AudioParameters::ALLOW_DSP_NOISE_SUPPRESSION;
}

inline bool CrasInputStream::DspBasedAgcIsAllowed() const {
  return params_.effects() & AudioParameters::ALLOW_DSP_AUTOMATIC_GAIN_CONTROL;
}

inline bool CrasInputStream::IgnoreUiGains() const {
  return params_.effects() & AudioParameters::IGNORE_UI_GAINS;
}

void CrasInputStream::Start(AudioInputCallback* callback) {
  DCHECK(client_);
  DCHECK(callback);

  // Channel map to CRAS_CHANNEL, values in the same order of
  // corresponding source in Chromium defined Channels.
  static const int kChannelMap[] = {
      CRAS_CH_FL,  CRAS_CH_FR,  CRAS_CH_FC, CRAS_CH_LFE, CRAS_CH_RL, CRAS_CH_RR,
      CRAS_CH_FLC, CRAS_CH_FRC, CRAS_CH_RC, CRAS_CH_SL,  CRAS_CH_SR};
  static_assert(std::size(kChannelMap) == CHANNELS_MAX + 1,
                "kChannelMap array size should match");

  // If already playing, stop before re-starting.
  if (started_) {
    return;
  }

  StartAgc();

  callback_ = callback;

  CRAS_STREAM_TYPE type = CRAS_STREAM_TYPE_DEFAULT;
  uint32_t flags = 0;
  if (params_.effects() & AudioParameters::PlatformEffectsMask::HOTWORD) {
    flags = HOTWORD_STREAM;
    type = CRAS_STREAM_TYPE_SPEECH_RECOGNITION;
  }

  unsigned int frames_per_packet = params_.frames_per_buffer();
  struct libcras_stream_params* stream_params = libcras_stream_params_create();
  if (!stream_params) {
    DLOG(ERROR) << "Error creating stream params";
    ReportStreamStartResult(
        StreamStartResult::kCallbackStartErrorCreatingStreamParameters);
    callback_->OnError();
    callback_ = NULL;
    return;
  }

  int rc = libcras_stream_params_set(
      stream_params, stream_direction_, frames_per_packet, frames_per_packet,
      type, audio_manager_->GetClientType(), flags, this,
      CrasInputStream::SamplesReady, CrasInputStream::StreamError,
      params_.sample_rate(), SND_PCM_FORMAT_S16, params_.channels());

  if (rc) {
    DLOG(WARNING) << "Error setting up stream parameters.";
    ReportStreamStartResult(
        StreamStartResult::kCallbackStartErrorSettingUpStreamParameters);
    callback_->OnError();
    callback_ = NULL;
    libcras_stream_params_destroy(stream_params);
    return;
  }

  // Initialize channel layout to all -1 to indicate that none of
  // the channels is set in the layout.
  int8_t layout[CRAS_CH_MAX];
  for (size_t i = 0; i < std::size(layout); ++i) {
    layout[i] = -1;
  }

  // Converts to CRAS defined channels. ChannelOrder will return -1
  // for channels that are not present in params_.channel_layout().
  for (size_t i = 0; i < std::size(kChannelMap); ++i) {
    layout[kChannelMap[i]] =
        ChannelOrder(params_.channel_layout(), static_cast<Channels>(i));
  }

  rc = libcras_stream_params_set_channel_layout(stream_params, CRAS_CH_MAX,
                                                layout);
  if (rc) {
    DLOG(WARNING) << "Error setting up the channel layout.";
    ReportStreamStartResult(
        StreamStartResult::kCallbackStartErrorSettingUpChannelLayout);
    callback_->OnError();
    callback_ = NULL;
    libcras_stream_params_destroy(stream_params);
    return;
  }

  if (UseCrasAec()) {
    libcras_stream_params_enable_aec(stream_params);
  }

  if (UseCrasNs()) {
    libcras_stream_params_enable_ns(stream_params);
  }

  if (UseCrasAgc()) {
    libcras_stream_params_enable_agc(stream_params);
  }

  if (base::FeatureList::IsEnabled(media::kCrOSSystemVoiceIsolationOption)) {
    if (UseClientControlledVoiceIsolation()) {
      if (UseCrasVoiceIsolation()) {
        libcras_stream_params_enable_voice_isolation(stream_params);
        RecordVoiceIsolationState(StreamEffectState::kForceEnable);
      } else {
        libcras_stream_params_disable_voice_isolation(stream_params);
        RecordVoiceIsolationState(StreamEffectState::kForceDisable);
      }
    } else {
      RecordVoiceIsolationState(StreamEffectState::kPlatformDefault);
    }
  }

  if (DspBasedAecIsAllowed()) {
    libcras_stream_params_allow_aec_on_dsp(stream_params);
  }

  if (DspBasedNsIsAllowed()) {
    libcras_stream_params_allow_ns_on_dsp(stream_params);
  }

  if (DspBasedAgcIsAllowed()) {
    libcras_stream_params_allow_agc_on_dsp(stream_params);
  }

  if (IgnoreUiGains()) {
    libcras_stream_params_ignore_ui_gains(stream_params);
  }

  // Adding the stream will start the audio callbacks.
  if (libcras_client_add_pinned_stream(client_, pin_device_, &stream_id_,
                                       stream_params)) {
    DLOG(WARNING) << "Failed to add the stream.";
    ReportStreamStartResult(
        StreamStartResult::kCallbackStartFailedAddingStream);
    callback_->OnError();
    callback_ = NULL;
  }

  // Mute system audio if requested.
  if (mute_system_audio_) {
    int muted;
    libcras_client_get_system_muted(client_, &muted);
    if (!muted) {
      libcras_client_set_system_mute(client_, 1);
    }
    mute_done_ = true;
  }

  // Done with config params.
  libcras_stream_params_destroy(stream_params);

  started_ = true;

  audio_manager_->RegisterSystemAecDumpSource(this);

  ReportStreamStartResult(StreamStartResult::kCallbackStartSuccess);
}

void CrasInputStream::Stop() {
  if (!client_) {
    return;
  }

  if (!callback_ || !started_) {
    return;
  }

  audio_manager_->DeregisterSystemAecDumpSource(this);

  if (mute_system_audio_ && mute_done_) {
    libcras_client_set_system_mute(client_, 0);
    mute_done_ = false;
  }

  StopAgc();

  // Removing the stream from the client stops audio.
  libcras_client_rm_stream(client_, stream_id_);

  ReportAndResetStats();

  started_ = false;
  callback_ = NULL;
}

// Static callback asking for samples.  Run on high priority thread.
int CrasInputStream::SamplesReady(struct libcras_stream_cb_data* data) {
  unsigned int frames;
  uint8_t* buf;
  struct timespec latency;
  void* usr_arg;
  uint32_t overrun_frames = 0;
  struct timespec dropped_samples_duration_ts;
  base::TimeDelta dropped_samples_duration;

  libcras_stream_cb_data_get_frames(data, &frames);
  libcras_stream_cb_data_get_buf(data, &buf);
  libcras_stream_cb_data_get_latency(data, &latency);
  libcras_stream_cb_data_get_usr_arg(data, &usr_arg);
  CrasInputStream* me = static_cast<CrasInputStream*>(usr_arg);
  me->ReadAudio(frames, buf, &latency);
  // Audio glitches are checked every callback.
  libcras_stream_cb_data_get_overrun_frames(data, &overrun_frames);
  libcras_stream_cb_data_get_dropped_samples_duration(
      data, &dropped_samples_duration_ts);
  dropped_samples_duration =
      base::TimeDelta::FromTimeSpec(dropped_samples_duration_ts);
  me->CalculateAudioGlitches(overrun_frames, dropped_samples_duration);
  return frames;
}

// Static callback for stream errors.
int CrasInputStream::StreamError(cras_client* client,
                                 cras_stream_id_t stream_id,
                                 int err,
                                 void* arg) {
  CrasInputStream* me = static_cast<CrasInputStream*>(arg);
  me->NotifyStreamError(err);
  return 0;
}

void CrasInputStream::ReadAudio(size_t frames,
                                uint8_t* buffer,
                                const timespec* latency_ts) {
  DCHECK(callback_);

  // Update the AGC volume level once every second. Note that, |volume| is
  // also updated each time SetVolume() is called through IPC by the
  // render-side AGC.
  double normalized_volume = 0.0;
  GetAgcVolume(&normalized_volume);

  const base::TimeDelta delay =
      std::max(base::TimeDelta::FromTimeSpec(*latency_ts), base::TimeDelta());

  // The delay says how long ago the capture was, so we subtract the delay from
  // Now() to find the capture time.
  const base::TimeTicks capture_time = base::TimeTicks::Now() - delay;

  audio_bus_->FromInterleaved<SignedInt16SampleTypeTraits>(
      reinterpret_cast<int16_t*>(buffer), audio_bus_->frames());

  peak_detector_.FindPeak(audio_bus_.get());

  callback_->OnData(audio_bus_.get(), capture_time, normalized_volume,
                    glitch_info_accumulator_.GetAndReset());
}

void CrasInputStream::NotifyStreamError(int err) {
  ReportNotifyStreamErrors(err);
  if (callback_) {
    callback_->OnError();
  }
}

double CrasInputStream::GetMaxVolume() {
  return 1.0f;
}

void CrasInputStream::SetVolume(double volume) {
  DCHECK(client_);

  // Set the volume ratio to CRAS's softare and stream specific gain.
  input_volume_ = volume;
  libcras_client_set_stream_volume(client_, stream_id_, input_volume_);

  // Update the AGC volume level based on the last setting above. Note that,
  // the volume-level resolution is not infinite and it is therefore not
  // possible to assume that the volume provided as input parameter can be
  // used directly. Instead, a new query to the audio hardware is required.
  // This method does nothing if AGC is disabled.
  UpdateAgcVolume();
}

double CrasInputStream::GetVolume() {
  if (!client_) {
    return 0.0;
  }

  return input_volume_;
}

bool CrasInputStream::IsMuted() {
  int muted = 0;
  libcras_client_get_system_capture_muted(client_, &muted);
  return static_cast<bool>(muted);
}

void CrasInputStream::SetOutputDeviceForAec(
    const std::string& output_device_id) {
  DCHECK(client_);

  int echo_ref_id;

  // Default device means to just use the system default output as AEC
  // reference. CRAS server side requires passing NO_DEVICE in that case.
  if (AudioDeviceDescription::IsDefaultDevice(output_device_id)) {
    echo_ref_id = NO_DEVICE;
  } else {
    uint64_t cras_node_id;
    base::StringToUint64(output_device_id, &cras_node_id);
    echo_ref_id = dev_index_of(cras_node_id);
  }
  libcras_client_set_aec_ref(client_, stream_id_, echo_ref_id);
}

void CrasInputStream::StartAecdump(base::File file) {
  FILE* stream = base::FileToFILE(std::move(file), "w");
  if (!client_) {
    return;
  }
#if DCHECK_IS_ON()
  DCHECK(!recording_enabled_);
  recording_enabled_ = true;
#endif

  libcras_client_set_aec_dump(client_, stream_id_, /*start=*/1, fileno(stream));
}

void CrasInputStream::StopAecdump() {
  if (!client_) {
    return;
  }
#if DCHECK_IS_ON()
  DCHECK(recording_enabled_);
  recording_enabled_ = false;
#endif
  libcras_client_set_aec_dump(client_, stream_id_, /*start=*/0, /*fd=*/-1);
}

void CrasInputStream::ReportAndResetStats() {
  SystemGlitchReporter::Stats stats =
      glitch_reporter_.GetLongTermStatsAndReset();

  std::string log_message = base::StringPrintf(
      "CRAS in: (num_glitches_detected=[%d], cumulative_audio_lost=[%" PRId64
      " ms],largest_glitch=[%" PRId64 " ms])",
      stats.glitches_detected, stats.total_glitch_duration.InMilliseconds(),
      stats.largest_glitch_duration.InMilliseconds());

  log_callback_.Run(log_message);
  if (stats.glitches_detected != 0) {
    DLOG(WARNING) << log_message;
  }
  last_overrun_frames_ = 0;
  last_dropped_samples_duration_ = base::TimeDelta();
}

void CrasInputStream::CalculateAudioGlitches(
    uint32_t overrun_frames,
    base::TimeDelta dropped_samples_duration) {
  // |overrun_frames| obtained from callback is the cumulative value of the
  // overwritten frames of the whole stream. Calculate the overrun frames this
  // callback and convert it to base::TimeDelta.
  DCHECK_GE(overrun_frames, last_overrun_frames_);
  uint32_t overrun_frames_this_callback = overrun_frames - last_overrun_frames_;
  base::TimeDelta overrun_glitch_duration = AudioTimestampHelper::FramesToTime(
      overrun_frames_this_callback, params_.sample_rate());

  // |dropped_samples_duration| obtained from callback is the cumulative value
  // of the dropped audio samples of the whole stream. Calculate the dropped
  // audio sample duration this callback.
  DCHECK_GE(dropped_samples_duration, last_dropped_samples_duration_);
  base::TimeDelta dropped_samples_glitch_duration =
      dropped_samples_duration - last_dropped_samples_duration_;

  base::TimeDelta glitch_duration =
      overrun_glitch_duration + dropped_samples_glitch_duration;
  glitch_reporter_.UpdateStats(glitch_duration);
  if (glitch_duration.is_positive()) {
    glitch_info_accumulator_.Add(AudioGlitchInfo::SingleBoundedSystemGlitch(
        glitch_duration, AudioGlitchInfo::Direction::kCapture));
  }

  last_overrun_frames_ = overrun_frames;
  last_dropped_samples_duration_ = dropped_samples_duration;
}

}  // namespace media