ash / display / touch_calibrator_controller.cc [blame]

// Copyright 2016 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "ash/display/touch_calibrator_controller.h"

#include <memory>

#include "ash/display/touch_calibrator_view.h"
#include "ash/display/window_tree_host_manager.h"
#include "ash/host/ash_window_tree_host.h"
#include "ash/shell.h"
#include "ash/touch/ash_touch_transform_controller.h"
#include "base/functional/bind.h"
#include "base/functional/callback_helpers.h"
#include "base/ranges/algorithm.h"
#include "base/task/sequenced_task_runner.h"
#include "base/task/single_thread_task_runner.h"
#include "ui/aura/window_tree_host.h"
#include "ui/display/display.h"
#include "ui/display/manager/display_manager.h"
#include "ui/display/screen.h"
#include "ui/events/devices/device_data_manager.h"
#include "ui/events/event.h"
#include "ui/events/types/event_type.h"
#include "ui/gfx/geometry/size_conversions.h"
#include "ui/views/view_utils.h"
#include "ui/views/widget/widget.h"

namespace ash {
namespace {

void InitInternalTouchDeviceIds(std::set<int>& internal_touch_device_ids) {
  internal_touch_device_ids.clear();
  const std::vector<ui::TouchscreenDevice>& device_list =
      ui::DeviceDataManager::GetInstance()->GetTouchscreenDevices();
  for (const auto& touchscreen_device : device_list) {
    if (touchscreen_device.type == ui::InputDeviceType::INPUT_DEVICE_INTERNAL) {
      internal_touch_device_ids.insert(touchscreen_device.id);
    }
  }
}

// Returns a transform to undo any transformations that are applied to events
// originating from the touch device identified with |touch_device_id|. This
// transform converts the event's location to the raw touch location.
gfx::Transform CalculateEventTransformer(int touch_device_id) {
  const display::DisplayManager* display_manager =
      Shell::Get()->display_manager();
  const std::vector<ui::TouchscreenDevice>& device_list =
      ui::DeviceDataManager::GetInstance()->GetTouchscreenDevices();

  auto device_it = base::ranges::find(device_list, touch_device_id,
                                      &ui::TouchscreenDevice::id);
  DCHECK(device_it != device_list.end())
      << "Device id " << touch_device_id
      << " is invalid. No such device connected to system";

  int64_t previous_display_id =
      display_manager->touch_device_manager()->GetAssociatedDisplay(*device_it);

  // If the touch device is not associated with any display. This may happen in
  // tests when the test does not setup the |ui::TouchDeviceTransform| before
  // generating a touch event.
  if (previous_display_id == display::kInvalidDisplayId) {
    return gfx::Transform();
  }

  // Undo the event transformations that the previous display applied on the
  // event location. We want to store the raw event location information.
  gfx::Transform tm =
      Shell::Get()
          ->window_tree_host_manager()
          ->GetAshWindowTreeHostForDisplayId(previous_display_id)
          ->AsWindowTreeHost()
          ->GetRootTransform();
  return tm;
}

}  // namespace

// Time interval after a touch event during which all other touch events are
// ignored during calibration.
const base::TimeDelta TouchCalibratorController::kTouchIntervalThreshold =
    base::Milliseconds(200);

TouchCalibratorController::TouchCalibratorController()
    : last_touch_timestamp_(base::Time::Now()) {}

TouchCalibratorController::~TouchCalibratorController() {
  touch_calibrator_widgets_.clear();
  already_mapped_display_ids_.clear();
  StopCalibrationAndResetParams();
}

void TouchCalibratorController::OnDidApplyDisplayChanges() {
  touch_calibrator_widgets_.clear();
  StopCalibrationAndResetParams();

  // Native touchscreen mapping state is not updated by
  // |StopCalibrationAndResetParam| since it would generally move on to the next
  // display afterwards. State must be reset in this case since display
  // configuration has changed and the current mapping instantiation is no
  // longer valid.
  if (state_ == CalibrationState::kNativeCalibrationTouchscreenMapping) {
    already_mapped_display_ids_.clear();
    state_ = CalibrationState::kInactive;
  }
}

void TouchCalibratorController::StartCalibration(
    const display::Display& target_display,
    bool is_custom_calibration,
    TouchCalibrationCallback opt_callback) {
  if (state_ != CalibrationState::kNativeCalibrationTouchscreenMapping) {
    state_ = is_custom_calibration ? CalibrationState::kCustomCalibration
                                   : CalibrationState::kNativeCalibration;
  }

  if (opt_callback) {
    opt_callback_ = std::move(opt_callback);
  }

  target_display_ = target_display;

  // Clear all touch calibrator views used in any previous calibration.
  touch_calibrator_widgets_.clear();

  // Set the touch device id as invalid so it can be set during calibration.
  touch_device_id_ = ui::InputDevice::kInvalidId;

  // Populate |internal_touch_device_ids_| with the ids of touch devices that
  // are currently associated with the internal display and are of type
  // |ui::InputDeviceType::INPUT_DEVICE_INTERNAL|.
  InitInternalTouchDeviceIds(internal_touch_device_ids_);

  // If this is a native touch calibration, then initialize the UX for it.
  if (state_ == CalibrationState::kNativeCalibration ||
      state_ == CalibrationState::kNativeCalibrationTouchscreenMapping) {
    Shell::Get()->display_manager()->AddDisplayManagerObserver(this);

    // Reset the calibration data.
    touch_point_quad_.fill(std::make_pair(gfx::Point(0, 0), gfx::Point(0, 0)));

    std::vector<display::Display> displays =
        display::Screen::GetScreen()->GetAllDisplays();

    for (const display::Display& display : displays) {
      bool is_primary_view = display.id() == target_display_.id();
      touch_calibrator_widgets_[display.id()] = TouchCalibratorView::Create(
          display, is_primary_view,
          state_ == CalibrationState::kNativeCalibrationTouchscreenMapping);
    }
  }

  Shell::Get()->touch_transformer_controller()->SetForCalibration(true);

  // Add self as an event handler target.
  Shell::Get()->AddPreTargetHandler(this);
}

void TouchCalibratorController::StartNativeTouchscreenMappingExperience(
    TouchCalibrationCallback opt_callback) {
  state_ = CalibrationState::kNativeCalibrationTouchscreenMapping;
  already_mapped_display_ids_.clear();
  CalibrateNextDisplay();
  opt_callback_all_displays_ = std::move(opt_callback);
}

void TouchCalibratorController::CalibrateNextDisplay() {
  CHECK(state_ == CalibrationState::kNativeCalibrationTouchscreenMapping);

  // Find the next external display to calibrate that we did not already handle.
  const auto& active_displays =
      Shell::Get()->display_manager()->active_display_list();
  const display::Display* next_display_to_map = nullptr;
  for (const auto& display : active_displays) {
    if (display.IsInternal() ||
        base::Contains(already_mapped_display_ids_, display.id())) {
      continue;
    }

    next_display_to_map = &display;
    break;
  }

  if (!next_display_to_map) {
    state_ = CalibrationState::kInactive;
    already_mapped_display_ids_.clear();
    if (opt_callback_all_displays_) {
      base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
          FROM_HERE, base::BindOnce(std::move(opt_callback_all_displays_),
                                    /*success=*/true));
    }

    for (const auto& it : touch_calibrator_widgets_) {
      if (auto* touch_calibrator_view = views::AsViewClass<TouchCalibratorView>(
              it.second->GetContentsView())) {
        touch_calibrator_view->SkipToFinalState();
      }
    }
    return;
  }

  already_mapped_display_ids_.emplace(next_display_to_map->id());
  StartCalibration(*next_display_to_map, /*is_custom_calibration=*/false,
                   base::DoNothing());
}

void TouchCalibratorController::StopCalibrationAndResetParams() {
  Shell::Get()->display_manager()->RemoveDisplayManagerObserver(this);

  Shell::Get()->touch_transformer_controller()->SetForCalibration(false);

  // Remove self as the event handler.
  Shell::Get()->RemovePreTargetHandler(this);

  // Transition all touch calibrator views to their final state for a graceful
  // exit if this is touch calibration with native UX.
  if (state_ == CalibrationState::kNativeCalibration ||
      state_ == CalibrationState::kNativeCalibrationTouchscreenMapping) {
    for (const auto& it : touch_calibrator_widgets_) {
      if (auto* touch_calibrator_view = views::AsViewClass<TouchCalibratorView>(
              it.second->GetContentsView())) {
        touch_calibrator_view->SkipToFinalState();
      }
    }
  }

  if (state_ != CalibrationState::kNativeCalibrationTouchscreenMapping) {
    state_ = CalibrationState::kInactive;
  }

  if (opt_callback_) {
    base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
        FROM_HERE,
        base::BindOnce(std::move(opt_callback_), false /* failure */));
    opt_callback_.Reset();
  }
}

void TouchCalibratorController::CompleteCalibration(
    const CalibrationPointPairQuad& pairs,
    const gfx::Size& display_size) {
  bool did_find_touch_device = false;
  const std::vector<ui::TouchscreenDevice>& device_list =
      ui::DeviceDataManager::GetInstance()->GetTouchscreenDevices();
  ui::TouchscreenDevice target_device;
  for (const auto& device : device_list) {
    if (device.id == touch_device_id_) {
      target_device = device;
      did_find_touch_device = true;
      break;
    }
  }

  if (!did_find_touch_device) {
    VLOG(1) << "No touch device with id: " << touch_device_id_ << " found to "
            << "complete touch calibration for display with id: "
            << target_display_.id() << ". Storing it as a fallback";
  }

  if (opt_callback_) {
    base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
        FROM_HERE,
        base::BindOnce(std::move(opt_callback_), true /* success */));
    opt_callback_.Reset();
  }
  StopCalibrationAndResetParams();

  const bool in_native_touchscreen_mapping =
      state_ == CalibrationState::kNativeCalibrationTouchscreenMapping;
  Shell::Get()->display_manager()->SetTouchCalibrationData(
      target_display_.id(), pairs, display_size, target_device,
      /*apply_spatial_calibration=*/!in_native_touchscreen_mapping);
  if (in_native_touchscreen_mapping) {
    CalibrateNextDisplay();
  }
}

bool TouchCalibratorController::IsCalibrating() const {
  return state_ != CalibrationState::kInactive;
}

// ui::EventHandler:
void TouchCalibratorController::OnKeyEvent(ui::KeyEvent* key) {
  if (state_ != CalibrationState::kNativeCalibration &&
      state_ != CalibrationState::kNativeCalibrationTouchscreenMapping) {
    return;
  }
  // Detect ESC key press.
  if (key->type() == ui::EventType::kKeyPressed &&
      key->key_code() == ui::VKEY_ESCAPE) {
    StopCalibrationAndResetParams();
    if (state_ == CalibrationState::kNativeCalibrationTouchscreenMapping) {
      CalibrateNextDisplay();
    }
  }

  key->StopPropagation();
}

void TouchCalibratorController::OnTouchEvent(ui::TouchEvent* touch) {
  if (!IsCalibrating())
    return;
  if (touch->type() != ui::EventType::kTouchReleased) {
    return;
  }
  if (base::Time::Now() - last_touch_timestamp_ < kTouchIntervalThreshold)
    return;
  last_touch_timestamp_ = base::Time::Now();

  // If the touch event originated from a touch device that is associated with
  // the internal display, then ignore it.
  if (internal_touch_device_ids_.count(touch->source_device_id())) {
    return;
  }

  if (touch_device_id_ == ui::InputDevice::kInvalidId) {
    touch_device_id_ = touch->source_device_id();
    event_transformer_ = CalculateEventTransformer(touch_device_id_);
  }

  // If this is a custom touch calibration, then everything else is managed
  // by the application responsible for the custom calibration UX.
  if (state_ == CalibrationState::kCustomCalibration) {
    return;
  }
  touch->StopPropagation();

  TouchCalibratorView* target_screen_calibration_view =
      views::AsViewClass<TouchCalibratorView>(
          touch_calibrator_widgets_[target_display_.id()]->GetContentsView());
  CHECK(target_screen_calibration_view);

  // If this is the final state, then store all calibration data and stop
  // calibration.
  if (state_ ==
          TouchCalibratorController::CalibrationState::kNativeCalibration &&
      target_screen_calibration_view->state() ==
          TouchCalibratorView::CALIBRATION_COMPLETE) {
    gfx::RectF calibration_bounds =
        Shell::Get()
            ->window_tree_host_manager()
            ->GetAshWindowTreeHostForDisplayId(target_display_.id())
            ->AsWindowTreeHost()
            ->GetRootTransform()
            .MapRect(
                gfx::RectF(target_screen_calibration_view->GetLocalBounds()));
    CompleteCalibration(touch_point_quad_,
                        gfx::ToRoundedSize(calibration_bounds.size()));
    return;
  }

  int state_index;
  // Maps the state to an integer value. Assigns a non negative integral value
  // for a state in which the user can interact with the the interface.
  switch (target_screen_calibration_view->state()) {
    case TouchCalibratorView::DISPLAY_POINT_1:
      state_index = 0;
      break;
    case TouchCalibratorView::DISPLAY_POINT_2:
      state_index = 1;
      break;
    case TouchCalibratorView::DISPLAY_POINT_3:
      state_index = 2;
      break;
    case TouchCalibratorView::DISPLAY_POINT_4:
      state_index = 3;
      break;
    default:
      // Return early if the interface is in a state that does not allow user
      // interaction.
      return;
  }

  // Store touch point corresponding to its display point.
  gfx::Point display_point;
  if (target_screen_calibration_view->GetDisplayPointLocation(&display_point)) {
    // If the screen has a root transform applied, the display point does not
    // correctly map to the touch point. This is specially evident if the
    // display is rotated or a device scale factor is applied. The display point
    // needs to have the root transform applied as well to correctly pair it
    // with the touch point.
    display_point = Shell::Get()
                        ->window_tree_host_manager()
                        ->GetAshWindowTreeHostForDisplayId(target_display_.id())
                        ->AsWindowTreeHost()
                        ->GetRootTransform()
                        .MapPoint(display_point);

    // Why do we need this? To understand this we need to know the life of an
    // event location. The event location undergoes the following
    // transformations along its path from the device to the event handler that
    // is this class.
    //
    // Touch Device -> EventFactoryEvdev -> DrmWindowHost
    //     -> WindowEventDispatcher -> EventHandler(this)
    //
    //  - The touch device dispatches the raw device event location. Lets assume
    //    this is (x, y).
    //  - The EventFactoryEvdev applies a touch transform that includes the
    //    calibration information as well as an offset of the native bounds
    //    of the display. This effectively converts the coordinates of the event
    //    from the raw device event location to the native screen coordinates.
    //    It gets the offset information from DrmWindowHost via the
    //    ManagedDisplayInfo class. If the offset of the PlatformWindow is (A,B)
    //    then the event location after this stage would be (x + A, y + B).
    //  - The DrmWindowHost removes the offset from the event location so that
    //    the location becomes relative to the platform window's origin. In
    //    Chrome OS it so happens that each display is its own platform window.
    //    So an offset equal to the display's origin in screen space is
    //    subtracted from the event location. This effectively undoes the
    //    previous step's transformation. Thus the event location after this
    //    step is (x, y) again.
    //  - WindowEventDispatcher applies an inverse root transform on the event
    //    location. This means that if the display is rotated or has a device
    //    scale factor, then those transformation are also applied to the event
    //    location. It effectively converts the coordinates from platform window
    //    coordinates to the aura's root window coordinates. The display in
    //    context here is the display that is associated with the touch device
    //    from which the event originated from.
    //
    // Up until the output of DrmWindowHost, everything is as expected. But
    // WindowEventDispatcher applies an inverse root transform which modifies
    // the raw event location that we wanted. Moreover, it modifies the raw
    // event location using the root transform of the display that the touch
    // device was previously associated with. To solve this, we need to undo the
    // changes made to the event location by WindowEventDispatcher. This is what
    // is achieved by |event_transformer_|.
    gfx::PointF event_location_f =
        event_transformer_.MapPoint(touch->location_f());

    touch_point_quad_[state_index] =
        std::make_pair(display_point, gfx::ToRoundedPoint(event_location_f));
  } else {
    // TODO(malaykeshav): Display some kind of error for the user.
    NOTREACHED() << "Touch calibration failed. Could not retrieve location for"
                    " display point. Retry calibration.";
  }

  // For calibrating all displays, skip the final state of showing a
  // "Calibration complete" screen.
  if (state_ == TouchCalibratorController::CalibrationState::
                    kNativeCalibrationTouchscreenMapping &&
      target_screen_calibration_view->state() ==
          TouchCalibratorView::DISPLAY_POINT_4) {
    gfx::RectF calibration_bounds =
        Shell::Get()
            ->window_tree_host_manager()
            ->GetAshWindowTreeHostForDisplayId(target_display_.id())
            ->AsWindowTreeHost()
            ->GetRootTransform()
            .MapRect(
                gfx::RectF(target_screen_calibration_view->GetLocalBounds()));
    CompleteCalibration(touch_point_quad_,
                        gfx::ToRoundedSize(calibration_bounds.size()));
    return;
  }

  target_screen_calibration_view->AdvanceToNextState();
}

}  // namespace ash