Implement proper touch screen support by emulating the stylus

Using a mouse for touch screen controls doesn't accurately encompass the
feel that one gets when using the 3ds stylus. This PR adds experimental
support for using a stylus for touch screen controls.

src/citra_qt/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(CMAKE_AUTOMOC ON)
+set(CMAKE_AUTORCC ON)
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/CMakeModules)
 
@@ -29,6 +30,8 @@ set(SRCS
             game_list.cpp
             hotkeys.cpp
             main.cpp
+            overlay.cpp
+            stylus.cpp
             ui_settings.cpp
             citra-qt.rc
             Info.plist
@@ -63,6 +66,8 @@ set(HEADERS
             game_list_p.h
             hotkeys.h
             main.h
+            overlay.h
+            stylus.h
             ui_settings.h
             )
 
@@ -92,10 +97,10 @@ endif()
 if (APPLE)
     set(MACOSX_ICON "../../dist/citra.icns")
     set_source_files_properties(${MACOSX_ICON} PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
-    add_executable(citra-qt MACOSX_BUNDLE ${SRCS} ${HEADERS} ${UI_HDRS} ${MACOSX_ICON})
+    add_executable(citra-qt MACOSX_BUNDLE ${SRCS} ${HEADERS} ${UI_HDRS} ${MACOSX_ICON} resources.qrc)
     set_target_properties(citra-qt PROPERTIES MACOSX_BUNDLE_INFO_PLIST ${CMAKE_CURRENT_SOURCE_DIR}/Info.plist)
 else()
-    add_executable(citra-qt ${SRCS} ${HEADERS} ${UI_HDRS})
+    add_executable(citra-qt ${SRCS} ${HEADERS} ${UI_HDRS} resources.qrc)
 endif()
 target_link_libraries(citra-qt core video_core audio_core common input_common)
 target_link_libraries(citra-qt ${OPENGL_gl_LIBRARY} ${CITRA_QT_LIBS})

src/citra_qt/bootmanager.cpp

@@ -100,13 +100,18 @@ private:
     bool do_painting;
 };
 
+Stylus* GRenderWindow::GetStylus() {
+    return stylus.get();
+}
+
 GRenderWindow::GRenderWindow(QWidget* parent, EmuThread* emu_thread)
     : QWidget(parent), child(nullptr), emu_thread(emu_thread) {
 
     std::string window_title = Common::StringFromFormat("Citra %s| %s-%s", Common::g_build_name,
                                                         Common::g_scm_branch, Common::g_scm_desc);
     setWindowTitle(QString::fromStdString(window_title));
-
+    setFocusPolicy(Qt::StrongFocus);
+    setMouseTracking(true);
     InputCommon::Init();
 }
 
@@ -148,6 +153,16 @@ void GRenderWindow::DoneCurrent() {
 
 void GRenderWindow::PollEvents() {}
 
+void GRenderWindow::UpdateStylusSize(unsigned width, unsigned height) {
+    if (overlay) {
+        QRect rect = QApplication::desktop()->screenGeometry();
+        LOG_WARNING(Frontend, "rect %d, %d", rect.width(), rect.height());
+        overlay->setFixedWidth(rect.width());
+        overlay->setFixedHeight(rect.height() - 1);
+        overlay->update();
+    }
+}
+
 // On Qt 5.0+, this correctly gets the size of the framebuffer (pixels).
 //
 // Older versions get the window size (density independent pixels),
@@ -159,6 +174,7 @@ void GRenderWindow::OnFramebufferSizeChanged() {
     qreal pixelRatio = windowPixelRatio();
     unsigned width = child->QPaintDevice::width() * pixelRatio;
     unsigned height = child->QPaintDevice::height() * pixelRatio;
+    UpdateStylusSize(width, height);
     UpdateCurrentFramebufferLayout(width, height);
 }
 
@@ -209,12 +225,22 @@ void GRenderWindow::keyReleaseEvent(QKeyEvent* event) {
     InputCommon::GetKeyboard()->ReleaseKey(event->key());
 }
 
+void GRenderWindow::UpdateStylusPosition(QMouseEvent* event) {
+    auto pos = event->pos();
+    stylus->UpdatePosition(pos.x(), pos.y());
+    overlay->update();
+}
+
 void GRenderWindow::mousePressEvent(QMouseEvent* event) {
     auto pos = event->pos();
-    if (event->button() == Qt::LeftButton) {
+    if (event->button() == Qt::LeftButton && stylus->Contains(pos)) {
-        qreal pixelRatio = windowPixelRatio();
+        stylus->SetHoldPoint(pos);
-        this->TouchPressed(static_cast<unsigned>(pos.x() * pixelRatio),
+        if (stylus->IsNearCenter(pos)) {
-                           static_cast<unsigned>(pos.y() * pixelRatio));
+            stylus->SetState(StylusState::HOLD);
+        } else {
+            stylus->SetState(StylusState::ROTATE);
+        }
+        QApplication::setOverrideCursor(Qt::ClosedHandCursor);
     } else if (event->button() == Qt::RightButton) {
         motion_emu->BeginTilt(pos.x(), pos.y());
     }
@@ -222,17 +248,44 @@ void GRenderWindow::mousePressEvent(QMouseEvent* event) {
 
 void GRenderWindow::mouseMoveEvent(QMouseEvent* event) {
     auto pos = event->pos();
+
+    if (stylus->GetState() == StylusState::HOLD) {
+        UpdateStylusPosition(event);
         qreal pixelRatio = windowPixelRatio();
-    this->TouchMoved(std::max(static_cast<unsigned>(pos.x() * pixelRatio), 0u),
+        QPoint p = stylus->GetTouchPoint();
-                     std::max(static_cast<unsigned>(pos.y() * pixelRatio), 0u));
+        QPoint screen = mapToGlobal(QWidget::geometry().topLeft());
+        this->TouchMoved(std::max(static_cast<unsigned>((p.x() - screen.x()) * pixelRatio), 0u),
+                         std::max(static_cast<unsigned>((p.y() - screen.y()) * pixelRatio), 0u));
+    } else if (stylus->GetState() == StylusState::ROTATE) {
+        stylus->Rotate(pos);
+        overlay->update();
+    }
+    if (event->button() == Qt::LeftButton) {
+    } else if (event->button() == Qt::RightButton) {
         motion_emu->Tilt(pos.x(), pos.y());
+    }
+}
+void GRenderWindow::mouseDoubleClickEvent(QMouseEvent* e) {
+    if (e->button() == Qt::LeftButton && stylus->Contains(e->pos())) {
+        qreal pixelRatio = windowPixelRatio();
+        QPoint p = stylus->GetTouchPoint();
+        QPoint screen = mapToGlobal(QWidget::geometry().topLeft());
+        LOG_ERROR(Frontend, "screen %d, %d", screen.x(), screen.y());
+        this->TouchPressed(static_cast<unsigned>((p.x() - screen.x()) * pixelRatio),
+                           static_cast<unsigned>((p.y() - screen.y()) * pixelRatio));
+        stylus->SetState(StylusState::HOLD);
+        stylus->SetHoldPoint(e->pos());
+    }
 }
 
 void GRenderWindow::mouseReleaseEvent(QMouseEvent* event) {
-    if (event->button() == Qt::LeftButton)
+    if (event->button() == Qt::LeftButton) {
         this->TouchReleased();
-    else if (event->button() == Qt::RightButton)
+        stylus->SetState(StylusState::DROP);
+        QApplication::restoreOverrideCursor();
+    } else if (event->button() == Qt::RightButton) {
         motion_emu->EndTilt();
+    }
 }
 
 void GRenderWindow::focusOutEvent(QFocusEvent* event) {
@@ -287,13 +340,18 @@ void GRenderWindow::OnMinimalClientAreaChangeRequest(
 
 void GRenderWindow::OnEmulationStarting(EmuThread* emu_thread) {
     motion_emu = std::make_unique<Motion::MotionEmu>(*this);
+    stylus = std::make_unique<Stylus>();
+    overlay = std::make_unique<Overlay>(this);
     this->emu_thread = emu_thread;
     child->DisablePainting();
+    overlay->show();
 }
 
 void GRenderWindow::OnEmulationStopping() {
     motion_emu = nullptr;
+    stylus = nullptr;
     emu_thread = nullptr;
+    overlay = nullptr;
     child->EnablePainting();
 }
 

src/citra_qt/bootmanager.h

@@ -9,10 +9,13 @@
 #include <mutex>
 #include <QGLWidget>
 #include <QThread>
+#include "citra_qt/overlay.h"
+#include "citra_qt/stylus.h"
 #include "common/thread.h"
 #include "core/frontend/emu_window.h"
 #include "core/frontend/motion_emu.h"
 
+
 class QKeyEvent;
 class QScreen;
 
@@ -127,12 +130,14 @@ public:
     void mousePressEvent(QMouseEvent* event) override;
     void mouseMoveEvent(QMouseEvent* event) override;
     void mouseReleaseEvent(QMouseEvent* event) override;
+    void GRenderWindow::mouseDoubleClickEvent(QMouseEvent* e) override;
 
     void focusOutEvent(QFocusEvent* event) override;
 
     void OnClientAreaResized(unsigned width, unsigned height);
 
     void InitRenderTarget();
+    Stylus* GRenderWindow::GetStylus();
 
 public slots:
     void moveContext(); // overridden
@@ -146,6 +151,8 @@ signals:
     void Closed();
 
 private:
+    void UpdateStylusPosition(QMouseEvent* event);
+    void UpdateStylusSize(unsigned width, unsigned height);
     void OnMinimalClientAreaChangeRequest(
         const std::pair<unsigned, unsigned>& minimal_size) override;
 
@@ -157,6 +164,10 @@ private:
 
     /// Motion sensors emulation
     std::unique_ptr<Motion::MotionEmu> motion_emu;
+    std::unique_ptr<Stylus> stylus;
+    std::unique_ptr<Overlay> overlay;
+
+    StylusState stylus_state;
 
 protected:
     void showEvent(QShowEvent* event) override;

src/citra_qt/citra-qt.rc

@@ -6,4 +6,3 @@
 // Icon with lowest ID value placed first to ensure application icon
 // remains consistent on all systems.
 CITRA_ICON              ICON                    "../../dist/citra.ico"
-

src/citra_qt/main.cpp

@@ -71,10 +71,13 @@ GMainWindow::GMainWindow() : config(new Config()), emu_thread(nullptr) {
 
     setWindowTitle(QString("Citra %1| %2-%3")
                        .arg(Common::g_build_name, Common::g_scm_branch, Common::g_scm_desc));
+    setWindowIcon(QIcon(":citra_icon.ico"));
+    setMouseTracking(true);
+    centralWidget()->setMouseTracking(true);
     show();
 
     game_list->PopulateAsync(UISettings::values.gamedir, UISettings::values.gamedir_deepscan);
-
+    game_list->setMouseTracking(true);
     QStringList args = QApplication::arguments();
     if (args.length() >= 2) {
         BootGame(args[1]);

src/citra_qt/overlay.cpp

@@ -0,0 +1,66 @@
+
+#include <QPainter>
+
+#include "citra_qt/bootmanager.h"
+#include "citra_qt/overlay.h"
+#include "citra_qt/stylus.h"
+
+Overlay::Overlay(GRenderWindow* parent) : QWidget(parent), parent(parent) {
+    // setWindowFlags(Qt::Widget | Qt::FramelessWindowHint);
+    // // setParent(0); // Create TopLevel-Widget
+    // setAttribute(Qt::WA_NoSystemBackground, true);
+    // setAttribute(Qt::WA_TranslucentBackground, true);
+    // setWindowFlags(Qt::FramelessWindowHint | Qt::ToolTip | Qt::WindowStaysOnTopHint);
+    setWindowFlags(Qt::FramelessWindowHint | Qt::ToolTip);
+    setAttribute(Qt::WA_TransparentForMouseEvents);
+    setAttribute(Qt::WA_NoSystemBackground);
+    setAttribute(Qt::WA_TranslucentBackground);
+    setFocusPolicy(Qt::NoFocus);
+    setMouseTracking(false);
+    move(0, 0);
+}
+
+void Overlay::keyPressEvent(QKeyEvent* event) {
+    parent->keyPressEvent(event);
+}
+
+void Overlay::keyReleaseEvent(QKeyEvent* event) {
+    parent->keyReleaseEvent(event);
+}
+
+void Overlay::mousePressEvent(QMouseEvent* event) {
+    parent->mousePressEvent(event);
+    QApplication::setActiveWindow(window());
+}
+void Overlay::mouseMoveEvent(QMouseEvent* event) {
+    parent->mouseMoveEvent(event);
+    // window()->setFocus();
+    QApplication::setActiveWindow(window());
+}
+void Overlay::mouseDoubleClickEvent(QMouseEvent* event) {
+    parent->mouseDoubleClickEvent(event);
+    // window()->setFocus();
+    QApplication::setActiveWindow(window());
+}
+void Overlay::mouseReleaseEvent(QMouseEvent* event) {
+    parent->mouseReleaseEvent(event);
+    // window()->setFocus();
+    QApplication::setActiveWindow(window());
+}
+
+void Overlay::paintEvent(QPaintEvent* ev) {
+    Stylus* stylus = parent->GetStylus();
+    QPainter painter(this);
+    // set the viewport to match the screen size
+    // painter.setWindow(QApplication::desktop()->screenGeometry());
+    painter.setRenderHint(QPainter::Antialiasing, true);
+    // painter.setPen(palette().dark().color());
+    // painter.setBrush(Qt::NoBrush);
+    // painter.drawRect(QRect(0, 0, width() - 1, height() - 1));
+    // QPoint p = stylus->GetTouchPoint();
+    // painter.drawEllipse(QRect(p.x() - 5, p.y() - 5, 10, 10));
+    QTransform t = stylus->GetTransform();
+    painter.setTransform(t);
+    painter.drawPixmap(0, 0, stylus->GetPix());
+    // painter.drawRect(0, 0, stylus->GetRect().width(), stylus->GetRect().height());
+}

src/citra_qt/overlay.h

@@ -0,0 +1,32 @@
+// Copyright 20XX Citra Emulator Project
+// Licensed under GPLv2 or any later version (but not GPLv824 cause that one sucked)
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <QPainter>
+#include <QtWidgets>
+
+class Stylus;
+class GRenderWindow;
+
+class Overlay : public QWidget {
+    Q_OBJECT
+public:
+    Overlay(GRenderWindow* parent);
+
+protected:
+    void paintEvent(QPaintEvent* ev) override;
+
+    void mousePressEvent(QMouseEvent* event) override;
+    void mouseReleaseEvent(QMouseEvent* event) override;
+    void mouseMoveEvent(QMouseEvent* event) override;
+    void mouseDoubleClickEvent(QMouseEvent* event) override;
+
+    void keyPressEvent(QKeyEvent* event);
+    void keyReleaseEvent(QKeyEvent* event);
+
+private:
+    GRenderWindow* parent;
+    Stylus* stylus;
+};

src/citra_qt/resources.qrc

@@ -0,0 +1,7 @@
+<!DOCTYPE RCC>
+<RCC version="1.0">
+<qresource>
+    <file alias="citra_icon.ico">../../dist/citra.ico</file>
+    <file alias="stylus.png">../../dist/stylus.png</file>
+</qresource>
+</RCC>

src/citra_qt/stylus.cpp

@@ -0,0 +1,84 @@
+#include <QApplication>
+#include <QtWidgets>
+
+#include "citra_qt/stylus.h"
+#include "common/logging/log.h"
+
+Stylus::Stylus() {
+    pic = QPixmap(":stylus.png");
+    x = QApplication::desktop()->screenGeometry().width() / 2;
+    y = QApplication::desktop()->screenGeometry().height() / 2;
+    rotation = 0;
+}
+
+Stylus::~Stylus() {}
+
+void Stylus::SetHoldPoint(QPoint p) {
+    // hold = QPoint(p.x() - x, p.y() - y);
+    diff = QPoint(p.x() - x, p.y() - y);
+    hold = p;
+}
+
+void Stylus::Rotate(QPoint n) {
+    QPoint center = GetCenter();
+    // get the angle from the center to the initial click point
+    qreal init_x = hold.x() - center.x();
+    qreal init_y = hold.y() - center.y();
+    qreal initial_angle = std::atan2(init_y, init_x);
+    qreal x = n.x() - center.x();
+    qreal y = n.y() - center.y();
+
+    qreal mv_angle = std::atan2(y, x);
+
+    // get the changed angle
+    rotation = (mv_angle - initial_angle) * 180 / M_PI;
+
+    if (std::fabs(rotation) > 360.0) {
+        rotation = fmod(rotation, 360);
+    }
+}
+
+bool Stylus::IsNearCenter(QPoint n) {
+    // QPoint c = GetCenter();
+    // return (n - c).manhattanLength() < 75;
+    // jokes on you i'm now checking to see if its the top half or the bottom half
+    QTransform t = GetTransform();
+    QRect top = QRect(0, 0, pic.width(), pic.height() / 2);
+    return t.mapRect(top).contains(n);
+}
+
+QPoint Stylus::GetCenter() {
+    return QRect(x, y, pic.width(), pic.height()).center();
+}
+
+void Stylus::UpdatePosition(unsigned x, unsigned y) {
+    this->x = x - diff.x();
+    this->y = y - diff.y();
+}
+
+QTransform Stylus::GetTransform() {
+    QTransform t;
+    t.translate(x, y);
+    t.rotate(rotation);
+    return t;
+}
+
+QRect Stylus::GetRect() {
+    return QRect(0, 0, pic.width(), pic.height());
+}
+
+QPolygon Stylus::GetPoly() {
+    QTransform t = GetTransform();
+    QRect r = GetRect();
+    return t.mapToPolygon(r);
+}
+
+bool Stylus::Contains(QPoint pos) {
+    QPolygon p = GetPoly();
+    return p.containsPoint(pos, Qt::OddEvenFill);
+}
+
+QPoint Stylus::GetTouchPoint() {
+    QTransform t = GetTransform();
+    return t.map(QPoint(pic.width() / 2, pic.height() - 10));
+}

src/citra_qt/stylus.h

@@ -0,0 +1,61 @@
+// Copyright 20XX Citra Emulator Project
+// Licensed under GPLv2 or any later version (but not GPLv824 cause that one sucked)
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <QPainter>
+#include <QVector2D>
+
+enum class StylusState : unsigned { ROTATE, HOLD, DROP, COUNT };
+
+class Stylus {
+public:
+    Stylus();
+    ~Stylus();
+    StylusState GetState() {
+        return state;
+    }
+
+    void Rotate(QPoint p);
+
+    void SetHoldPoint(QPoint p);
+
+    void SetState(StylusState s) {
+        state = s;
+    }
+    void UpdatePosition(unsigned x, unsigned y);
+    unsigned GetX() {
+        return x;
+    }
+    unsigned GetY() {
+        return y;
+    }
+
+    QPixmap GetPix() {
+        return pic;
+    }
+
+    QTransform GetTransform();
+
+    QPolygon GetPoly();
+
+    bool Contains(QPoint p);
+
+    QPoint GetTouchPoint();
+
+    bool IsNearCenter(QPoint p);
+
+    QPoint GetCenter();
+
+    QRect GetRect();
+
+private:
+    unsigned x;
+    unsigned y;
+    qreal rotation;
+    QPixmap pic;
+    StylusState state;
+    QPoint hold;
+    QPoint diff;
+};