diff --git a/src/yuzu/configuration/configure_input_player.cpp b/src/yuzu/configuration/configure_input_player.cpp
index 9a61a6e159..8c5921eb6e 100644
--- a/src/yuzu/configuration/configure_input_player.cpp
+++ b/src/yuzu/configuration/configure_input_player.cpp
@@ -256,6 +256,11 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
         ui->buttonSL,       ui->buttonSR,     ui->buttonHome,      ui->buttonScreenshot,
     };
 
+    mod_buttons = {
+        ui->buttonLStickMod,
+        ui->buttonRStickMod,
+    };
+
     analog_map_buttons = {{
         {
             ui->buttonLStickUp,
@@ -311,11 +316,25 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
 
     for (int button_id = 0; button_id < Settings::NativeButton::NumButtons; ++button_id) {
         auto* const button = button_map[button_id];
+
         if (button == nullptr) {
             continue;
         }
+
         ConfigureButtonClick(button_map[button_id], &buttons_param[button_id],
                              Config::default_buttons[button_id]);
+
+        button->setContextMenuPolicy(Qt::CustomContextMenu);
+
+        connect(button, &QPushButton::customContextMenuRequested,
+                [=, this](const QPoint& menu_location) {
+                    QMenu context_menu;
+                    context_menu.addAction(tr("Clear"), [&] {
+                        buttons_param[button_id].Clear();
+                        button_map[button_id]->setText(tr("[not set]"));
+                    });
+                    context_menu.exec(button_map[button_id]->mapToGlobal(menu_location));
+                });
     }
 
     for (int motion_id = 0; motion_id < Settings::NativeMotion::NumMotions; ++motion_id) {
@@ -324,15 +343,11 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
             continue;
         }
 
+        ConfigureButtonClick(motion_map[motion_id], &motions_param[motion_id],
+                             Config::default_motions[motion_id]);
+
         button->setContextMenuPolicy(Qt::CustomContextMenu);
-        connect(button, &QPushButton::clicked, [=, this] {
-            HandleClick(
-                motion_map[motion_id],
-                [=, this](Common::ParamPackage params) {
-                    motions_param[motion_id] = std::move(params);
-                },
-                InputCommon::Polling::DeviceType::Motion);
-        });
+
         connect(button, &QPushButton::customContextMenuRequested,
                 [=, this](const QPoint& menu_location) {
                     QMenu context_menu;
@@ -344,10 +359,6 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
                 });
     }
 
-    // Handle clicks for the modifier buttons as well.
-    ConfigureButtonClick(ui->buttonLStickMod, &lstick_mod, Config::default_stick_mod[0]);
-    ConfigureButtonClick(ui->buttonRStickMod, &rstick_mod, Config::default_stick_mod[1]);
-
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
             auto* const analog_button = analog_map_buttons[analog_id][sub_button_id];
@@ -365,8 +376,37 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
                     },
                     InputCommon::Polling::DeviceType::AnalogPreferred);
             });
+
+            analog_button->setContextMenuPolicy(Qt::CustomContextMenu);
+
+            connect(analog_button, &QPushButton::customContextMenuRequested,
+                    [=, this](const QPoint& menu_location) {
+                        QMenu context_menu;
+                        context_menu.addAction(tr("Clear"), [&] {
+                            analogs_param[analog_id].Clear();
+                            analog_map_buttons[analog_id][sub_button_id]->setText(tr("[not set]"));
+                        });
+                        context_menu.exec(analog_map_buttons[analog_id][sub_button_id]->mapToGlobal(
+                            menu_location));
+                    });
         }
 
+        // Handle clicks for the modifier buttons as well.
+        ConfigureButtonClick(mod_buttons[analog_id], &stick_mod_param[analog_id],
+                             Config::default_stick_mod[analog_id]);
+
+        mod_buttons[analog_id]->setContextMenuPolicy(Qt::CustomContextMenu);
+
+        connect(mod_buttons[analog_id], &QPushButton::customContextMenuRequested,
+                [=, this](const QPoint& menu_location) {
+                    QMenu context_menu;
+                    context_menu.addAction(tr("Clear"), [&] {
+                        stick_mod_param[analog_id].Clear();
+                        mod_buttons[analog_id]->setText(tr("[not set]"));
+                    });
+                    context_menu.exec(mod_buttons[analog_id]->mapToGlobal(menu_location));
+                });
+
         connect(analog_map_range_spinbox[analog_id], qOverload<int>(&QSpinBox::valueChanged),
                 [=, this] {
                     const auto spinbox_value = analog_map_range_spinbox[analog_id]->value();
@@ -585,19 +625,16 @@ void ConfigureInputPlayer::RestoreDefaults() {
             InputCommon::GenerateKeyboardParam(Config::default_buttons[button_id])};
     }
 
-    // Reset Modifier Buttons
-    lstick_mod =
-        Common::ParamPackage(InputCommon::GenerateKeyboardParam(Config::default_stick_mod[0]));
-    rstick_mod =
-        Common::ParamPackage(InputCommon::GenerateKeyboardParam(Config::default_stick_mod[1]));
-
-    // Reset Analogs
+    // Reset Analogs and Modifier Buttons
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
             Common::ParamPackage params{InputCommon::GenerateKeyboardParam(
                 Config::default_analogs[analog_id][sub_button_id])};
             SetAnalogParam(params, analogs_param[analog_id], analog_sub_buttons[sub_button_id]);
         }
+
+        stick_mod_param[analog_id] = Common::ParamPackage(
+            InputCommon::GenerateKeyboardParam(Config::default_stick_mod[analog_id]));
     }
 
     for (int motion_id = 0; motion_id < Settings::NativeMotion::NumMotions; ++motion_id) {
@@ -613,30 +650,29 @@ void ConfigureInputPlayer::RestoreDefaults() {
 void ConfigureInputPlayer::ClearAll() {
     for (int button_id = 0; button_id < Settings::NativeButton::NumButtons; ++button_id) {
         const auto* const button = button_map[button_id];
-        if (button == nullptr || !button->isEnabled()) {
+        if (button == nullptr) {
             continue;
         }
 
         buttons_param[button_id].Clear();
     }
 
-    lstick_mod.Clear();
-    rstick_mod.Clear();
-
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
             const auto* const analog_button = analog_map_buttons[analog_id][sub_button_id];
-            if (analog_button == nullptr || !analog_button->isEnabled()) {
+            if (analog_button == nullptr) {
                 continue;
             }
 
             analogs_param[analog_id].Clear();
         }
+
+        stick_mod_param[analog_id].Clear();
     }
 
     for (int motion_id = 0; motion_id < Settings::NativeMotion::NumMotions; ++motion_id) {
-        const auto* const button = motion_map[motion_id];
-        if (button == nullptr || !button->isEnabled()) {
+        const auto* const motion_button = motion_map[motion_id];
+        if (motion_button == nullptr) {
             continue;
         }
 
@@ -656,9 +692,6 @@ void ConfigureInputPlayer::UpdateUI() {
         motion_map[motion_id]->setText(ButtonToText(motions_param[motion_id]));
     }
 
-    ui->buttonLStickMod->setText(ButtonToText(lstick_mod));
-    ui->buttonRStickMod->setText(ButtonToText(rstick_mod));
-
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
             auto* const analog_button = analog_map_buttons[analog_id][sub_button_id];
@@ -671,6 +704,8 @@ void ConfigureInputPlayer::UpdateUI() {
                 AnalogToText(analogs_param[analog_id], analog_sub_buttons[sub_button_id]));
         }
 
+        mod_buttons[analog_id]->setText(ButtonToText(stick_mod_param[analog_id]));
+
         const auto deadzone_label = analog_map_deadzone_label[analog_id];
         const auto deadzone_slider = analog_map_deadzone_slider[analog_id];
         const auto modifier_groupbox = analog_map_modifier_groupbox[analog_id];
diff --git a/src/yuzu/configuration/configure_input_player.h b/src/yuzu/configuration/configure_input_player.h
index b343f2c1d8..ce443dec59 100644
--- a/src/yuzu/configuration/configure_input_player.h
+++ b/src/yuzu/configuration/configure_input_player.h
@@ -131,6 +131,7 @@ private:
 
     std::array<Common::ParamPackage, Settings::NativeButton::NumButtons> buttons_param;
     std::array<Common::ParamPackage, Settings::NativeAnalog::NumAnalogs> analogs_param;
+    std::array<Common::ParamPackage, Settings::NativeAnalog::NumAnalogs> stick_mod_param;
     std::array<Common::ParamPackage, Settings::NativeMotion::NumMotions> motions_param;
 
     static constexpr int ANALOG_SUB_BUTTONS_NUM = 4;
@@ -140,8 +141,7 @@ private:
     /// Each motion input is represented by a QPushButton.
     std::array<QPushButton*, Settings::NativeMotion::NumMotions> motion_map;
     /// Extra buttons for the modifiers.
-    Common::ParamPackage lstick_mod;
-    Common::ParamPackage rstick_mod;
+    std::array<QPushButton*, Settings::NativeAnalog::NumAnalogs> mod_buttons;
 
     /// A group of four QPushButtons represent one analog input. The buttons each represent up,
     /// down, left, right, respectively.