Services/HLE: Implement PrepareToCloseLibraryApplet and CloseLibraryApplet.

This allows LLE library applets (like swkbd) to properly close and return to the application instead of hanging.

There is still a bug in our rasterizer cache that may cause crashes some time after an applet is closed, but that is tangential to this change and should be tackled separately.

src/core/hle/service/apt/applet_manager.cpp

@@ -390,6 +390,49 @@ ResultCode AppletManager::StartLibraryApplet(AppletId applet_id,
     }
 }
 
+ResultCode AppletManager::PrepareToCloseLibraryApplet(bool not_pause, bool exiting,
+                                                      bool jump_home) {
+    if (next_parameter) {
+        return ResultCode(ErrCodes::ParameterPresent, ErrorModule::Applet,
+                          ErrorSummary::InvalidState, ErrorLevel::Status);
+    }
+
+    if (!not_pause)
+        library_applet_closing_command = SignalType::WakeupByPause;
+    else if (jump_home)
+        library_applet_closing_command = SignalType::WakeupToJumpHome;
+    else if (exiting)
+        library_applet_closing_command = SignalType::WakeupByCancel;
+    else
+        library_applet_closing_command = SignalType::WakeupByExit;
+
+    return RESULT_SUCCESS;
+}
+
+ResultCode AppletManager::CloseLibraryApplet(Kernel::SharedPtr<Kernel::Object> object,
+                                             std::vector<u8> buffer) {
+    auto& slot = applet_slots[static_cast<size_t>(AppletSlot::LibraryApplet)];
+
+    MessageParameter param;
+    // TODO(Subv): The destination id should be the "current applet slot id", which changes
+    // constantly depending on what is going on in the system. Most of the time it is the running
+    // application, but it could be something else if a system applet is launched.
+    param.destination_id = AppletId::Application;
+    param.sender_id = slot.applet_id;
+    param.object = std::move(object);
+    param.signal = library_applet_closing_command;
+    param.buffer = std::move(buffer);
+
+    ResultCode result = SendParameter(param);
+
+    if (library_applet_closing_command != SignalType::WakeupByPause) {
+        // TODO(Subv): Terminate the running applet title
+        slot.Reset();
+    }
+
+    return result;
+}
+
 ResultVal<AppletManager::AppletInfo> AppletManager::GetAppletInfo(AppletId app_id) {
     const auto* slot = GetAppletSlotData(app_id);
 

src/core/hle/service/apt/applet_manager.h

@@ -128,6 +128,8 @@ public:
     ResultCode FinishPreloadingLibraryApplet(AppletId applet_id);
     ResultCode StartLibraryApplet(AppletId applet_id, Kernel::SharedPtr<Kernel::Object> object,
                                   const std::vector<u8>& buffer);
+    ResultCode PrepareToCloseLibraryApplet(bool not_pause, bool exiting, bool jump_home);
+    ResultCode CloseLibraryApplet(Kernel::SharedPtr<Kernel::Object> object, std::vector<u8> buffer);
 
     struct AppletInfo {
         u64 title_id;
@@ -164,6 +166,12 @@ private:
         AppletAttributes attributes;
         Kernel::SharedPtr<Kernel::Event> notification_event;
         Kernel::SharedPtr<Kernel::Event> parameter_event;
+
+        void Reset() {
+            applet_id = AppletId::None;
+            registered = false;
+            attributes.raw = 0;
+        }
     };
 
     // Holds data about the concurrently running applets in the system.
@@ -172,6 +180,9 @@ private:
     // This overload returns nullptr if no applet with the specified id has been started.
     AppletSlotData* GetAppletSlotData(AppletId id);
     AppletSlotData* GetAppletSlotData(AppletAttributes attributes);
+
+    // Command that will be sent to the application when a library applet calls CloseLibraryApplet.
+    SignalType library_applet_closing_command;
 };
 
 } // namespace APT

src/core/hle/service/apt/apt.cpp

@@ -552,6 +552,31 @@ void Module::Interface::CancelLibraryApplet(Kernel::HLERequestContext& ctx) {
     LOG_WARNING(Service_APT, "(STUBBED) called exiting={}", exiting);
 }
 
+void Module::Interface::PrepareToCloseLibraryApplet(Kernel::HLERequestContext& ctx) {
+    IPC::RequestParser rp(ctx, 0x25, 3, 0); // 0x002500C0
+    bool not_pause = rp.Pop<bool>();
+    bool exiting = rp.Pop<bool>();
+    bool jump_to_home = rp.Pop<bool>();
+
+    LOG_DEBUG(Service_APT, "called not_pause={} exiting={} jump_to_home={}", not_pause, exiting,
+              jump_to_home);
+
+    IPC::RequestBuilder rb = rp.MakeBuilder(1, 0);
+    rb.Push(apt->applet_manager->PrepareToCloseLibraryApplet(not_pause, exiting, jump_to_home));
+}
+
+void Module::Interface::CloseLibraryApplet(Kernel::HLERequestContext& ctx) {
+    IPC::RequestParser rp(ctx, 0x28, 1, 4); // 0x00280044
+    u32 parameter_size = rp.Pop<u32>();
+    auto object = rp.PopGenericObject();
+    std::vector<u8> buffer = rp.PopStaticBuffer();
+
+    LOG_DEBUG(Service_APT, "called size={}", parameter_size);
+
+    IPC::RequestBuilder rb = rp.MakeBuilder(1, 0);
+    rb.Push(apt->applet_manager->CloseLibraryApplet(std::move(object), std::move(buffer)));
+}
+
 void Module::Interface::SendCaptureBufferInfo(Kernel::HLERequestContext& ctx) {
     IPC::RequestParser rp(ctx, 0x40, 1, 2); // 0x00400042
     u32 size = rp.Pop<u32>();

src/core/hle/service/apt/apt.h

@@ -420,6 +420,34 @@ public:
          */
         void CancelLibraryApplet(Kernel::HLERequestContext& ctx);
 
+        /**
+         * APT::PrepareToCloseLibraryApplet service function
+         *  Inputs:
+         *      0 : Command header [0x002500C0]
+         *      1 : bool, Not pause
+         *      2 : bool, Caller exiting
+         *      3 : bool, Jump to home
+         *  Outputs:
+         *      0 : Header code
+         *      1 : Result code
+         */
+        void PrepareToCloseLibraryApplet(Kernel::HLERequestContext& ctx);
+
+        /**
+         * APT::CloseLibraryApplet service function
+         *  Inputs:
+         *      0 : Command header [0x00280044]
+         *      1 : Buffer size
+         *      2 : 0x0
+         *      3 : Object handle
+         *      4 : (Size << 14) | 2
+         *      5 : Input buffer virtual address
+         *  Outputs:
+         *      0 : Header code
+         *      1 : Result code
+         */
+        void CloseLibraryApplet(Kernel::HLERequestContext& ctx);
+
         /**
          * APT::SendCaptureBufferInfo service function
          *  Inputs:

src/core/hle/service/apt/apt_a.cpp

@@ -46,10 +46,10 @@ APT_A::APT_A(std::shared_ptr<Module> apt)
         {0x00220040, nullptr, "PrepareToCloseApplication"},
         {0x00230040, nullptr, "PrepareToJumpToApplication"},
         {0x00240044, nullptr, "JumpToApplication"},
-        {0x002500C0, nullptr, "PrepareToCloseLibraryApplet"},
+        {0x002500C0, &APT_A::PrepareToCloseLibraryApplet, "PrepareToCloseLibraryApplet"},
         {0x00260000, nullptr, "PrepareToCloseSystemApplet"},
         {0x00270044, nullptr, "CloseApplication"},
-        {0x00280044, nullptr, "CloseLibraryApplet"},
+        {0x00280044, &APT_A::CloseLibraryApplet, "CloseLibraryApplet"},
         {0x00290044, nullptr, "CloseSystemApplet"},
         {0x002A0000, nullptr, "OrderToCloseSystemApplet"},
         {0x002B0000, nullptr, "PrepareToJumpToHomeMenu"},

src/core/hle/service/apt/apt_u.cpp

@@ -46,10 +46,10 @@ APT_U::APT_U(std::shared_ptr<Module> apt)
         {0x00220040, nullptr, "PrepareToCloseApplication"},
         {0x00230040, nullptr, "PrepareToJumpToApplication"},
         {0x00240044, nullptr, "JumpToApplication"},
-        {0x002500C0, nullptr, "PrepareToCloseLibraryApplet"},
+        {0x002500C0, &APT_U::PrepareToCloseLibraryApplet, "PrepareToCloseLibraryApplet"},
         {0x00260000, nullptr, "PrepareToCloseSystemApplet"},
         {0x00270044, nullptr, "CloseApplication"},
-        {0x00280044, nullptr, "CloseLibraryApplet"},
+        {0x00280044, &APT_U::CloseLibraryApplet, "CloseLibraryApplet"},
         {0x00290044, nullptr, "CloseSystemApplet"},
         {0x002A0000, nullptr, "OrderToCloseSystemApplet"},
         {0x002B0000, nullptr, "PrepareToJumpToHomeMenu"},