sync(upstream): integrate Batch 42 relinker, AGC driver, libc/libkernel and PRX exports

CMakeLists.txt

@@ -101,6 +101,10 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
             ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND})
         set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20)
+        add_test(NAME shader_cache_runtime_sources COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND})
+        set_tests_properties(shader_cache_runtime_sources PROPERTIES TIMEOUT 180)
         add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
         add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}
@@ -117,6 +121,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_unwrap_self.py)
         add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py)
+        add_test(NAME conventions_debt_length COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_debt_length.py)
         add_test(NAME test_registration COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_registration.py)
         add_test(NAME check_skips COMMAND ${Python3_EXECUTABLE}

CONTRIBUTING.md

@@ -8,7 +8,7 @@
 - A silent stub is allowed only when it unblocks a title and only affects the UI; add it to [silent stubs](docs/dev/TechnicalDebt.md#silent-stubs).
 - Missing imports fail at startup with a clear error; don't replace them with fallbacks that keep running.
 - Implement the general behaviour of a function, not what one title happens to need.
-- Don't add replacements for modules that titles ship themselves in `sce_module/`, `sce_modules/` or `prx/`: engine or middleware modules (Cohtml, FMOD, GOG Galaxy) and SDK libraries that only wrap other system libraries (NpCppWebApi over NpWebApi2). The relinker converts and loads the title's own module, and a host library with the same name is left out of `DT_NEEDED`. Only system libraries, which reach the kernel or the hardware, are reimplemented in [core/libs/prx](core/libs/prx).
+- Don't add replacements for modules that titles ship themselves in `sce_module/`, `sce_modules/` or `prx/`: engine or middleware modules (Cohtml, FMOD, GOG Galaxy) and SDK libraries that only wrap other system libraries. The relinker converts and loads the title's own module, and a host library with the same name is left out of `DT_NEEDED`. Only system libraries, which reach the kernel or the hardware, are reimplemented in [core/libs/prx](core/libs/prx). A host copy may still exist for a wrapped library that some dumps lack entirely, like libSceNpCppWebApi: the title's own module wins whenever one is shipped, so it is only ever consulted by titles that ship nothing.
 - Avoid non-standard extensions (`__attribute__`, etc.) where standard C++ is enough. Helper symbols that must not become NIDs use the `_nid_no_patch` or `_nid_no_patch_cut` suffix.
 - The relinker uses only the C++20 standard library.
 - Third-party code is added as a submodule under `3rdparty/` and built from source, not found on the system. With `ANYPS5_USE_SYSTEM_DEPS=ON` the packages packaged by the distributions (SDL2, freetype, Vulkan-Headers, SPIRV-Headers, glslang and SPIRV-Tools) come from the system instead; that resolution lives in [cmake/SystemDeps.cmake](cmake/SystemDeps.cmake) and the default build never reads the system.

core/libs/CMakeLists.txt

@@ -429,6 +429,16 @@ if(BUILD_TESTING)
     add_test(NAME host_thread_local COMMAND host_thread_local_tests)
     set_tests_properties(host_thread_local PROPERTIES TIMEOUT 15)
 
+    add_test_executable(guest_np_cpp_web_api_intrusive_ptr_tests tests/GuestNpCppWebApiIntrusivePtr.cpp)
+    target_include_directories(guest_np_cpp_web_api_intrusive_ptr_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_cpp_web_api_intrusive_ptr_tests PRIVATE libSceNpCppWebApi libc)
+    configure_windows_unwind(guest_np_cpp_web_api_intrusive_ptr_tests)
+    add_test(NAME guest_np_cpp_web_api_intrusive_ptr COMMAND guest_np_cpp_web_api_intrusive_ptr_tests)
+    if(WIN32)
+        set_tests_properties(guest_np_cpp_web_api_intrusive_ptr PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCppWebApi>")
+    endif()
+
     add_test_executable(savedata_core_write_replacement_tests tests/SaveDataWriteReplacement.cpp)
     target_include_directories(savedata_core_write_replacement_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_link_libraries(savedata_core_write_replacement_tests PRIVATE libSceSaveData)
@@ -498,4 +508,24 @@ if(BUILD_TESTING)
         set_tests_properties(guest_acm PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceAcm>")
     endif()
+
+    add_test_executable(guest_voice_chat_tests tests/GuestVoiceChat.cpp)
+    target_include_directories(guest_voice_chat_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_voice_chat_tests PRIVATE libSceVoiceChat libc)
+    configure_windows_unwind(guest_voice_chat_tests)
+    add_test(NAME guest_voice_chat COMMAND guest_voice_chat_tests)
+    if(WIN32)
+        set_tests_properties(guest_voice_chat PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVoiceChat>")
+    endif()
+
+    add_test_executable(guest_np_game_intent_tests tests/GuestNpGameIntent.cpp)
+    target_include_directories(guest_np_game_intent_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_link_libraries(guest_np_game_intent_tests PRIVATE libSceNpGameIntent libc)
+    configure_windows_unwind(guest_np_game_intent_tests)
+    add_test(NAME guest_np_game_intent COMMAND guest_np_game_intent_tests)
+    if(WIN32)
+        set_tests_properties(guest_np_game_intent PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpGameIntent>")
+    endif()
 endif()

core/libs/SceTypes.hpp

@@ -1410,6 +1410,11 @@ struct NpGameIntentData {
     std::uint8_t padding[7];
 };
 
+struct NpGameIntentInitParam {
+    std::size_t size;
+    std::uint8_t reserved[32];
+};
+
 struct NpGameIntentInfo {
     std::size_t size;
     std::int32_t user_id;

core/libs/nid/src/main.cpp

@@ -18,6 +18,8 @@ void _writeFile(const std::string& path, const std::vector<std::uint8_t>& data)
     std::ofstream f(path, std::ios::binary);
     if (!f) throw std::runtime_error("cannot write: " + path);
     f.write(reinterpret_cast<const char*>(data.data()), static_cast<std::streamsize>(data.size()));
+    f.close();
+    if (!f) throw std::runtime_error("cannot write: " + path);
 }
 
 }

core/libs/prx/libSceAbstractLocal/Export.cpp

@@ -0,0 +1,10 @@
+#include <cstddef>
+#include <cstdint>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+APS5_DUMMY_FUN
+
+}

core/libs/prx/libSceAbstractStorage/Export.cpp

@@ -0,0 +1,10 @@
+#include <cstddef>
+#include <cstdint>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+APS5_DUMMY_FUN
+
+}

core/libs/prx/libSceAbstractTcs/Export.cpp

@@ -0,0 +1,10 @@
+#include <cstddef>
+#include <cstdint>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+APS5_DUMMY_FUN
+
+}

core/libs/prx/libSceAbstractTwitter/Export.cpp

@@ -0,0 +1,10 @@
+#include <cstddef>
+#include <cstdint>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+APS5_DUMMY_FUN
+
+}

core/libs/prx/libSceAbstractYoutube/Export.cpp

@@ -0,0 +1,10 @@
+#include <cstddef>
+#include <cstdint>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+APS5_DUMMY_FUN
+
+}

core/libs/prx/libSceAc3Enc/Export.cpp

@@ -0,0 +1,10 @@
+#include <cstdint>
+#include <cstddef>
+#include "SceTypes.hpp"
+#include "prx/libc/include/General.hpp"
+
+extern "C" {
+
+APS5_DUMMY_FUN
+
+}

core/libs/prx/libSceAgc/Cb/include/Sync.hpp

@@ -7,7 +7,7 @@
 struct CommandBuffer;
 
 extern "C" {
-std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address);
+std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint32_t operation, std::uint32_t modifier, std::uint32_t value);
 }
 
 #endif

core/libs/prx/libSceAgc/Cb/src/Sync.cpp

@@ -1,6 +1,5 @@
 #include "prx/libSceAgc/Cb/include/Sync.hpp"
 
-#include "prx/libSceAgc/Command/include/Control.hpp"
 #include "prx/libSceAgc/Command/include/Packet.hpp"
 #include <cstdio>
 #include <cstdlib>
@@ -59,11 +58,13 @@ std::uint32_t* APS5_VABI sceAgcCbReleaseMem(CommandBuffer* buf, std::uint8_t act
     return packet;
 }
 
-std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address) {
-    const std::uint8_t operation = action != 0 ? 7u : 6u;
-    const std::uint8_t signalType = clientCode & 1u;
-    const std::uint8_t waitForMailbox = (clientCode >> 1u) & 1u;
-    return Agc::Command::WriteMemSemaphore(buf, address, waitForMailbox, signalType, operation, __func__);
+std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint32_t operation, std::uint32_t modifier, std::uint32_t value) {
+    (void)operation;
+    (void)modifier;
+    (void)value;
+    const auto guestAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(address));
+    const auto maskedAddress = guestAddress & ~0xfULL;
+    return Agc::Command::Emit(buf, 0x39u, {static_cast<std::uint32_t>(maskedAddress), static_cast<std::uint32_t>(maskedAddress >> 32u), 0x80100000u}, __func__);
 }
 
 }

core/libs/prx/libSceAgc/tests/Semaphore.cpp

@@ -14,7 +14,7 @@
 
 extern "C" int APS5_VABI sceAgcSetAmmSemaphoreMemory(void* memory, std::uint64_t sizeInBytes);
 extern "C" int APS5_VABI sceAgcGetSemaphoreLabel(std::uint32_t index, void** label);
-extern "C" std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address);
+extern "C" std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint32_t operation, std::uint32_t modifier, std::uint32_t value);
 
 namespace {
 
@@ -145,34 +145,49 @@ void testMemSemaphore() {
     check(sceAgcGetSemaphoreLabel(0, &label) == 0 && label != nullptr, "failed to get label for memory semaphore test");
     const auto guestAddress = reinterpret_cast<std::uintptr_t>(label);
 
-    for (std::uint8_t action = 0; action <= 1; ++action) {
-        for (std::uint8_t clientCode = 0; clientCode <= 3; ++clientCode) {
-            std::array<std::uint32_t, 16> words{};
-            words.fill(0xdeadbeefu);
-            CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
-
-            auto* packet = sceAgcCbMemSemaphore(&buffer, action, clientCode, label);
-            check(packet == words.data(), "packet address mismatch");
-            check(packet[0] == 0xc0023900u, "MEM_SEMAPHORE header opcode mismatch");
-            check(packet[1] == static_cast<std::uint32_t>(guestAddress), "MEM_SEMAPHORE address low mismatch");
-            check(packet[2] == static_cast<std::uint32_t>(guestAddress >> 32u), "MEM_SEMAPHORE address high mismatch");
-
-            const std::uint32_t expectedOp = action != 0 ? 7u : 6u;
-            const std::uint32_t expectedSignal = clientCode & 1u;
-            const std::uint32_t expectedMailbox = (clientCode >> 1u) & 1u;
-            const std::uint32_t expectedControl = (expectedOp << 29u) | (expectedSignal << 20u) | (expectedMailbox << 16u);
-            check(packet[3] == expectedControl, "MEM_SEMAPHORE control word mismatch");
-            check(buffer.cursor_up == words.data() + 4, "cursor did not advance by 4 words");
-            check(words[4] == 0xdeadbeefu, "words past packet modified");
-        }
+    {
+        std::array<std::uint32_t, 16> words{};
+        words.fill(0xdeadbeefu);
+        CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+        const auto* packet = sceAgcCbMemSemaphore(&buffer, reinterpret_cast<const volatile void*>(0x1111111122222222ull), 0x33333333u, 0x44444444u, 0);
+        check(packet == words.data(), "packet address mismatch");
+        check(packet[0] == 0xc0023900u, "MEM_SEMAPHORE header opcode mismatch");
+        check(packet[1] == 0x22222220u, "MEM_SEMAPHORE address low mismatch");
+        check(packet[2] == 0x11111111u, "MEM_SEMAPHORE address high mismatch");
+        check(packet[3] == 0x80100000u, "MEM_SEMAPHORE control word mismatch");
+        check(buffer.cursor_up == words.data() + 4, "cursor did not advance by 4 words");
+        check(words[4] == 0xdeadbeefu, "words past packet modified");
     }
 
-    std::array<std::uint32_t, 16> words{};
-    CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
-    expectFailure([&] { sceAgcCbMemSemaphore(nullptr, 0, 0, label); });
-    expectFailure([&] { sceAgcCbMemSemaphore(&buffer, 0, 0, nullptr); });
-    auto* unalignedLabel = reinterpret_cast<void*>(guestAddress + 1);
-    expectFailure([&] { sceAgcCbMemSemaphore(&buffer, 0, 0, unalignedLabel); });
+    {
+        std::array<std::uint32_t, 16> words{};
+        words.fill(0xdeadbeefu);
+        CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+        const auto* packet = sceAgcCbMemSemaphore(&buffer, reinterpret_cast<const volatile void*>(1ull), 2, 3, 4);
+        check(packet == words.data(), "packet address mismatch");
+        check(packet[0] == 0xc0023900u, "MEM_SEMAPHORE header opcode mismatch");
+        check(packet[1] == 0u, "MEM_SEMAPHORE address low mismatch");
+        check(packet[2] == 0u, "MEM_SEMAPHORE address high mismatch");
+        check(packet[3] == 0x80100000u, "MEM_SEMAPHORE control word mismatch");
+        check(buffer.cursor_up == words.data() + 4, "cursor did not advance by 4 words");
+        check(words[4] == 0xdeadbeefu, "words past packet modified");
+    }
+
+    {
+        std::array<std::uint32_t, 16> words{};
+        words.fill(0xdeadbeefu);
+        CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
+        const auto* packet = sceAgcCbMemSemaphore(&buffer, label, 0x12345678u, 0x9abcdef0u, 0x55aa55aau);
+        check(packet == words.data(), "packet address mismatch");
+        check(packet[0] == 0xc0023900u, "MEM_SEMAPHORE header opcode mismatch");
+        check(packet[1] == static_cast<std::uint32_t>(guestAddress & ~static_cast<std::uintptr_t>(0xfu)), "MEM_SEMAPHORE address low mismatch");
+        check(packet[2] == static_cast<std::uint32_t>(guestAddress >> 32u), "MEM_SEMAPHORE address high mismatch");
+        check(packet[3] == 0x80100000u, "MEM_SEMAPHORE control word mismatch");
+        check(buffer.cursor_up == words.data() + 4, "cursor did not advance by 4 words");
+        check(words[4] == 0xdeadbeefu, "words past packet modified");
+    }
+
+    expectFailure([&] { sceAgcCbMemSemaphore(nullptr, label, 0, 0, 0); });
 }
 
 }