fix(libc): check allocation result in vasprintf and guard null pointer in strndup

core/libs/prx/libc/CMakeLists.txt

@@ -61,6 +61,15 @@ if(WIN32)
 endif()
 
 if(BUILD_TESTING)
+    add_test_executable(guest_allocating_strings_tests ${LIBS_INCLUDE_DIR}/tests/GuestAllocatingStrings.cpp)
+    target_include_directories(guest_allocating_strings_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_allocating_strings_tests PRIVATE libc)
+    configure_windows_unwind(guest_allocating_strings_tests)
+    add_test(NAME guest_allocating_strings COMMAND guest_allocating_strings_tests)
+    if(WIN32)
+        set_tests_properties(guest_allocating_strings PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test_executable(guest_array_callbacks_tests ${LIBS_INCLUDE_DIR}/tests/GuestArrayCallbacks.cpp)
     target_include_directories(guest_array_callbacks_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_array_callbacks_tests PRIVATE libc)

core/libs/prx/libc/src/AllocatingStrings.cpp

@@ -11,6 +11,7 @@ extern "C" {
 int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaList* args);
 
 char* APS5_VABI strndup_nid_postfix(const char* source, std::size_t limit) {
+    if (!source) { errno = 22; return nullptr; }
     std::size_t length = 0;
     while (length < limit && source[length] != '\0') ++length;
     try {

core/libs/prx/libc/src/Formatting.cpp

@@ -229,6 +229,10 @@ int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaLi
         const char* source = guestWide ? guestText.c_str() : text;
 #endif
         auto* output = static_cast<char*>(ApplicationHeapAllocate_nid_no_patch(static_cast<size_t>(count) + 1));
+        if (!output) {
+            errno = 12;
+            return -1;
+        }
         std::memcpy(output, source, static_cast<size_t>(count) + 1);
         *destination = output;
         return count;

core/libs/tests/GuestAllocatingStrings.cpp

@@ -0,0 +1,138 @@
+#include "prx/libc/include/ApplicationHeap.hpp"
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include "SceTypes.hpp"
+#include <array>
+#include <cstdarg>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+
+extern "C" {
+char* APS5_VABI strndup_nid_postfix(const char*, std::size_t);
+char* APS5_VABI strdup_nid_postfix(const char*);
+int APS5_VABI asprintf_nid_postfix(char**, const char*, ...);
+int APS5_VABI vasprintf_nid_postfix(char**, const char*, VaList*);
+void APS5_VABI free_nid_postfix(void*);
+int* APS5_VABI __error_nid_postfix();
+}
+
+namespace {
+
+void Require(bool value) {
+    if (!value) std::abort();
+}
+
+bool failAllocation = false;
+
+void* APS5_VABI Allocate(std::size_t size) { return failAllocation ? nullptr : std::malloc(size); }
+void APS5_VABI Free(void* ptr) { std::free(ptr); }
+void* APS5_VABI Reallocate(void* ptr, std::size_t size) { return std::realloc(ptr, size); }
+void* APS5_VABI Calloc(std::size_t count, std::size_t size) { return std::calloc(count, size); }
+void* APS5_VABI Align(std::size_t, std::size_t size) { return std::malloc(size); }
+void* APS5_VABI Realign(void* ptr, std::size_t size, std::size_t) { return std::realloc(ptr, size); }
+int APS5_VABI PosixAlign(void** ptr, std::size_t, std::size_t size) {
+    *ptr = std::malloc(size);
+    return *ptr ? 0 : 12;
+}
+
+void InitializeHeap() {
+    alignas(16) static std::array<std::byte, 0x40> process{};
+    alignas(16) static std::array<std::byte, 0x38> libc{};
+    alignas(16) static std::array<std::byte, 0x78> replacement{};
+    const auto write = [](auto& data, std::size_t offset, auto value) {
+        std::memcpy(data.data() + offset, &value, sizeof(value));
+    };
+    write(process, 0, std::uint64_t{0x40});
+    write(process, 8, std::uint32_t{0x4942524f});
+    write(process, 0x38, libc.data());
+    write(libc, 0, std::uint64_t{0x38});
+    write(libc, 0x30, replacement.data());
+    write(replacement, 0, std::uint64_t{0x78});
+    write(replacement, 8, std::uint64_t{2});
+    write(replacement, 0x20, reinterpret_cast<void*>(&Allocate));
+    write(replacement, 0x28, reinterpret_cast<void*>(&Free));
+    write(replacement, 0x30, reinterpret_cast<void*>(&Calloc));
+    write(replacement, 0x38, reinterpret_cast<void*>(&Reallocate));
+    write(replacement, 0x40, reinterpret_cast<void*>(&Align));
+    write(replacement, 0x48, reinterpret_cast<void*>(&Realign));
+    write(replacement, 0x50, reinterpret_cast<void*>(&PosixAlign));
+    ApplicationHeapInitialize_nid_no_patch(process.data());
+}
+
+int CallVasprintf(char** destination, const char* format, ...) {
+#ifdef _WIN32
+    __builtin_sysv_va_list args;
+    __builtin_sysv_va_start(args, format);
+#else
+    std::va_list args;
+    va_start(args, format);
+#endif
+    const int result = vasprintf_nid_postfix(destination, format, reinterpret_cast<VaList*>(args));
+#ifdef _WIN32
+    __builtin_sysv_va_end(args);
+#else
+    va_end(args);
+#endif
+    return result;
+}
+
+}
+
+int main() {
+    *__error_nid_postfix() = 0;
+    Require(strndup_nid_postfix(nullptr, 0) == nullptr);
+    Require(*__error_nid_postfix() == 22);
+
+    *__error_nid_postfix() = 0;
+    Require(strndup_nid_postfix(nullptr, 32) == nullptr);
+    Require(*__error_nid_postfix() == 22);
+
+    *__error_nid_postfix() = 0;
+    Require(strdup_nid_postfix(nullptr) == nullptr);
+    Require(*__error_nid_postfix() == 22);
+
+    *__error_nid_postfix() = 0;
+    Require(vasprintf_nid_postfix(nullptr, "format", nullptr) == -1);
+    Require(*__error_nid_postfix() == 22);
+
+    char* allocated = nullptr;
+    *__error_nid_postfix() = 0;
+    Require(CallVasprintf(&allocated, nullptr) == -1);
+    Require(*__error_nid_postfix() == 22);
+    Require(allocated == nullptr);
+
+    *__error_nid_postfix() = 0;
+    Require(vasprintf_nid_postfix(&allocated, "format", nullptr) == -1);
+    Require(*__error_nid_postfix() == 22);
+    Require(allocated == nullptr);
+
+    InitializeHeap();
+
+    char* dup1 = strndup_nid_postfix("hello world", 5);
+    Require(dup1 != nullptr);
+    Require(std::strcmp(dup1, "hello") == 0);
+    free_nid_postfix(dup1);
+
+    char* dup2 = strdup_nid_postfix("testing 1 2 3");
+    Require(dup2 != nullptr);
+    Require(std::strcmp(dup2, "testing 1 2 3") == 0);
+    free_nid_postfix(dup2);
+
+    allocated = nullptr;
+    const int count = asprintf_nid_postfix(&allocated, "value=%d %s", 42, "ok");
+    Require(count == 11);
+    Require(allocated != nullptr);
+    Require(std::strcmp(allocated, "value=42 ok") == 0);
+    free_nid_postfix(allocated);
+
+    failAllocation = true;
+    allocated = nullptr;
+    *__error_nid_postfix() = 0;
+    Require(asprintf_nid_postfix(&allocated, "value=%d", 1) == -1);
+    Require(*__error_nid_postfix() == 12);
+    Require(allocated == nullptr);
+    failAllocation = false;
+
+    return 0;
+}

docs/dev/debt/libc.md

@@ -26,6 +26,7 @@
 - [swprintf_s and vswprintf_s](../../../core/libs/prx/libc/src/FormattingWide.cpp) (libc) - follow C11 K.3.9.1.4 with `RSIZE_MAX` assumed to be `SIZE_MAX >> 1`: the complete output, including the terminating null, must fit in the buffer, otherwise it is a runtime-constraint violation that sets `buffer[0]` to zero and returns a negative value, and the buffer is not partially written. Argument and encoding checks match `snwprintf_s`
 - [vsnwprintf_s](../../../core/libs/prx/libc/src/FormattingWide.cpp) (libc) - `va_list` counterpart of `snwprintf_s` (C11 K.3.9.1.3): truncates to the buffer and returns the full length, with the same constraints and argument checks
 - [vsnprintf_s and vsscanf_s](../../../core/libs/prx/libc/src/Formatting.cpp) (libc) - `vsnprintf_s` forwards to `vsnprintf` semantics: the C11 K.3.7.1.5 runtime constraints (null buffer, `size == 0`, `%n`, null `%s`) are not checked, so `size == 0` returns the would-be length instead of a negative value, and no constraint handler is invoked. `vsscanf_s` returns the count of items converted before an undersized `%s`/`%c`/`%[` destination (0 when it is the first), not EOF. Neither is confirmed on a PS5 title
+- [strndup and strdup](../../../core/libs/prx/libc/src/AllocatingStrings.cpp) (libc) return null with `errno` set to `EINVAL` (22) when `source` is null, preventing a host segfault on invalid guest input. POSIX.1-2008 and FreeBSD 11.4 libc leave `strndup(NULL)` and `strdup(NULL)` undefined (faulting); exact console behavior on hardware is unverified.
 - [localeconv](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, re-exported by libSceAmpr) - the 0x90-byte Dinkumware struct (empty monetary strings, `CHAR_MAX` for the 14 `char` fields, `decimal_point` ".", `_False`/`_True` "false"/"true") is read from the `libc.prx` that Astro's Playroom ships. It is a fixed C-locale table: whether the console changes it with `setlocale` is not known, and `_Frac_grouping`, `_Frac_sep`, `_No`, `_Yes` are empty strings as in that dump
 - [set_constraint_handler_s](../../../core/libs/prx/libc/src/WideConversion.cpp) (libc, re-exported by libSceAmpr) - the default handler is assumed to abort; each module keeps its own handler state, so a handler set through one library is not seen by the other. `ctime_s` and `asctime_s` follow the Dinkumware 26-byte format with `EINVAL` for a buffer under 26 bytes, a null pointer or an out-of-range weekday/month; `ctime_s` uses the guest time zone of `localtime_s`, and the console's exact checks are not verified
 - [__udivti3](../../../core/libs/prx/libc/src/RuntimeSupport.cpp) (libc, re-exported by libSceAmpr) - the hardware traps on a zero divisor; this throws instead