fix(relinker): discover required modules in nested standard directories

core/relinker/CMakeLists.txt

@@ -272,6 +272,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_absolute_symbols.py $<TARGET_FILE:relinker>)
         add_test(NAME guest_module_identity COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_module_identity.py $<TARGET_FILE:relinker>)
+        add_test(NAME guest_needed_nested_modules COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_needed_nested_modules.py $<TARGET_FILE:relinker>)
         add_test(NAME windows_import_modules COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_import_modules.py $<TARGET_FILE:relinker>)
         add_test(NAME elf_header COMMAND ${Python3_EXECUTABLE}

core/relinker/relinker/src/guest/GuestModuleBuilder.cpp

@@ -95,19 +95,22 @@ std::vector<GuestArtifact> GuestModuleBuilder::Build(const std::filesystem::path
         if (alias != paths.end()) neededAliases.emplace(name, *alias);
         else missingNeeded.insert(name);
     }
-    if (!missingNeeded.empty()) {
+    if (!missingNeeded.empty() || !unmatchedExclusions.empty()) {
         std::map<std::string, std::filesystem::path> found;
         std::map<std::string, std::filesystem::path> foundByStem;
         const auto record = [](std::map<std::string, std::filesystem::path>& matches, const std::string& name, const std::filesystem::path& path) {
             if (!matches.emplace(name, path).second) throw Domain::RelinkerException("Ambiguous needed module: " + matches.at(name).string() + " and " + path.string());
         };
         for (auto it = std::filesystem::recursive_directory_iterator(root); it != std::filesystem::recursive_directory_iterator(); ++it) {
-            if (it->is_directory() && std::find(directories.begin(), directories.end(), it->path()) != directories.end()) {
-                it.disable_recursion_pending();
-                continue;
-            }
             if (!it->is_regular_file()) continue;
             const auto name = it->path().filename().string();
+            if (name.ends_with(GuestModuleSuffix)) continue;
+            if (excludedModules.contains(name)) {
+                for (auto parent = it->path().parent_path(); parent != root && !parent.empty(); parent = parent.parent_path()) {
+                    if (std::any_of(directories.begin(), directories.end(), [&](const auto& directory) { return std::filesystem::equivalent(parent, directory); })) unmatchedExclusions.erase(name);
+                }
+                continue;
+            }
             const auto stem = ModuleStem(name, windows);
             std::vector<std::string> stemMatches;
             if (!stem.empty()) {
@@ -149,10 +152,6 @@ std::vector<GuestArtifact> GuestModuleBuilder::Build(const std::filesystem::path
     if (!unresolved.empty()) {
         std::map<std::string, std::filesystem::path> identities;
         for (auto it = std::filesystem::recursive_directory_iterator(root); it != std::filesystem::recursive_directory_iterator(); ++it) {
-            if (it->is_directory() && std::find(directories.begin(), directories.end(), it->path()) != directories.end()) {
-                it.disable_recursion_pending();
-                continue;
-            }
             const auto& path = it->path();
             const auto extension = path.extension().string();
             if (!it->is_regular_file() || (extension != ".prx" && extension != ".sprx" && extension != ".suprx") ||

core/relinker/relinker/tests/test_guest_needed_modules.py

@@ -36,7 +36,8 @@ def main():
         work = Path(directory)
 
         def convert(case, windows, needed=NEEDED):
-            (case / "sce_module").mkdir(parents=True, exist_ok=True)
+            if not (case / "sce_modules").exists():
+                (case / "sce_module").mkdir(parents=True, exist_ok=True)
             source = case / "input.elf"
             source.write_bytes(executable_with_needed(needed))
             output = case / ("output.exe" if windows else "output.elf")
@@ -63,7 +64,7 @@ def convert(case, windows, needed=NEEDED):
                 needed = needed_libraries(output.read_bytes())
                 assert needed == ["$ORIGIN/app0/Media/Modules/needed.prx.guest.prx"], needed
 
-            for directory in ("Media/Modules", "sce_module"):
+            for directory in ("Media/Modules", "sce_module", "sce_module/nested", "sce_modules/nested", "prx/shipping"):
                 case = work / f"{windows}-debug-name-{directory.replace('/', '-')}"
                 (case / directory).mkdir(parents=True)
                 (case / directory / "needed.prx").write_bytes(module_with_symbol(True))

core/relinker/relinker/tests/test_guest_needed_nested_modules.py

@@ -0,0 +1,146 @@
+from pathlib import Path
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+
+from test_guest_module_directories import needed_libraries
+from test_guest_module_identity import declared_provider
+from test_windows_import_modules import executable
+
+
+def main():
+    relinker = Path(sys.argv[1]).resolve()
+    with tempfile.TemporaryDirectory(prefix='anyps5-needed-nested-') as directory:
+        work = Path(directory)
+
+        def convert(label, windows, request, files, excluded=()):
+            case = work / label
+            standard = next((p.split('/')[0] for p in files if p.split('/')[0].lower() in ('sce_module', 'sce_modules')), 'sce_module')
+            (case / standard).mkdir(parents=True)
+            for relative, image in files.items():
+                target = case / relative
+                target.parent.mkdir(parents=True, exist_ok=True)
+                target.write_bytes(image)
+            source = case / 'input.elf'
+            source.write_bytes(executable(request, module_name='libGuest'))
+            output = case / ('output.exe' if windows else 'output.elf')
+            options = [arg for name in excluded for arg in ('--exclude-sce-module', name)]
+            result = subprocess.run([str(relinker), *(['--windows'] if windows else []), *options,
+                                     '--rpath', '$ORIGIN/custom-hosts', str(source), str(output)],
+                                    capture_output=True, text=True, timeout=30)
+            return result, output
+
+        def success(result, output, relative, windows):
+            assert result.returncode == 0, (result.stdout, result.stderr)
+            expected = output.parent / 'app0' / (relative + '.guest.prx')
+            artifacts = list((output.parent / 'app0').rglob('*.guest.prx'))
+            assert artifacts == [expected], artifacts
+            if windows and os.name == 'nt':
+                run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30)
+                assert run.returncode == 22, (run.returncode, run.stdout, run.stderr)
+            elif not windows:
+                actual = needed_libraries(output.read_bytes())
+                expected = ['$ORIGIN/app0/' + relative + '.guest.prx']
+                assert actual == expected, (actual, expected)
+
+        unsupported = declared_provider(77, ('libUnused',))
+        struct.pack_into('<qQ', unsupported, 0x600 + 5 * 16, 0x70000000, 0)
+        for windows in (True, False):
+            for directory in ('prx/shipping', 'sce_module/nested', 'sce_modules/nested'):
+                label = f'{windows}-{directory.replace("/", "-")}'
+                for identity in (False, True):
+                    filename = 'renamed.prx' if identity else 'libGuest.suprx'
+                    relative = directory + '/' + filename
+                    files = {relative: declared_provider(22, ('libGuest' if identity else 'other',)),
+                             directory + '/unused.prx': unsupported,
+                             directory + '/opaque.sprx': b'not an ELF',
+                             directory + '/old.prx.guest.prx': b'\x7fELF'}
+                    result, output = convert(f'{label}-{identity}', windows, 'libGuest.suprx', files)
+                    success(result, output, relative, windows)
+
+                    other = directory + '/deeper/' + ('duplicate.prx' if identity else filename)
+                    files[other] = declared_provider(33, ('libGuest',))
+                    result, output = convert(f'{label}-{identity}-ambiguous', windows, 'libGuest.suprx', files)
+                    diagnostic = 'Ambiguous needed module identity' if identity else 'Ambiguous needed module:'
+                    assert result.returncode == 2 and diagnostic in result.stderr, result.stderr
+                    assert not output.exists()
+
+                for soname in (False, True):
+                    standard = directory.split('/')[0]
+                    relative = standard + ('/physical.prx' if soname else '/libGuest.suprx')
+                    files = {relative: declared_provider(22, ('other',), 'libGuest.suprx' if soname else None),
+                             directory + '/renamed.prx': declared_provider(11, ('libGuest',))}
+                    result, output = convert(f'{label}-precedence-{soname}', windows, 'libGuest.suprx', files)
+                    success(result, output, relative, windows)
+
+                files = {directory + '/kept.prx': declared_provider(22, ('libGuest',)),
+                         directory + '/omitted.prx': declared_provider(11, ('libGuest',))}
+                result, output = convert(f'{label}-excluded-identity', windows, 'libGuest.suprx', files, ('omitted.prx',))
+                success(result, output, directory + '/kept.prx', windows)
+
+                files = {directory + '/libGuest.prx': declared_provider(22, ('libGuest',)),
+                         directory + '/libGuest.sprx': declared_provider(33, ('libGuest',))}
+                result, output = convert(f'{label}-excluded-stem', windows, 'libGuest.debug_prx', files, ('libGuest.sprx',))
+                success(result, output, directory + '/libGuest.prx', windows)
+
+                result, output = convert(f'{label}-ambiguous-stem', windows, 'libGuest.debug_prx', files)
+                assert result.returncode == 2 and 'Ambiguous needed module:' in result.stderr, result.stderr
+                assert not output.exists()
+
+                files = {directory + '/libGuest.prx': declared_provider(22, ('libGuest',))}
+                result, output = convert(f'{label}-excluded-only-stem', windows, 'libGuest.debug_prx', files, ('libGuest.prx',))
+                assert result.returncode == 0, result.stderr
+                assert not list((output.parent / 'app0').rglob('*.guest.prx'))
+                if not windows:
+                    assert 'libGuest.debug_prx' in needed_libraries(output.read_bytes())
+
+                files = {directory + '/libGuest.suprx': declared_provider(22, ('libGuest',))}
+                result, output = convert(f'{label}-excluded-required', windows, 'libGuest.suprx', files, ('libGuest.suprx',))
+                assert result.returncode == 0, result.stderr
+                assert not list((output.parent / 'app0').rglob('*.guest.prx'))
+
+                files = {directory + '/kept.prx': declared_provider(22, ('libGuest',)),
+                         'unrelated/omitted.prx': declared_provider(11, ('other',))}
+                for excluded in ('absent.prx', 'omitted.prx'):
+                    result, output = convert(f'{label}-unknown-{excluded}', windows, 'libGuest.suprx', files, (excluded,))
+                    assert result.returncode == 2 and 'Excluded guest module file not found' in result.stderr, result.stderr
+                    assert not output.exists()
+
+                generated = 'libGuest.prx.guest.prx'
+                files = {directory + '/' + generated: declared_provider(22, ('libGuest',))}
+                result, output = convert(f'{label}-generated-required', windows, generated, files)
+                assert result.returncode == 0, result.stderr
+                assert not list((output.parent / 'app0').rglob('*.guest.prx'))
+                if not windows:
+                    assert generated in needed_libraries(output.read_bytes())
+                result, output = convert(f'{label}-generated-excluded', windows, generated, files, (generated,))
+                assert result.returncode == 2 and 'Excluded guest module file not found' in result.stderr, result.stderr
+                assert not output.exists()
+
+        if os.name == 'nt':
+            for windows in (True, False):
+                for directory in ('PrX/shipping', 'Sce_Module/nested', 'Sce_Modules/nested'):
+                    label = f'host-case-{windows}-{directory.replace("/", "-")}'
+                    files = {directory + '/kept.prx': declared_provider(22, ('libGuest',)),
+                             directory + '/omitted.prx': declared_provider(11, ('libGuest',))}
+                    result, output = convert(label, windows, 'libGuest.suprx', files, ('omitted.prx',))
+                    success(result, output, directory + '/kept.prx', windows)
+
+                    files = {directory + '/libGuest.suprx': declared_provider(22, ('libGuest',))}
+                    result, output = convert(label + '-required', windows, 'libGuest.suprx', files, ('libGuest.suprx',))
+                    assert result.returncode == 0, result.stderr
+                    assert not list((output.parent / 'app0').rglob('*.guest.prx'))
+
+        for filename, identity in (('party.prx', 'Party'), ('libplayfabmultiplayer.prx', 'libPlayFabMultiplayer')):
+            relative = 'prx/shipping/' + filename
+            for windows in (True, False):
+                result, output = convert(f'case-identity-{windows}-{filename}', windows, identity + '.prx',
+                    {relative: declared_provider(22, (identity, 'libGuest'))})
+                success(result, output, relative, windows)
+    print('Nested required guest module tests passed')
+
+
+if __name__ == '__main__':
+    main()