Upstream sync 11E: AGC driver onto Metal (29 commits)

core/libs/prx/libSceAgc/DcbState/src/Init.cpp

@@ -10,7 +10,7 @@
 extern "C" {
 
 int APS5_VABI sceAgcInit(std::uint32_t version) {
-    if (version >= 14) version = 13;
+    if (version >= 14) return 0;
     Agc::Command::GetRegisterDefaults(version, false, __func__);
     Agc::Command::GetRegisterDefaults(version, true, __func__);
     return 0;

core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp

@@ -61,7 +61,7 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader*
         throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32");
     }
 
-    AgcDriver::ShaderPreparationScope transaction;
+    AgcDriver::ShaderPreparationScope transaction({ps != nullptr ? gs : nullptr, ps});
     std::array<ShaderRegister, 32> values{};
     auto* output = regs;
     regs = values.data();

core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp

@@ -22,7 +22,7 @@ extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegist
 
     // The simple vertex/pixel pipeline combines two primitive context registers
     // and 32 interpolant registers. Publish only after both helpers succeed.
-    AgcDriver::ShaderPreparationScope transaction;
+    AgcDriver::ShaderPreparationScope transaction({vertex, pixel});
     std::array<ShaderRegister, 34> contextValues{};
     std::array<ShaderRegister, 3> primitiveValues{};
     sceAgcCreatePrimState(contextValues.data(), primitiveValues.data(), nullptr, vertex, primitiveType);

core/libs/prx/libSceAgc/Shader/src/PrimState.cpp

@@ -8,6 +8,7 @@
 
 #include "SceShaders.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp"
 #include "prx/libSceAgc/Shader/include/ShaderUtils.hpp"
 #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp"
 
@@ -48,7 +49,9 @@ int APS5_VABI sceAgcCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_
     const std::array<const Shader*, 2> stages{hs, gs};
     const bool patch = prim_type == static_cast<std::uint32_t>(ShaderRegs::PrimitiveType::Patch);
     if (patch == ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_HS_BIT) != 0)) {
+        AgcDriver::ShaderPreparationScope transaction({hs, gs});
         AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues);
+        transaction.Commit();
     }
     if (cx_regs != nullptr) std::copy(contextValues.begin(), contextValues.end(), cx_regs);
     if (uc_regs != nullptr) std::copy(primitiveValues.begin(), primitiveValues.end(), uc_regs);

core/libs/prx/libSceAgc/tests/Commands.cpp

@@ -541,9 +541,9 @@ void testDefaults() {
     check(sceAgcInit_0090(&state, 14) == 0 && state == 0x12345678, "out-of-range AGC initialization version changed caller state");
     check(sceAgcInit(8) == 0, "AGC version initialization failed");
     check(sceAgcInit(13) == 0, "AGC version 13 initialization failed");
-    check(sceAgcInit(14) == 0, "out-of-range AGC version did not fall back");
-    check(sceAgcInit(0xffffffffu) == 0, "maximum AGC version did not fall back");
-    check(sceAgcInit(0xffffc278u) == 0, "pointer-valued AGC argument did not fall back");
+    check(sceAgcInit(14) == 0, "out-of-range AGC version did not return success");
+    check(sceAgcInit(0xffffffffu) == 0, "maximum AGC version did not return success");
+    check(sceAgcInit(0xffffc278u) == 0, "pointer-valued AGC argument did not return success");
     for (std::uint32_t version = 0; version < 14; ++version) {
         for (const bool internal : {false, true}) {
             auto* first = Agc::Command::GetRegisterDefaults(version, internal, __func__);

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -892,6 +892,12 @@ if(BUILD_TESTING)
         set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_storage_kept_unit_settle COMMAND agc_driver_recorder_tests --kept-unit-settle-only)
+    set_tests_properties(agc_storage_kept_unit_settle PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_storage_kept_unit_settle PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_storage_baseline_settled COMMAND agc_driver_recorder_tests --settled-baseline-only)
     add_test(NAME agc_storage_integer_clear COMMAND agc_driver_recorder_tests --integer-clear-only)
     set_tests_properties(agc_storage_integer_clear PROPERTIES SKIP_RETURN_CODE 77)
@@ -1092,6 +1098,8 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
     list(APPEND agcExecutionTests ${agcExecutionTest})
 endforeach()
 
+target_link_libraries(agc_driver_preparation_scope_settle_tests PRIVATE libSceAgc)
+
 target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(agc_driver_dual_source_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -1153,6 +1161,11 @@ if(BUILD_TESTING)
     add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization)
     set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     add_test(NAME agc_shader_frontend_pair COMMAND agc_shader_memory_tests --frontend-pair)
+    add_test(NAME agc_shader_quad_mask_no_demote COMMAND agc_shader_memory_tests --quad-mask-no-demote)
+    set_tests_properties(agc_shader_quad_mask_no_demote PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(agc_shader_quad_mask_no_demote PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_shader_frontend_pair_off COMMAND agc_shader_memory_tests --frontend-pair)
     set_tests_properties(agc_shader_frontend_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
     set_tests_properties(agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_FRONTEND_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -1240,14 +1253,17 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_pm4_priority_refused PROPERTIES ENVIRONMENT
             "ANYPS5_NO_SHADER_CACHE=1;APS5_THREAD_PRIORITY=high;DBUS_SYSTEM_BUS_ADDRESS=unix:path=/nonexistent/aps5-no-rtkit" SKIP_RETURN_CODE 77)
     endif()
+    add_test(NAME agc_driver_worker_failure_log COMMAND agc_driver_flip_tests failure-log)
+    set_tests_properties(agc_driver_worker_failure_log PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 20)
     add_test(NAME agc_driver_flip_compute COMMAND agc_driver_flip_tests compute)
     set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_driver_flip_compute agc_driver_worker_failure_log PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
     add_test(NAME agc_driver_draw_cache_reuse_off COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_GPL=1" SKIP_RETURN_CODE 77)
+    set_property(TEST agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off APPEND PROPERTY ENVIRONMENT "APS5_VERIFY_DRAW_RECIPE=1" "APS5_VERIFY_REGISTER_KEYS=1")
     if(WIN32)
         set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")

core/libs/prx/libSceAgcDriver/Execution/include/Driver/DeviceAccess.hpp

@@ -7,6 +7,7 @@
 #include <cstddef>
 #include <memory>
 #include <mutex>
+#include "prx/libc/include/general/AtomicSharedPtr.hpp"
 
 namespace AgcDriver::DriverDetail {
 
@@ -21,7 +22,7 @@ class DevicePointer {
     bool operator==(std::nullptr_t) const;
 
 private:
-    std::atomic<std::shared_ptr<VulkanDevice>> pointer;
+    AtomicSharedPtr<VulkanDevice> pointer;
 };
 
 class DeviceUseGate {

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp

@@ -17,6 +17,7 @@
 #include <span>
 #include <utility>
 #include <vector>
+#include "prx/libc/include/general/AtomicSharedPtr.hpp"
 
 namespace AgcDriver::DriverDetail {
 
@@ -31,7 +32,7 @@ struct DispatchVariant {
 
     std::shared_ptr<const ShaderSnapshot> shader;
 
-    std::atomic<std::shared_ptr<const Recipe>> recipe;
+    AtomicSharedPtr<const Recipe> recipe;
 
     static constexpr std::uint32_t NoFlatBinding = std::numeric_limits<std::uint32_t>::max();
     std::vector<std::uint32_t> dataPositions;

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -11,6 +11,7 @@
 #include <list>
 #include <memory>
 #include <vector>
+#include "prx/libc/include/general/AtomicSharedPtr.hpp"
 
 namespace AgcDriver::DriverDetail {
 
@@ -54,7 +55,7 @@ struct DrawEntry {
 
     std::vector<std::vector<std::shared_ptr<DispatchVariant>>> stages;
 
-    std::atomic<std::shared_ptr<const std::vector<DrawRecipeRecord>>> recipes;
+    AtomicSharedPtr<const std::vector<DrawRecipeRecord>> recipes;
     std::uint64_t touched = 0;
     std::list<std::uint64_t>::iterator order;
 };

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -60,6 +60,7 @@ class Driver {
     void ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
     void ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
     void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
+    void SettleShaderPreparations(std::span<const Shader* const> stages);
     static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters);
     static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
     static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request);

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp

@@ -1,9 +1,14 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_BUFFERCOPY_HPP
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_BUFFERCOPY_HPP
 
+#include "Recompiler.hpp"
+#include <array>
 #include <atomic>
 #include <cstddef>
 #include <cstdint>
+#include <optional>
+#include <span>
+#include <vector>
 
 namespace AgcDriver::DriverDetail {
 
@@ -13,6 +18,17 @@ extern std::atomic<std::uint64_t> knownValueEntries, knownValueReads, knownValue
 
 enum CopyPath { CopyCpu = 0, CopyGpu = 1, CopyNotImported = 2, CopyAliased = 3, CopyRecordPending = 4, CopyShape = 5, CopyOverlapping = 6, CopyInaccessible = 7, CopyOverGpuMax = 8, CopyPaths = 9 };
 
+extern std::array<std::atomic<std::uint64_t>, CopyPaths> copyPathCounts;
+
+struct StreamCopy {
+    std::uint64_t source = 0;
+    std::uint64_t destination = 0;
+    std::size_t bytes = 0;
+};
+
+bool MatchesStreamCopyKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute);
+std::optional<StreamCopy> StreamCopyRange(const std::vector<std::uint32_t>& userData, std::uint32_t groups);
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -100,6 +100,7 @@ std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnosti
 std::uint64_t NullPixelProgramAddress();
 ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device);
 std::optional<ShaderRecompiler::ShaderFloatMode> RegisteredFloatMode(const ShaderSnapshot& snapshot);
+std::optional<ShaderRecompiler::ShaderFloatMode> DispatchFloatMode(const ShaderSnapshot& snapshot, const Registers& live);
 std::shared_ptr<const ShaderSnapshot> RegisteredHeader(const ShaderRegistry& registry, const Shader* shader);
 // Inline so the Metal driver, which does not link ShaderRegistry.cpp, resolves programs identically.
 inline std::shared_ptr<const ShaderSnapshot> RegisteredProgram(const ShaderRegistry& registry, std::uint64_t address, std::initializer_list<std::uint8_t> types) {

core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp

@@ -1,19 +1,24 @@
 #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERPREPARATIONSCOPE_HPP
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERPREPARATIONSCOPE_HPP
 
+#include <cstddef>
+#include <initializer_list>
+
+struct Shader;
+
 namespace AgcDriver::DriverDetail {
 class ShaderPreparationTransaction;
 }
 
-extern "C" AgcDriver::DriverDetail::ShaderPreparationTransaction* AgcDriverBeginShaderPreparation_nid_postfix();
+extern "C" AgcDriver::DriverDetail::ShaderPreparationTransaction* AgcDriverBeginShaderPreparation_nid_postfix(const Shader* const* shaders, std::size_t count);
 extern "C" void AgcDriverCommitShaderPreparation_nid_postfix(AgcDriver::DriverDetail::ShaderPreparationTransaction* transaction);
 extern "C" void AgcDriverEndShaderPreparation_nid_postfix(AgcDriver::DriverDetail::ShaderPreparationTransaction* transaction) noexcept;
 
 namespace AgcDriver {
 
 class ShaderPreparationScope {
 public:
-    ShaderPreparationScope() : transaction(AgcDriverBeginShaderPreparation_nid_postfix()) {}
+    explicit ShaderPreparationScope(std::initializer_list<const Shader*> shaders) : transaction(AgcDriverBeginShaderPreparation_nid_postfix(shaders.begin(), shaders.size())) {}
     ~ShaderPreparationScope() { AgcDriverEndShaderPreparation_nid_postfix(transaction); }
     ShaderPreparationScope(const ShaderPreparationScope&) = delete;
     ShaderPreparationScope& operator=(const ShaderPreparationScope&) = delete;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp

@@ -2,6 +2,7 @@
 #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
@@ -79,7 +80,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     const bool partialLastGroup = partialGroups != std::array<std::uint32_t, 3>{};
     if (resolving) {
         if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
-        if (!partialLastGroup && indirectArguments == 0 && matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
+        if (!partialLastGroup && indirectArguments == 0 && (matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute))) return;
     } else {
         if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) Graphics::NoteHtileDepthClear(htile);
         if (!partialLastGroup && fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) {
@@ -94,7 +95,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
     if (partialLastGroup) compute.partialThreads = partialGroups;
     ShaderRecompiler::RecompileRequest request{
         {ShaderRecompiler::ShaderStage::Compute, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header},
-        {decoded.waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory, RegisteredFloatMode(snapshot)},
+        {decoded.waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory, DispatchFloatMode(snapshot, queue.shader)},
         localDevice->ComputeTarget(decoded.waveSize),
         {0, 0, 0, 128}
     };

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp

@@ -8,6 +8,8 @@
 #include "Optimization/ResourceProgram.hpp"
 #include <cstdlib>
 #include <cstring>
+#include <sstream>
+#include <stdexcept>
 
 namespace AgcDriver::DriverDetail {
 
@@ -24,9 +26,27 @@ std::shared_ptr<const ShaderRecompiler::RecompileResult> Driver::materializeDraw
     auto& stageCapture = stageCaptures[i];
     stageCapture.forgetSerial = GuestMemory::ForgetSerial();
     stageCapture.pushOffset = pushOffset;
+    const auto named = [&](const std::exception& error) {
+        static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr;
+        std::string dump;
+        if (dumpShaders) {
+            try {
+                dump = dumpRequest(program.binary.codeAddress, request);
+            } catch (...) {}
+        }
+        std::ostringstream where;
+        where << "shader 0x" << std::hex << program.binary.codeAddress << " stage " << std::dec << static_cast<unsigned>(program.binary.stage);
+        if (!dump.empty()) where << " (" << dump << ")";
+        where << ": ";
+        return std::runtime_error(where.str() + error.what());
+    };
     const auto capture = [&] {
         const SampledReadScope sampling(evidenceReads);
-        return shaderMemory.Capture(invocation);
+        try {
+            return shaderMemory.Capture(invocation);
+        } catch (const std::exception& error) {
+            throw named(error);
+        }
     }();
 
     stageCapture.regions = shaderMemory.TakeRecentRegions();
@@ -70,7 +90,11 @@ std::shared_ptr<const ShaderRecompiler::RecompileResult> Driver::materializeDraw
     phaseTiming.Phase(DrawRowCapture);
 
     timing.Mark("capture_resources");
-    stageCapture.compiled = invocation.Materialize(*capture);
+    try {
+        stageCapture.compiled = invocation.Materialize(*capture);
+    } catch (const std::exception& error) {
+        throw named(error);
+    }
     timing.Mark("materialize");
     phaseTiming.Phase(DrawRowRecompile);
     return stageCapture.compiled;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -7,9 +7,25 @@
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include <cstdio>
+#include <exception>
 
 namespace AgcDriver::DriverDetail {
 
+namespace {
+
+void PrintFailure(std::exception_ptr error) {
+    try {
+        std::rethrow_exception(error);
+    } catch (const std::exception& cause) {
+        std::fprintf(stderr, "[gpu] driver failure: %s\n", cause.what());
+    } catch (...) {
+        std::fprintf(stderr, "[gpu] driver failure with a non-standard exception\n");
+    }
+    std::fflush(stderr);
+}
+
+}
+
 Driver& Driver::Get() {
     static Driver driver;
     return driver;
@@ -96,9 +112,11 @@ void Driver::CheckFailure() {
 
 void Driver::ReportFailure(std::exception_ptr error) {
     require(error != nullptr, "null asynchronous failure");
+    bool first = false;
     {
         std::lock_guard lock(mutex);
-        if (!failure) failure = error;
+        first = !failure;
+        if (first) failure = error;
         failed.store(true, std::memory_order_release);
         for (const auto& [handle, output] : outputs) output->Fail(failure);
         for (auto& [queue, worker] : workers) {
@@ -109,6 +127,7 @@ void Driver::ReportFailure(std::exception_ptr error) {
             worker.unfinishedWrites.clear();
             worker.queued.store(0, std::memory_order_release);
         }
+        if (first) PrintFailure(error);
     }
     changed.notify_all();
 }