Keep startup workflow binds recoverable

src/workflow/Aevatar.Workflow.Infrastructure/Workflows/FileBackedWorkflowCatalogPort.cs

@@ -279,11 +279,11 @@ private async Task<long> WaitForCommittedBindAsync(
         string actorId,
         CancellationToken ct)
     {
-        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
-        timeoutCts.CancelAfter(_options.BindCommitTimeout);
+        using var bindTimeoutCts = new CancellationTokenSource(_options.BindCommitTimeout);
+        using var readCts = CancellationTokenSource.CreateLinkedTokenSource(ct, bindTimeoutCts.Token);
         try
         {
-            await foreach (var observed in sink.ReadAllAsync(timeoutCts.Token))
+            await foreach (var observed in sink.ReadAllAsync(readCts.Token))
             {
                 if (TryObserveCommittedBind(
                         observed,
@@ -296,7 +296,7 @@ private async Task<long> WaitForCommittedBindAsync(
                 }
             }
         }
-        catch (OperationCanceledException ex) when (!ct.IsCancellationRequested && timeoutCts.IsCancellationRequested)
+        catch (OperationCanceledException ex) when (bindTimeoutCts.IsCancellationRequested)
         {
             throw BindNotCommitted(definition, actorId, ex);
         }

src/workflow/Aevatar.Workflow.Infrastructure/Workflows/WorkflowDefinitionBootstrapHostedService.cs

@@ -12,6 +12,8 @@ internal sealed class WorkflowDefinitionBootstrapHostedService : IHostedService
     private readonly FileBackedWorkflowCatalogPort _definitionMaterializer;
     private readonly IOptions<WorkflowDefinitionFileSourceOptions> _options;
     private readonly ILogger<WorkflowDefinitionBootstrapHostedService> _logger;
+    private CancellationTokenSource? _retryCts;
+    private Task? _retryTask;
 
     public WorkflowDefinitionBootstrapHostedService(
         IWorkflowDefinitionCatalog registry,
@@ -40,8 +42,127 @@ public async Task StartAsync(CancellationToken cancellationToken)
             .Where(definition => definition != null)
             .Select(definition => definition!)
             .ToList();
-        await _definitionMaterializer.MaterializeAsync(definitions, cancellationToken);
+        try
+        {
+            await _definitionMaterializer.MaterializeAsync(definitions, cancellationToken);
+        }
+        catch (WorkflowDefinitionMaterializationException ex)
+            when (ex.Code == WorkflowDefinitionMaterializationException.BindNotCommittedCode &&
+                  !cancellationToken.IsCancellationRequested)
+        {
+            _logger.LogWarning(
+                ex,
+                "Startup workflow definition bind was not observed before timeout; host startup will continue and retry materialization in the background. workflow_name={WorkflowName} actor_id={ActorId} expected_execution_mode={ExpectedExecutionMode}",
+                ex.WorkflowName,
+                ex.ActorId,
+                ex.ExpectedExecutionMode);
+            StartBackgroundRetry(definitions, ex.WorkflowName);
+        }
     }
 
-    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        if (_retryCts == null || _retryTask == null)
+            return;
+
+        await _retryCts.CancelAsync();
+        try
+        {
+            await _retryTask.WaitAsync(cancellationToken);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (OperationCanceledException)
+        {
+        }
+        finally
+        {
+            _retryCts.Dispose();
+            _retryCts = null;
+            _retryTask = null;
+        }
+    }
+
+    private void StartBackgroundRetry(
+        IReadOnlyList<WorkflowDefinitionRegistration> definitions,
+        string timedOutWorkflowName)
+    {
+        var retryDelay = _options.Value.BindCommitRetryDelay;
+        if (retryDelay <= TimeSpan.Zero)
+        {
+            throw new ArgumentOutOfRangeException(
+                nameof(WorkflowDefinitionFileSourceOptions.BindCommitRetryDelay),
+                retryDelay,
+                "Workflow definition bind commit retry delay must be positive.");
+        }
+
+        var remainingDefinitions = GetDefinitionsAfter(definitions, timedOutWorkflowName);
+        if (remainingDefinitions.Count == 0)
+        {
+            _logger.LogInformation(
+                "Startup workflow definition materialization has no remaining definitions after timed-out bind observation. workflow_name={WorkflowName}",
+                timedOutWorkflowName);
+            return;
+        }
+
+        _retryCts = new CancellationTokenSource();
+        _retryTask = RetryMaterializationAsync(remainingDefinitions, retryDelay, _retryCts.Token);
+    }
+
+    private async Task RetryMaterializationAsync(
+        IReadOnlyList<WorkflowDefinitionRegistration> definitions,
+        TimeSpan retryDelay,
+        CancellationToken ct)
+    {
+        var remainingDefinitions = definitions;
+        while (!ct.IsCancellationRequested && remainingDefinitions.Count > 0)
+        {
+            try
+            {
+                await Task.Delay(retryDelay, ct);
+                await _definitionMaterializer.MaterializeAsync(remainingDefinitions, ct);
+                _logger.LogInformation("Startup workflow definition materialization completed in background retry.");
+                return;
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                return;
+            }
+            catch (WorkflowDefinitionMaterializationException ex)
+                when (ex.Code == WorkflowDefinitionMaterializationException.BindNotCommittedCode)
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Startup workflow definition bind still has not been observed; materialization will continue with remaining definitions. workflow_name={WorkflowName} actor_id={ActorId} expected_execution_mode={ExpectedExecutionMode}",
+                    ex.WorkflowName,
+                    ex.ActorId,
+                    ex.ExpectedExecutionMode);
+                remainingDefinitions = GetDefinitionsAfter(remainingDefinitions, ex.WorkflowName);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Startup workflow definition materialization retry failed; materialization will retry.");
+            }
+        }
+    }
+
+    private static IReadOnlyList<WorkflowDefinitionRegistration> GetDefinitionsAfter(
+        IReadOnlyList<WorkflowDefinitionRegistration> definitions,
+        string workflowName)
+    {
+        var orderedDefinitions = definitions
+            .OrderBy(definition => definition.WorkflowName, StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+        var failedIndex = Array.FindIndex(
+            orderedDefinitions,
+            definition => string.Equals(
+                definition.WorkflowName,
+                workflowName,
+                StringComparison.OrdinalIgnoreCase));
+        return failedIndex < 0 || failedIndex + 1 >= orderedDefinitions.Length
+            ? []
+            : orderedDefinitions[(failedIndex + 1)..];
+    }
 }

src/workflow/Aevatar.Workflow.Infrastructure/Workflows/WorkflowDefinitionFileSourceOptions.cs

@@ -15,4 +15,6 @@ public sealed class WorkflowDefinitionFileSourceOptions
         WorkflowDefinitionDuplicatePolicy.Throw;
 
     public TimeSpan BindCommitTimeout { get; set; } = TimeSpan.FromSeconds(30);
+
+    public TimeSpan BindCommitRetryDelay { get; set; } = TimeSpan.FromSeconds(30);
 }

test/Aevatar.Workflow.Host.Api.Tests/FileBackedWorkflowCatalogAdmissionTests.cs

@@ -487,6 +487,56 @@ public async Task Bootstrap_ShouldLoadConfiguredDirectories_AndHonorCancellation
         }
     }
 
+    [Fact]
+    public async Task Bootstrap_WhenStartupBindIsNotObserved_ShouldContinueHostStartup()
+    {
+        var tempDir = Path.Combine(Path.GetTempPath(), "wf-bootstrap-" + Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(tempDir);
+        try
+        {
+            File.WriteAllText(Path.Combine(tempDir, "studio.yaml"), "name: studio");
+            var registry = new WorkflowDefinitionCatalog();
+            var options = new WorkflowDefinitionFileSourceOptions
+            {
+                DuplicatePolicy = WorkflowDefinitionDuplicatePolicy.Override,
+                BindCommitTimeout = TimeSpan.FromMilliseconds(1),
+                BindCommitRetryDelay = TimeSpan.FromMilliseconds(1),
+            };
+            options.WorkflowDirectories.Add(tempDir);
+            var materializerOptions = new WorkflowDefinitionFileSourceOptions
+            {
+                BindCommitTimeout = TimeSpan.FromMilliseconds(1),
+            };
+            var observations = new RecordingWorkflowDefinitionBindObservationRuntime();
+            var dispatch = new RecordingActorDispatchPort(observations)
+            {
+                PublishCommittedBind = false,
+            };
+            var service = new WorkflowDefinitionBootstrapHostedService(
+                registry,
+                new WorkflowDefinitionFileLoader(),
+                new FileBackedWorkflowCatalogPort(
+                    new RecordingActorRuntime(),
+                    dispatch,
+                    observations,
+                    observations,
+                    new RecordingWorkflowCapabilityAdmissionService(),
+                    Options.Create(materializerOptions),
+                    NullLogger<FileBackedWorkflowCatalogPort>.Instance),
+                Options.Create(options),
+                NullLogger<WorkflowDefinitionBootstrapHostedService>.Instance);
+
+            await service.StartAsync(CancellationToken.None);
+
+            dispatch.Envelopes.Should().ContainSingle();
+            await service.StopAsync(CancellationToken.None);
+        }
+        finally
+        {
+            Directory.Delete(tempDir, recursive: true);
+        }
+    }
+
     private static ServiceProvider CreateLocalRuntimeProvider(TimeSpan bindCommitTimeout)
     {
         var services = new ServiceCollection();
@@ -653,13 +703,16 @@ private sealed class RecordingActorDispatchPort(
     {
         public List<(string ActorId, EventEnvelope Envelope)> Envelopes { get; } = [];
 
+        public bool PublishCommittedBind { get; init; } = true;
+
         public async Task<DispatchAdmission> DispatchAsync(
             string actorId,
             EventEnvelope envelope,
             CancellationToken ct = default)
         {
             Envelopes.Add((actorId, envelope));
-            await observations.PublishCommittedBindAsync(actorId, envelope, ct);
+            if (PublishCommittedBind)
+                await observations.PublishCommittedBindAsync(actorId, envelope, ct);
             return DispatchAdmissionFactory.Create(actorId, envelope);
         }
     }