diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs index 98322f07..8be14691 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs @@ -91,22 +91,55 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) using IServiceScope scope = _serviceScopeFactory.CreateScope(); var engines = scope.ServiceProvider.GetRequiredService>(); - // Subscriptions are created before recovery so no changes are missed during the recovery window. - using ISubscription engineSub = await engines.SubscribeAsync( - e => - e.CurrentBuild != null - && e.CurrentBuild.BuildJobRunner == BuildJobRunnerType.Local - && e.CurrentBuild.JobState == BuildJobState.Pending, - mode: SubscriptionMode.Repository, - cancellationToken: stoppingToken - ); + while (!stoppingToken.IsCancellationRequested) + { + try + { + // Subscriptions are created before recovery so no changes are missed during the recovery window. + using ISubscription engineSub = await engines.SubscribeAsync( + e => + e.CurrentBuild != null + && e.CurrentBuild.BuildJobRunner == BuildJobRunnerType.Local + && e.CurrentBuild.JobState == BuildJobState.Pending, + mode: SubscriptionMode.Repository, + cancellationToken: stoppingToken + ); - await RecoverPendingJobsAsync(scope.ServiceProvider, stoppingToken); + await RecoverPendingJobsAsync(scope.ServiceProvider, stoppingToken); - await Task.WhenAll( - WatchEngineGroupAsync(engineSub, engineGroup, stoppingToken), - ProcessJobsAsync(engineGroup, stoppingToken) - ); + // Allow either the watch or process task to complete first + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + Task watchTask = WatchEngineGroupAsync(engineSub, engineGroup, linkedCts.Token); + Task processTask = ProcessJobsAsync(engineGroup, linkedCts.Token); + Task completedTask = await Task.WhenAny(watchTask, processTask); + + // If one of the tasks has faulted, cancel the other + if (completedTask.IsFaulted) + { + await linkedCts.CancelAsync(); + _logger.LogError( + completedTask.Exception, + "Exception while executing task on local build runner for {EngineGroup} engines.", + engineGroup + ); + } + + // Wait for both jobs to have completed (faulted or otherwise) + await Task.WhenAll(watchTask, processTask); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Exception while executing local build runner for {EngineGroup} engines.", + engineGroup + ); + } + } } private async Task RecoverPendingJobsAsync(IServiceProvider sp, CancellationToken cancellationToken) @@ -239,7 +272,7 @@ CancellationToken cancellationToken } } - private async Task ProcessJobsAsync(EngineGroup engineGroup, CancellationToken stoppingToken) + protected virtual async Task ProcessJobsAsync(EngineGroup engineGroup, CancellationToken stoppingToken) { Channel channel = _jobChannels[engineGroup]; while (!stoppingToken.IsCancellationRequested) diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Services/TranslationEngineLocalBuildJobRunnerTests.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Services/TranslationEngineLocalBuildJobRunnerTests.cs new file mode 100644 index 00000000..d2887856 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Services/TranslationEngineLocalBuildJobRunnerTests.cs @@ -0,0 +1,86 @@ +namespace Serval.Machine.Translation.Services; + +[TestFixture] +public class TranslationEngineLocalBuildJobRunnerTests +{ + [Test] + public async Task FaultsAreLogged() + { + using var env = new TestEnvironment(); + + // SUT + await env.StartAsync(); + await Task.Delay(100); + await env.CancelAsync(); + + // Verify exceptions were logged + env.Logger.Received() + .Log( + LogLevel.Error, + Arg.Any(), + Arg.Is(v => v.ToString()!.Contains("Exception while executing task on local build runner")), + Arg.Any(), + Arg.Any>()! + ); + env.Logger.Received() + .Log( + LogLevel.Error, + Arg.Any(), + Arg.Is(v => v.ToString()!.Contains("Exception while executing local build runner")), + Arg.Any(), + Arg.Any>()! + ); + } + + private class TestEnvironment : DisposableBase + { + private readonly TranslationEngineLocalBuildJobRunner _jobRunner; + private readonly CancellationTokenSource _runnerCts = new(); + private readonly ServiceProvider _serviceProvider; + + public TestEnvironment() + { + var platformService = Substitute.For(); + platformService.EngineGroup.Returns(EngineGroup.Translation); + var engines = new MemoryRepository(); + var services = new ServiceCollection(); + services.AddKeyedSingleton(EngineGroup.Translation, (_, _) => platformService); + services.AddScoped>(_ => new BuildJobService( + [], + engines + )); + services.AddScoped(_ => new MemoryDataAccessContext()); + services.AddSingleton>(engines); + _serviceProvider = services.BuildServiceProvider(); + Logger = Substitute.For>(); + _jobRunner = new FaultedTranslationEngineLocalBuildJobRunner( + factories: [], + _serviceProvider.GetRequiredService(), + Logger + ); + } + + public ILogger Logger { get; } + + public Task StartAsync() => _jobRunner.StartAsync(_runnerCts.Token); + + public Task CancelAsync() => _runnerCts.CancelAsync(); + + protected override void DisposeManagedResources() + { + _runnerCts.Cancel(); + _serviceProvider.Dispose(); + _runnerCts.Dispose(); + } + } + + private class FaultedTranslationEngineLocalBuildJobRunner( + IEnumerable factories, + IServiceScopeFactory serviceScopeFactory, + ILogger logger + ) : TranslationEngineLocalBuildJobRunner(factories, serviceScopeFactory, logger) + { + protected override Task ProcessJobsAsync(EngineGroup engineGroup, CancellationToken stoppingToken) => + Task.FromException(new InvalidOperationException()); + } +}