问题
I have a mediator which I have recently needed to synchronize one at a time message dispatch on a background thread but it is locking, demonstrated below.
I post a command to a queue and return a task from a TaskCompletionSource:
public Task<object> Send(object command, CancellationToken cancellationToken)
{
var item = new CommandItem() { Command = request, Tcs = new TaskCompletionSource<object>(), Ct = cancellationToken };
this.queue.Writer.WriteAsync(item); // just write and immediatly return the tcs
return item.Tcs.Task;
}
It then is picked up from the background worker, and handler created:
var item = await this.queue.Reader.ReadAsync(cancellationToken);
// work out command type snipped
var command = item.Command as LockMeGoodCommand;
var handler = new LockMeGoodCommandHandler();
var result = await handler.Handle(command, item.Ct);
item.Tcs.SetResult(result);
It is then handled, with the below locking up when the command handler is send to within the command handler (when using a background thread, but within thread it is OK):
public async Task<int> Handle(LockMeGoodCommand command, CancellationToken cancellationToken)
{
Console.WriteLine(command.GetType().Name);
// this would get the result but will lock forever when using background worker bus implementation
var otherResult = await this.commandBus.Send(new BoringCommand(), cancellationToken);
// perform some action based on the result - but we never get here
Console.WriteLine("otherResult is " + otherResult);
return 3;
}
** Question and potential fix **
I believe we can avoid a deadlock by detecting if the background thread is posting to it itself from within its thread (via the command handler which then calls Send() to post a new command), and if so it should not use any thread mechanics (post to the command queue or TaskCompletionSource) and should instead simply handle the task directly.
I have tried to detect the thread but it is not working, so i set the manual flag isSameThread to true within my handler above var otherResult = await this.commandBus.Send(new BoringCommand(), cancellationToken, true) and I can confirm it works and the deadlock is avoided.
Any caveats in this fix? How would one detect if the background thread is requesting to send a command (how can a thread detect itself) and how would one finish off the below code (from DispatchOnBackgroundThread.Send() to include this self-calling detection (so I can do away with the isSameThread flag)?
It would seem this is more involved as each await will give a different thread ID.
// in thread start we set the thread id of the background thread
this.workerThreadId = System.Threading.Thread.CurrentThread.ManagedThreadId;
public Task<object> Send(object command, CancellationToken cancellationToken, bool isSameThread = false)
{
Console.WriteLine($"this.workerThreadId: {this.workerThreadId}, Thread.CurrentThread.ManagedThreadId: {Thread.CurrentThread.ManagedThreadId}");
// below doesnt work gives different numbers so i use flag instead
// this.workerThreadId == Thread.CurrentThread.ManagedThreadId
if (isSameThread == true)
{
if (command is BoringCommand boringCommand)
{
var handler = new BoringCommandHandler();
return handler.Handle(boringCommand, cancellationToken).ContinueWith(t => (object)t);
}
else if (command is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this);
return handler.Handle(lockMeGoodCommand, cancellationToken).ContinueWith(t => (object)t);
}
else
throw new Exception("unknown");
}
else
{
var item = new CommandItem() { Command = command, Tcs = new TaskCompletionSource<object>(), Ct = cancellationToken };
this.queue.Writer.WriteAsync(item); // just write and immediatly return the cts
return item.Tcs.Task;
}
}
** Code demonstrating issue **
using System;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
namespace TestDeadlock
{
class BoringCommand { }
class LockMeGoodCommand { }
class BoringCommandHandler
{
public Task<int> Handle(BoringCommand command, CancellationToken cancellationToken)
{
Console.WriteLine(command.GetType().Name);
return Task.FromResult(1);
}
}
class LockMeGoodCommandHandler
{
private readonly DispatchOnBackgroundThread commandBus;
public LockMeGoodCommandHandler(DispatchOnBackgroundThread commandBus) => this.commandBus = commandBus;
public async Task<int> Handle(LockMeGoodCommand command, CancellationToken cancellationToken)
{
Console.WriteLine(command.GetType().Name);
// this locks forever
var otherResult = await this.commandBus.Send(new BoringCommand(), cancellationToken);
Console.WriteLine("otherResult is " + otherResult);
return 3;
}
}
public class DispatchOnBackgroundThread
{
private readonly Channel<CommandItem> queue = Channel.CreateUnbounded<CommandItem>();
private Task worker = null;
class CommandItem
{
public object Command { get; set; }
public CancellationToken Ct { get; set; }
public TaskCompletionSource<object> Tcs { get; set; }
}
public Task<object> Send(object command, CancellationToken cancellationToken)
{
var item = new CommandItem()
{ Command = command, Tcs = new TaskCompletionSource<object>(), Ct = cancellationToken };
this.queue.Writer.WriteAsync(item); // just write and immediatly return the tcs
return item.Tcs.Task;
}
public void Start(CancellationToken cancellationToken)
{
this.worker = Task.Factory.StartNew(async () =>
{
try
{
while (cancellationToken.IsCancellationRequested == false)
{
var item = await this.queue.Reader.ReadAsync(cancellationToken);
// simplified DI container magic to static invocation
if (item.Command is BoringCommand boringCommand)
{
var handler = new BoringCommandHandler();
var result = await handler.Handle(boringCommand, item.Ct);
item.Tcs.SetResult(result);
}
if (item.Command is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this);
var result = await handler.Handle(lockMeGoodCommand, item.Ct);
item.Tcs.SetResult(result);
}
}
}
catch (TaskCanceledException) { }
},
TaskCreationOptions.LongRunning)
.Unwrap();
}
public async Task StopAsync()
{
this.queue.Writer.Complete();
await this.worker;
}
}
class Program
{
static async Task Main(string[] args)
{
var cts = new CancellationTokenSource();
var threadStrategy = new DispatchOnBackgroundThread();
threadStrategy.Start(cts.Token);
var result1 = await threadStrategy.Send(new BoringCommand(), cts.Token);
var result2 = await threadStrategy.Send(new LockMeGoodCommand(), cts.Token);
cts.Cancel();
await threadStrategy.StopAsync();
}
}
}
** Simple non-threaded mediator implementation that works without locking **
public class DispatchInCallingThread
{
public async Task<object> Send(object request, CancellationToken cancellationToken)
{
// simplified DI container magic to static invocation
if (request is BoringCommand boringCommand)
{
var handler = new BoringCommandHandler();
return await handler.Handle(boringCommand, cancellationToken);
}
else if (request is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this);
return await handler.Handle(lockMeGoodCommand, cancellationToken);
}
else
throw new Exception("unknown");
}
}
回答1:
The reason for the deadlock is rather simple:
- There is one code loop (not a specific thread; see below) that is responsible for processing the queue. As it processes each command, it
awaits that command's handler. - There is a command handler that
awaits another command to be handled. However, this cannot work because no further commands will be processed; the code loop will not dequeue the next command until this one completes.
Put another way, it is not logically possible for one command to execute another command if commands can only be executed one at a time.
There's a few possible approaches to solving this problem. I do not recommend the "re-entrant" approach; reentrancy is the cause of many subtle logic bugs. The approaches I would recommend are one of:
- Change the
Sendsemantics so that they're a "queue" semantic. This means it's not possible to get command results; results would have to be sent as a message through some mediator. - Have the code loop not
awaitthe command handler, allowing it to loop back and pick up the next command. This means it doesn't "synchronize one at a time" any more. - Redefine "synchronize one at a time" to mean "one at a time but if it's
awaiting then it doesn't count as one". In that case, you can probably use something likeConcurrentExclusiveSchedulerPairorNito.AsyncEx.AsyncContextto run the method chunks one at a time.
Side note: LongRunning doesn't do what you think it's doing. StartNew is not async-aware, so the LongRunning flag only applies to the code up to the first await; after that, the code in that lambda will run on arbitrary thread pool threads (without LongRunning set). Replacing StartNew with Task.Run will make the code more clear.
回答2:
Thanks for Stephen for the answer and Peter for the comments, it indeed blindingly clear when stated thank you,
There is one code loop (not a specific thread; see below) that is responsible for processing the queue. As it processes each command, it awaits that command's handler.
There is a command handler that awaits another command to be handled. However, this cannot work because no further commands will be processed; the code loop will not dequeue the next command until this one completes.
With the above in mind I have found a way to handle without any threading hacks (detecting stack/re-entrance depth etc) or schedulers.
In the example below I "inject" into the handler not the looping calling class, but a different type of command handler dispatcher which does not do any queuing, it instead processes directly within the thread.
The below is called from within the thread loop, then there is no inter-dependency:
public class DispatchInCallingThread: ICommandBus
{
public async Task<object> Send(object request, CancellationToken cancellationToken)
{
// simplified DI container magic to static invocation
if (request is BoringCommand boringCommand)
{
var handler = new BoringCommandHandler();
return await handler.Handle(boringCommand, cancellationToken);
}
else if (request is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this);
return await handler.Handle(lockMeGoodCommand, cancellationToken);
}
else
throw new Exception("cough furball");
}
public void Start(CancellationToken cancellationToken) { }
public Task StopAsync() { return Task.CompletedTask; }
}
And within the background thread, this is the injection into the instantiated command handler:
else if (item.Command is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this.dispatchInCallingThread);
var result = await handler.Handle(lockMeGoodCommand, item.Ct);
item.Tcs.SetResult(result);
}
Now the code runs forever (will need to implement proper shutdown logic for cancellation token source being set):
using System;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
namespace TestDeadlock
{
class BoringCommand { }
class LockMeGoodCommand { }
class BoringCommandHandler
{
public Task<int> Handle(BoringCommand command, CancellationToken cancellationToken)
{
Console.WriteLine(command.GetType().Name);
return Task.FromResult(1);
}
}
class LockMeGoodCommandHandler
{
private readonly ICommandBus commandBus;
public LockMeGoodCommandHandler(ICommandBus commandBus) => this.commandBus = commandBus;
public async Task<int> Handle(LockMeGoodCommand command, CancellationToken cancellationToken)
{
Console.WriteLine(command.GetType().Name);
var otherResult = await this.commandBus.Send(new BoringCommand(), cancellationToken);
var otherResult2 = await this.commandBus.Send(new BoringCommand(), cancellationToken);
return 3;
}
}
public interface ICommandBus
{
Task<object> Send(object request, CancellationToken cancellationToken);
void Start(CancellationToken cancellationToken);
Task StopAsync();
}
public class DispatchOnBackgroundThread : ICommandBus
{
private readonly Channel<CommandItem> queue = Channel.CreateUnbounded<CommandItem>();
private Task worker = null;
private readonly DispatchInCallingThread dispatchInCallingThread = new DispatchInCallingThread();
class CommandItem
{
public object Command { get; set; }
public CancellationToken Ct { get; set; }
public TaskCompletionSource<object> Tcs { get; set; }
}
public Task<object> Send(object command, CancellationToken cancellationToken)
{
var item = new CommandItem() { Command = command, Tcs = new TaskCompletionSource<object>(), Ct = cancellationToken };
this.queue.Writer.WriteAsync(item, cancellationToken); // just write and immediatly return the cts
return item.Tcs.Task;
}
public void Start(CancellationToken cancellationToken)
{
var scheduler = new ConcurrentExclusiveSchedulerPair();
this.worker = Task.Factory.StartNew(async () =>
{
CommandItem item = null;
try
{
while (cancellationToken.IsCancellationRequested == false)
{
item = await this.queue.Reader.ReadAsync(cancellationToken);
// simplified DI container magic to static invocation
if (item.Command is BoringCommand boringCommand)
{
var handler = new BoringCommandHandler();
var result = handler.Handle(boringCommand, item.Ct);
item.Tcs.SetResult(result);
}
else if (item.Command is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this.dispatchInCallingThread);
var result = await handler.Handle(lockMeGoodCommand, item.Ct);
item.Tcs.SetResult(result);
}
else
throw new Exception("unknown");
}
}
catch (TaskCanceledException)
{
if (item != null)
item.Tcs.SetCanceled();
}
Console.WriteLine("exit background thread");
})
.Unwrap();
}
public async Task StopAsync()
{
this.queue.Writer.Complete();
await this.worker;
}
}
public class DispatchInCallingThread: ICommandBus
{
public async Task<object> Send(object request, CancellationToken cancellationToken)
{
// simplified DI container magic to static invocation
if (request is BoringCommand boringCommand)
{
var handler = new BoringCommandHandler();
return await handler.Handle(boringCommand, cancellationToken);
}
else if (request is LockMeGoodCommand lockMeGoodCommand)
{
var handler = new LockMeGoodCommandHandler(this);
return await handler.Handle(lockMeGoodCommand, cancellationToken);
}
else
throw new Exception("unknown");
}
public void Start(CancellationToken cancellationToken) { }
public Task StopAsync() { return Task.CompletedTask; }
}
class Program
{
static async Task Main(string[] args)
{
await TestDispatchOnBackgroundThread();
}
static async Task TestDispatchOnBackgroundThread()
{
var cts = new CancellationTokenSource();
Console.CancelKeyPress += delegate {
Console.WriteLine("setting cts.Cancel()");
cts.Cancel();
};
var threadStrategy = new DispatchOnBackgroundThread();
threadStrategy.Start(cts.Token);
while (cts.IsCancellationRequested == false)
{
Console.WriteLine("***************** sending new batch ****************");
var result1 = await threadStrategy.Send(new BoringCommand(), cts.Token);
var result3 = await threadStrategy.Send(new LockMeGoodCommand(), cts.Token);
Thread.Sleep(1000);
}
await threadStrategy.StopAsync();
}
}
}
For further info, the actual implementation with dependency injection is here https://stackoverflow.com/a/61791817/915839 which was able to dynamically switch to in-thread dispatch within the worker thread
来源：https://stackoverflow.com/questions/61758446/mediator-deadlock-on-async-await-within-background-worker-how-to-detect-thread