python thread not exiting with atexit

问题

Here is my script. When I run it in a shell it just hangs indefinitely whereas I would expect it to terminate cleanly.

import logging
from logging import StreamHandler
import pymsteams
import queue
import threading
import atexit


class TeamsHandler(StreamHandler):
    def __init__(self, channel_url):
        super().__init__()
        self.channel_url = channel_url
        self.queue = queue.Queue()
        self.thread = threading.Thread(target=self._worker)
        self.thread.start()
        atexit.register(self.queue.put, None)

    def _worker(self):
        while True:
            record = self.queue.get()
            if record is None:
                break
            msg = self.format(record)
            print(msg)

    def emit(self, record):
        # enqueue the record to log and return control to the caller
        self.queue.put(record)


if __name__ == "__main__":
    my_logger = logging.getLogger('TestLogging')
    my_logger.setLevel(logging.DEBUG)
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.DEBUG)
    my_logger.addHandler(console_handler)

    CHANNEL_ID = "not_used_anyway"
    teamshandler = TeamsHandler(CHANNEL_ID)
    teamshandler.setFormatter(logging.Formatter('%(levelname)s %(message)s'))
    teamshandler.setLevel(logging.DEBUG)
    my_logger.addHandler(teamshandler)
    for i in range(1, 2):
        my_logger.error(f"this is an error [{i}]")
        my_logger.info(f"this is an info [{i}]")

The None record that should be sent by atexit (line 28) never arrives so the thread stays open forever.

How to make sure that the program exits cleanly by modifying the TeamsHandler only ?

回答1:

I got something working, have a look:

import queue
import threading


class Worker:
    def __init__(self):
        self.queue = queue.Queue()
        threading.Thread(target=self._worker).start()

    def _worker(self):
        print("starting thread")
        while True:
            record = self.queue.get()
            if record is None:
                print("exiting")
                break
            print(f"Got message: {record}")

    def emit(self, record):
        self.queue.put(record)


class Wrapper:
    def __init__(self):
        self._worker = Worker()

    def __del__(self):
        print("Wrapper is being deleted")
        self._worker.emit(None)

    def emit(self, record):
        self._worker.emit(record)

def main():
    worker = Wrapper()
    worker.emit("foo")
    worker.emit("bar")
    print("main exits")


if __name__ == "__main__":
    main()

The point here is that when main exits, worker (which is an instance of Wrapper) goes out of scope, and its __del__ method is called, and it sends stop message to a real worker object.

The results of running this code ("Got message" lines can be in different places, of course):

starting thread
main exits
Wrapper is being deleted
Got message: foo
Got message: bar
exiting

回答2:

As pointed out by avysk, the problem is likely that atexit handlers fire too late, after the waiting for the non-daemon threads is already (supposed to be) done, which leads to deadlock.

If I were you, I'd just add a call like TeamsHandler.finish() at the end of if __name__ == '__main__' block, and modify TeamsHandler along these lines (untested):

_queues = []

class TeamsHandler(StreamHandler):
    def __init__(self, channel_url):
        super().__init__()
        self.channel_url = channel_url
        self.queue = queue.Queue()
        self.thread = threading.Thread(target=self._worker)
        self.thread.start()
        _queues.append(self.queue)

    def _worker(self):
        while True:
            record = self.queue.get()
            if record is None:
                break
            msg = self.format(record)
            print(msg)

    def emit(self, record):
        # enqueue the record to log and return control to the caller
        self.queue.put(record)

    @staticmethod
    def finish(self):
        for q in _queues:
            q.put(None)
        del _queues[:]

来源：https://stackoverflow.com/questions/62812511/python-thread-not-exiting-with-atexit