QueueListener never starts when configuring logging using dictConfig.

[Digoya]

Bug report

Bug description:

import logging
import logging.handlers
import queue
import time
from logging.config import dictConfig

# Initially making sure it works
logging.basicConfig(level='INFO')
log = logging.getLogger('bug_test_1')
log.info('Start test')

# Make config with queue handler
config = {
    'version': 1,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'INFO',
        },
        'test_queue_handler': {
            'class': 'logging.handlers.QueueHandler',
            'handlers': [
                'console',
            ],
        },
    },
    'loggers': {
        '': {
            'level': 'INFO',
            'handlers': ['test_queue_handler'],
        },
    }
}
dictConfig(config)

# Currently won't log anything,
# queue will have a message,
# but it won't be processed by the listener
log = logging.getLogger('bug_test_2')
log.info('Not working!')

# As an option to fix we need to add code into two places:
class FixedDictConfigurator(logging.config.DictConfigurator):
    def _configure_queue_handler(self, klass, **kwargs):
        if 'queue' in kwargs:
            q = kwargs['queue']
        else:
            q = queue.Queue()  # unbounded
        rhl = kwargs.get('respect_handler_level', False)
        if 'listener' in kwargs:
            lklass = kwargs['listener']
        else:
            lklass = logging.handlers.QueueListener
        listener = lklass(q, *kwargs.get('handlers', []), respect_handler_level=rhl)
        handler = klass(q)
        handler.listener = listener
        # Start listener here, I am not sure about the best place to start.
        handler.listener.start()
        return handler

# Need to make sure to stop listener to avoid losing unhandled messages in queue
class FixedQueueHandler(logging.handlers.QueueHandler):
    def close(self):
        self.listener.stop()
        super().close()

FixedDictConfigurator(config).configure()

# After a fix listener works as intended
log = logging.getLogger('bug_test_3')
log.error('Working now!')

# To wait for listener to react, because I am not using fixed queue handler
time.sleep(1)

CPython versions tested on:

3.12

Operating systems tested on:

Linux

[solaikannanpandiyan]

There isn't QueueListenerHandler(reading) like QueueHandler(writing) in Python. Also its sensible design decision from python side. may be you can try the below approach.

import logging
from logging.config import dictConfig
from logging.handlers import QueueHandler, QueueListener
import queue

# Create a queue
log_queue = queue.Queue(-1)  # no limit on size

# Configuration dictionary
config = {
    'version': 1,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'INFO',
            'formatter': 'standard',
        },
        'queue_handler': {
            'class': 'logging.handlers.QueueHandler',
            'queue': log_queue,
        },
    },
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
    },
    'loggers': {
        '': {  # root logger
            'handlers': ['queue_handler'],
            'level': 'INFO',
            'propagate': False
        },
    },
}

# Apply the configuration
dictConfig(config)

# Set up and start the queue listener
console_handler = logging.getLogger('').handlers[0].handlers[0]  # Get the console handler
queue_listener = QueueListener(log_queue, console_handler)
queue_listener.start()

# Now you can use logging as normal
log = logging.getLogger('test_logger')
log.info('This should work now!')

# When you're done, stop the listener
# ### queue_listener.stop()

or this one if you are hell bend on using a dict based config:

import logging
from logging.config import dictConfig
from logging.handlers import QueueHandler, QueueListener
import queue
import atexit

# Custom class to set up QueueListener
class QueueListenerHandler(logging.Handler):
    def __init__(self, handlers, queue, respect_handler_level=False):
        super().__init__()
        self.listener = QueueListener(queue, *handlers, respect_handler_level=respect_handler_level)
        self.queue = queue
        self.start()
        atexit.register(self.stop)

    def handle(self, record):
        self.queue.put_nowait(record)

    def start(self):
        self.listener.start()

    def stop(self):
        self.listener.stop()

    def emit(self, record):
        pass

# Create a queue
log_queue = queue.Queue(-1)  # no limit on size

# Configuration dictionary
config = {
    'version': 1,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'INFO',
            'formatter': 'standard',
        },
        'queue_handler': {
            'class': 'logging.handlers.QueueHandler',
            'queue': log_queue,
        },
        'queue_listener': {
            'class': __name__ + '.QueueListenerHandler',
            'handlers': ['console'],
            'queue': log_queue,
        },
    },
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
    },
    'loggers': {
        '': {  # root logger
            'handlers': ['queue_handler'],
            'level': 'INFO',
            'propagate': False
        },
    },
}

# Apply the configuration
dictConfig(config)

# Get the queue listener handler and start it
listener_handler = logging.getLogger('')

# Now you can use logging as normal
log = logging.getLogger('test_logger')
log.info('This should work now!')