Not working with Django

[gquittet]

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "formatters": {
        "logstash": {
            "()": "logstash_async.formatter.DjangoLogstashFormatter",
            "message_type": "django-logstash",
            "fqdn": False,
            "extra_prefix": None,
            "extra": {
                "application": "my-app",
                "project_path": os.getcwd(),
                "environment": "test",
            },
        },
    },
    "handlers": {
        "logstash": {
            "level": "DEBUG",
            "class": "logstash_async.handler.AsynchronousLogstashHandler",
            "formatter": "logstash",
            "transport": "logstash_async.transport.TcpTransport",
            "host": "xxx.xxx.xxx.xxx",
            "port": xxxx,
            "database_path": "{}/logstash.db".format(os.getcwd()),
        },
    },
    "loggers": {
        "django.request": {
            "handlers": ["logstash"],
            "level": "DEBUG",
            "propagate": True,
        },
    },
}

Hello, I tried your library in my Django Rest Framework application following the documentation, but nothing is working.

I know that my elastic and logstash is fully working because I'm using it with a lot of other application.

Do you have an idea why it's not working?

[eht16]

The configuration looks good to me, I don't see an error.

You could try adding a root logger with a console handler to check if there are any errors logged.

You don't have any TLS settings configured, are you sure your Logstash instance expects plain text JSON via TCP?

Do you see any events in the database file? If it is a transport problem, then you should see events in the database. If it is a logging configuration problem, the database is probably empty.

[gquittet]

My database file is not empty, so looks like there are events in it. see: #issuecomment-1283747520

I just switched the protocol to send information to Logstash using the HTTP Protocol (#usage-with-httptransport)

But same issue, I've got only the WARNING and ERROR logging levels.

[gquittet]

From what I see, my SQLite database is empty:

So I also think it's a logging configuration problem.

But, requests are logged in the console with the INFO logging level, so I don't understand why the INFO logging level is not handled by Logstash 🤔

It's maybe an issue with gunicorn logging 🤔

[eht16]

python-logstash-async logs any errors it encounters to STDERR if no specific logger is configured. So, either check the STDERR of your application for errors or configure a root logger as suggested in my post above. With a configured root logger, or the specific logger named "LogProcessingWorker", you should see the errors logged to whatever handler you specify.

Example:

...
    'loggers': {
        'root': {
            'handlers':['console'],
            'level':'DEBUG',
            'propagate': False,
        },
...

[gquittet]

I finally made it working! 🚀

If you want, you can add the following documentation to yours.

You can close this issue if you have no question on my work 👍

Click to see the full explanation

My LOGGING settings.py looks like this: ```python LOGGING = { "version": 1, "disable_existing_loggers": False, "formatters": { "logstash": { "()": MyLogstashFormatter, "fqdn": False, # To disable grouping of the extra items and have them on the top level of the log event message, simply set this option to None or the empty string. "extra_prefix": None, "metadata": {"beat": "django-api", "version": "1"}, "extra": { "application": "my-django-app", "version": "v1", "project_path": os.getcwd(), "environment": os.environ["ENVIRONMENT"], }, }, }, "handlers": { "console": { "class": "logging.StreamHandler", }, "logstash": { "level": "DEBUG", "class": "logstash_async.handler.AsynchronousLogstashHandler", "formatter": "logstash", "transport": "logstash_async.transport.TcpTransport", "host": "my_logstash_host.local", "port": 1234, "database_path": "{}/logstash.db".format(os.getcwd()), }, }, "filters": { "require_request": { "()": RequireRequestFilter, }, "require_response": { "()": RequireResponseFilter, }, }, "root": { "handlers": ["console"], "level": "INFO", }, "loggers": { "django": { "handlers": ["console"], "level": "INFO", "propagate": False, }, "django.request": { "handlers": ["logstash"], "level": "WARNING", "propagate": True, "filters": ["require_request", "require_response"], }, }, } ``` I had to create one logging middleware that looks like this: ```python import contextlib import logging import traceback LOGGER = logging.getLogger("django.request") class LoggingMiddleware: def __init__(self, get_response): self.get_response = get_response self.stacktrace = None def process_exception(self, _request, _exception): self.stacktrace = traceback.format_exc() return None def __call__(self, request): with contextlib.suppress(UnicodeDecodeError, AttributeError): body = request.body.decode("utf-8") response = self.get_response(request) log_func = LOGGER.info if response.status_code in range(400, 499): log_func = LOGGER.warning if response.status_code in range(500, 599): log_func = LOGGER.error log_func( "%s", request, extra={ "request": request, "response": response, "body": body or "", "stack_trace": self.stacktrace or "", }, ) return response ``` Two filters to remove message that has no request or response object: ```python import logging class RequireRequestFilter(logging.Filter): def filter(self, record): return hasattr(record, "request") and hasattr(record.request, "META") class RequireResponseFilter(logging.Filter): def filter(self, record): return hasattr(record, "response") ``` And then I have a custom formatter to display the data I want: ```python import json from typing import Any, Dict from django.core.exceptions import DisallowedHost from django.http import HttpRequest from logstash_async.formatter import DjangoLogstashFormatter def get_host(request: HttpRequest) -> str: try: return request.get_host() except DisallowedHost: if "HTTP_HOST" in request.META: return request.META["HTTP_HOST"] return request.META["SERVER_NAME"] def get_interesting_fields(record) -> Dict[str, Any]: request: HttpRequest = record.request body: str = record.body if hasattr(record, "body") else "" fields = { "request": { "method": request.method, "path": request.get_full_path(), "body": body, "headers": json.dumps(dict(request.headers)), "user_agent": request.META.get("HTTP_USER_AGENT", ""), "host": get_host(request), "uri": request.build_absolute_uri(), "remote_address": request.META.get("REMOTE_ADDR", ""), "referer": request.META.get("HTTP_REFERER", ""), } } response = record.response fields["response"] = { "status_code": response.status_code, "headers": json.dumps(dict(response.headers)), } return fields UNWANTED_KEYS = [ "body", "request", "response", "req_user", "req_method", "req_useragent", "req_remote_address", "req_host", "req_uri", "req_user", "req_method", "req_referer", ] def filter_unwanted_fields(message: Dict[str, Any]) -> Dict[str, Any]: return {key: value for key, value in message.items() if key not in UNWANTED_KEYS} class MyLogstashFormatter(DjangoLogstashFormatter): def format(self, record): message = json.loads(super().format(record)) # noinspection PyTypeChecker django_fields = get_interesting_fields(record) message.update({"rest": django_fields}) if hasattr(record, "stack_trace") and record.stack_trace: message.update({"stack_trace": record.stack_trace}) message = filter_unwanted_fields(message) return json.dumps(message, ensure_ascii=True) ```