How to make Sentry with OTel integration work in gunicorn + gevent ?

[awoimbee]

Environment

SaaS (https://sentry.io/)

What are you trying to accomplish?

I want to make Sentry with gunicorn work (see https://github.com/benoitc/gunicorn/issues/1855), with the additional complexity that I want the Sentry <> OTel integration.

How are you getting stuck?

Following the python OTel docs, I setup OTel in gunicorn.conf.py, in post_fork().

But I can't import sentry_sdk and sentry_sdk.init() in post_fork() because sentry uses sockets and gunicorn patches sockets with patch_all() after post_fork(), right before post_worker_init(). So I need to put sentry_sdk.init() in post_worker_init().

But I can't move my OTLPSpanExporter() to post_worker_init() because then, when the OTel endpoint is down, my API stops responding (because of Transient error StatusCode.UNAVAILABLE encountered while exporting traces to <endpoint>, retrying in <many>s.).

All in all, my current gunicorn.conf.py is:

import os
from importlib.metadata import version

from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import set_global_textmap
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

bind = "0.0.0.0:8080"
wsgi_app = "myapp.entrypoint"

worker_class = "gevent"
workers = int(os.getenv("GUNICORN_WORKERS", "2"))
worker_connections = int(os.getenv("GUNICORN_WORKER_CONNECTIONS", "1000"))
max_requests = 5000
max_requests_jitter = 1000
preload_app = False
timeout = 120
keepalive = 60
graceful_timeout = 75

errorlog = "-"

# Setup tracing
# https://opentelemetry-python.readthedocs.io/en/latest/examples/fork-process-model/README.html?highlight=gunicorn#working-with-fork-process-models
# https://docs.sentry.io/platforms/python/tracing/instrumentation/opentelemetry/
def post_fork(server, worker):
    resource = Resource.create({SERVICE_NAME: "myapp", "worker": worker.pid})
    tracer_provider = TracerProvider(resource=resource)

    if otlp_endpoint := os.getenv("TRACING_OTLP_ENDPOINT"):
        otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
        span_processor = BatchSpanProcessor(otlp_exporter)
        tracer_provider.add_span_processor(span_processor)

    print(f"Sentry enabled: {os.getenv('SENTRY_DSN') is not None}")  # noqa: T201
    print(f"OTel export enabled: {otlp_endpoint is not None}")  # noqa: T201
    trace.set_tracer_provider(tracer_provider)

def post_worker_init(worker):
    import sentry_sdk
    from sentry_sdk.integrations.opentelemetry import (
        SentryPropagator,
        SentrySpanProcessor,
    )

    if sentry_dsn := os.getenv("SENTRY_DSN"):
        sentry_env = os.getenv("SENTRY_ENVIRONMENT")
        release = version("myapp")
        sentry_sdk.init(
            dsn=sentry_dsn,
            environment=sentry_env,
            traces_sample_rate=1.0,
            instrumenter="otel",
            release=f"{release}@{os.environ.get('GIT_REF', '(none)')}",
        )
        tracer_provider = trace.get_tracer_provider()
        tracer_provider.add_span_processor(SentrySpanProcessor())
        set_global_textmap(SentryPropagator())

The final issue here is that I keep adding span processors without removing them...

Where in the product are you?

Issues

Link

No response

DSN

https://3503250dac9844d091ff31832b78ebb1@o456214.ingest.sentry.io/5465401

Version

No response

[getsantry[bot]]

Assigning to @getsentry/support for routing ⏲️

[awoimbee]

But I can't import sentry_sdk and sentry_sdk.init() in post_fork() because sentry uses sockets and gunicorn patches sockets

Here is the stack trace

```python /usr/local/lib/python3.12/site-packages/gunicorn/workers/ggevent.py:38: MonkeyPatchWarning: Monkey-patching ssl after ssl has already been imported may lead to errors, including RecursionError on Python 3.6. It may also silently lead to incorrect behaviour on Python 3.7. Please monkey-patch earlier. See https://github.com/gevent/gevent/issues/1016. Modules that had direct imports (NOT patched): ['urllib3.util (/usr/local/lib/python3.12/site-packages/urllib3/util/__init__.py)', 'urllib3.util.ssl_ (/usr/local/lib/python3.12/site-packages/urllib3/util/ssl_.py)']. monkey.patch_all() [2024-11-13 18:46:39 +0000] [7] [ERROR] Exception in worker process Traceback (most recent call last): File "/usr/local/lib/python3.12/site-packages/gunicorn/arbiter.py", line 608, in spawn_worker worker.init_process() File "/usr/local/lib/python3.12/site-packages/gunicorn/workers/ggevent.py", line 146, in init_process super().init_process() File "/usr/local/lib/python3.12/site-packages/gunicorn/workers/base.py", line 135, in init_process self.load_wsgi() File "/usr/local/lib/python3.12/site-packages/gunicorn/workers/base.py", line 147, in load_wsgi self.wsgi = self.app.wsgi() ^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/site-packages/gunicorn/app/base.py", line 66, in wsgi self.callable = self.load() ^^^^^^^^^^^ File "/usr/local/lib/python3.12/site-packages/gunicorn/app/wsgiapp.py", line 57, in load return self.load_wsgiapp() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/site-packages/gunicorn/app/wsgiapp.py", line 47, in load_wsgiapp return util.import_app(self.app_uri) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/site-packages/gunicorn/util.py", line 370, in import_app mod = importlib.import_module(module) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/importlib/__init__.py", line 90, in import_module return _bootstrap._gcd_import(name[level:], package, level) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1387, in _gcd_import File "", line 1360, in _find_and_load File "", line 1331, in _find_and_load_unlocked File "", line 935, in _load_unlocked File "", line 995, in exec_module File "", line 488, in _call_with_frames_removed File "/usr/local/lib/python3.12/site-packages/myapp/entrypoint.py", line 4, in import requests File "/usr/local/lib/python3.12/site-packages/requests/__init__.py", line 164, in from .api import delete, get, head, options, patch, post, put, request File "/usr/local/lib/python3.12/site-packages/requests/api.py", line 11, in from . import sessions File "/usr/local/lib/python3.12/site-packages/requests/sessions.py", line 15, in from .adapters import HTTPAdapter File "/usr/local/lib/python3.12/site-packages/requests/adapters.py", line 80, in _preloaded_ssl_context = create_urllib3_context() ^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/site-packages/urllib3/util/ssl_.py", line 290, in create_urllib3_context context.minimum_version = TLSVersion.TLSv1_2 ^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.12/ssl.py", line 546, in minimum_version super(SSLContext, SSLContext).minimum_version.__set__(self, value) File "/usr/local/lib/python3.12/ssl.py", line 546, in minimum_version super(SSLContext, SSLContext).minimum_version.__set__(self, value) File "/usr/local/lib/python3.12/ssl.py", line 546, in minimum_version super(SSLContext, SSLContext).minimum_version.__set__(self, value) [Previous line repeated 925 more times] File "/usr/local/lib/python3.12/ssl.py", line 544, in minimum_version if value == TLSVersion.SSLv3: ^^^^^^^^^^^^^^^^ RecursionError: maximum recursion depth exceeded [2024-11-13 18:46:39 +0000] [7] [INFO] Worker exiting (pid: 7) Exception ignored in thread started by: > Traceback (most recent call last): File "/usr/local/lib/python3.12/threading.py", line 1030, in _bootstrap self._bootstrap_inner() File "/usr/local/lib/python3.12/threading.py", line 1077, in _bootstrap_inner self._delete() File "/usr/local/lib/python3.12/threading.py", line 1109, in _delete del _active[get_ident()] ~~~~~~~^^^^^^^^^^^^^ KeyError: 140065124234112 ```

[getsantry[bot]]

Routing to @getsentry/product-owners-settings-integrations for triage ⏲️