1import asyncio2import logging3import threading4from contextlib import asynccontextmanager5from unittest.mock import Mock67import pytest8from fastapi import FastAPI9from fastapi.exceptions import FastAPIError10from fastapi.telemetry import _runtime as runtime11from fastapi.testclient import TestClient1213from ._otlp import otlp_collector14from ._subprocess import run_in_subprocess151617def test_otlp_collector_does_not_resolve_hostname(monkeypatch):18    # Reverse DNS during HTTPServer binding caused macOS CI timeouts.19    lookup = Mock()20    monkeypatch.setattr("socket.getfqdn", lookup)21    with otlp_collector():22        pass23    lookup.assert_not_called()242526@pytest.mark.parametrize("base_path", ["", "/collector/"])27@pytest.mark.parametrize("configuration", ["environment", "explicit"])28@run_in_subprocess29def test_real_otlp_export_and_repeated_lifespans(base_path, configuration):30    import os31    from contextlib import asynccontextmanager3233    from fastapi import FastAPI34    from fastapi.telemetry import _runtime35    from fastapi.testclient import TestClient36    from opentelemetry import _logs, trace37    from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import (38        ExportLogsServiceRequest,39    )40    from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (41        ExportMetricsServiceRequest,42    )43    from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (44        ExportTraceServiceRequest,45    )46    from opentelemetry.proto.trace.v1.trace_pb2 import Span4748    with otlp_collector() as (base, received):49        prefix = base_path.rstrip("/")50        os.environ["FASTAPI_OTEL_AUTO_CONFIGURE"] = (51            "TRUE" if configuration == "environment" else "false"52        )53        os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = base + base_path54        os.environ["OTEL_SERVICE_NAME"] = "native-test"55        os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = "x-test=value"56        os.environ["OTEL_RESOURCE_ATTRIBUTES"] = "test.resource=example"57        os.environ["OTEL_BSP_SCHEDULE_DELAY"] = "60000"58        os.environ["OTEL_BLRP_SCHEDULE_DELAY"] = "60000"59        os.environ["OTEL_METRIC_EXPORT_INTERVAL"] = "60000"60        seen = []6162        @asynccontextmanager63        async def lifespan(app):64            seen.append(trace.get_tracer_provider())65            yield6667        app = FastAPI(68            lifespan=lifespan,69            telemetry={"auto_configure": True} if configuration == "explicit" else {},70        )7172        @app.get("/items/{value}")73        def endpoint(value: int):74            _logs.get_logger("test").emit(body="request processed")75            return value7677        try:78            for _ in range(2):79                with TestClient(app) as client:80                    assert client.get("/items/3").json() == 381            assert len(_runtime._owned) == 382            assert seen[0] is seen[1]83            assert all(headers.get("x-test") == "value" for _, _, headers in received)84            traces = [85                ExportTraceServiceRequest.FromString(body)86                for path, body, _ in received87                if path == prefix + "/v1/traces"88            ]89            spans = [90                span91                for request in traces92                for resource in request.resource_spans93                for scope in resource.scope_spans94                for span in scope.spans95            ]96            assert len(spans) == 897            servers = [span for span in spans if span.kind == Span.SPAN_KIND_SERVER]98            assert len(servers) == 299            assert all(span.name == "GET /items/{value}" for span in servers)100            metric_requests = [101                ExportMetricsServiceRequest.FromString(body)102                for path, body, _ in received103                if path == prefix + "/v1/metrics"104            ]105            assert metric_requests106            resources = metric_requests[-1].resource_metrics107            resource_attributes = {108                a.key: a.value.string_value for a in resources[0].resource.attributes109            }110            assert resource_attributes["service.name"] == "native-test"111            assert resource_attributes["test.resource"] == "example"112            histograms = [113                metric.histogram114                for resource in resources115                for scope in resource.scope_metrics116                for metric in scope.metrics117                if metric.name == "http.server.request.duration"118            ]119            assert histograms[0].data_points[0].count == 2120            log_requests = [121                ExportLogsServiceRequest.FromString(body)122                for path, body, _ in received123                if path == prefix + "/v1/logs"124            ]125            records = [126                record127                for request in log_requests128                for resource in request.resource_logs129                for scope in resource.scope_logs130                for record in scope.log_records131            ]132            assert len(records) == 2133            assert all(134                record.body.string_value == "request processed" for record in records135            )136        finally:137            _runtime._shutdown()138139140@pytest.mark.parametrize(141    "env",142    [143        {144            "OTEL_TRACES_EXPORTER": "",145            "OTEL_METRICS_EXPORTER": "",146            "OTEL_LOGS_EXPORTER": "",147            "OTEL_EXPORTER_OTLP_PROTOCOL": "",148            "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "",149            "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "",150            "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL": "",151        },152        {153            "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "",154            "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "",155            "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "",156        },157    ],158)159def test_empty_environment_uses_defaults(monkeypatch, env):160    monkeypatch.setenv(161        "OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318/collector/"162    )163    for name, value in env.items():164        monkeypatch.setenv(name, value)165    for signal in ("TRACES", "METRICS", "LOGS"):166        assert runtime._export_endpoint(signal) == (167            f"http://127.0.0.1:4318/collector/v1/{signal.lower()}"168        )169170171@pytest.mark.parametrize("protocol", ["http/protobuf", "grpc"])172def test_empty_signal_protocol_uses_general_protocol(monkeypatch, protocol):173    monkeypatch.setenv(174        "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://127.0.0.1:4318/v1/traces"175    )176    monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", protocol)177    monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "")178    if protocol == "http/protobuf":179        assert runtime._export_endpoint("TRACES") == "http://127.0.0.1:4318/v1/traces"180    else:181        with pytest.raises(FastAPIError, match="http/protobuf"):182            runtime._export_endpoint("TRACES")183184185@pytest.mark.parametrize("signal", ["TRACES", "METRICS", "LOGS"])186def test_signal_endpoint_overrides_general_endpoint(monkeypatch, signal):187    endpoint = "http://127.0.0.1:4318/custom/"188    monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "invalid")189    monkeypatch.setenv(f"OTEL_EXPORTER_OTLP_{signal}_ENDPOINT", endpoint)190    assert runtime._export_endpoint(signal) == endpoint191192193@pytest.mark.parametrize(194    "env,message",195    [196        ({"OTEL_EXPORTER_OTLP_ENDPOINT": "not-a-url"}, "absolute HTTP"),197        (198            {199                "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",200                "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc",201            },202            "http/protobuf",203        ),204        (205            {206                "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",207                "OTEL_TRACES_EXPORTER": "console",208            },209            "otlp or none",210        ),211    ],212)213def test_invalid_configuration_warns_without_disabling_providers(214    monkeypatch, caplog, telemetry, env, message215):216    for name, value in env.items():217        monkeypatch.setenv(name, value)218    config, exporter, _ = telemetry219    config["auto_configure"] = True220    events = []221222    @asynccontextmanager223    async def lifespan(app):224        events.append("startup")225        yield226        events.append("shutdown")227228    app = FastAPI(telemetry=config, lifespan=lifespan)229230    @app.get("/health")231    async def health():232        return {"status": "ok"}233234    with TestClient(app) as client:235        assert client.get("/health").json() == {"status": "ok"}236    assert events == ["startup", "shutdown"]237    assert exporter.get_finished_spans()238    assert not runtime._owned239    assert len(caplog.records) == 1240    assert caplog.records[0].levelno == logging.WARNING241    assert message in caplog.messages[0]242243244@pytest.mark.parametrize("stage", ["startup", "shutdown"])245def test_application_lifespan_errors_still_propagate(monkeypatch, stage):246    monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "not-a-url")247248    @asynccontextmanager249    async def lifespan(app):250        if stage == "startup":251            raise RuntimeError("application startup failed")252        yield253        raise RuntimeError("application shutdown failed")254255    with pytest.raises(RuntimeError, match=f"application {stage} failed"):256        with TestClient(257            FastAPI(lifespan=lifespan, telemetry={"auto_configure": True})258        ) as client:259            assert client.get("/").status_code == 404260261262@pytest.mark.parametrize(263    "env",264    [265        {"FASTAPI_OTEL_AUTO_CONFIGURE": "true"},266        {"FASTAPI_OTEL_AUTO_CONFIGURE": "true", "OTEL_SERVICE_NAME": "no-endpoint"},267        {268            "FASTAPI_OTEL_AUTO_CONFIGURE": "true",269            "OTEL_TRACES_EXPORTER": "otlp",270            "OTEL_METRICS_EXPORTER": "otlp",271        },272        {"OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1"},273        {274            "FASTAPI_OTEL_AUTO_CONFIGURE": "",275            "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",276        },277        {278            "FASTAPI_OTEL_AUTO_CONFIGURE": "false",279            "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",280        },281        {282            "FASTAPI_OTEL_AUTO_CONFIGURE": "1",283            "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",284        },285        {286            "FASTAPI_OTEL_AUTO_CONFIGURE": "true",287            "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",288            "OTEL_TRACES_EXPORTER": "none",289            "OTEL_METRICS_EXPORTER": "none",290            "OTEL_LOGS_EXPORTER": "none",291        },292        {293            "FASTAPI_OTEL_AUTO_CONFIGURE": "true",294            "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1",295            "OTEL_SDK_DISABLED": "true",296        },297    ],298)299@run_in_subprocess300def test_no_implicit_export(env):301    import os302303    os.environ.update(env)304305    from fastapi import FastAPI306    from fastapi.telemetry import _runtime307    from fastapi.testclient import TestClient308309    with TestClient(FastAPI()) as client:310        assert client.get("/").status_code == 404311    assert _runtime._owned == []312313314@run_in_subprocess315def test_missing_sdk_warns_without_preventing_startup():316    import os317318    os.environ.update(319        {"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://127.0.0.1:1/metrics"}320    )321322    import sys323    from importlib.abc import MetaPathFinder324325    class BlockSDK(MetaPathFinder):326        def find_spec(self, fullname, path=None, target=None):327            if fullname.startswith(("opentelemetry.sdk", "opentelemetry.exporter")):328                raise ImportError("SDK absent")329330    sys.meta_path.insert(0, BlockSDK())331    from unittest import TestCase332333    from fastapi import FastAPI334    from fastapi.testclient import TestClient335336    with TestCase().assertLogs("fastapi", level="WARNING") as logs:337        with TestClient(FastAPI(telemetry={"auto_configure": True})) as client:338            assert client.get("/").status_code == 404339    assert len(logs.records) == 1340    assert logs.records[0].levelno == logging.WARNING341    assert "fastapi[opentelemetry]" in logs.output[0]342    assert not runtime._owned343344345@pytest.mark.parametrize(346    "env,config",347    [348        ({}, {}),349        ({"FASTAPI_OTEL_AUTO_CONFIGURE": "false"}, {}),350        ({"FASTAPI_OTEL_AUTO_CONFIGURE": "true"}, {"auto_configure": False}),351    ],352)353@run_in_subprocess354def test_external_globals_are_unchanged_when_auto_configuration_is_disabled(355    env, config356):357    import os358359    os.environ.update({"OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:1"})360    os.environ.update(env)361362    from fastapi import FastAPI363    from fastapi.telemetry import _runtime364    from fastapi.testclient import TestClient365    from opentelemetry import _logs, metrics, trace366    from opentelemetry.sdk._logs import LoggerProvider367    from opentelemetry.sdk.metrics import MeterProvider368    from opentelemetry.sdk.trace import TracerProvider369    from opentelemetry.sdk.trace.export import SimpleSpanProcessor370    from opentelemetry.sdk.trace.export.in_memory_span_exporter import (371        InMemorySpanExporter,372    )373374    exporter = InMemorySpanExporter()375    tp = TracerProvider()376    tp.add_span_processor(SimpleSpanProcessor(exporter))377    mp = MeterProvider()378    lp = LoggerProvider()379    _logs.set_logger_provider(lp)380    trace.set_tracer_provider(tp)381    metrics.set_meter_provider(mp)382    for _ in range(2):383        with TestClient(FastAPI(telemetry=config)) as client:384            client.get("/")385    assert trace.get_tracer_provider() is tp386    assert metrics.get_meter_provider() is mp387    assert _logs.get_logger_provider() is lp388    assert not _runtime._owned389    assert len(exporter.get_finished_spans()) == 2390391392def test_owned_flush_failure_does_not_break_shutdown(monkeypatch, caplog):393    calls = []394395    class FailingProvider:396        def force_flush(self):397            raise RuntimeError("flush failed")398399        def shutdown(self):400            raise RuntimeError("shutdown failed")401402    class Provider:403        def force_flush(self):404            calls.append("flush")405406        def shutdown(self):407            calls.append("shutdown")408409    monkeypatch.setattr(runtime, "_owned", [FailingProvider(), Provider()])410    with TestClient(FastAPI()) as client:411        assert client.get("/").status_code == 404412    runtime._shutdown()413    runtime._shutdown()414    assert calls == ["flush", "shutdown"]415    assert caplog.text.count("FastAPI telemetry cleanup failed") == 2416417418@pytest.mark.parametrize(419    "message_type",420    [421        "lifespan.startup.failed",422        "lifespan.shutdown.failed",423        "lifespan.shutdown.complete",424    ],425)426def test_lifespan_flush_runs_off_event_loop(monkeypatch, message_type):427    calls = []428    flush_threads = []429    event_loop_thread = threading.get_ident()430431    class Provider:432        def force_flush(self):433            flush_threads.append(threading.get_ident())434            calls.append("flush")435436        def shutdown(self):437            calls.append("shutdown")438439    monkeypatch.setattr(runtime, "_owned", [Provider()])440441    async def app(scope, receive, send):442        assert await receive() == {"type": "lifespan.startup"}443        await send({"type": message_type})444445    async def receive():446        return {"type": "lifespan.startup"}447448    async def send(message):449        calls.append(message["type"])450451    asyncio.run(452        runtime.lifespan(453            config=FastAPI()._telemetry, app=app, scope={}, receive=receive, send=send454        )455    )456    assert calls == ["flush", message_type]457    assert len(flush_threads) == 1458    assert flush_threads[0] != event_loop_thread459    runtime._shutdown()460    runtime._shutdown()461    assert calls == ["flush", message_type, "shutdown"]462463464def test_registration_provider_prefers_public_metric_reader(monkeypatch):465    from logfire._internal.metrics import ProxyMeterProvider466    from opentelemetry.metrics import NoOpMeterProvider467468    provider = NoOpMeterProvider()469    proxy = ProxyMeterProvider(provider=provider)470    assert runtime._registration_provider(proxy) is provider471472    monkeypatch.setattr(proxy, "add_metric_reader", lambda reader: None, raising=False)473    assert runtime._registration_provider(proxy) is proxy474475476@pytest.mark.parametrize("wrapped_meter", [False, True])477def test_concurrent_provider_owner_wins(monkeypatch, wrapped_meter):478    from opentelemetry import _logs, metrics, trace479    from opentelemetry.sdk import _logs as sdk_logs480    from opentelemetry.sdk import metrics as sdk_metrics481    from opentelemetry.sdk import trace as sdk_trace482483    stopped = []484    winners = []485    for name, api, sdk, cls_name in [486        ("tracer", trace, sdk_trace, "TracerProvider"),487        ("meter", metrics, sdk_metrics, "MeterProvider"),488        ("logger", _logs, sdk_logs, "LoggerProvider"),489    ]:490        original = getattr(sdk, cls_name)491        winner = original(shutdown_on_exit=False)492        winners.append(winner)493494        class Provider(original):495            def shutdown(self, _name=name):496                stopped.append(_name)497                super().shutdown()498499        if wrapped_meter and name == "meter":500            from logfire._internal.metrics import ProxyMeterProvider501502            winner = ProxyMeterProvider(winner)503        current = [getattr(api, f"get_{name}_provider")()]504        monkeypatch.setattr(sdk, cls_name, Provider)505        monkeypatch.setattr(api, f"get_{name}_provider", lambda c=current: c[0])506        monkeypatch.setattr(507            api,508            f"set_{name}_provider",509            lambda provider, c=current, w=winner: c.__setitem__(0, w),510        )511    monkeypatch.setattr(runtime, "_owned", [])512    monkeypatch.setattr(runtime, "_configured", [])513    monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:1")514    runtime._configure_from_environment(515        FastAPI(telemetry={"auto_configure": True})._telemetry516    )517    assert stopped == ["tracer", "meter", "logger"]518    assert [provider for _, provider in runtime._configured] == winners519    assert len(runtime._owned) == 3520    assert all(component not in winners for component in runtime._owned)521    runtime._shutdown()522    for winner in winners:523        winner.shutdown()524525526@pytest.mark.skipif(527    not hasattr(__import__("os"), "fork"), reason="POSIX worker lifecycle"528)529@run_in_subprocess530def test_environment_export_initializes_after_fork():531    import os532    import threading533    from http.server import BaseHTTPRequestHandler534535    from fastapi import FastAPI536    from fastapi.telemetry import _runtime537    from fastapi.testclient import TestClient538539    from tests.test_telemetry._otlp import _CollectorServer540541    received = []542543    class Handler(BaseHTTPRequestHandler):544        def do_POST(self):545            self.rfile.read(int(self.headers["Content-Length"]))546            received.append(self.path)547            self.send_response(200)548            self.end_headers()549550        def log_message(self, format, *args):551            pass552553    server = _CollectorServer(("127.0.0.1", 0), Handler)554    os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = (555        f"http://127.0.0.1:{server.server_port}/traces"556    )557    os.environ["FASTAPI_OTEL_AUTO_CONFIGURE"] = "true"558    # Importing and constructing before fork must not create providers.559    app = FastAPI()560    assert not _runtime._owned561    pid = os.fork()562    if pid == 0:563        try:564            with TestClient(app) as client:565                assert client.get("/").status_code == 404566            assert len(_runtime._owned) == 1567            _runtime._shutdown()568        except BaseException:  # pragma: no cover569            os._exit(1)570        os._exit(0)571    thread = threading.Thread(target=server.serve_forever, daemon=True)572    thread.start()573    _, status = os.waitpid(pid, 0)574    assert status == 0, status575    assert received == ["/traces"], received576    assert not _runtime._owned577    server.shutdown()578    server.server_close()579580581@run_in_subprocess582def test_real_otlp_exception_export_without_traces_or_metrics():583    import os584585    from fastapi import FastAPI586    from fastapi.telemetry import _runtime587    from fastapi.testclient import TestClient588    from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import (589        ExportLogsServiceRequest,590    )591592    with otlp_collector() as (base, received):593        os.environ["OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"] = base + "/logs"594        os.environ["OTEL_SERVICE_NAME"] = "errors-only"595        app = FastAPI(telemetry={"auto_configure": True})596597        @app.get("/items/{item_id}")598        def endpoint(item_id: int):599            raise ValueError("exported exception")600601        try:602            for _ in range(2):603                with TestClient(app, raise_server_exceptions=False) as client:604                    assert client.get("/items/1").status_code == 500605            assert len(_runtime._owned) == 1606            assert all(path == "/logs" for path, _, _ in received)607            requests = [608                ExportLogsServiceRequest.FromString(body) for _, body, _ in received609            ]610            records = [611                record612                for request in requests613                for resource in request.resource_logs614                for scope in resource.scope_logs615                for record in scope.log_records616            ]617            assert len(records) == 2618            for record in records:619                attributes = {a.key: a.value.string_value for a in record.attributes}620                assert attributes["exception.type"] == "ValueError"621                assert attributes["exception.message"] == "exported exception"622                assert attributes["http.route"] == "/items/{item_id}"623                assert "in endpoint" in attributes["exception.stacktrace"]624                assert (625                    "ValueError: exported exception"626                    in attributes["exception.stacktrace"]627                )628        finally:629            _runtime._shutdown()630631632@pytest.mark.parametrize("name", ["tracer", "meter", "logger"])633def test_unsupported_provider_warns_without_preventing_startup(634    monkeypatch, caplog, name635):636    from opentelemetry._logs import NoOpLoggerProvider637    from opentelemetry.metrics import NoOpMeterProvider638    from opentelemetry.trace import NoOpTracerProvider639640    signal, provider = {641        "tracer": ("TRACES", NoOpTracerProvider),642        "meter": ("METRICS", NoOpMeterProvider),643        "logger": ("LOGS", NoOpLoggerProvider),644    }[name]645    monkeypatch.setenv(f"OTEL_EXPORTER_OTLP_{signal}_ENDPOINT", "http://127.0.0.1:1")646    app = FastAPI(telemetry={f"{name}_provider": provider(), "auto_configure": True})647    with TestClient(app) as client:648        assert client.get("/").status_code == 404649    assert "does not support adding an OTLP exporter" in caplog.text650    assert "auto_configure" in caplog.text651    assert not runtime._owned652653654@pytest.mark.parametrize("error_type", [ValueError, AttributeError])655def test_failed_registration_closes_new_exporter(monkeypatch, caplog, error_type):656    from opentelemetry.exporter.otlp.proto.http import trace_exporter657    from opentelemetry.sdk.trace import TracerProvider658    from opentelemetry.sdk.trace.export.in_memory_span_exporter import (659        InMemorySpanExporter,660    )661662    stopped = []663664    class Exporter(InMemorySpanExporter):665        def __init__(self, *, endpoint):666            super().__init__()667668        def shutdown(self):669            stopped.append(True)670            super().shutdown()671672    provider = TracerProvider(shutdown_on_exit=False)673674    def fail(component):675        raise error_type("registration failed")676677    monkeypatch.setattr(provider, "add_span_processor", fail)678    monkeypatch.setattr(trace_exporter, "OTLPSpanExporter", Exporter)679    monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://127.0.0.1:1")680    app = FastAPI(telemetry={"tracer_provider": provider, "auto_configure": True})681    with TestClient(app) as client:682        assert client.get("/").status_code == 404683    assert "registration failed" in caplog.text684    assert stopped == [True]685    provider.shutdown()