using GB5Shared.Telemetry.Logging; using GB5Shared.Telemetry.Observability; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Exporter; using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; using System.Diagnostics; namespace GB5Shared.Telemetry; /// /// Extension methods for registering the unified GB5 OpenTelemetry stack. /// /// Call once per service in Program.cs — it replaces the /// old per-service Zipkin block and the manual Tracer singleton registration. /// /// /// Configuration is driven entirely by the "OpenTelemetry" appsettings.json section /// (see ). When the section is absent the service defaults to /// Zipkin at localhost:9411, always-sample — identical to the previous behaviour. /// /// /// To switch a service to OTLP (Jaeger / Grafana Tempo) in production, set: /// /// "OpenTelemetry": { /// "Exporter": "OTLP", /// "OtlpEndpoint": "http://otel-collector:4317", /// "Environment": "Production", /// "SamplingRatio": 0.2 /// } /// /// /// public static class GB5TelemetryExtensions { /// /// Registers OpenTelemetry tracing and (optionally) metrics for a GB5 service. /// /// The DI container from WebApplicationBuilder.Services. /// Application configuration (appsettings.json + environment overrides). /// /// Unique service identifier emitted in every span, e.g. "GB5-FRAMEWORK", /// "GB5-ADMIN", "GB5-DMS". Use the existing name your service already /// reports to Zipkin so existing dashboards continue to work without renaming. /// public static IServiceCollection AddGB5Telemetry( this IServiceCollection services, IConfiguration configuration, string serviceName) { var section = configuration.GetSection(TelemetryOptions.SectionName); var options = section.Get() ?? new TelemetryOptions(); // Always the real per-service name — never the appsettings section default. // Consumed by RequestTracingMiddleware so TOBSERVABILITY reports each service's // own identity instead of a shared "GB5" literal. options.ServiceName = serviceName; // Resolve effective sampling ratio — explicit appsettings value wins; // fall back to environment-aware defaults (Production: 5 %, UAT: 10 %, QA: 50 %, Dev: 100 %) if (options.UseEnvironmentSamplingDefaults && section[nameof(TelemetryOptions.SamplingRatio)] is null) { var aspnetEnv = System.Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); options.SamplingRatio = TelemetryOptions.GetEnvironmentSamplingDefault(aspnetEnv); } // Always register as singleton — RequestTracingMiddleware and other components // inject TelemetryOptions regardless of the Enabled flag. services.AddSingleton(options); if (!options.Enabled) { // Register a no-op Tracer so BaseEndpoint.Resolve() never throws // even when telemetry is administratively disabled. services.AddSingleton(_ => TracerProvider.Default.GetTracer(serviceName)); return services; } // ── OTel resource — identifies this service in every span ───────────────── var resource = ResourceBuilder.CreateDefault() .AddService( serviceName: serviceName, serviceVersion: options.ServiceVersion) .AddAttributes(new Dictionary { ["deployment.environment"] = options.Environment, ["host.name"] = System.Environment.MachineName, ["process.id"] = System.Environment.ProcessId, }); var otel = services.AddOpenTelemetry(); // ── Tracing ─────────────────────────────────────────────────────────────── otel.WithTracing(tracing => { tracing .SetResourceBuilder(resource) // ASP.NET Core — instrument every inbound HTTP request .AddAspNetCoreInstrumentation(o => { o.RecordException = options.RecordExceptions; // Skip infrastructure endpoints that generate noise with no business value o.Filter = static ctx => { var path = ctx.Request.Path.Value; return path is not null && !path.StartsWith("/health", StringComparison.OrdinalIgnoreCase) && !path.StartsWith("/healthz", StringComparison.OrdinalIgnoreCase) && !path.StartsWith("/swagger", StringComparison.OrdinalIgnoreCase) && path != "/"; }; // Enrich inbound spans with GB5 correlation context from request headers o.EnrichWithHttpRequest = static (activity, request) => { var clientIp = request.HttpContext.Connection.RemoteIpAddress?.ToString(); if (clientIp is not null) activity.SetTag("http.client_ip", clientIp); if (request.Headers.TryGetValue("X-Correlation-Id", out var correlId) && !string.IsNullOrEmpty(correlId)) activity.SetTag("gb5.correlation.id", correlId.ToString()); }; o.EnrichWithHttpResponse = static (activity, response) => { activity.SetTag("http.response.status_code", response.StatusCode); if (response.StatusCode >= 400) activity.SetStatus(ActivityStatusCode.Error, $"HTTP {response.StatusCode}"); if (response.ContentLength.HasValue) activity.SetTag("http.response_content_length", response.ContentLength.Value); }; // Surface exception type in the span for quick triage without reading // full stack traces in the trace viewer o.EnrichWithException = static (activity, ex) => { activity.SetTag("exception.escaped", true); activity.SetTag("exception.full_type", ex.GetType().FullName); if (ex.InnerException is not null) activity.SetTag("exception.inner_type", ex.InnerException.GetType().FullName); }; }) // HttpClient — instrument all outbound HTTP calls from named clients .AddHttpClientInstrumentation(o => { o.RecordException = options.RecordExceptions; // Exclude Dapr sidecar HTTP API calls (ports 3500/3501 on localhost) // — Dapr gRPC calls are not captured by this instrumentation anyway, // this guard prevents any stray Dapr HTTP health/metadata calls. o.FilterHttpRequestMessage = static msg => { var host = msg.RequestUri?.Host; var port = msg.RequestUri?.Port ?? 0; return !(host is "localhost" or "127.0.0.1" && (port == 3500 || port == 3501)); }; o.EnrichWithHttpRequestMessage = static (activity, req) => { activity.SetTag("http.outbound.url", req.RequestUri?.AbsoluteUri); activity.SetTag("http.outbound.method", req.Method.Method); }; o.EnrichWithHttpResponseMessage = static (activity, res) => activity.SetTag("http.outbound.status_code", (int)res.StatusCode); o.EnrichWithException = static (activity, ex) => activity.SetTag("http.outbound.error", ex.GetType().Name); }); // SqlClient auto-instrumentation is intentionally omitted. // QueryExecutor emits its own GB5.Database spans with full SQL text, // typed parameters (db.params), caller identity (db.caller), and row counts. // The SqlClient layer would create a duplicate span per Dapper call whose // name is just a list of FROM/JOIN table names with no parameter values. // Register all GB5 custom ActivitySources so their spans are captured // (Endpoints, Dapr.Publish, Dapr.Subscribe, Database, Scheduler, etc.) foreach (var sourceName in GB5ActivitySources.AllSourceNames) tracing.AddSource(sourceName); // Tags every span with gb5.call.origin (Interactive vs Automated) so traces // can be filtered by who/what triggered them, independent of module or Host. tracing.AddProcessor(new GB5CallOriginProcessor()); // ── Sampling ────────────────────────────────────────────────────────── if (options.SamplingRatio >= 1.0) tracing.SetSampler(new AlwaysOnSampler()); else if (options.SamplingRatio <= 0.0) tracing.SetSampler(new AlwaysOffSampler()); else tracing.SetSampler(new TraceIdRatioBasedSampler(options.SamplingRatio)); ApplyTraceExporter(tracing, options); }); // ── Metrics ─────────────────────────────────────────────────────────────── if (options.EnableMetrics) { otel.WithMetrics(metrics => { metrics .SetResourceBuilder(resource) .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation(); ApplyMetricExporter(metrics, options); }); } // ── Log-to-span bridge ──────────────────────────────────────────────────── // SpanEventLoggerProvider intercepts every ILogger call in GB5 application code // and adds it as a span event (annotation) on Activity.Current. // This makes BLL, DAL, and service log statements visible in the trace timeline // without any per-class changes anywhere in the codebase. if (options.CaptureLogEvents) { services.AddLogging(logging => logging.AddProvider(new SpanEventLoggerProvider(options))); } // ── Tracer singleton — backward compatibility with BaseEndpoint.Resolve() ─ // BaseEndpoint resolves Tracer from DI for span operations. The provider is // registered by AddOpenTelemetry() above; we just expose a named Tracer from it. services.AddSingleton(sp => sp.GetRequiredService().GetTracer(serviceName)); // ── ObservabilityWriter — non-blocking TOBSERVABILITY sink ──────────────── // Registered as singleton + hosted service so it starts automatically. // RequestTracingMiddleware resolves IObservabilityWriter (optional) and // calls Enqueue() fire-and-forget after every request. services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); services.AddHostedService(sp => sp.GetRequiredService()); return services; } // ── Private helpers ─────────────────────────────────────────────────────── private static void ApplyTraceExporter(TracerProviderBuilder tracing, TelemetryOptions opts) { switch (opts.Exporter.ToUpperInvariant()) { case "OTLP": tracing.AddOtlpExporter(o => { o.Endpoint = new Uri(opts.OtlpEndpoint); o.Protocol = OtlpExportProtocol.Grpc; if (opts.TraceExportProcessor.Equals("Simple", StringComparison.OrdinalIgnoreCase)) { o.ExportProcessorType = ExportProcessorType.Simple; } else { o.ExportProcessorType = ExportProcessorType.Batch; o.BatchExportProcessorOptions.ScheduledDelayMilliseconds = opts.BatchScheduledDelayMilliseconds; } }); break; case "ZIPKIN": tracing.AddZipkinExporter(o => { o.Endpoint = new Uri(opts.ZipkinEndpoint); // Real-time trace visibility: each phase of a long-running request gets its // own child span (see AnalysisDAL.DynamicOutput / ReportExport.PdfExportCore // for examples using GB5Trace.BeginSection). Those child spans end — and // become exportable — as soon as their phase finishes, well before the whole // request completes. Batch keeps exporting off the request thread; shortening // the flush delay (BatchScheduledDelayMilliseconds, default 5000ms in the SDK) // is what makes those completed phases actually show up in Zipkin promptly // instead of sitting buffered until the 5s timer or a full 512-span batch. if (opts.TraceExportProcessor.Equals("Simple", StringComparison.OrdinalIgnoreCase)) { o.ExportProcessorType = ExportProcessorType.Simple; } else { o.ExportProcessorType = ExportProcessorType.Batch; o.BatchExportProcessorOptions.ScheduledDelayMilliseconds = opts.BatchScheduledDelayMilliseconds; } }); break; case "CONSOLE": tracing.AddConsoleExporter(); break; // "NONE" or unrecognised: instrumentation active but no export // (useful for CPU profiling or local smoke-testing without a collector) } // Optional secondary console output alongside the primary exporter if (opts.EnableConsoleExporter && !opts.Exporter.Equals("CONSOLE", StringComparison.OrdinalIgnoreCase)) { tracing.AddConsoleExporter(); } } private static void ApplyMetricExporter(MeterProviderBuilder metrics, TelemetryOptions opts) { switch (opts.Exporter.ToUpperInvariant()) { case "OTLP": metrics.AddOtlpExporter(o => { o.Endpoint = new Uri(opts.OtlpEndpoint); o.Protocol = OtlpExportProtocol.Grpc; }); break; case "CONSOLE": metrics.AddConsoleExporter(); break; // ZIPKIN does not support the OpenTelemetry metrics protocol. // When Exporter=ZIPKIN, metrics are collected in-process but not exported // unless EnableConsoleExporter is also true. } if (opts.EnableConsoleExporter && !opts.Exporter.Equals("CONSOLE", StringComparison.OrdinalIgnoreCase)) { metrics.AddConsoleExporter(); } } }