using GB5Shared.Telemetry.Logging;
using GB5Shared.Telemetry.Observability;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using OpenTelemetry;
using OpenTelemetry.Exporter;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using System.Diagnostics;
namespace GB5Shared.Telemetry;
///
/// Extension methods for registering the unified GB5 OpenTelemetry stack.
///
/// Call once per service in Program.cs — it replaces the
/// old per-service Zipkin block and the manual Tracer singleton registration.
///
///
/// Configuration is driven entirely by the "OpenTelemetry" appsettings.json section
/// (see ). When the section is absent the service defaults to
/// Zipkin at localhost:9411, always-sample — identical to the previous behaviour.
///
///
/// To switch a service to OTLP (Jaeger / Grafana Tempo) in production, set:
///
/// "OpenTelemetry": {
/// "Exporter": "OTLP",
/// "OtlpEndpoint": "http://otel-collector:4317",
/// "Environment": "Production",
/// "SamplingRatio": 0.2
/// }
///
///
///
public static class GB5TelemetryExtensions
{
///
/// Registers OpenTelemetry tracing and (optionally) metrics for a GB5 service.
///
/// The DI container from WebApplicationBuilder.Services.
/// Application configuration (appsettings.json + environment overrides).
///
/// Unique service identifier emitted in every span, e.g. "GB5-FRAMEWORK",
/// "GB5-ADMIN", "GB5-DMS". Use the existing name your service already
/// reports to Zipkin so existing dashboards continue to work without renaming.
///
public static IServiceCollection AddGB5Telemetry(
this IServiceCollection services,
IConfiguration configuration,
string serviceName)
{
var section = configuration.GetSection(TelemetryOptions.SectionName);
var options = section.Get() ?? new TelemetryOptions();
// Always the real per-service name — never the appsettings section default.
// Consumed by RequestTracingMiddleware so TOBSERVABILITY reports each service's
// own identity instead of a shared "GB5" literal.
options.ServiceName = serviceName;
// Resolve effective sampling ratio — explicit appsettings value wins;
// fall back to environment-aware defaults (Production: 5 %, UAT: 10 %, QA: 50 %, Dev: 100 %)
if (options.UseEnvironmentSamplingDefaults
&& section[nameof(TelemetryOptions.SamplingRatio)] is null)
{
var aspnetEnv = System.Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
options.SamplingRatio = TelemetryOptions.GetEnvironmentSamplingDefault(aspnetEnv);
}
// Always register as singleton — RequestTracingMiddleware and other components
// inject TelemetryOptions regardless of the Enabled flag.
services.AddSingleton(options);
if (!options.Enabled)
{
// Register a no-op Tracer so BaseEndpoint.Resolve() never throws
// even when telemetry is administratively disabled.
services.AddSingleton(_ => TracerProvider.Default.GetTracer(serviceName));
return services;
}
// ── OTel resource — identifies this service in every span ─────────────────
var resource = ResourceBuilder.CreateDefault()
.AddService(
serviceName: serviceName,
serviceVersion: options.ServiceVersion)
.AddAttributes(new Dictionary
{
["deployment.environment"] = options.Environment,
["host.name"] = System.Environment.MachineName,
["process.id"] = System.Environment.ProcessId,
});
var otel = services.AddOpenTelemetry();
// ── Tracing ───────────────────────────────────────────────────────────────
otel.WithTracing(tracing =>
{
tracing
.SetResourceBuilder(resource)
// ASP.NET Core — instrument every inbound HTTP request
.AddAspNetCoreInstrumentation(o =>
{
o.RecordException = options.RecordExceptions;
// Skip infrastructure endpoints that generate noise with no business value
o.Filter = static ctx =>
{
var path = ctx.Request.Path.Value;
return path is not null
&& !path.StartsWith("/health", StringComparison.OrdinalIgnoreCase)
&& !path.StartsWith("/healthz", StringComparison.OrdinalIgnoreCase)
&& !path.StartsWith("/swagger", StringComparison.OrdinalIgnoreCase)
&& path != "/";
};
// Enrich inbound spans with GB5 correlation context from request headers
o.EnrichWithHttpRequest = static (activity, request) =>
{
var clientIp = request.HttpContext.Connection.RemoteIpAddress?.ToString();
if (clientIp is not null)
activity.SetTag("http.client_ip", clientIp);
if (request.Headers.TryGetValue("X-Correlation-Id", out var correlId)
&& !string.IsNullOrEmpty(correlId))
activity.SetTag("gb5.correlation.id", correlId.ToString());
};
o.EnrichWithHttpResponse = static (activity, response) =>
{
activity.SetTag("http.response.status_code", response.StatusCode);
if (response.StatusCode >= 400)
activity.SetStatus(ActivityStatusCode.Error, $"HTTP {response.StatusCode}");
if (response.ContentLength.HasValue)
activity.SetTag("http.response_content_length", response.ContentLength.Value);
};
// Surface exception type in the span for quick triage without reading
// full stack traces in the trace viewer
o.EnrichWithException = static (activity, ex) =>
{
activity.SetTag("exception.escaped", true);
activity.SetTag("exception.full_type", ex.GetType().FullName);
if (ex.InnerException is not null)
activity.SetTag("exception.inner_type",
ex.InnerException.GetType().FullName);
};
})
// HttpClient — instrument all outbound HTTP calls from named clients
.AddHttpClientInstrumentation(o =>
{
o.RecordException = options.RecordExceptions;
// Exclude Dapr sidecar HTTP API calls (ports 3500/3501 on localhost)
// — Dapr gRPC calls are not captured by this instrumentation anyway,
// this guard prevents any stray Dapr HTTP health/metadata calls.
o.FilterHttpRequestMessage = static msg =>
{
var host = msg.RequestUri?.Host;
var port = msg.RequestUri?.Port ?? 0;
return !(host is "localhost" or "127.0.0.1"
&& (port == 3500 || port == 3501));
};
o.EnrichWithHttpRequestMessage = static (activity, req) =>
{
activity.SetTag("http.outbound.url", req.RequestUri?.AbsoluteUri);
activity.SetTag("http.outbound.method", req.Method.Method);
};
o.EnrichWithHttpResponseMessage = static (activity, res) =>
activity.SetTag("http.outbound.status_code", (int)res.StatusCode);
o.EnrichWithException = static (activity, ex) =>
activity.SetTag("http.outbound.error", ex.GetType().Name);
});
// SqlClient auto-instrumentation is intentionally omitted.
// QueryExecutor emits its own GB5.Database spans with full SQL text,
// typed parameters (db.params), caller identity (db.caller), and row counts.
// The SqlClient layer would create a duplicate span per Dapper call whose
// name is just a list of FROM/JOIN table names with no parameter values.
// Register all GB5 custom ActivitySources so their spans are captured
// (Endpoints, Dapr.Publish, Dapr.Subscribe, Database, Scheduler, etc.)
foreach (var sourceName in GB5ActivitySources.AllSourceNames)
tracing.AddSource(sourceName);
// Tags every span with gb5.call.origin (Interactive vs Automated) so traces
// can be filtered by who/what triggered them, independent of module or Host.
tracing.AddProcessor(new GB5CallOriginProcessor());
// ── Sampling ──────────────────────────────────────────────────────────
if (options.SamplingRatio >= 1.0)
tracing.SetSampler(new AlwaysOnSampler());
else if (options.SamplingRatio <= 0.0)
tracing.SetSampler(new AlwaysOffSampler());
else
tracing.SetSampler(new TraceIdRatioBasedSampler(options.SamplingRatio));
ApplyTraceExporter(tracing, options);
});
// ── Metrics ───────────────────────────────────────────────────────────────
if (options.EnableMetrics)
{
otel.WithMetrics(metrics =>
{
metrics
.SetResourceBuilder(resource)
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation();
ApplyMetricExporter(metrics, options);
});
}
// ── Log-to-span bridge ────────────────────────────────────────────────────
// SpanEventLoggerProvider intercepts every ILogger call in GB5 application code
// and adds it as a span event (annotation) on Activity.Current.
// This makes BLL, DAL, and service log statements visible in the trace timeline
// without any per-class changes anywhere in the codebase.
if (options.CaptureLogEvents)
{
services.AddLogging(logging =>
logging.AddProvider(new SpanEventLoggerProvider(options)));
}
// ── Tracer singleton — backward compatibility with BaseEndpoint.Resolve() ─
// BaseEndpoint resolves Tracer from DI for span operations. The provider is
// registered by AddOpenTelemetry() above; we just expose a named Tracer from it.
services.AddSingleton(sp =>
sp.GetRequiredService().GetTracer(serviceName));
// ── ObservabilityWriter — non-blocking TOBSERVABILITY sink ────────────────
// Registered as singleton + hosted service so it starts automatically.
// RequestTracingMiddleware resolves IObservabilityWriter (optional) and
// calls Enqueue() fire-and-forget after every request.
services.AddSingleton();
services.AddSingleton(sp =>
sp.GetRequiredService());
services.AddHostedService(sp =>
sp.GetRequiredService());
return services;
}
// ── Private helpers ───────────────────────────────────────────────────────
private static void ApplyTraceExporter(TracerProviderBuilder tracing, TelemetryOptions opts)
{
switch (opts.Exporter.ToUpperInvariant())
{
case "OTLP":
tracing.AddOtlpExporter(o =>
{
o.Endpoint = new Uri(opts.OtlpEndpoint);
o.Protocol = OtlpExportProtocol.Grpc;
if (opts.TraceExportProcessor.Equals("Simple", StringComparison.OrdinalIgnoreCase))
{
o.ExportProcessorType = ExportProcessorType.Simple;
}
else
{
o.ExportProcessorType = ExportProcessorType.Batch;
o.BatchExportProcessorOptions.ScheduledDelayMilliseconds = opts.BatchScheduledDelayMilliseconds;
}
});
break;
case "ZIPKIN":
tracing.AddZipkinExporter(o =>
{
o.Endpoint = new Uri(opts.ZipkinEndpoint);
// Real-time trace visibility: each phase of a long-running request gets its
// own child span (see AnalysisDAL.DynamicOutput / ReportExport.PdfExportCore
// for examples using GB5Trace.BeginSection). Those child spans end — and
// become exportable — as soon as their phase finishes, well before the whole
// request completes. Batch keeps exporting off the request thread; shortening
// the flush delay (BatchScheduledDelayMilliseconds, default 5000ms in the SDK)
// is what makes those completed phases actually show up in Zipkin promptly
// instead of sitting buffered until the 5s timer or a full 512-span batch.
if (opts.TraceExportProcessor.Equals("Simple", StringComparison.OrdinalIgnoreCase))
{
o.ExportProcessorType = ExportProcessorType.Simple;
}
else
{
o.ExportProcessorType = ExportProcessorType.Batch;
o.BatchExportProcessorOptions.ScheduledDelayMilliseconds = opts.BatchScheduledDelayMilliseconds;
}
});
break;
case "CONSOLE":
tracing.AddConsoleExporter();
break;
// "NONE" or unrecognised: instrumentation active but no export
// (useful for CPU profiling or local smoke-testing without a collector)
}
// Optional secondary console output alongside the primary exporter
if (opts.EnableConsoleExporter
&& !opts.Exporter.Equals("CONSOLE", StringComparison.OrdinalIgnoreCase))
{
tracing.AddConsoleExporter();
}
}
private static void ApplyMetricExporter(MeterProviderBuilder metrics, TelemetryOptions opts)
{
switch (opts.Exporter.ToUpperInvariant())
{
case "OTLP":
metrics.AddOtlpExporter(o =>
{
o.Endpoint = new Uri(opts.OtlpEndpoint);
o.Protocol = OtlpExportProtocol.Grpc;
});
break;
case "CONSOLE":
metrics.AddConsoleExporter();
break;
// ZIPKIN does not support the OpenTelemetry metrics protocol.
// When Exporter=ZIPKIN, metrics are collected in-process but not exported
// unless EnableConsoleExporter is also true.
}
if (opts.EnableConsoleExporter
&& !opts.Exporter.Equals("CONSOLE", StringComparison.OrdinalIgnoreCase))
{
metrics.AddConsoleExporter();
}
}
}