using System.Diagnostics; using System.Reflection; using FastEndpoints; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace GB5Shared.Hosting; public sealed class ModuleLoadResult { public IReadOnlyList HealthyModules { get; } public Assembly[] FastEndpointsAssemblies { get; } public IReadOnlyList<(string ModuleName, string Reason)> Excluded { get; } private readonly HashSet _brokenEndpointTypes; internal ModuleLoadResult( IReadOnlyList healthyModules, Assembly[] fastEndpointsAssemblies, HashSet brokenEndpointTypes, IReadOnlyList<(string, string)> excluded) { HealthyModules = healthyModules; FastEndpointsAssemblies = fastEndpointsAssemblies; _brokenEndpointTypes = brokenEndpointTypes; Excluded = excluded; } /// /// Pass as o.Filter in AddFastEndpoints(o => ...). FastEndpoints calls this once /// per discovered type during startup and skips registering it when this returns false - the /// only mechanism that actually works for excluding a single broken endpoint, since /// o.Assemblies does NOT restrict discovery (FastEndpoints scans every assembly already loaded /// into the process regardless of what's listed there). /// public bool EndpointFilter(Type type) => !_brokenEndpointTypes.Contains(type); /// Assembly name -> display name, for . public Dictionary BuildModuleByAssembly() => HealthyModules.ToDictionary(m => m.OwnerAssembly.GetName().Name ?? m.DisplayName, m => m.DisplayName); /// " | "-joined display names, for Swagger descriptions / the host's root banner text. public string BannerText() => string.Join(" | ", HealthyModules.Select(m => m.DisplayName).OrderBy(n => n, StringComparer.Ordinal)); public void MapEndpoints(WebApplication app, ILogger logger) { foreach (var m in HealthyModules) { try { m.Module.MapEndpoints(app); } catch (Exception ex) { logger.LogError(ex, "Module '{Module}' threw during MapEndpoints() - its non-FastEndpoints routes " + "(SignalR hubs/controllers) may be missing, but the rest of the host keeps running.", m.DisplayName); } } } /// /// Maps GET /health on this host, reporting every module discovered at startup individually /// (healthy vs excluded, with the reason for exclusion) instead of a single process-wide /// up/down flag. Excluded entries whose name contains "." are a broken endpoint within an /// otherwise-healthy module (format "{ModuleDisplayName}.{EndpointTypeName}"), so those are /// folded into their owning module's record as a partial-degradation note rather than listed /// as a separate top-level module. /// public void MapModuleHealth(WebApplication app, string hostName) { app.MapGet("/health", () => { var moduleIssues = Excluded .GroupBy(e => e.ModuleName.Contains('.') ? e.ModuleName[..e.ModuleName.IndexOf('.')] : e.ModuleName) .ToDictionary(g => g.Key, g => g.Select(e => e.Reason).ToList(), StringComparer.OrdinalIgnoreCase); var modules = HealthyModules .Select(m => new { Name = m.DisplayName, Status = moduleIssues.ContainsKey(m.DisplayName) ? "degraded" : "healthy", Issues = moduleIssues.TryGetValue(m.DisplayName, out var issues) ? issues : new List(), DiskMB = GetModuleDiskMB(m) }) .Concat(Excluded .Where(e => !e.ModuleName.Contains('.')) // whole-module failures not already folded above .Select(e => new { Name = e.ModuleName, Status = "failed", Issues = new List { e.Reason }, DiskMB = 0.0 })) .OrderBy(m => m.Name, StringComparer.Ordinal) .ToList(); var proc = Process.GetCurrentProcess(); return Results.Ok(new { Host = hostName, Status = modules.Any(m => m.Status == "failed") ? "degraded" : "healthy", Timestamp = DateTime.UtcNow, ModuleCount = modules.Count, WorkingSetMB = Math.Round(proc.WorkingSet64 / 1_048_576.0, 1), DiskTotalMB = Math.Round(modules.Sum(m => m.DiskMB), 1), Modules = modules }); }).AllowAnonymous(); } /// /// Sum of every DLL belonging to this module's own project layer (e.g. AccountsSL.dll + /// AccountsBLL.dll + AccountsDAL.dll) sitting next to the Host's executable. RAM can't be split /// per-module (all modules share one process's working set - see WorkingSetMB above, reported /// once per host), but on-disk size genuinely is per-assembly and safe to sum here. /// private static double GetModuleDiskMB(DiscoveredModule m) { try { var baseName = (m.OwnerAssembly.GetName().Name ?? m.DisplayName); // Strip a trailing SL/BLL/DAL layer suffix to get the module's root name (e.g. "AccountsSL" -> "Accounts"), // then sum every sibling assembly in the same output folder that starts with that root. var root = baseName; foreach (var suffix in new[] { "SL", "BLL", "DAL" }) if (root.EndsWith(suffix, StringComparison.Ordinal)) { root = root[..^suffix.Length]; break; } var dir = Path.GetDirectoryName(m.OwnerAssembly.Location); if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return 0.0; var bytes = Directory.EnumerateFiles(dir, $"{root}*.dll") .Where(f => Path.GetFileNameWithoutExtension(f).StartsWith(root, StringComparison.Ordinal)) .Sum(f => new FileInfo(f).Length); return Math.Round(bytes / 1_048_576.0, 2); } catch { return 0.0; // disk size is a display nicety, never worth failing /health over } } } /// /// Registers every discovered module's DI services and finds anything that can't load safely, so /// one broken module can never take the rest of the host down at startup. This is the historical /// failure mode behind JobEngine (missing config threw inside Register()) and IDMS/OKR (an /// endpoint's constructor dependency FastEndpoints' own eager validation can't resolve) — /// previously all three had to be excluded from the .csproj entirely because a single broken /// module crashed FastEndpoints' one combined startup scan for every OTHER module on the host too. /// /// Proven live (see git history around 2026-07-17): excluding a broken assembly from /// AddFastEndpoints' o.Assemblies is NOT sufficient — FastEndpoints scans every assembly already /// loaded into the process regardless of that list, so merely loading an assembly during discovery /// is enough for FastEndpoints to try constructing its endpoints anyway. The only mechanism that /// actually prevents a specific broken endpoint from being registered is /// EndpointDiscoveryOptions.Filter, a per-TYPE predicate FastEndpoints itself calls during startup — /// see , which must be wired into /// AddFastEndpoints(o => o.Filter = ...) by every Host's Program.cs. /// /// Three independent checks run here, all fail-safe (exclude on doubt) for endpoint construction, /// since a resolution failure here is a 100% reliable predictor of the identical failure crashing /// the whole host later — excluding one endpoint type is a strictly better outcome than that: /// 1. Register() is wrapped in try/catch - catches config-driven startup throws. Every endpoint /// type owned by a module whose Register() failed is marked broken (its DI setup never ran). /// 2. Every FastEndpoints endpoint type in every discovered, cleanly-loadable assembly has its /// constructor dependencies test-resolved against a throwaway ServiceProvider snapshot of the /// container built so far. Both a null result and a thrown exception mark that endpoint type /// broken. /// 3. An assembly whose own types can't be enumerated at all (ReflectionTypeLoadException — e.g. /// an incidental legacy dependency referencing assemblies that don't exist in .NET 9) is /// excluded wholesale; this is why also restricts which .dll /// files ever get loaded in the first place to GB5's own SL/BLL/DAL naming convention, so /// random third-party/legacy DLLs sitting in the output folder never reach this point. /// public static class ModuleLoader { public static ModuleLoadResult LoadAll( IServiceCollection services, ConfigurationManager configuration, ILogger logger, params string[] alwaysIncludeAssemblies) { LayerModuleConfigOverrides(configuration, logger); var (discoveredAssemblies, discoveredModules) = ModuleDiscovery.ScanBaseDirectory(logger); var excluded = new List<(string, string)>(); var registeredModules = new List(); var brokenOwnerAssemblies = new HashSet(); foreach (var dm in discoveredModules) { try { dm.Module.Register(services, configuration); registeredModules.Add(dm); } catch (Exception ex) { logger.LogError(ex, "Module '{Module}' disabled - Register() threw at startup. It will not be " + "available on this host until the underlying issue is fixed.", dm.DisplayName); excluded.Add((dm.DisplayName, $"Register() threw: {ex.GetType().Name}: {ex.Message}")); brokenOwnerAssemblies.Add(dm.OwnerAssembly); } } ServiceProvider? probe = null; try { probe = services.BuildServiceProvider(); } catch (Exception ex) { // Fail-open only here: if the throwaway container itself won't even build, skip this // safety net entirely rather than exclude every endpoint over it. logger.LogWarning(ex, "Module endpoint DI pre-flight check skipped - throwaway ServiceProvider failed to build."); } var brokenEndpointTypes = new HashSet(); var cleanAssemblies = new List(); foreach (var candidate in discoveredAssemblies) { Type[] types; try { types = candidate.Assembly.GetTypes(); } catch (ReflectionTypeLoadException ex) { var missing = ex.LoaderExceptions.Where(e => e is not null).Select(e => e!.Message).Distinct().Take(3); var reason = $"assembly types could not be fully loaded - {string.Join("; ", missing)}"; var owner = registeredModules.FirstOrDefault(m => m.OwnerAssembly == candidate.Assembly); logger.LogError( "Module '{Module}' disabled - {Reason}. It will not be available on this host until the underlying issue is fixed.", owner?.DisplayName ?? candidate.FileName, reason); excluded.Add((owner?.DisplayName ?? candidate.FileName, reason)); continue; // whole assembly excluded from o.Assemblies - can't safely enumerate its types at all } cleanAssemblies.Add(candidate.Assembly); var registerFailed = brokenOwnerAssemblies.Contains(candidate.Assembly); foreach (var type in types) { if (type.IsAbstract || !typeof(IEndpoint).IsAssignableFrom(type)) continue; if (registerFailed) { brokenEndpointTypes.Add(type); // this module's Register() never ran - its DI setup is incomplete continue; } if (probe is null) continue; // pre-flight unavailable - fail-open, leave endpoint in var problem = TryResolveEndpointDependencies(probe, type); if (problem is null) continue; brokenEndpointTypes.Add(type); var owner = registeredModules.FirstOrDefault(m => m.OwnerAssembly == candidate.Assembly); var label = owner?.DisplayName ?? candidate.FileName; logger.LogError( "Endpoint '{Endpoint}' in module '{Module}' disabled - {Problem}. " + "The rest of the module keeps running; only this endpoint is unavailable until fixed.", type.Name, label, problem); excluded.Add(($"{label}.{type.Name}", problem)); } } probe?.Dispose(); var healthyModules = registeredModules .Where(m => cleanAssemblies.Contains(m.OwnerAssembly)) .ToList(); var extraAssemblies = alwaysIncludeAssemblies .Select(TryLoadByName) .Where(a => a is not null) .Cast(); var fastEndpointsAssemblies = cleanAssemblies .Concat(extraAssemblies) .Distinct() .ToArray(); return new ModuleLoadResult(healthyModules, fastEndpointsAssemblies, brokenEndpointTypes, excluded); } private static string? TryResolveEndpointDependencies(ServiceProvider probe, Type endpointType) { var ctor = endpointType.GetConstructors().OrderByDescending(c => c.GetParameters().Length).FirstOrDefault(); if (ctor is null) return null; using var scope = probe.CreateScope(); foreach (var param in ctor.GetParameters()) { var pt = param.ParameterType; if (pt.IsValueType || pt == typeof(string) || param.HasDefaultValue) continue; // not a DI-resolved service parameter object? resolved; try { resolved = scope.ServiceProvider.GetService(pt); } catch (Exception resolveEx) { // A DI resolution exception here is a 100% reliable signal that the exact same // construction throws in production too (proven live: FastEndpoints eagerly // constructs every endpoint at MapFastEndpoints() time, and a factory throwing here // - e.g. GB5Shared.Storage.StorageProviderFactory.Create() with no NetworkBasePath // configured - crashed the entire host, not just that one endpoint). return $"needs '{pt.Name}' which throws while resolving: {resolveEx.GetType().Name}: {resolveEx.Message}"; } if (resolved is null) return $"needs '{pt.Name}' which is not registered in DI"; } return null; } private static Assembly? TryLoadByName(string name) { try { return Assembly.Load(name); } catch { return null; } } /// /// Every module's own appsettings.json (copied by Hosts/Directory.Build.targets into /// ModuleConfigs/<ModuleName>.appsettings.json) is layered into this Host's configuration /// at the LOWEST priority - inserted at Sources[0], so the Host's own appsettings.json/ /// environment variables/command-line args (already added by WebApplication.CreateBuilder /// before this runs) always win for any key both define. This only fills gaps: a module /// that reads a config section only present in its own standalone appsettings.json (e.g. /// StorageConfiguration, Vault) gets that value here too instead of silently missing it - /// the exact class of bug behind "works standalone, breaks inside the Host". /// private static void LayerModuleConfigOverrides(ConfigurationManager configuration, ILogger logger) { var dir = Path.Combine(AppContext.BaseDirectory, "ModuleConfigs"); if (!Directory.Exists(dir)) return; foreach (var file in Directory.EnumerateFiles(dir, "*.json").OrderBy(f => f, StringComparer.Ordinal)) { try { configuration.Sources.Insert(0, new Microsoft.Extensions.Configuration.Json.JsonConfigurationSource { FileProvider = new Microsoft.Extensions.FileProviders.PhysicalFileProvider(dir), Path = Path.GetFileName(file), Optional = true, ReloadOnChange = false, }); } catch (Exception ex) { // Fail-open: a module config file that can't be layered in is a lost gap-fill, // not a reason to prevent the host from starting. logger.LogWarning(ex, "Could not layer module config file '{File}' - skipped.", file); } } } }