From cf081fc3f4424f14c819509a6f49df136371d6d6 Mon Sep 17 00:00:00 2001 From: Justin Middler Date: Tue, 28 Jul 2026 22:44:24 +1000 Subject: [PATCH] feat(dcr): Data Collection Rule optimiser and health assessment --- AGENTS.md | 1 + Bower.sln | 15 + src/Bower.Dcr/AGENTS.md | 5 + src/Bower.Dcr/Bower.Dcr.csproj | 1 + src/Bower.Dcr/DcrOptimiser.cs | 379 +++++++++++++++++++ src/Bower.Dcr/packages.lock.json | 6 + tests/Bower.UnitTests/Bower.UnitTests.csproj | 1 + tests/Bower.UnitTests/DcrOptimiserTests.cs | 78 ++++ tests/Bower.UnitTests/packages.lock.json | 3 + 9 files changed, 489 insertions(+) create mode 100644 src/Bower.Dcr/AGENTS.md create mode 100644 src/Bower.Dcr/Bower.Dcr.csproj create mode 100644 src/Bower.Dcr/DcrOptimiser.cs create mode 100644 src/Bower.Dcr/packages.lock.json create mode 100644 tests/Bower.UnitTests/DcrOptimiserTests.cs diff --git a/AGENTS.md b/AGENTS.md index c28c9da..74c5f6a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,7 @@ control plane, or runtime AI filter. - `src/Bower.Pipeline`: declarative telemetry pipeline model, templates and validation. - `src/Bower.Analytics`: telemetry quality and coverage scoring. - `src/Bower.Source.Ama`: Azure Monitor Agent companion discovery and custom log mapping. +- `src/Bower.Dcr`: Data Collection Rule optimiser and health assessment. - `schemas`, `policies`, `deploy`, `docs`, `tests`: versioned product assets. Inspect nearest `AGENTS.md` before editing. diff --git a/Bower.sln b/Bower.sln index eba0f57..9076a20 100644 --- a/Bower.sln +++ b/Bower.sln @@ -47,6 +47,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Analytics", "src\Bowe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Source.Ama", "src\Bower.Source.Ama\Bower.Source.Ama.csproj", "{A876B3F7-3209-4E9E-8993-2D9FF55637F2}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Dcr", "src\Bower.Dcr\Bower.Dcr.csproj", "{C6FDDD4F-022B-4431-85DF-839A7DEF1263}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -309,6 +311,18 @@ Global {A876B3F7-3209-4E9E-8993-2D9FF55637F2}.Release|x64.Build.0 = Release|Any CPU {A876B3F7-3209-4E9E-8993-2D9FF55637F2}.Release|x86.ActiveCfg = Release|Any CPU {A876B3F7-3209-4E9E-8993-2D9FF55637F2}.Release|x86.Build.0 = Release|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Debug|x64.ActiveCfg = Debug|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Debug|x64.Build.0 = Debug|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Debug|x86.ActiveCfg = Debug|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Debug|x86.Build.0 = Debug|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|Any CPU.Build.0 = Release|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x64.ActiveCfg = Release|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x64.Build.0 = Release|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x86.ActiveCfg = Release|Any CPU + {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -322,5 +336,6 @@ Global {E125F241-5F0D-43FD-8781-092995E1D309} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {F1BFA9A3-4762-47C4-B97D-578A105A8D6C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {A876B3F7-3209-4E9E-8993-2D9FF55637F2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C6FDDD4F-022B-4431-85DF-839A7DEF1263} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/src/Bower.Dcr/AGENTS.md b/src/Bower.Dcr/AGENTS.md new file mode 100644 index 0000000..ab67a88 --- /dev/null +++ b/src/Bower.Dcr/AGENTS.md @@ -0,0 +1,5 @@ +# DCR optimiser instructions + +Analyse DCR documents offline. Never call live Azure APIs from unit tests. +Recommendations must be deterministic and include estimated savings only when +inputs provide volume signals. diff --git a/src/Bower.Dcr/Bower.Dcr.csproj b/src/Bower.Dcr/Bower.Dcr.csproj new file mode 100644 index 0000000..6b512ec --- /dev/null +++ b/src/Bower.Dcr/Bower.Dcr.csproj @@ -0,0 +1 @@ + diff --git a/src/Bower.Dcr/DcrOptimiser.cs b/src/Bower.Dcr/DcrOptimiser.cs new file mode 100644 index 0000000..f2d93d8 --- /dev/null +++ b/src/Bower.Dcr/DcrOptimiser.cs @@ -0,0 +1,379 @@ +using System.Globalization; +using System.Text.Json; + +namespace Bower.Dcr; + +public sealed record DcrDataSource( + string Name, + string Kind, + IReadOnlyList Streams, + IReadOnlyList XPathQueries, + bool Enabled); + +public sealed record DcrDocument( + string Id, + string Name, + string? WorkspaceId, + IReadOnlyList DataSources, + IReadOnlyList Destinations); + +public sealed record DcrRecommendation( + string Code, + string Severity, + string Title, + string Detail, + double? EstimatedMonthlyGbSaved); + +public sealed record DcrAssessmentReport( + string DcrId, + string DcrName, + int CoverageScore, + int HealthScore, + double? EstimatedMonthlyIngestionGb, + double? EstimatedMonthlySavingsGb, + IReadOnlyList Recommendations, + DateTimeOffset AssessedAt); + +public static class DcrDocumentParser +{ + public static DcrDocument Parse(string json) + { + ArgumentException.ThrowIfNullOrWhiteSpace(json); + using JsonDocument document = JsonDocument.Parse(json); + JsonElement root = document.RootElement; + string id = ReadString(root, "id") ?? ReadString(root, "name") ?? "unknown"; + string name = ReadString(root, "name") ?? id; + string? workspace = ReadString(root, "workspaceId") + ?? ReadNested(root, "destinations", "logAnalytics", "workspaceResourceId"); + + List sources = []; + if (root.TryGetProperty("dataSources", out JsonElement dataSources)) + { + if (dataSources.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement item in dataSources.EnumerateArray()) + { + sources.Add(ParseSource(item)); + } + } + else if (dataSources.ValueKind == JsonValueKind.Object) + { + foreach (JsonProperty property in dataSources.EnumerateObject()) + { + if (property.Value.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement item in property.Value.EnumerateArray()) + { + sources.Add(ParseSource(item, property.Name)); + } + } + } + } + } + + List destinations = []; + if (root.TryGetProperty("destinations", out JsonElement destNode)) + { + if (destNode.ValueKind == JsonValueKind.Array) + { + destinations.AddRange( + destNode.EnumerateArray() + .Select(item => ReadString(item, "name") ?? item.ToString()) + .Where(item => !string.IsNullOrWhiteSpace(item))!); + } + else if (destNode.ValueKind == JsonValueKind.Object) + { + destinations.AddRange(destNode.EnumerateObject().Select(item => item.Name)); + } + } + + return new DcrDocument(id, name, workspace, sources, destinations); + } + + private static DcrDataSource ParseSource(JsonElement item, string? fallbackKind = null) + { + string name = ReadString(item, "name") ?? ReadString(item, "streams") ?? "source"; + string kind = ReadString(item, "kind") + ?? ReadString(item, "type") + ?? fallbackKind + ?? "unknown"; + List streams = ReadStringArray(item, "streams"); + List xpaths = ReadStringArray(item, "xPathQueries"); + if (xpaths.Count == 0) + { + xpaths = ReadStringArray(item, "xpathQueries"); + } + + bool enabled = !item.TryGetProperty("enabled", out JsonElement enabledNode) + || enabledNode.ValueKind != JsonValueKind.False; + return new DcrDataSource(name, kind, streams, xpaths, enabled); + } + + private static string? ReadString(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement property)) + { + return null; + } + + return property.ValueKind == JsonValueKind.String ? property.GetString() : property.ToString(); + } + + private static string? ReadNested(JsonElement element, params string[] path) + { + JsonElement current = element; + foreach (string segment in path) + { + if (current.ValueKind != JsonValueKind.Object || + !current.TryGetProperty(segment, out current)) + { + return null; + } + } + + return current.ValueKind == JsonValueKind.String ? current.GetString() : current.ToString(); + } + + private static List ReadStringArray(JsonElement element, string name) + { + if (!element.TryGetProperty(name, out JsonElement property) || + property.ValueKind != JsonValueKind.Array) + { + return []; + } + + return property.EnumerateArray() + .Select(item => item.GetString() ?? item.ToString()) + .Where(item => !string.IsNullOrWhiteSpace(item)) + .ToList()!; + } +} + +public static class DcrOptimiser +{ + private static readonly string[] RecommendedSecurityEventIds = + [ + "4624", "4625", "4648", "4672", "4688", "4720", "4728", "4732", "1102" + ]; + + private static readonly string[] RecommendedSysmonIds = ["1", "3", "11"]; + + public static DcrAssessmentReport Assess( + DcrDocument document, + double? currentMonthlyIngestionGb = null, + DateTimeOffset? now = null) + { + ArgumentNullException.ThrowIfNull(document); + DateTimeOffset assessedAt = now ?? DateTimeOffset.UtcNow; + List recommendations = []; + + // Duplicate names + IEnumerable> duplicates = document.DataSources + .GroupBy(item => item.Name, StringComparer.OrdinalIgnoreCase) + .Where(group => group.Count() > 1); + foreach (IGrouping group in duplicates) + { + recommendations.Add( + new DcrRecommendation( + "duplicate-source", + "medium", + "Duplicate collection sources", + $"Data source name '{group.Key}' appears {group.Count()} times.", + currentMonthlyIngestionGb is null ? null : currentMonthlyIngestionGb * 0.05)); + } + + bool hasWindowsEvents = document.DataSources.Any(item => + item.Kind.Contains("windows", StringComparison.OrdinalIgnoreCase) + || item.Streams.Any(stream => stream.Contains("WindowsEvent", StringComparison.OrdinalIgnoreCase))); + + bool hasWildcardSecurity = document.DataSources.Any(item => + item.XPathQueries.Any(query => + query.Contains("Security!*", StringComparison.OrdinalIgnoreCase) + || query.Contains("*[System[(EventID='*')]]", StringComparison.OrdinalIgnoreCase))); + + if (hasWildcardSecurity) + { + recommendations.Add( + new DcrRecommendation( + "broad-security-events", + "high", + "Replace Security Event * with targeted Event IDs", + "Broad Security channel collection inflates ingestion. Prefer targeted Event IDs.", + currentMonthlyIngestionGb is null ? null : currentMonthlyIngestionGb * 0.25)); + } + + string joinedXPath = string.Join(' ', document.DataSources.SelectMany(item => item.XPathQueries)); + string[] missingSecurityIds = RecommendedSecurityEventIds + .Where(id => !joinedXPath.Contains(id, StringComparison.Ordinal)) + .ToArray(); + if (hasWindowsEvents && missingSecurityIds.Length > 0) + { + recommendations.Add( + new DcrRecommendation( + "missing-security-event-ids", + "medium", + "Missing high-value Windows Event IDs", + $"Consider collecting Event IDs: {string.Join(", ", missingSecurityIds)}.", + null)); + } + + bool hasSysmon = document.DataSources.Any(item => + item.Name.Contains("sysmon", StringComparison.OrdinalIgnoreCase) + || item.XPathQueries.Any(query => query.Contains("Sysmon", StringComparison.OrdinalIgnoreCase)) + || item.Streams.Any(stream => stream.Contains("Sysmon", StringComparison.OrdinalIgnoreCase))); + if (!hasSysmon) + { + recommendations.Add( + new DcrRecommendation( + "missing-sysmon", + "high", + "Missing Sysmon configuration", + "Collect Sysmon Event IDs 1, 3 and 11 for process, network and file visibility.", + null)); + } + else + { + string[] missingSysmon = RecommendedSysmonIds + .Where(id => !joinedXPath.Contains($"EventID={id}", StringComparison.OrdinalIgnoreCase) + && !joinedXPath.Contains($"EventID='{id}'", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + if (missingSysmon.Length > 0) + { + recommendations.Add( + new DcrRecommendation( + "incomplete-sysmon", + "medium", + "Incomplete Sysmon Event IDs", + $"Ensure Sysmon Event IDs {string.Join(", ", RecommendedSysmonIds)} are collected.", + null)); + } + } + + bool hasDefender = document.DataSources.Any(item => + item.Name.Contains("defender", StringComparison.OrdinalIgnoreCase) + || item.Streams.Any(stream => stream.Contains("Microsoft-Windows-Windows Defender", StringComparison.OrdinalIgnoreCase))); + if (!hasDefender) + { + recommendations.Add( + new DcrRecommendation( + "missing-defender", + "medium", + "Disabled or missing Defender telemetry", + "Enable Defender / XDR connector or collect Defender operational channels.", + null)); + } + + bool hasIis = document.DataSources.Any(item => + item.Name.Contains("iis", StringComparison.OrdinalIgnoreCase) + || item.Streams.Any(stream => stream.Contains("IIS", StringComparison.OrdinalIgnoreCase))); + if (!hasIis) + { + recommendations.Add( + new DcrRecommendation( + "missing-iis", + "low", + "Missing IIS logs", + "If web workloads exist, collect IIS W3C logs with path filters.", + null)); + } + + if (document.DataSources.Any(item => !item.Enabled)) + { + recommendations.Add( + new DcrRecommendation( + "disabled-sources", + "medium", + "Disabled data sources present", + "Review disabled sources for accidental coverage gaps or stale configuration.", + null)); + } + + if (document.Destinations.Count == 0) + { + recommendations.Add( + new DcrRecommendation( + "missing-destination", + "critical", + "No destinations configured", + "DCR has no Log Analytics / destination association.", + null)); + } + + int healthScore = 100; + healthScore -= recommendations.Count(item => item.Severity == "critical") * 30; + healthScore -= recommendations.Count(item => item.Severity == "high") * 15; + healthScore -= recommendations.Count(item => item.Severity == "medium") * 8; + healthScore -= recommendations.Count(item => item.Severity == "low") * 3; + healthScore = Math.Clamp(healthScore, 0, 100); + + int coveragePoints = 0; + if (hasWindowsEvents) coveragePoints += 30; + if (hasSysmon) coveragePoints += 30; + if (hasDefender) coveragePoints += 20; + if (hasIis) coveragePoints += 10; + if (document.Destinations.Count > 0) coveragePoints += 10; + int coverageScore = Math.Clamp(coveragePoints, 0, 100); + + double? savings = recommendations + .Where(item => item.EstimatedMonthlyGbSaved is not null) + .Select(item => item.EstimatedMonthlyGbSaved!.Value) + .DefaultIfEmpty() + .Sum(); + if (savings == 0) + { + savings = null; + } + + return new DcrAssessmentReport( + document.Id, + document.Name, + coverageScore, + healthScore, + currentMonthlyIngestionGb, + savings, + recommendations, + assessedAt); + } + + public static string ExportMarkdown(DcrAssessmentReport report) + { + ArgumentNullException.ThrowIfNull(report); + System.Text.StringBuilder builder = new(); + builder.AppendLine(CultureInfo.InvariantCulture, $"# DCR Assessment: {report.DcrName}"); + builder.AppendLine(); + builder.AppendLine(CultureInfo.InvariantCulture, $"- DCR id: `{report.DcrId}`"); + builder.AppendLine(CultureInfo.InvariantCulture, $"- Coverage score: **{report.CoverageScore}**"); + builder.AppendLine(CultureInfo.InvariantCulture, $"- Health score: **{report.HealthScore}**"); + if (report.EstimatedMonthlyIngestionGb is not null) + { + builder.AppendLine( + CultureInfo.InvariantCulture, + $"- Current monthly ingestion: **{report.EstimatedMonthlyIngestionGb:0.##} GB**"); + } + + if (report.EstimatedMonthlySavingsGb is not null) + { + builder.AppendLine( + CultureInfo.InvariantCulture, + $"- Estimated monthly savings: **{report.EstimatedMonthlySavingsGb:0.##} GB**"); + } + + builder.AppendLine(); + builder.AppendLine("## Recommendations"); + if (report.Recommendations.Count == 0) + { + builder.AppendLine("- No issues detected."); + } + else + { + foreach (DcrRecommendation recommendation in report.Recommendations) + { + builder.AppendLine( + CultureInfo.InvariantCulture, + $"- **[{recommendation.Severity}] {recommendation.Title}** ({recommendation.Code}): {recommendation.Detail}"); + } + } + + return builder.ToString(); + } +} diff --git a/src/Bower.Dcr/packages.lock.json b/src/Bower.Dcr/packages.lock.json new file mode 100644 index 0000000..6afd678 --- /dev/null +++ b/src/Bower.Dcr/packages.lock.json @@ -0,0 +1,6 @@ +{ + "version": 2, + "dependencies": { + "net10.0": {} + } +} \ No newline at end of file diff --git a/tests/Bower.UnitTests/Bower.UnitTests.csproj b/tests/Bower.UnitTests/Bower.UnitTests.csproj index 1ea2f2a..6d9ec96 100644 --- a/tests/Bower.UnitTests/Bower.UnitTests.csproj +++ b/tests/Bower.UnitTests/Bower.UnitTests.csproj @@ -38,5 +38,6 @@ + diff --git a/tests/Bower.UnitTests/DcrOptimiserTests.cs b/tests/Bower.UnitTests/DcrOptimiserTests.cs new file mode 100644 index 0000000..925ea98 --- /dev/null +++ b/tests/Bower.UnitTests/DcrOptimiserTests.cs @@ -0,0 +1,78 @@ +using Bower.Dcr; + +namespace Bower.UnitTests; + +public sealed class DcrOptimiserTests +{ + [Fact] + public void Assess_FlagsBroadSecurityAndMissingSysmon() + { + const string json = + """ + { + "id": "dcr-1", + "name": "servers", + "workspaceId": "/subscriptions/x/resourceGroups/rg/providers/Microsoft.OperationalInsights/workspaces/law", + "dataSources": [ + { + "name": "windows-security", + "kind": "windowsEventLogs", + "streams": ["Microsoft-WindowsEvent"], + "xPathQueries": ["Security!*"] + } + ], + "destinations": [{ "name": "law" }] + } + """; + + DcrDocument document = DcrDocumentParser.Parse(json); + DcrAssessmentReport report = DcrOptimiser.Assess(document, currentMonthlyIngestionGb: 100); + + Assert.Contains(report.Recommendations, item => item.Code == "broad-security-events"); + Assert.Contains(report.Recommendations, item => item.Code == "missing-sysmon"); + Assert.True(report.EstimatedMonthlySavingsGb is > 0); + Assert.True(report.HealthScore < 100); + Assert.Contains("DCR Assessment", DcrOptimiser.ExportMarkdown(report)); + } + + [Fact] + public void Assess_HealthyDcr_ScoresHigh() + { + DcrDocument document = new( + "dcr-good", + "good", + "law", + [ + new DcrDataSource( + "security", + "windowsEventLogs", + ["Microsoft-WindowsEvent"], + ["Security!*[System[(EventID=4624 or EventID=4625 or EventID=4688)]]"], + true), + new DcrDataSource( + "sysmon", + "windowsEventLogs", + ["Microsoft-WindowsEvent"], + ["Microsoft-Windows-Sysmon/Operational!*[System[(EventID=1 or EventID=3 or EventID=11)]]"], + true), + new DcrDataSource( + "defender", + "windowsEventLogs", + ["Microsoft-Windows-Windows Defender/Operational"], + [], + true), + new DcrDataSource( + "iis", + "iisLogs", + ["Microsoft-IIS"], + [], + true) + ], + ["law"]); + + DcrAssessmentReport report = DcrOptimiser.Assess(document); + + Assert.True(report.CoverageScore >= 90); + Assert.DoesNotContain(report.Recommendations, item => item.Code == "missing-sysmon"); + } +} diff --git a/tests/Bower.UnitTests/packages.lock.json b/tests/Bower.UnitTests/packages.lock.json index 4abe549..ec52e2f 100644 --- a/tests/Bower.UnitTests/packages.lock.json +++ b/tests/Bower.UnitTests/packages.lock.json @@ -432,6 +432,9 @@ "Bower.Redaction": "[1.0.0, )" } }, + "bower.dcr": { + "type": "Project" + }, "bower.detection": { "type": "Project", "dependencies": {