From 67658f1862bbe5dbc1f01c83eb2952a727a198b3 Mon Sep 17 00:00:00 2001 From: Justin Middler Date: Tue, 28 Jul 2026 22:34:26 +1000 Subject: [PATCH] feat(ui): add visual telemetry pipeline builder foundation Declarative pipeline model with templates, cycle/secret validation, CLI validate/template commands, management API endpoints and console pipelines page. --- AGENTS.md | 1 + Bower.sln | 15 + src/Bower.Cli/Bower.Cli.csproj | 1 + src/Bower.Cli/Program.cs | 40 +++ src/Bower.Cli/packages.lock.json | 6 + .../Bower.Management.Api.csproj | 3 + src/Bower.Management.Api/Program.cs | 28 ++ src/Bower.Management.Api/packages.lock.json | 12 + src/Bower.Pipeline/AGENTS.md | 5 + src/Bower.Pipeline/Bower.Pipeline.csproj | 5 + src/Bower.Pipeline/PipelineDocument.cs | 113 ++++++++ src/Bower.Pipeline/PipelineModels.cs | 47 ++++ src/Bower.Pipeline/PipelineValidator.cs | 263 ++++++++++++++++++ src/Bower.Pipeline/packages.lock.json | 13 + tests/Bower.UnitTests/Bower.UnitTests.csproj | 3 + .../Bower.UnitTests/PipelineValidatorTests.cs | 58 ++++ tests/Bower.UnitTests/packages.lock.json | 7 + ui/Bower.Management.Web/src/App.tsx | 52 +++- ui/Bower.Management.Web/src/types.ts | 21 ++ 19 files changed, 692 insertions(+), 1 deletion(-) create mode 100644 src/Bower.Pipeline/AGENTS.md create mode 100644 src/Bower.Pipeline/Bower.Pipeline.csproj create mode 100644 src/Bower.Pipeline/PipelineDocument.cs create mode 100644 src/Bower.Pipeline/PipelineModels.cs create mode 100644 src/Bower.Pipeline/PipelineValidator.cs create mode 100644 src/Bower.Pipeline/packages.lock.json create mode 100644 tests/Bower.UnitTests/PipelineValidatorTests.cs diff --git a/AGENTS.md b/AGENTS.md index 3a47627..e6c99d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ control plane, or runtime AI filter. - `src/Bower.Source.Aws`: AWS security telemetry parsers (CloudTrail, GuardDuty, Security Hub, CloudWatch). - `src/Bower.Ocsf`: OCSF normalisation engine and source mappers. - `src/Bower.Detection`: Sigma-compatible detection rules engine. +- `src/Bower.Pipeline`: declarative telemetry pipeline model, templates and validation. - `schemas`, `policies`, `deploy`, `docs`, `tests`: versioned product assets. Inspect nearest `AGENTS.md` before editing. diff --git a/Bower.sln b/Bower.sln index 57f1994..dd8c82b 100644 --- a/Bower.sln +++ b/Bower.sln @@ -41,6 +41,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Ocsf", "src\Bower.Ocs EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Detection", "src\Bower.Detection\Bower.Detection.csproj", "{C63ADD85-CA8A-49DC-9366-30E9426C7062}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Pipeline", "src\Bower.Pipeline\Bower.Pipeline.csproj", "{E125F241-5F0D-43FD-8781-092995E1D309}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -267,6 +269,18 @@ Global {C63ADD85-CA8A-49DC-9366-30E9426C7062}.Release|x64.Build.0 = Release|Any CPU {C63ADD85-CA8A-49DC-9366-30E9426C7062}.Release|x86.ActiveCfg = Release|Any CPU {C63ADD85-CA8A-49DC-9366-30E9426C7062}.Release|x86.Build.0 = Release|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Debug|x64.ActiveCfg = Debug|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Debug|x64.Build.0 = Debug|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Debug|x86.ActiveCfg = Debug|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Debug|x86.Build.0 = Debug|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Release|Any CPU.Build.0 = Release|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x64.ActiveCfg = Release|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x64.Build.0 = Release|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x86.ActiveCfg = Release|Any CPU + {E125F241-5F0D-43FD-8781-092995E1D309}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -277,5 +291,6 @@ Global {EA3A7CAF-51CE-4A81-A808-2239893A5332} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {F95A8CF7-7C6D-49DF-853D-F845DB65C26F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {C63ADD85-CA8A-49DC-9366-30E9426C7062} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E125F241-5F0D-43FD-8781-092995E1D309} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/src/Bower.Cli/Bower.Cli.csproj b/src/Bower.Cli/Bower.Cli.csproj index 6650829..d6b475b 100644 --- a/src/Bower.Cli/Bower.Cli.csproj +++ b/src/Bower.Cli/Bower.Cli.csproj @@ -10,5 +10,6 @@ + diff --git a/src/Bower.Cli/Program.cs b/src/Bower.Cli/Program.cs index 3bc4f04..8d1d6f6 100644 --- a/src/Bower.Cli/Program.cs +++ b/src/Bower.Cli/Program.cs @@ -2,6 +2,7 @@ using System.Text.Json; using Bower.Contracts; using Bower.Persistence; +using Bower.Pipeline; using Bower.PolicyEngine; return await RunAsync(args); @@ -14,6 +15,8 @@ static async Task RunAsync(string[] args) { ["version"] => WriteVersion(), ["validate", .. string[] rest] => Validate(rest), + ["pipeline", "validate", .. string[] rest] => ValidatePipeline(rest), + ["pipeline", "template", .. string[] rest] => PipelineTemplate(rest), ["queue", "inspect", .. string[] rest] => await InspectQueueAsync(rest), ["test", "emit", .. string[] rest] => await EmitCanaryAsync(rest), ["developer", "init", .. string[] rest] => DeveloperInit(rest), @@ -55,6 +58,41 @@ static int Validate(string[] args) return 0; } +static int ValidatePipeline(string[] args) +{ + string? path = GetOption(args, "--file") ?? args.FirstOrDefault(item => !item.StartsWith('-')); + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + return Fail("Provide an existing pipeline file with --file PATH."); + } + + TelemetryPipeline pipeline = PipelineDocument.ParseYaml(File.ReadAllText(path)); + PipelineValidationResult validation = PipelineValidator.Validate(pipeline); + PipelinePerformanceEstimate estimate = PipelineValidator.Estimate(pipeline); + Console.WriteLine( + JsonSerializer.Serialize( + new + { + valid = validation.IsValid, + pipeline.Id, + pipeline.Name, + pipeline.Version, + order = validation.TopologicalOrder, + estimate, + issues = validation.Issues + }, + new JsonSerializerOptions { WriteIndented = true })); + return validation.IsValid ? 0 : 2; +} + +static int PipelineTemplate(string[] args) +{ + string templateId = GetOption(args, "--id") ?? args.FirstOrDefault(item => !item.StartsWith('-')) ?? "sentinel-app"; + TelemetryPipeline pipeline = PipelineValidator.CreateTemplate(templateId); + Console.WriteLine(PipelineDocument.ToYaml(pipeline)); + return 0; +} + static async Task InspectQueueAsync(string[] args) { string path = GetOption(args, "--database") @@ -191,6 +229,8 @@ Bower security telemetry Commands: bower validate [--policy-directory PATH] + bower pipeline validate --file PATH + bower pipeline template [--id sentinel-app|aws-security] bower queue inspect [--database PATH] bower test emit [--endpoint URL] bower developer init [--path PATH] diff --git a/src/Bower.Cli/packages.lock.json b/src/Bower.Cli/packages.lock.json index 51f82d6..024637f 100644 --- a/src/Bower.Cli/packages.lock.json +++ b/src/Bower.Cli/packages.lock.json @@ -168,6 +168,12 @@ "SQLitePCLRaw.bundle_e_sqlite3": "[3.0.4, )" } }, + "bower.pipeline": { + "type": "Project", + "dependencies": { + "YamlDotNet": "[18.1.0, )" + } + }, "bower.policyengine": { "type": "Project", "dependencies": { diff --git a/src/Bower.Management.Api/Bower.Management.Api.csproj b/src/Bower.Management.Api/Bower.Management.Api.csproj index b599453..f4f7ab6 100644 --- a/src/Bower.Management.Api/Bower.Management.Api.csproj +++ b/src/Bower.Management.Api/Bower.Management.Api.csproj @@ -7,4 +7,7 @@ + + + diff --git a/src/Bower.Management.Api/Program.cs b/src/Bower.Management.Api/Program.cs index f99ad08..4247730 100644 --- a/src/Bower.Management.Api/Program.cs +++ b/src/Bower.Management.Api/Program.cs @@ -1,6 +1,7 @@ using System.Security.Claims; using System.Text.Json.Serialization; using Bower.Management.Api; +using Bower.Pipeline; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; @@ -229,6 +230,33 @@ await store.GetAsync(id, cancellationToken) is { } record }) .RequireAuthorization("Collector"); +api.MapGet( + "/pipelines/templates", + () => Results.Ok( + new[] + { + PipelineValidator.CreateTemplate("sentinel-app"), + PipelineValidator.CreateTemplate("aws-security") + })) + .RequireAuthorization("View"); + +api.MapPost( + "/pipelines/validate", + (TelemetryPipeline pipeline) => + { + PipelineValidationResult validation = PipelineValidator.Validate(pipeline); + PipelinePerformanceEstimate estimate = PipelineValidator.Estimate(pipeline); + return Results.Ok( + new + { + validation.IsValid, + validation.Issues, + validation.TopologicalOrder, + estimate + }); + }) + .RequireAuthorization("Operate"); + api.MapGet( "/approvals", (ManagementStore store, CancellationToken cancellationToken) => diff --git a/src/Bower.Management.Api/packages.lock.json b/src/Bower.Management.Api/packages.lock.json index 07d9a4a..035ed10 100644 --- a/src/Bower.Management.Api/packages.lock.json +++ b/src/Bower.Management.Api/packages.lock.json @@ -126,6 +126,18 @@ "Microsoft.IdentityModel.JsonWebTokens": "8.19.2", "Microsoft.IdentityModel.Tokens": "8.19.2" } + }, + "bower.pipeline": { + "type": "Project", + "dependencies": { + "YamlDotNet": "[18.1.0, )" + } + }, + "YamlDotNet": { + "type": "CentralTransitive", + "requested": "[18.1.0, )", + "resolved": "18.1.0", + "contentHash": "5K+9KFg2TdTl7VXv88Qzi/0lqK6JFoNP3lRuImPYGRV7K/QYklDyTrj4+A+KAki1JsQi6qKY+hDyY7d6WRqjrw==" } } } diff --git a/src/Bower.Pipeline/AGENTS.md b/src/Bower.Pipeline/AGENTS.md new file mode 100644 index 0000000..5463a84 --- /dev/null +++ b/src/Bower.Pipeline/AGENTS.md @@ -0,0 +1,5 @@ +# Pipeline builder instructions + +Pipeline documents are declarative graphs only. Validation must reject cycles, +unknown node kinds, unbounded fan-out and secret material in node config. +Version every pipeline document and keep import/export pure. diff --git a/src/Bower.Pipeline/Bower.Pipeline.csproj b/src/Bower.Pipeline/Bower.Pipeline.csproj new file mode 100644 index 0000000..4fdad98 --- /dev/null +++ b/src/Bower.Pipeline/Bower.Pipeline.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Bower.Pipeline/PipelineDocument.cs b/src/Bower.Pipeline/PipelineDocument.cs new file mode 100644 index 0000000..43fe518 --- /dev/null +++ b/src/Bower.Pipeline/PipelineDocument.cs @@ -0,0 +1,113 @@ +using System.Text; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Bower.Pipeline; + +public static class PipelineDocument +{ + private static readonly IDeserializer Deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + private static readonly ISerializer Serializer = new SerializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull) + .Build(); + + public static TelemetryPipeline ParseYaml(string yaml) + { + ArgumentException.ThrowIfNullOrWhiteSpace(yaml); + if (Encoding.UTF8.GetByteCount(yaml) > 512 * 1024) + { + throw new InvalidDataException("Pipeline document exceeds 512 KiB."); + } + + PipelineDto dto = Deserializer.Deserialize(yaml) + ?? throw new InvalidDataException("Pipeline document was empty."); + return dto.ToModel(); + } + + public static string ToYaml(TelemetryPipeline pipeline) + { + ArgumentNullException.ThrowIfNull(pipeline); + return Serializer.Serialize(PipelineDto.FromModel(pipeline)); + } + + private sealed class PipelineDto + { + public string? Id { get; set; } + public string? Name { get; set; } + public string? Version { get; set; } + public string? Description { get; set; } + public List? Nodes { get; set; } + public List? Edges { get; set; } + public List? Tags { get; set; } + + public TelemetryPipeline ToModel() + { + return new TelemetryPipeline( + Id ?? string.Empty, + Name ?? string.Empty, + Version ?? string.Empty, + Description ?? string.Empty, + (Nodes ?? []).Select(node => node.ToModel()).ToArray(), + (Edges ?? []).Select(edge => edge.ToModel()).ToArray(), + Tags); + } + + public static PipelineDto FromModel(TelemetryPipeline pipeline) + { + return new PipelineDto + { + Id = pipeline.Id, + Name = pipeline.Name, + Version = pipeline.Version, + Description = pipeline.Description, + Nodes = pipeline.Nodes.Select(NodeDto.FromModel).ToList(), + Edges = pipeline.Edges.Select(EdgeDto.FromModel).ToList(), + Tags = pipeline.Tags?.ToList() + }; + } + } + + private sealed class NodeDto + { + public string? Id { get; set; } + public string? Kind { get; set; } + public string? Type { get; set; } + public Dictionary? Config { get; set; } + + public PipelineNode ToModel() + { + if (!Enum.TryParse(Kind, ignoreCase: true, out PipelineNodeKind kind)) + { + throw new InvalidDataException($"Unknown node kind '{Kind}'."); + } + + return new PipelineNode(Id ?? string.Empty, kind, Type ?? string.Empty, Config); + } + + public static NodeDto FromModel(PipelineNode node) + { + return new NodeDto + { + Id = node.Id, + Kind = node.Kind.ToString(), + Type = node.Type, + Config = node.Config is null ? null : new Dictionary(node.Config) + }; + } + } + + private sealed class EdgeDto + { + public string? From { get; set; } + public string? To { get; set; } + + public PipelineEdge ToModel() => new(From ?? string.Empty, To ?? string.Empty); + + public static EdgeDto FromModel(PipelineEdge edge) => new() { From = edge.From, To = edge.To }; + } +} diff --git a/src/Bower.Pipeline/PipelineModels.cs b/src/Bower.Pipeline/PipelineModels.cs new file mode 100644 index 0000000..bbbdc72 --- /dev/null +++ b/src/Bower.Pipeline/PipelineModels.cs @@ -0,0 +1,47 @@ +namespace Bower.Pipeline; + +public enum PipelineNodeKind +{ + Source, + Filter, + Redact, + Normalise, + Detect, + Enrich, + Output +} + +public sealed record PipelineNode( + string Id, + PipelineNodeKind Kind, + string Type, + IReadOnlyDictionary? Config = null); + +public sealed record PipelineEdge(string From, string To); + +public sealed record TelemetryPipeline( + string Id, + string Name, + string Version, + string Description, + IReadOnlyList Nodes, + IReadOnlyList Edges, + IReadOnlyList? Tags = null); + +public sealed record PipelineValidationIssue( + string Code, + string Message, + string? NodeId = null); + +public sealed record PipelineValidationResult( + bool IsValid, + IReadOnlyList Issues, + IReadOnlyList TopologicalOrder, + int EstimatedStageCount); + +public sealed record PipelinePerformanceEstimate( + int NodeCount, + int EdgeCount, + int SourceCount, + int OutputCount, + int MaxDepth); diff --git a/src/Bower.Pipeline/PipelineValidator.cs b/src/Bower.Pipeline/PipelineValidator.cs new file mode 100644 index 0000000..9934d76 --- /dev/null +++ b/src/Bower.Pipeline/PipelineValidator.cs @@ -0,0 +1,263 @@ +namespace Bower.Pipeline; + +public static class PipelineValidator +{ + private static readonly HashSet AllowedSourceTypes = new(StringComparer.OrdinalIgnoreCase) + { + "http-collector", "sqlserver", "aws-cloudtrail", "aws-guardduty", "aws-securityhub", + "aws-cloudwatch", "file-tail", "ama-companion", "ec2-host" + }; + + private static readonly HashSet AllowedOutputTypes = new(StringComparer.OrdinalIgnoreCase) + { + "azure-logs-ingestion", "ama-spool", "http", "kafka", "syslog", "security-lake", "opensearch" + }; + + private static readonly HashSet SecretKeys = new(StringComparer.OrdinalIgnoreCase) + { + "password", "secret", "token", "apikey", "connectionstring", "privatekey" + }; + + public static PipelineValidationResult Validate(TelemetryPipeline pipeline) + { + ArgumentNullException.ThrowIfNull(pipeline); + List issues = []; + + if (string.IsNullOrWhiteSpace(pipeline.Id) || pipeline.Id.Length > 128) + { + issues.Add(new PipelineValidationIssue("pipeline-id", "Pipeline id is required and must be <= 128 chars.")); + } + + if (string.IsNullOrWhiteSpace(pipeline.Name)) + { + issues.Add(new PipelineValidationIssue("pipeline-name", "Pipeline name is required.")); + } + + if (string.IsNullOrWhiteSpace(pipeline.Version)) + { + issues.Add(new PipelineValidationIssue("pipeline-version", "Pipeline version is required.")); + } + + if (pipeline.Nodes.Count == 0) + { + issues.Add(new PipelineValidationIssue("nodes-empty", "Pipeline must contain at least one node.")); + } + + if (pipeline.Nodes.Count > 64) + { + issues.Add(new PipelineValidationIssue("nodes-too-many", "Pipeline cannot exceed 64 nodes.")); + } + + Dictionary nodes = new(StringComparer.Ordinal); + foreach (PipelineNode node in pipeline.Nodes) + { + if (string.IsNullOrWhiteSpace(node.Id) || !nodes.TryAdd(node.Id, node)) + { + issues.Add(new PipelineValidationIssue("node-id", "Node ids must be unique and non-empty.", node.Id)); + continue; + } + + ValidateNode(node, issues); + } + + foreach (PipelineEdge edge in pipeline.Edges) + { + if (!nodes.ContainsKey(edge.From) || !nodes.ContainsKey(edge.To)) + { + issues.Add( + new PipelineValidationIssue( + "edge-unknown-node", + $"Edge '{edge.From}' -> '{edge.To}' references unknown node.")); + } + + if (string.Equals(edge.From, edge.To, StringComparison.Ordinal)) + { + issues.Add(new PipelineValidationIssue("edge-self", "Self-edges are not allowed.", edge.From)); + } + } + + List order = []; + if (nodes.Count == pipeline.Nodes.Count && + pipeline.Edges.All(edge => nodes.ContainsKey(edge.From) && nodes.ContainsKey(edge.To))) + { + if (!TryTopologicalSort(pipeline, out order, out string? cycleNode)) + { + issues.Add(new PipelineValidationIssue("cycle", "Pipeline graph contains a cycle.", cycleNode)); + } + } + + bool hasSource = pipeline.Nodes.Any(node => node.Kind == PipelineNodeKind.Source); + bool hasOutput = pipeline.Nodes.Any(node => node.Kind == PipelineNodeKind.Output); + if (!hasSource) + { + issues.Add(new PipelineValidationIssue("missing-source", "Pipeline needs at least one source node.")); + } + + if (!hasOutput) + { + issues.Add(new PipelineValidationIssue("missing-output", "Pipeline needs at least one output node.")); + } + + return new PipelineValidationResult(issues.Count == 0, issues, order, order.Count); + } + + public static PipelinePerformanceEstimate Estimate(TelemetryPipeline pipeline) + { + ArgumentNullException.ThrowIfNull(pipeline); + Dictionary depth = pipeline.Nodes.ToDictionary(node => node.Id, _ => 0, StringComparer.Ordinal); + foreach (PipelineEdge edge in pipeline.Edges) + { + if (depth.TryGetValue(edge.From, out int fromDepth) && + depth.TryGetValue(edge.To, out int toDepth)) + { + depth[edge.To] = Math.Max(toDepth, fromDepth + 1); + } + } + + return new PipelinePerformanceEstimate( + pipeline.Nodes.Count, + pipeline.Edges.Count, + pipeline.Nodes.Count(node => node.Kind == PipelineNodeKind.Source), + pipeline.Nodes.Count(node => node.Kind == PipelineNodeKind.Output), + depth.Count == 0 ? 0 : depth.Values.Max()); + } + + public static TelemetryPipeline CreateTemplate(string templateId) + { + return templateId.ToLowerInvariant() switch + { + "sentinel-app" => new TelemetryPipeline( + "template-sentinel-app", + "Application to Sentinel", + "1.0.0", + "HTTP collector → redact → policy filter → Azure Logs Ingestion", + [ + new PipelineNode("source", PipelineNodeKind.Source, "http-collector"), + new PipelineNode("redact", PipelineNodeKind.Redact, "json-redactor"), + new PipelineNode("filter", PipelineNodeKind.Filter, "policy-engine"), + new PipelineNode("out", PipelineNodeKind.Output, "azure-logs-ingestion") + ], + [ + new PipelineEdge("source", "redact"), + new PipelineEdge("redact", "filter"), + new PipelineEdge("filter", "out") + ], + ["template", "sentinel"]), + "aws-security" => new TelemetryPipeline( + "template-aws-security", + "AWS Security to multi-destination", + "1.0.0", + "AWS sources → normalise → detect → dual output", + [ + new PipelineNode("cloudtrail", PipelineNodeKind.Source, "aws-cloudtrail"), + new PipelineNode("guardduty", PipelineNodeKind.Source, "aws-guardduty"), + new PipelineNode("normalise", PipelineNodeKind.Normalise, "ocsf"), + new PipelineNode("detect", PipelineNodeKind.Detect, "sigma"), + new PipelineNode("sentinel", PipelineNodeKind.Output, "azure-logs-ingestion"), + new PipelineNode("lake", PipelineNodeKind.Output, "security-lake") + ], + [ + new PipelineEdge("cloudtrail", "normalise"), + new PipelineEdge("guardduty", "normalise"), + new PipelineEdge("normalise", "detect"), + new PipelineEdge("detect", "sentinel"), + new PipelineEdge("detect", "lake") + ], + ["template", "aws"]), + _ => throw new ArgumentException($"Unknown pipeline template '{templateId}'.") + }; + } + + private static void ValidateNode(PipelineNode node, List issues) + { + if (string.IsNullOrWhiteSpace(node.Type)) + { + issues.Add(new PipelineValidationIssue("node-type", "Node type is required.", node.Id)); + } + + if (node.Kind == PipelineNodeKind.Source && + !AllowedSourceTypes.Contains(node.Type)) + { + issues.Add(new PipelineValidationIssue("source-type", $"Unknown source type '{node.Type}'.", node.Id)); + } + + if (node.Kind == PipelineNodeKind.Output && + !AllowedOutputTypes.Contains(node.Type)) + { + issues.Add(new PipelineValidationIssue("output-type", $"Unknown output type '{node.Type}'.", node.Id)); + } + + if (node.Config is null) + { + return; + } + + foreach ((string key, string value) in node.Config) + { + string normalized = string.Concat(key.Where(char.IsLetterOrDigit)); + if (SecretKeys.Contains(normalized)) + { + issues.Add( + new PipelineValidationIssue( + "secret-in-config", + $"Node config key '{key}' looks like a secret and is not allowed.", + node.Id)); + } + + if (value.Length > 4_096) + { + issues.Add( + new PipelineValidationIssue( + "config-too-large", + $"Config value for '{key}' exceeds 4 KiB.", + node.Id)); + } + } + } + + private static bool TryTopologicalSort( + TelemetryPipeline pipeline, + out List order, + out string? cycleNode) + { + Dictionary> adjacency = pipeline.Nodes.ToDictionary( + node => node.Id, + _ => new List(), + StringComparer.Ordinal); + Dictionary indegree = pipeline.Nodes.ToDictionary( + node => node.Id, + _ => 0, + StringComparer.Ordinal); + + foreach (PipelineEdge edge in pipeline.Edges) + { + adjacency[edge.From].Add(edge.To); + indegree[edge.To]++; + } + + Queue queue = new(indegree.Where(pair => pair.Value == 0).Select(pair => pair.Key)); + order = []; + while (queue.Count > 0) + { + string current = queue.Dequeue(); + order.Add(current); + foreach (string next in adjacency[current]) + { + indegree[next]--; + if (indegree[next] == 0) + { + queue.Enqueue(next); + } + } + } + + if (order.Count != pipeline.Nodes.Count) + { + cycleNode = indegree.First(pair => pair.Value > 0).Key; + return false; + } + + cycleNode = null; + return true; + } +} diff --git a/src/Bower.Pipeline/packages.lock.json b/src/Bower.Pipeline/packages.lock.json new file mode 100644 index 0000000..677693c --- /dev/null +++ b/src/Bower.Pipeline/packages.lock.json @@ -0,0 +1,13 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "YamlDotNet": { + "type": "Direct", + "requested": "[18.1.0, )", + "resolved": "18.1.0", + "contentHash": "5K+9KFg2TdTl7VXv88Qzi/0lqK6JFoNP3lRuImPYGRV7K/QYklDyTrj4+A+KAki1JsQi6qKY+hDyY7d6WRqjrw==" + } + } + } +} \ No newline at end of file diff --git a/tests/Bower.UnitTests/Bower.UnitTests.csproj b/tests/Bower.UnitTests/Bower.UnitTests.csproj index e1a5ef7..02e4547 100644 --- a/tests/Bower.UnitTests/Bower.UnitTests.csproj +++ b/tests/Bower.UnitTests/Bower.UnitTests.csproj @@ -34,4 +34,7 @@ PreserveNewest + + + diff --git a/tests/Bower.UnitTests/PipelineValidatorTests.cs b/tests/Bower.UnitTests/PipelineValidatorTests.cs new file mode 100644 index 0000000..1114b55 --- /dev/null +++ b/tests/Bower.UnitTests/PipelineValidatorTests.cs @@ -0,0 +1,58 @@ +using Bower.Pipeline; + +namespace Bower.UnitTests; + +public sealed class PipelineValidatorTests +{ + [Fact] + public void Template_SentinelApp_IsValid() + { + TelemetryPipeline pipeline = PipelineValidator.CreateTemplate("sentinel-app"); + PipelineValidationResult result = PipelineValidator.Validate(pipeline); + + Assert.True(result.IsValid, string.Join("; ", result.Issues.Select(issue => issue.Message))); + Assert.Equal(4, result.TopologicalOrder.Count); + Assert.Equal("source", result.TopologicalOrder[0]); + } + + [Fact] + public void Validate_RejectsCycleAndSecrets() + { + TelemetryPipeline pipeline = new( + "bad", + "Bad", + "1.0.0", + "cycle", + [ + new PipelineNode("a", PipelineNodeKind.Source, "http-collector"), + new PipelineNode( + "b", + PipelineNodeKind.Output, + "http", + new Dictionary { ["apiKey"] = "super-secret" }) + ], + [ + new PipelineEdge("a", "b"), + new PipelineEdge("b", "a") + ]); + + PipelineValidationResult result = PipelineValidator.Validate(pipeline); + + Assert.False(result.IsValid); + Assert.Contains(result.Issues, issue => issue.Code == "cycle"); + Assert.Contains(result.Issues, issue => issue.Code == "secret-in-config"); + } + + [Fact] + public void Yaml_RoundTripsTemplate() + { + TelemetryPipeline original = PipelineValidator.CreateTemplate("aws-security"); + string yaml = PipelineDocument.ToYaml(original); + TelemetryPipeline parsed = PipelineDocument.ParseYaml(yaml); + + Assert.Equal(original.Id, parsed.Id); + Assert.Equal(original.Nodes.Count, parsed.Nodes.Count); + Assert.Equal(original.Edges.Count, parsed.Edges.Count); + Assert.True(PipelineValidator.Validate(parsed).IsValid); + } +} diff --git a/tests/Bower.UnitTests/packages.lock.json b/tests/Bower.UnitTests/packages.lock.json index 2d7c07e..f7aca87 100644 --- a/tests/Bower.UnitTests/packages.lock.json +++ b/tests/Bower.UnitTests/packages.lock.json @@ -436,6 +436,7 @@ "bower.management.api": { "type": "Project", "dependencies": { + "Bower.Pipeline": "[1.0.0, )", "Microsoft.AspNetCore.Authentication.JwtBearer": "[10.0.10, )", "Microsoft.Data.Sqlite": "[10.0.10, )", "SQLitePCLRaw.bundle_e_sqlite3": "[3.0.4, )" @@ -462,6 +463,12 @@ "SQLitePCLRaw.bundle_e_sqlite3": "[3.0.4, )" } }, + "bower.pipeline": { + "type": "Project", + "dependencies": { + "YamlDotNet": "[18.1.0, )" + } + }, "bower.policyengine": { "type": "Project", "dependencies": { diff --git a/ui/Bower.Management.Web/src/App.tsx b/ui/Bower.Management.Web/src/App.tsx index 0d3a195..c5d177c 100644 --- a/ui/Bower.Management.Web/src/App.tsx +++ b/ui/Bower.Management.Web/src/App.tsx @@ -3,6 +3,7 @@ import { Boxes, CheckCircle2, ClipboardCheck, + GitBranch, KeyRound, Menu, Moon, @@ -15,11 +16,12 @@ import { type ReactNode, useCallback, useEffect, useState } from "react"; import { NavLink, Navigate, Route, Routes } from "react-router-dom"; import { useApi } from "./api"; import { useAuth } from "./auth"; -import type { Access, Approval, Audit, Collector, Overview } from "./types"; +import type { Access, Approval, Audit, Collector, Overview, PipelineTemplate } from "./types"; const navItems = [ { to: "/", label: "Overview", icon: Activity }, { to: "/collectors", label: "Collectors", icon: Boxes }, + { to: "/pipelines", label: "Pipelines", icon: GitBranch }, { to: "/approvals", label: "Approvals", icon: ClipboardCheck }, { to: "/access", label: "Access", icon: KeyRound }, { to: "/audit", label: "Audit", icon: ScrollText } @@ -124,6 +126,7 @@ export function App() { } /> } /> + } /> } /> } /> } /> @@ -261,6 +264,53 @@ function CollectorsPage() { ); } +function PipelinesPage() { + const { data, loading, error } = useResource( + "/api/pipelines/templates" + ); + return ( + + + {(templates) => + templates.length ? ( +
+ {templates.map((template) => ( +
+
+

{template.name}

+ {template.version} +
+

{template.description}

+

+ {template.nodes.length} nodes · {template.edges.length} edges +

+
    + {template.nodes.map((node) => ( +
  1. + {node.id} · {node.kind} ·{" "} + {node.type} +
    2. + ))} +
+
+ ))} +
+ ) : ( + + + ); +} + function ApprovalsPage() { const api = useApi(); const [refresh, setRefresh] = useState(0); diff --git a/ui/Bower.Management.Web/src/types.ts b/ui/Bower.Management.Web/src/types.ts index 16e03f9..c828543 100644 --- a/ui/Bower.Management.Web/src/types.ts +++ b/ui/Bower.Management.Web/src/types.ts @@ -75,3 +75,24 @@ export interface Access { roles: string[]; developmentAuthentication: boolean; } + +export interface PipelineNode { + id: string; + kind: string; + type: string; +} + +export interface PipelineEdge { + from: string; + to: string; +} + +export interface PipelineTemplate { + id: string; + name: string; + version: string; + description: string; + nodes: PipelineNode[]; + edges: PipelineEdge[]; + tags?: string[]; +}