From 5f8c0cb6b1334984c8c8ade4be273703c5a37e36 Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 17 Jul 2026 08:41:42 -0500 Subject: [PATCH] feat(cli): K2 merged download smoke with selective install --download install --download now fetches only IsSelected components so --select mod:NAME works for single-mod network smoke. Adds LongRunning test, agent script, and docs. --- docs/knowledgebase/cli-selection-semantics.md | 10 + .../k2_ingested_merge_download_smoke.sh | 113 ++++++++++ src/ModSync.Core/CLI/ModBuildConverter.cs | 23 ++- src/ModSync.Tests/K2FullGuideDownloadTests.cs | 193 ++++++++++++++++++ 4 files changed, 333 insertions(+), 6 deletions(-) create mode 100755 scripts/agents/k2_ingested_merge_download_smoke.sh create mode 100644 src/ModSync.Tests/K2FullGuideDownloadTests.cs diff --git a/docs/knowledgebase/cli-selection-semantics.md b/docs/knowledgebase/cli-selection-semantics.md index 07ec87d1..c8a845e8 100644 --- a/docs/knowledgebase/cli-selection-semantics.md +++ b/docs/knowledgebase/cli-selection-semantics.md @@ -18,6 +18,16 @@ Only components already marked `IsSelected = true` in the file are installed. Us Repeatable filters: `category:Name`, `tier:Name`, `mod:Name` (case-insensitive exact or substring match on component `Name`). Only matching components stay selected. Filters combine with AND semantics when multiple types are used together. Repeat `--select` for multiple mod names (e.g. `--select mod:A --select mod:B`). +## `install` with `--download` + +`[REPO]` On `install`, `--select` (or `--use-file-selection` / TOML `IsSelected`) is applied **before** downloads. Only components with `IsSelected == true` are fetched when `--download` is set. Use this for single-mod network smoke instead of `convert --download`, which downloads all URL-bearing components before selection filters run. + +Example (merged neocities K2 + golden TOML): + +```bash +./scripts/agents/k2_ingested_merge_download_smoke.sh --download-mod "Silent Sion Restoration" +``` + ## `validate` without `--select` (default) Validates **all loaded components**. TOML `IsSelected` is **not** used unless `--use-file-selection` or `--select` is provided. diff --git a/scripts/agents/k2_ingested_merge_download_smoke.sh b/scripts/agents/k2_ingested_merge_download_smoke.sh new file mode 100755 index 00000000..72303a6e --- /dev/null +++ b/scripts/agents/k2_ingested_merge_download_smoke.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Merge neocities K2 Full + golden TOML, then download one mod via install --download --select mod:NAME. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +FIXTURE="${FIXTURE:-$ROOT/src/ModSync.Tests/Fixtures/k2_full_guide.md}" +GOLDEN="${GOLDEN:-$ROOT/mod-builds/TOMLs/KOTOR2_Full.toml}" +OUT_DIR="${OUT_DIR:-$ROOT/tmp/k2_merge_download_smoke}" +MERGED="${MERGED:-$OUT_DIR/k2_merged.toml}" +KOTOR_DIR="${KOTOR_DIR:-$OUT_DIR/kotor}" +MOD_DIR="${MOD_DIR:-$OUT_DIR/mods}" +DOWNLOAD_MOD="${DOWNLOAD_MOD:-Silent Sion Restoration}" +SKIP_DOWNLOAD=false + +usage() { + cat <<'EOF' +Usage: k2_ingested_merge_download_smoke.sh [options] + +Merges the neocities K2 Full markdown fixture with golden KOTOR2_Full.toml, then +downloads one mod with install --download --select mod:NAME (network required). + +Options: + --fixture PATH Neocities markdown fixture (default: k2_full_guide.md) + --golden PATH Golden TOML (default: ./mod-builds/TOMLs/KOTOR2_Full.toml) + --out-dir PATH Output directory (default: ./tmp/k2_merge_download_smoke) + --download-mod NAME Mod name for --select mod:NAME (default: Silent Sion Restoration) + --skip-download Merge + validate dry-run only; do not download + -h, --help Show this help + +Environment: + FIXTURE, GOLDEN, OUT_DIR, MOD_DIR, KOTOR_DIR, MERGED, DOWNLOAD_MOD +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --fixture) FIXTURE="$2"; shift 2 ;; + --golden) GOLDEN="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; MERGED="$OUT_DIR/k2_merged.toml"; KOTOR_DIR="$OUT_DIR/kotor"; MOD_DIR="$OUT_DIR/mods"; shift 2 ;; + --download-mod) DOWNLOAD_MOD="$2"; shift 2 ;; + --skip-download) SKIP_DOWNLOAD=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +if [[ ! -f "$FIXTURE" ]]; then + echo "Fixture not found: $FIXTURE" >&2 + exit 1 +fi + +if [[ ! -f "$GOLDEN" ]]; then + echo "Golden TOML not found: $GOLDEN" >&2 + echo "Clone mod-builds: git clone https://github.com/th3w1zard1/mod-builds ./mod-builds" >&2 + exit 1 +fi + +mkdir -p "$OUT_DIR" "$MOD_DIR" +"$ROOT/scripts/agents/create_template_kotor_install.sh" "$KOTOR_DIR" "$MOD_DIR" >/dev/null + +strip_dependencies_for_mods() { + local toml_path="$1" + shift + python3 - "$toml_path" "$@" <<'PY' +import re, sys +path = sys.argv[1] +names = sys.argv[2:] +text = open(path, encoding="utf-8").read() +for name in names: + pattern = rf'(\[\[thisMod\]\]\nGuid = "[^"]+"\nName = "{re.escape(name)}"[\s\S]*?)(?=\n\[\[thisMod\]\]|\Z)' + match = re.search(pattern, text) + if not match: + print(f"warning: mod block not found for {name!r}", file=sys.stderr) + continue + block = match.group(1) + block = re.sub(r'\nDependencies = \[.*?\]\n', '\n', block) + block = re.sub(r'\nRestrictions = \[.*?\]\n', '\n', block) + text = text.replace(match.group(1), block) +open(path, "w", encoding="utf-8").write(text) +PY +} + +dotnet build "$ROOT/src/ModSync.Tests/ModSync.Tests.csproj" -c Debug -f net9.0 -v q + +echo "==> merge (golden existing + neocities incoming)" +dotnet run --project "$ROOT/src/ModSync.Tests/ModSync.Tests.csproj" -f net9.0 --no-build -- \ + merge --existing "$GOLDEN" --incoming "$FIXTURE" \ + --use-existing-order --prefer-existing-instructions --prefer-existing-options --prefer-existing-modlinks \ + -f toml -o "$MERGED" --plaintext + +echo "==> validate --dry-run-only (structure + VFS; archives may be missing locally)" +dotnet run --project "$ROOT/src/ModSync.Tests/ModSync.Tests.csproj" -f net9.0 --no-build -- \ + validate --input "$MERGED" --game-dir "$KOTOR_DIR" --source-dir "$MOD_DIR" \ + --dry-run-only --errors-only --best-effort || true + +if [[ "$SKIP_DOWNLOAD" == true ]]; then + echo "Merge download smoke complete (merge + validate). Skipping network download." + echo " Merged TOML: $MERGED" + exit 0 +fi + +echo "==> install --download --select mod:$DOWNLOAD_MOD" +strip_dependencies_for_mods "$MERGED" "$DOWNLOAD_MOD" +dotnet run --project "$ROOT/src/ModSync.Tests/ModSync.Tests.csproj" -f net9.0 --no-build -- \ + install --input "$MERGED" --game-dir "$KOTOR_DIR" --source-dir "$MOD_DIR" \ + --select "mod:$DOWNLOAD_MOD" --download --skip-validation --best-effort -y + +echo "Download smoke complete." +echo " Merged TOML: $MERGED" +echo " Mod workspace: $MOD_DIR" +ls -la "$MOD_DIR"/*.zip 2>/dev/null || echo " (no .zip files at mod workspace root yet)" diff --git a/src/ModSync.Core/CLI/ModBuildConverter.cs b/src/ModSync.Core/CLI/ModBuildConverter.cs index 1e3939c0..d9e74109 100644 --- a/src/ModSync.Core/CLI/ModBuildConverter.cs +++ b/src/ModSync.Core/CLI/ModBuildConverter.cs @@ -1021,9 +1021,22 @@ private static async Task RunFomodPostDownloadHookAsync( await FomodPostDownloadOrchestrator.ProcessAsync(components, modDirectory, host).ConfigureAwait(false); } - private static async Task DownloadAllModFilesAsync(List components, string destinationDirectory, bool verbose, bool sequential = true, CancellationToken cancellationToken = default) + private static async Task DownloadAllModFilesAsync( + List components, + string destinationDirectory, + bool verbose, + bool sequential = true, + CancellationToken cancellationToken = default, + bool downloadSelectedOnly = false) { - int componentCount = components.Count(c => c.ResourceRegistry != null && c.ResourceRegistry.Count > 0); + IEnumerable candidates = components.Where(c => c.ResourceRegistry != null && c.ResourceRegistry.Count > 0); + if (downloadSelectedOnly) + { + candidates = candidates.Where(c => c.IsSelected); + } + + var componentsToProcess = candidates.ToList(); + int componentCount = componentsToProcess.Count; if (componentCount == 0) { if (s_progressDisplay != null) @@ -1145,9 +1158,6 @@ private static async Task DownloadAllModFilesAsync(List c.ResourceRegistry != null && c.ResourceRegistry.Count > 0).ToList(); - - await Logger.LogVerboseAsync($"[Download] Processing {componentsToProcess.Count} components with concurrency limit of 10").ConfigureAwait(false); using (var semaphore = new SemaphoreSlim(10)) @@ -3228,7 +3238,8 @@ await Logger.LogWarningAsync( sourceDir, opts.Verbose, sequential: !opts.Concurrent, - downloadCts.Token).ConfigureAwait(false); + downloadCts.Token, + downloadSelectedOnly: true).ConfigureAwait(false); } LogAllErrors(s_globalDownloadCache, forceConsoleOutput: true); diff --git a/src/ModSync.Tests/K2FullGuideDownloadTests.cs b/src/ModSync.Tests/K2FullGuideDownloadTests.cs new file mode 100644 index 00000000..7d58d148 --- /dev/null +++ b/src/ModSync.Tests/K2FullGuideDownloadTests.cs @@ -0,0 +1,193 @@ +// Copyright 2021-2025 ModSync +// Licensed under the Business Source License 1.1 (BSL 1.1). +// See LICENSE.txt file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using ModSync.Core; +using ModSync.Core.CLI; +using ModSync.Core.Services; + +using NUnit.Framework; + +namespace ModSync.Tests +{ + /// + /// Network download smoke for neocities K2 Full merged with golden mod-builds TOML. + /// + [TestFixture] + [Category("Integration")] + [Category("LongRunning")] + public sealed class K2FullGuideDownloadTests + { + private static readonly string GoldenTomlRelative = Path.Combine("mod-builds", "TOMLs", "KOTOR2_Full.toml"); + private const string SilentSionModName = "Silent Sion Restoration"; + private const string SilentSionArchive = "Silent Sion Restoration.zip"; + + private string _testDirectory; + private string _modDirectory; + private string _kotorDirectory; + private MainConfig _previousMainConfig; + + [SetUp] + public void SetUp() + { + _testDirectory = Path.Combine(Path.GetTempPath(), "ModSync_K2Download_" + Guid.NewGuid()); + _modDirectory = Path.Combine(_testDirectory, "Mods"); + _kotorDirectory = Path.Combine(_testDirectory, "KOTOR"); + Directory.CreateDirectory(_modDirectory); + Directory.CreateDirectory(_kotorDirectory); + Directory.CreateDirectory(Path.Combine(_kotorDirectory, "Override")); + File.WriteAllText(Path.Combine(_kotorDirectory, "swkotor.exe"), "fake exe"); + File.WriteAllText(Path.Combine(_kotorDirectory, "dialog.tlk"), "fake dialog"); + + _previousMainConfig = MainConfig.Instance; + MainConfig.Instance = new MainConfig + { + sourcePath = new DirectoryInfo(_modDirectory), + destinationPath = new DirectoryInfo(_kotorDirectory), + }; + } + + [TearDown] + public void TearDown() + { + MainConfig.Instance = _previousMainConfig; + + try + { + if (Directory.Exists(_testDirectory)) + { + Directory.Delete(_testDirectory, recursive: true); + } + } + catch + { + // Ignore cleanup errors + } + } + + [Test] + public void K2FullGuideFixture_MergedSilentSion_DownloadsViaInstallSelect_LongRunning() + { + (string fixturePath, string goldenTomlPath) = ResolveInputs(); + string mergedToml = Path.Combine(_testDirectory, "k2_merged.toml"); + + int mergeExit = ModBuildConverter.Run(new[] + { + "merge", + "--existing", goldenTomlPath, + "--incoming", fixturePath, + "--use-existing-order", + "--prefer-existing-instructions", + "--prefer-existing-options", + "--prefer-existing-modlinks", + "-f", "toml", + "-o", mergedToml, + "--plaintext", + }); + + Assert.That(mergeExit, Is.EqualTo(0), "merge neocities fixture + golden TOML should succeed"); + StripDependenciesForDownloadSmoke(mergedToml, SilentSionModName); + + int installExit = ModBuildConverter.Run(new[] + { + "install", + "--input", mergedToml, + "--game-dir", _kotorDirectory, + "--source-dir", _modDirectory, + "--select", "mod:" + SilentSionModName, + "--download", + "--skip-validation", + "--best-effort", + "-y", + }); + + string archivePath = Path.Combine(_modDirectory, SilentSionArchive); + bool archiveExists = File.Exists(archivePath); + bool anyZipInModDir = Directory.GetFiles(_modDirectory, "*.zip", SearchOption.TopDirectoryOnly).Length > 0; + + Assert.Multiple(() => + { + Assert.That(installExit, Is.EqualTo(0), + "install --download --select mod:Silent Sion should complete in best-effort mode"); + Assert.That(archiveExists || anyZipInModDir, Is.True, + $"Expected {SilentSionArchive} (or another downloaded archive) under the mod workspace"); + }); + } + + private static (string fixturePath, string goldenTomlPath) ResolveInputs() + { + string repoRoot = ResolveRepoRoot(); + string fixturePath = Path.Combine(repoRoot, "src", "ModSync.Tests", "Fixtures", "k2_full_guide.md"); + string goldenTomlPath = Path.Combine(repoRoot, GoldenTomlRelative); + + if (!File.Exists(fixturePath)) + { + Assert.Fail($"Expected neocities fixture at {fixturePath}"); + } + + if (!File.Exists(goldenTomlPath)) + { + Assert.Ignore($"Golden TOML not found at {goldenTomlPath} (clone mod-builds at repo root for this test)."); + } + + return (fixturePath, goldenTomlPath); + } + + private static string ResolveRepoRoot() + { + string[] candidates = + { + Path.GetFullPath(Path.Combine(TestContext.CurrentContext.TestDirectory, "..", "..", "..", "..")), + Path.GetFullPath(Path.Combine(TestContext.CurrentContext.TestDirectory, "..", "..", "..", "..", "..")), + Path.GetFullPath(Environment.CurrentDirectory), + }; + + foreach (string candidate in candidates.Distinct(StringComparer.Ordinal)) + { + if (File.Exists(Path.Combine(candidate, "ModSync.sln"))) + { + return candidate; + } + } + + throw new DirectoryNotFoundException("Could not locate repository root containing ModSync.sln"); + } + + private static void StripDependenciesForDownloadSmoke(string tomlPath, params string[] modNameFragments) + { + List components = FileLoadingService.LoadFromFile(tomlPath).ToList(); + bool changed = false; + + foreach (ModComponent component in components) + { + if (!modNameFragments.Any(fragment => + component.Name.IndexOf(fragment, StringComparison.OrdinalIgnoreCase) >= 0)) + { + continue; + } + + if (component.Dependencies.Count > 0) + { + component.Dependencies.Clear(); + changed = true; + } + + if (component.Restrictions.Count > 0) + { + component.Restrictions.Clear(); + changed = true; + } + } + + if (changed) + { + FileLoadingService.SaveToFile(components, tomlPath); + } + } + } +}