From e7c6a83ff3e4311372a2b0f07a1e043f0564fd7e Mon Sep 17 00:00:00 2001 From: Eike Waldt Date: Thu, 2 Jul 2026 15:21:30 +0200 Subject: [PATCH] refactor(aggregation): remove docs/projects/ mirror layer and dead config The aggregation pipeline previously mirrored fetched docs into docs/projects// via target_path, transform_directory_structure, and /projects/-scoped link rewrites. These were never the canonical output path for VitePress; removing them simplifies the pipeline to a single copy step. Removed: - target_path field from RepoConfig, repos-config.json, and repos-config.local.json - transform_directory_structure() (mirror step) and fix_broken_project_links() - base_path="/projects" parameter and all /projects/-prefixed link rewrites from rewrite_links() - docs/projects from .gitignore, Makefile clean, sidebar excludePattern, and ignoreDeadLinks - Project Mirrors section from contributing docs Dead config eliminated alongside: - special_files (was always {} in every repo entry) - dict-style structure (was an identity map for gardenlinux; replaced by the existing "flat" default) - unused media_dirs parameter from transform_directory_structure() - ./same-dir, bare-filename, inside-docs ../, and numbered-prefix link rewrites (VitePress resolves these natively) Hardened and simplified: - RepoConfig.from_dict() now raises ValueError for non-string structure values - from_dict() gains sensible defaults: docs_path="docs", media_directories=[".media", "assets"] - save_config() omits fields at their default values Signed-off-by: Eike Waldt On-behalf-of: SAP Assisted-by: Kilo:claude-opus-4-7 Assisted-by: Kilo:claude-sonnet-4-6 --- .gitignore | 1 - Makefile | 1 - docs/.vitepress/config.mts | 1 - docs/.vitepress/sidebar.ts | 1 - .../documentation/adding-repos.md | 107 +++----- .../documentation/aggregation-architecture.md | 48 +--- .../documentation/configuration.md | 92 +------ docs/contributing/documentation/testing.md | 9 +- .../documentation/working-locally.md | 4 - repos-config.json | 62 +---- repos-config.local.json | 48 +--- src/aggregate.py | 39 +-- src/aggregation/__init__.py | 4 +- src/aggregation/config.py | 9 +- src/aggregation/models.py | 32 ++- src/aggregation/structure.py | 250 ++++++++---------- src/aggregation/transformer.py | 156 ++--------- tests/integration/test_aggregation.py | 6 +- tests/unit/test_config.py | 184 +++++++++---- tests/unit/test_models.py | 98 +++++-- tests/unit/test_structure.py | 244 ++++++++++++----- tests/unit/test_transformer.py | 111 ++++---- 22 files changed, 677 insertions(+), 830 deletions(-) diff --git a/.gitignore b/.gitignore index 50f6cd5..bf81099 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ shell.nix .envrc # added by aggregation -docs/projects docs/**/.media docs/**/_static docs/**/assets diff --git a/Makefile b/Makefile index e19721a..1ad0bdd 100644 --- a/Makefile +++ b/Makefile @@ -138,7 +138,6 @@ clean: @echo "Cleaning build artifacts and aggregated docs..." rm -rf docs/.vitepress/dist rm -rf docs/.vitepress/cache - rm -rf docs/projects @# Clean aggregated (untracked) content from section directories, preserving git-tracked files @if [ -d .git ]; then \ git clean -fdX docs/contributing/ docs/explanation/ docs/how-to/ docs/reference/ docs/tutorials/ 2>/dev/null || true; \ diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 1f1fd4f..a07c6ec 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -81,7 +81,6 @@ export default defineConfig({ // Ignore dead links in legacy documentation /\/how-to\/troubleshooting\//, /localhost/, - /\/projects\/gardenlinux\/02_operators\/deployment/, // Ignore section index links without trailing slash (VitePress internal) /\/tutorials\/index$/, /\/how-to\/index$/, diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 6c82cb9..a7da552 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -13,7 +13,6 @@ export function generateDocumentationSidebar(): any { useTitleFromFrontmatter: true, useFolderLinkFromIndexFile: true, useFolderTitleFromIndexFile: true, - excludePattern: ['projects'], sortMenusByFrontmatterOrder: true, frontmatterOrderDefaultValue: 999, prefixSeparator: '/', diff --git a/docs/contributing/documentation/adding-repos.md b/docs/contributing/documentation/adding-repos.md index 72d3045..f5d5ede 100644 --- a/docs/contributing/documentation/adding-repos.md +++ b/docs/contributing/documentation/adding-repos.md @@ -35,7 +35,6 @@ Edit `repos-config.json` and add a new entry to the `repos` array: "name": "new-repo", "url": "https://github.com/gardenlinux/new-repo", "docs_path": "docs", - "target_path": "projects/new-repo", "ref": "main", "structure": "flat" } @@ -46,7 +45,6 @@ Edit `repos-config.json` and add a new entry to the `repos` array: - **`name`**: Unique identifier for the repository - **`url`**: Git URL or `file://` path for local development - **`docs_path`**: Path to docs directory within the repository -- **`target_path`**: Where to place mirrored docs (usually `projects/`) - **`ref`**: Git branch or tag to fetch from ### Optional Fields @@ -54,41 +52,10 @@ Edit `repos-config.json` and add a new entry to the `repos` array: - **`commit`**: Lock to a specific commit hash for reproducibility - **`root_files`**: List of root-level files to copy (e.g., `["README.md", "CONTRIBUTING.md"]`) -- **`structure`**: Directory mapping strategy (see below) +- **`structure`**: Directory mapping strategy (`"flat"` or `"sphinx"`; see below) - **`media_directories`**: List of media directories to copy (e.g., `[".media", "_static", "assets"]`) -## Step 2: Choose a Structure Strategy - -### Flat Structure - -Copy all files as-is without reorganization: - -```json -"structure": "flat" -``` - -### Sphinx Structure - -For Sphinx-generated documentation: - -```json -"structure": "sphinx" -``` - -### Custom Mapping - -Map source directories to Diataxis categories: - -```json -"structure": { - "tutorials": "tutorials", - "guides": "how-to", - "concepts": "explanation", - "api": "reference" -} -``` - ## Step 3: Configure Targeted Documentation To have files automatically placed into the main Diataxis structure, add @@ -105,10 +72,29 @@ github_target_path: "docs/how-to/example-guide.md" Content here... ``` -Files with `github_target_path` will be copied to that exact location, not to -`projects//`. +Files with `github_target_path` will be copied to that exact location in the +site. Source-repo files without `github_target_path` are excluded from the +built site entirely. + +## Step 2: Choose a Structure Strategy + +### Flat Structure -## Step 4: Test with Local Configuration +Copy all docs files as-is without reorganization: + +```json +"structure": "flat" +``` + +### Sphinx Structure + +For Sphinx-generated documentation: + +```json +"structure": "sphinx" +``` + +## Step 3: Test with Local Configuration Create or edit `repos-config.local.json` for local testing: @@ -117,10 +103,7 @@ Create or edit `repos-config.local.json` for local testing: "repos": [ { "name": "new-repo", - "url": "file://../new-repo", - "docs_path": "docs", - "target_path": "projects/new-repo", - "structure": "flat" + "url": "file://../new-repo" } ] } @@ -132,22 +115,16 @@ Then test aggregation: make aggregate-local ``` -## Step 5: Verify the Output +## Step 4: Verify the Output -Check that files are in the correct locations: - -```bash -ls -la docs/projects/new-repo/ -``` - -If using `github_target_path`, verify targeted files: +Check that targeted files landed in the correct Diataxis section: ```bash ls -la docs/tutorials/ ls -la docs/how-to/ ``` -## Step 6: Lock the Commit (Production) +## Step 5: Lock the Commit (Production) For production, lock to a specific commit: @@ -163,7 +140,6 @@ Or manually add the commit hash: "name": "new-repo", "url": "https://github.com/gardenlinux/new-repo", "docs_path": "docs", - "target_path": "projects/new-repo", "ref": "main", "commit": "abc123def456...", "structure": "flat" @@ -204,20 +180,6 @@ Copy root-level files (like README.md or CONTRIBUTING.md): These files can also have `github_target_path` front-matter for targeted placement. -### Special Files - -Handle non-standard files: - -```json -{ - "name": "new-repo", - "special_files": { - "GUIDE.md": "how-to", - "CONCEPTS.md": "explanation" - } -} -``` - ## Complete Example Here's a complete configuration: @@ -227,23 +189,16 @@ Here's a complete configuration: "name": "example-tool", "url": "https://github.com/gardenlinux/example-tool", "docs_path": "documentation", - "target_path": "projects/example-tool", "ref": "docs-ng", "commit": "1234567890abcdef", "root_files": ["README.md"], - "structure": { - "getting-started": "tutorials", - "guides": "how-to", - "concepts": "explanation", - "api-reference": "reference" - }, - "media_directories": [".media", "images"], - "special_files": { - "CHANGELOG.md": "reference" - } + "structure": "flat", + "media_directories": [".media", "images"] } ``` +The `docs_path` field defaults to `"docs"` and may be omitted when the documentation lives in a `docs/` directory. + ## Troubleshooting ### Files Not Appearing diff --git a/docs/contributing/documentation/aggregation-architecture.md b/docs/contributing/documentation/aggregation-architecture.md index 54ae542..6ac8c30 100644 --- a/docs/contributing/documentation/aggregation-architecture.md +++ b/docs/contributing/documentation/aggregation-architecture.md @@ -90,8 +90,9 @@ source repositories into a unified VitePress documentation site. 1. **Link Rewriting:** Transform relative links to work across repository boundaries - - Intra-repo links: Maintained relative to project mirror - - Cross-repo links: Rewritten to absolute paths + - Links that escape the docs tree via `../`: Redirected to GitHub + - Absolute `/` links: Redirected to GitHub + - Relative and `./` links: Left unchanged for VitePress to resolve natively - External links: Preserved as-is 2. **Front-matter Handling:** Ensure all documents have proper front-matter @@ -99,8 +100,6 @@ source repositories into a unified VitePress documentation site. - Quote YAML values safely - Preserve existing metadata -3. **Project Link Validation:** Fix broken links to project mirrors - ### 3. Structure Stage (`structure.py`) **Purpose:** Organize documentation into the final directory structure @@ -109,16 +108,11 @@ source repositories into a unified VitePress documentation site. 1. **Targeted Documentation:** Copy files with `github_target_path` to specified locations -2. **Directory Mapping:** Transform source directories according to `structure` - config +2. **Internal Link Verification:** Fail aggregation if any shipped file links + to a source-repo file that was not itself shipped (hard-fail to catch + unmigrated links early) 3. **Media Copying:** Discover and copy media directories -4. **Markdown Processing:** Apply transformations to all markdown files - -**Structure Types:** - -- **Flat:** Copy all files as-is -- **Sphinx:** Handle Sphinx documentation structure -- **Custom Mapping:** Map source directories to Diataxis categories +4. **Markdown Processing:** Apply front-matter fixes to all copied files ## Key Mechanisms @@ -141,17 +135,8 @@ github_target_path: "docs/how-to/example.md" 4. Apply markdown transformations This allows fine-grained control over where content appears in the final site. - -### Project Mirrors - -In addition to targeted docs, the entire `docs/` directory from each repo is -mirrored under `docs/projects//`: - -**Purpose:** - -- Preserve complete repository documentation -- Provide fallback for untargeted content -- Enable browsing of raw source structure +All source-repo files that are not tagged with `github_target_path` are +excluded from the built site entirely. ### Media Directory Handling @@ -215,12 +200,6 @@ This fetches the latest from `ref` and updates commit locks. - **Testability:** Easy to test individual stages - **Extensibility:** New transformations added without affecting fetch/structure -### Why Project Mirrors? - -- **Completeness:** No documentation is lost -- **Development:** Easier to debug and understand source structure -- **Backwards Compatibility:** Existing links to source repos still work - ## Data Flow ### Repository → Temporary Directory @@ -243,11 +222,8 @@ Temp Directory Docs Output ├── tutorials/ docs/ │ └── guide.md ├── tutorials/ │ (github_target_path) │ └── guide.md (targeted) -├── how-to/ ├── how-to/ -└── reference/ └── projects/repo-name/ - ├── tutorials/ (mirror) - ├── how-to/ (mirror) - └── reference/ (mirror) +├── how-to/ └── how-to/ +└── reference/ (targeted files only) ``` ## Performance Characteristics @@ -265,7 +241,7 @@ Temp Directory Docs Output ### Structure Stage - **Targeted copy:** O(n) where n = files with github_target_path -- **Directory mapping:** O(n) where n = total files +- **Link verification:** O(n * l) where l = avg links per file - **Media copy:** O(m) where m = media files ### Overall diff --git a/docs/contributing/documentation/configuration.md b/docs/contributing/documentation/configuration.md index 16196d2..f22b5cd 100644 --- a/docs/contributing/documentation/configuration.md +++ b/docs/contributing/documentation/configuration.md @@ -43,16 +43,11 @@ git operations. { "name": "repository-name", "url": "https://github.com/org/repo", - "docs_path": "docs", - "target_path": "projects/repository-name", "ref": "main", "commit": "abc123...", "root_files": ["README.md"], "structure": "flat", - "media_directories": [".media", "assets"], - "special_files": { - "GUIDE.md": "how-to" - } + "media_directories": [".media", "assets"] } ] } @@ -84,15 +79,9 @@ git operations. - **Type:** String - **Description:** Path to documentation directory within the repository - **Examples:** `"docs"`, `"documentation"`, `"."` (for root) +- **Default:** `"docs"` (may be omitted when the value is `"docs"`) - **Notes:** Relative to repository root; content of this directory is copied -#### `target_path` - -- **Type:** String -- **Description:** Destination path in the docs directory -- **Example:** `"projects/gardenlinux"` -- **Notes:** Usually `projects/` for project mirrors - #### `ref` - **Type:** String @@ -121,54 +110,25 @@ git operations. #### `structure` -- **Type:** String or Object +- **Type:** String - **Description:** How to reorganize directory structure - **Options:** - `"flat"` — Copy all files as-is - `"sphinx"` — Sphinx documentation structure - - Object — Custom directory mapping (see below) - **Default:** `"flat"` -**Custom Structure Example:** - -```json -"structure": { - "tutorials": "tutorials", - "guides": "how-to", - "concepts": "explanation", - "api-reference": "reference" -} -``` - -This maps source directories to Diataxis categories. - #### `media_directories` - **Type:** Array of strings - **Description:** Directory names to treat as media/assets - **Example:** `[".media", "assets", "_static", "images"]` -- **Default:** `[]` +- **Default:** `[".media", "assets"]` - **Notes:** - Searched recursively in source repository - Nested media dirs (e.g., `tutorials/assets/`) copied to same relative path - Root-level media dirs (e.g., `_static/`) copied to common ancestor of targeted files -#### `special_files` - -- **Type:** Object (filename → category mapping) -- **Description:** Map non-standard files to Diataxis categories -- **Example:** - ```json - { - "GUIDE.md": "how-to", - "CONCEPTS.md": "explanation", - "CHANGELOG.md": "reference" - } - ``` -- **Default:** `{}` -- **Notes:** Used when files don't follow standard naming conventions - ## Complete Example ```json @@ -177,39 +137,26 @@ This maps source directories to Diataxis categories. { "name": "gardenlinux", "url": "https://github.com/gardenlinux/gardenlinux", - "docs_path": "docs", - "target_path": "projects/gardenlinux", "ref": "docs-ng", "commit": "c4b1d8d7f878fcb3e779315d28e35fcb19ae4dfb", - "root_files": ["CONTRIBUTING.md", "SECURITY.md"], - "structure": { - "tutorials": "tutorials", - "how-to": "how-to", - "explanation": "explanation", - "reference": "reference", - "contributing": "contributing" - }, - "media_directories": [".media", "assets", "_static"] + "root_files": ["CONTRIBUTING.md", "SECURITY.md"] }, { "name": "builder", "url": "https://github.com/gardenlinux/builder", - "docs_path": "docs", - "target_path": "projects/builder", "ref": "docs-ng", - "commit": "b10476ad8c46130f310e36daa42c6e2c14fb51a9", - "structure": "flat", - "media_directories": [".media", "assets", "_static"] + "commit": "b10476ad8c46130f310e36daa42c6e2c14fb51a9" }, { "name": "python-gardenlinux-lib", "url": "https://github.com/gardenlinux/python-gardenlinux-lib", - "docs_path": "docs", - "target_path": "projects/python-gardenlinux-lib", "ref": "docs-ng", "commit": "9142fccc3d83ab51759db7d328fa19166bc1df63", "structure": "sphinx", - "media_directories": [".media", "assets", "_static"] + "target_map": { + "cli.md": "reference/python-gardenlinux-lib-cli.md", + "api.md": "reference/python-gardenlinux-lib-api.md" + } } ] } @@ -238,16 +185,7 @@ This maps source directories to Diataxis categories. { "name": "gardenlinux", "url": "file://../gardenlinux", - "docs_path": "docs", - "target_path": "projects/gardenlinux", - "root_files": ["CONTRIBUTING.md", "SECURITY.md"], - "structure": { - "tutorials": "tutorials", - "how-to": "how-to", - "explanation": "explanation", - "reference": "reference" - }, - "media_directories": [".media", "assets"] + "root_files": ["CONTRIBUTING.md", "SECURITY.md"] } ] } @@ -263,10 +201,7 @@ Simplest configuration for a flat repository: { "name": "my-repo", "url": "https://github.com/org/my-repo", - "docs_path": "docs", - "target_path": "projects/my-repo", - "ref": "main", - "structure": "flat" + "ref": "main" } ``` @@ -278,10 +213,7 @@ Repository using `github_target_path` front-matter: { "name": "my-repo", "url": "https://github.com/org/my-repo", - "docs_path": "docs", - "target_path": "projects/my-repo", "ref": "main", - "structure": "flat", "media_directories": ["assets", "_static"] } ``` diff --git a/docs/contributing/documentation/testing.md b/docs/contributing/documentation/testing.md index 4cb5cd3..77b139f 100644 --- a/docs/contributing/documentation/testing.md +++ b/docs/contributing/documentation/testing.md @@ -30,6 +30,7 @@ tests/ ├── unit/ # Unit tests (pure functions) │ ├── test_config.py │ ├── test_models.py +│ ├── test_structure.py │ └── test_transformer.py └── integration/ # Integration tests (filesystem) └── test_aggregation.py @@ -117,10 +118,10 @@ Example: ```python def test_rewrite_links(): - """Test that links are properly rewritten.""" - content = "[link](../other/file.md)" - result = rewrite_links(content, "repo-name", "path/to/file.md") - assert "[link](/projects/repo-name/other/file.md)" in result + """Test that out-of-docs links are redirected to GitHub.""" + content = "[link](../../README.md)" + result = rewrite_links(content, "repo-name", "subdir/file.md") + assert "github.com/gardenlinux/repo-name/blob/main" in result ``` ### Adding an Integration Test diff --git a/docs/contributing/documentation/working-locally.md b/docs/contributing/documentation/working-locally.md index f6cb6fc..b29ee0f 100644 --- a/docs/contributing/documentation/working-locally.md +++ b/docs/contributing/documentation/working-locally.md @@ -142,10 +142,6 @@ After aggregation, your docs directory will look like: ``` docs/ -├── projects/ # Mirrored repository docs -│ ├── gardenlinux/ -│ ├── builder/ -│ └── python-gardenlinux-lib/ ├── tutorials/ # Aggregated tutorials ├── how-to/ # Aggregated guides ├── explanation/ # Aggregated explanations diff --git a/repos-config.json b/repos-config.json index f9982fc..a9ad788 100644 --- a/repos-config.json +++ b/repos-config.json @@ -3,54 +3,28 @@ { "name": "gardenlinux", "url": "https://github.com/gardenlinux/gardenlinux", - "docs_path": "docs", - "target_path": "projects/gardenlinux", "ref": "docs-ng", - "commit": "71a86404897f709975969b7c1489429a61f70f35", + "commit": "b6ea9704d22871d1893a9e4567f43eca94201d52", "root_files": [ "CONTRIBUTING.md", "SECURITY.md", "features/*/README.md", "features/*/info.yaml", "flavors.yaml" - ], - "structure": { - "tutorials": "tutorials", - "how-to": "how-to", - "explanation": "explanation", - "reference": "reference", - "contributing": "contributing" - }, - "media_directories": [ - ".media", - "assets" ] }, { "name": "builder", "url": "https://github.com/gardenlinux/builder", - "docs_path": "docs", - "target_path": "projects/builder", "ref": "docs-ng", - "commit": "6326277fe220553b6ac8fcf87956a8fcc6639064", - "media_directories": [ - ".media", - "assets", - "_static" - ] + "commit": "6326277fe220553b6ac8fcf87956a8fcc6639064" }, { "name": "python-gardenlinux-lib", "url": "https://github.com/gardenlinux/python-gardenlinux-lib", - "docs_path": "docs", - "target_path": "projects/python-gardenlinux-lib", "ref": "docs-ng", "commit": "61a370280ff6c726f91b3d65f7ed172448841858", "structure": "sphinx", - "media_directories": [ - ".media", - "assets" - ], "target_map": { "cli.md": "reference/python-gardenlinux-lib-cli.md", "api.md": "reference/python-gardenlinux-lib-api.md" @@ -59,50 +33,26 @@ { "name": "package-linux", "url": "https://github.com/gardenlinux/package-linux", - "docs_path": "docs", - "target_path": "projects/package-linux", "ref": "docs-ng", - "commit": "45f0d1a62a723d0052bcfbf9eb40e26366595164", - "media_directories": [ - ".media", - "assets" - ] + "commit": "45f0d1a62a723d0052bcfbf9eb40e26366595164" }, { "name": "package-build", "url": "https://github.com/gardenlinux/package-build", - "docs_path": "docs", - "target_path": "projects/package-build", "ref": "docs-ng", - "commit": "2021e13e55a73d060450c295307c9a996f08ba91", - "media_directories": [ - ".media", - "assets" - ] + "commit": "2021e13e55a73d060450c295307c9a996f08ba91" }, { "name": "repo", "url": "https://github.com/gardenlinux/repo", - "docs_path": "docs", - "target_path": "projects/repo", "ref": "docs-ng", - "commit": "9adfc48e8933c6f6a66f00bbfb28756259d04aa1", - "media_directories": [ - ".media", - "assets" - ] + "commit": "9adfc48e8933c6f6a66f00bbfb28756259d04aa1" }, { "name": "glrd", "url": "https://github.com/gardenlinux/glrd", - "docs_path": "docs", - "target_path": "projects/glrd", "ref": "docs-ng", - "commit": "403361e0462e95666fd5528efd1217a7f64b3569", - "media_directories": [ - ".media", - "assets" - ] + "commit": "c51d0a8f902322848aa17bdee8ddd3a86bf86bf3" } ] } diff --git a/repos-config.local.json b/repos-config.local.json index 2bbaff6..9a8228d 100644 --- a/repos-config.local.json +++ b/repos-config.local.json @@ -3,76 +3,42 @@ { "name": "gardenlinux", "url": "file://../gardenlinux", - "docs_path": "docs", "root_files": [ "CONTRIBUTING.md", "SECURITY.md", "features/*/README.md", "features/*/info.yaml", "flavors.yaml" - ], - "target_path": "projects/gardenlinux", - "structure": { - "tutorials": "tutorials", - "how-to": "how-to", - "explanation": "explanation", - "reference": "reference", - "contributing": "contributing" - }, - "special_files": {}, - "media_directories": [".media", "assets"] + ] }, { "name": "builder", - "url": "file://../builder", - "docs_path": "docs", - "target_path": "projects/builder", - "structure": "flat", - "media_directories": [".media", "assets"] + "url": "file://../builder" }, { "name": "python-gardenlinux-lib", "url": "file://../python-gardenlinux-lib", - "docs_path": "docs", - "target_path": "projects/python-gardenlinux-lib", "structure": "sphinx", "target_map": { "cli.md": "reference/python-gardenlinux-lib-cli.md", "api.md": "reference/python-gardenlinux-lib-api.md" - }, - "media_directories": [".media", "assets"] + } }, { "name": "package-linux", - "url": "file://../package-linux", - "docs_path": "docs", - "target_path": "projects/package-linux", - "structure": "flat", - "media_directories": [".media", "assets"] + "url": "file://../package-linux" }, { "name": "package-build", - "url": "file://../package-build", - "docs_path": "docs", - "target_path": "projects/package-build", - "structure": "flat", - "media_directories": [".media", "assets"] + "url": "file://../package-build" }, { "name": "repo", - "url": "file://../repo", - "docs_path": "docs", - "target_path": "projects/repo", - "structure": "flat", - "media_directories": [".media", "assets"] + "url": "file://../repo" }, { "name": "glrd", - "url": "file://../glrd", - "docs_path": "docs", - "target_path": "projects/glrd", - "structure": "flat", - "media_directories": [".media", "assets"] + "url": "file://../glrd" } ] } diff --git a/src/aggregate.py b/src/aggregate.py index 48b8a40..736fddd 100755 --- a/src/aggregate.py +++ b/src/aggregate.py @@ -12,8 +12,8 @@ from pathlib import Path from aggregation import (DocsFetcher, copy_targeted_docs, load_config, - process_all_markdown, save_config, - transform_directory_structure) + save_config) +from aggregation.structure import verify_internal_links from aggregation.flavor_matrix import generate_flavor_matrix_docs from aggregation.release_notes import generate_release_notes_docs from aggregation.releases import generate_release_docs @@ -31,7 +31,6 @@ def transform_repo_docs( print(f"{'='*60}") source_dir = temp_dir / repo_name - target_dir = docs_dir / repo.target_path # Step 1: Copy files with 'github_target_path:' frontmatter print(f"\nStep 1: Processing targeted files...") @@ -43,19 +42,19 @@ def transform_repo_docs( repo.root_files, ) - # Step 2: Transform project structure - print(f"\nStep 2: Transforming project structure...") - transform_directory_structure( + # Step 2: Verify internal links in shipped files + print(f"\nStep 2: Verifying internal links...") + link_errors = verify_internal_links( str(source_dir), - str(target_dir), - repo.structure, - repo.special_files, - repo.media_directories, + str(docs_dir), + repo_name, ) - - # Step 3: Process markdown files - print(f"\nStep 3: Processing markdown files...") - process_all_markdown(str(target_dir), repo_name) + if link_errors > 0: + print( + f" [ERROR] {link_errors} unshipped link(s) found in {repo_name}. " + "Add github_target_path to the linked files or remove the links." + ) + return False print(f"\n✓ Transformation complete for {repo_name}") return True @@ -247,6 +246,7 @@ def main() -> int: fail_count = 0 # Aggregate each repository + gardenlinux_temp_dir = None for repo in repos: # Filter to a single repo when --single is set if args.single and repo.name != args.repo: @@ -263,6 +263,8 @@ def main() -> int: success_count += 1 if resolved_commit: resolved_commits[repo.name] = resolved_commit + if repo.name == "gardenlinux": + gardenlinux_temp_dir = temp_dir / repo.name else: fail_count += 1 @@ -281,13 +283,12 @@ def main() -> int: print(f"\n✓ Config updated: {config_path}") # Generate flavor matrix documentation after all repos are aggregated - # Use docs/projects/gardenlinux path since temp_dir is cleaned up - gardenlinux_docs_path = docs_dir / "projects" / "gardenlinux" - if gardenlinux_docs_path.exists(): + # Use the gardenlinux temp dir (still available inside the with block) + if gardenlinux_temp_dir and gardenlinux_temp_dir.exists(): print(f"\n{'='*60}") print("Generating flavor matrix documentation...") print(f"{'='*60}\n") - generate_flavor_matrix_docs(docs_dir, gardenlinux_docs_path) + generate_flavor_matrix_docs(docs_dir, gardenlinux_temp_dir) # Generate release documentation from GLRD print(f"\n{'='*60}") @@ -309,7 +310,7 @@ def main() -> int: print(f"Failed: {fail_count}") print("\nNext steps:") - print(" 1. Review the changes in docs/projects/") + print(" 1. Review the changes in docs/") print(" 2. Run 'make run' or 'pnpm run docs:dev' to preview") print(" 3. Commit the changes if satisfied") diff --git a/src/aggregation/__init__.py b/src/aggregation/__init__.py index a25e926..9f765c6 100644 --- a/src/aggregation/__init__.py +++ b/src/aggregation/__init__.py @@ -8,7 +8,7 @@ from .releases import generate_release_docs from .sphinx_builder import build_sphinx_markdown from .structure import (copy_targeted_docs, process_all_markdown, - transform_directory_structure) + verify_internal_links) from .transformer import (ensure_frontmatter, parse_frontmatter, quote_yaml_value, rewrite_links) @@ -27,9 +27,9 @@ "quote_yaml_value", "parse_frontmatter", # Structure - "transform_directory_structure", "copy_targeted_docs", "process_all_markdown", + "verify_internal_links", # Releases "generate_release_docs", # Flavor Matrix diff --git a/src/aggregation/config.py b/src/aggregation/config.py index 2f8e7ee..e6b278a 100644 --- a/src/aggregation/config.py +++ b/src/aggregation/config.py @@ -4,7 +4,7 @@ import sys from typing import List -from .models import RepoConfig +from .models import RepoConfig, _DEFAULT_MEDIA_DIRECTORIES def load_config(config_path: str) -> List[RepoConfig]: @@ -53,16 +53,15 @@ def save_config(config_path: str, repos: List[RepoConfig]) -> None: { "name": repo.name, "url": repo.url, - "docs_path": repo.docs_path, - "target_path": repo.target_path, + **({"docs_path": repo.docs_path} if repo.docs_path != "docs" else {}), + **({"target_path": repo.target_path} if repo.target_path else {}), **({"ref": repo.ref} if repo.ref else {}), **({"commit": repo.commit} if repo.commit else {}), **({"root_files": repo.root_files} if repo.root_files else {}), **({"structure": repo.structure} if repo.structure != "flat" else {}), - **({"special_files": repo.special_files} if repo.special_files else {}), **( {"media_directories": repo.media_directories} - if repo.media_directories + if repo.media_directories != list(_DEFAULT_MEDIA_DIRECTORIES) else {} ), **({"target_map": repo.target_map} if repo.target_map else {}), diff --git a/src/aggregation/models.py b/src/aggregation/models.py index f09dc86..e75faff 100644 --- a/src/aggregation/models.py +++ b/src/aggregation/models.py @@ -1,7 +1,10 @@ """Data models for documentation aggregation.""" from dataclasses import dataclass, field -from typing import Dict, List, Union +from typing import Dict, List + + +_DEFAULT_MEDIA_DIRECTORIES = [".media", "assets"] @dataclass @@ -11,15 +14,14 @@ class RepoConfig: name: str url: str docs_path: str - target_path: str ref: str commit: str | None = None + target_path: str = "" root_files: List[str] = field(default_factory=list) - structure: Union[str, Dict[str, str]] = "flat" - special_files: Dict[str, str] = field(default_factory=dict) + structure: str = "flat" media_directories: List[str] = field(default_factory=list) target_map: Dict[str, str] = field(default_factory=dict) - + @property def is_local(self) -> bool: """Check if this is a local file:// repository.""" @@ -46,17 +48,27 @@ def validate(self) -> None: @classmethod def from_dict(cls, data: Dict) -> "RepoConfig": """Create RepoConfig from dictionary.""" + media_directories = data.get("media_directories") + if media_directories is None: + media_directories = list(_DEFAULT_MEDIA_DIRECTORIES) + + structure = data.get("structure", "flat") + if not isinstance(structure, str): + raise ValueError( + f"Repository '{data.get('name')}': 'structure' must be a string " + f"('flat' or 'sphinx'), got {type(structure).__name__}" + ) + return cls( name=data["name"], url=data["url"], - docs_path=data["docs_path"], - target_path=data["target_path"], + docs_path=data.get("docs_path", "docs"), + target_path=data.get("target_path", ""), ref=data.get("ref") or "main", commit=data.get("commit"), root_files=data.get("root_files", []), - structure=data.get("structure", "flat"), - special_files=data.get("special_files", {}), - media_directories=data.get("media_directories", []), + structure=structure, + media_directories=media_directories, target_map=data.get("target_map", {}), ) diff --git a/src/aggregation/structure.py b/src/aggregation/structure.py index 3be0225..9f42fce 100644 --- a/src/aggregation/structure.py +++ b/src/aggregation/structure.py @@ -4,105 +4,7 @@ from pathlib import Path from typing import List -from .transformer import (ensure_frontmatter, fix_broken_project_links, - parse_frontmatter, rewrite_links) - - -def transform_directory_structure( - source_dir: str, - target_dir: str, - structure_map, - special_files: dict | None = None, - media_dirs: List[str] | None = None, -) -> None: - """ - Transform directory structure based on mapping. - - Args: - source_dir: Source directory with fetched docs - target_dir: Target directory in docs/projects/ - structure_map: Directory structure mapping or copy mode - special_files: Map of files to move to specific locations - media_dirs: List of media directories to preserve - """ - source_path = Path(source_dir) - target_path = Path(target_dir) - - # For sphinx structure: clean the target directory first so stale source - # files from previous runs do not accumulate alongside the built Markdown. - if structure_map == "sphinx" and target_path.exists(): - print(f" Cleaning target directory for sphinx build: {target_path}") - shutil.rmtree(target_path) - - target_path.mkdir(parents=True, exist_ok=True) - - special_files = special_files or {} - media_dirs = media_dirs or [] - - if isinstance(structure_map, dict): - # Structured transformation with subdirectories specified - for old_name, new_name in structure_map.items(): - old_path = source_path / old_name - new_path = target_path / new_name - - if old_path.exists(): - print(f" Transforming: {old_name} -> {new_name}") - shutil.copytree(old_path, new_path, dirs_exist_ok=True) - - # Handle special files - for item in source_path.iterdir(): - if item.name in structure_map: - continue - - if item.name in special_files: - target_subdir = target_path / special_files[item.name] - target_subdir.mkdir(parents=True, exist_ok=True) - if item.is_file(): - print(f" Moving {item.name} to {special_files[item.name]}") - shutil.copy2(item, target_subdir / item.name) - elif item.is_dir(): - print(f" Moving {item.name} to {special_files[item.name]}") - shutil.copytree(item, target_subdir / item.name, dirs_exist_ok=True) - elif item.name in media_dirs: - print(f" Copying media directory: {item.name}") - shutil.copytree(item, target_path / item.name, dirs_exist_ok=True) - elif item.is_file() and not item.name.startswith("_"): - shutil.copy2(item, target_path / item.name) - elif ( - item.is_dir() - and not item.name.startswith("_") - and not item.name.startswith(".") - ): - shutil.copytree(item, target_path / item.name, dirs_exist_ok=True) - - else: - # Flat/sphinx structure - copy all files as-is (merged logic) - print(f" Copying {structure_map} structure") - # Files generated by Sphinx that are not useful in VitePress - _sphinx_artifacts = {"genindex.md", "py-modindex.md", "search.md"} - for item in source_path.glob("*"): - if structure_map == "sphinx" and item.name in _sphinx_artifacts: - print(f" Skipping Sphinx artifact: {item.name}") - continue - # For sphinx structure: skip Markdown files that already have a - # github_target_path frontmatter field — they were placed at their - # canonical VitePress location by copy_targeted_docs (Step 1) and - # must not be duplicated here, as duplicate HTML anchor IDs across - # two pages break VitePress's MiniSearch indexer. - if structure_map == "sphinx" and item.is_file() and item.suffix == ".md": - try: - content = item.read_text(encoding="utf-8") - frontmatter, _ = parse_frontmatter(content) - if frontmatter and "github_target_path" in frontmatter: - print(f" Skipping targeted file (already placed): {item.name}") - continue - except Exception: - pass - target_item = target_path / item.name - if item.is_file(): - shutil.copy2(item, target_item) - elif item.is_dir(): - shutil.copytree(item, target_item, dirs_exist_ok=True) +from .transformer import (ensure_frontmatter, parse_frontmatter, rewrite_links) def copy_targeted_docs( @@ -229,6 +131,14 @@ def copy_targeted_docs( common_parent = p break + # Build mapping from source parent dir to set of target parent dirs. + # Used to colocate nested media dirs with retargeted markdown files. + source_to_target_parents: dict[Path, set[Path]] = {} + for src_rel, target_rel in targeted_files: + src_parent = Path(src_rel).parent # e.g. Path("overview") + target_parent = Path(target_rel).parent # e.g. Path("reference/supporting_tools") + source_to_target_parents.setdefault(src_parent, set()).add(target_parent) + for media_dir_name in media_dirs: # Recursively find all instances of this media directory in the source for media_dir in source_path.rglob(media_dir_name): @@ -251,61 +161,125 @@ def copy_targeted_docs( f" ✓ Copied media: {common_parent / media_dir_name}" ) else: - # Nested media directory: copy to same relative path - target_media = docs_path / rel_path - target_media.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(media_dir, target_media, dirs_exist_ok=True) - print(f" ✓ Copied media: {rel_path}") + # Nested media directory: look up source parent in mapping + # to colocate media with the retargeted markdown file(s). + media_source_parent = rel_path.parent # e.g. Path("overview") + if media_source_parent in source_to_target_parents: + for target_parent in source_to_target_parents[media_source_parent]: + target_media = docs_path / target_parent / media_dir_name + target_media.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(media_dir, target_media, dirs_exist_ok=True) + print(f" ✓ Copied media: {target_parent / media_dir_name}") + else: + # No mapping found: fall back to source-relative placement + target_media = docs_path / rel_path + target_media.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(media_dir, target_media, dirs_exist_ok=True) + print(f" ✓ Copied media: {rel_path}") else: print(" No files with 'github_target_path:' frontmatter found") -def process_markdown_file( - file_path: Path, +def verify_internal_links( + source_dir: str, + docs_dir: str, repo_name: str, - target_dir: str, - base_path: str = "/projects", -) -> bool: +) -> int: """ - Process a single markdown file: rewrite links, fix frontmatter. + Verify that all internal relative links in shipped markdown files resolve to + files that were also shipped (have github_target_path). + + Exits with a non-zero count of errors when any shipped file links to a + source-repo file that was not itself shipped. Args: - file_path: Path to markdown file + source_dir: Source directory with fetched docs (temp dir) + docs_dir: Docs root directory (where targeted files were placed) repo_name: Repository name - target_dir: Target directory path - base_path: Base path for projects Returns: - Success status + Number of broken links found (0 = success) """ - try: - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() + import re - # Calculate relative path from target_dir - file_path_obj = Path(file_path) - target_path_obj = Path(target_dir) + source_path = Path(source_dir) + docs_path = Path(docs_dir) + + # Build set of shipped source paths (files with github_target_path) + shipped_source_paths = set() + md_files = list(source_path.rglob("*.md")) + + for md_file in md_files: try: - file_rel_path = str(file_path_obj.relative_to(target_path_obj)) - except ValueError: - file_rel_path = "" + content = md_file.read_text(encoding="utf-8") + frontmatter, _ = parse_frontmatter(content) + if frontmatter and "github_target_path" in frontmatter: + shipped_source_paths.add(md_file.resolve()) + except Exception: + pass - content = rewrite_links(content, repo_name, file_rel_path, base_path) - content = fix_broken_project_links(content, repo_name, target_dir, base_path) - content = ensure_frontmatter(content) + if not shipped_source_paths: + return 0 - with open(file_path, "w", encoding="utf-8") as f: - f.write(content) + errors = 0 - return True - except Exception as e: - print(f" [Warning] Error processing {file_path}: {e}") - return False + # For each shipped file, check its internal links + for md_file in md_files: + if md_file.resolve() not in shipped_source_paths: + continue + + try: + content = md_file.read_text(encoding="utf-8") + except Exception: + continue + + # Find all markdown links + for match in re.finditer(r"\[([^\]]*)\]\(([^)]+)\)", content): + link = match.group(2) + + # Skip external links, anchors, special protocols + if ( + link.startswith("http://") + or link.startswith("https://") + or link.startswith("#") + or link.startswith("mailto:") + or (":" in link and not link.startswith("/") and not link.startswith("./") and not link.startswith("../")) + ): + continue + + # Skip absolute links starting with / (these are VitePress absolute paths + # or GitHub-redirected links — not source-repo relative links) + if link.startswith("/"): + continue + + # Resolve relative link from the file's directory + file_dir = md_file.parent + link_path_str = link.split("#")[0] # Strip fragment + if not link_path_str: + continue + + # Resolve the link target + resolved = (file_dir / link_path_str).resolve() + + # If it's a .md file, check if it's in the shipped set + if resolved.suffix == ".md": + if resolved.is_relative_to(source_path.resolve()): + if resolved not in shipped_source_paths and resolved.exists(): + rel_file = md_file.relative_to(source_path) + print( + f" [ERROR] Unshipped link in {repo_name}/{rel_file}: " + f"'{link}' → {resolved.relative_to(source_path.resolve())} " + f"(file exists but has no github_target_path)" + ) + errors += 1 + + return errors def process_all_markdown(target_dir: str, repo_name: str) -> None: """ Process all markdown files in target directory. + Renames README.md to index.md for VitePress compatibility. Args: target_dir: Target directory path @@ -326,7 +300,17 @@ def process_all_markdown(target_dir: str, repo_name: str) -> None: success_count = 0 for md_file in md_files: - if process_markdown_file(md_file, repo_name, target_dir): + try: + with open(md_file, "r", encoding="utf-8") as f: + content = f.read() + + content = ensure_frontmatter(content) + + with open(md_file, "w", encoding="utf-8") as f: + f.write(content) + success_count += 1 + except Exception as e: + print(f" [Warning] Error processing {md_file}: {e}") print(f" ✓ Processed {success_count}/{len(md_files)} files successfully") diff --git a/src/aggregation/transformer.py b/src/aggregation/transformer.py index 275784d..9d6386b 100644 --- a/src/aggregation/transformer.py +++ b/src/aggregation/transformer.py @@ -9,17 +9,20 @@ def rewrite_links( content: str, repo_name: str, file_rel_path: str = "", - base_path: str = "/projects", github_base: str = "https://github.com/gardenlinux", ) -> str: """ Rewrite internal markdown links to work with VitePress structure. + Only rewrites links that escape the docs tree (going too many levels up + via ../), redirecting them to the corresponding GitHub URL. All other + links — relative, absolute /..., or ./ — are left unchanged for VitePress + to resolve natively. + Args: content: The markdown content repo_name: Name of the repository file_rel_path: Relative path of the file within the repo - base_path: Base path for projects github_base: Base URL for GitHub organization Returns: @@ -50,96 +53,34 @@ def replace_link(match): if link.startswith("#"): return match.group(0) - # Skip if already a /projects/ link - if link.startswith(f"{base_path}/"): - return match.group(0) - - # Handle relative paths for .media directory - if ".media/" in link: - media_part = link - while media_part.startswith("../"): - media_part = media_part[3:] - media_part = media_part.replace("./", "") - new_link = f"{base_path}/{repo_name}/{media_part}" - return f"[{text}]({new_link})" - - # Handle relative links - if link.startswith("../") or link.startswith("./"): - stripped_link = link.replace(".md", "") - - # For ./ links (same directory) - if link.startswith("./"): - stripped_link = stripped_link.replace("./", "") - if file_dir: - new_link = f"{base_path}/{repo_name}/{file_dir}/{stripped_link}" - else: - new_link = f"{base_path}/{repo_name}/{stripped_link}" - else: - # For ../ links, check if they go outside docs/ - levels_up = link.count("../") - stripped_link = stripped_link.replace("../", "") - - # Check if we go outside docs/ - if file_dir: - dir_depth = len(file_dir.split("/")) - if levels_up > dir_depth: - # Link to GitHub - new_link = ( - f"{github_base}/{repo_name}/blob/main/{stripped_link}" - ) - return f"[{text}]({new_link})" - - # Remove numbered prefixes - stripped_link = re.sub(r"\d+_(\w+)", r"\1", stripped_link) - new_link = f"{base_path}/{repo_name}/{stripped_link}" + # Handle ../ links that go outside docs/ — redirect to GitHub + if link.startswith("../"): + levels_up = link.count("../") + stripped_link = link.replace("../", "").replace(".md", "") - return f"[{text}]({new_link})" + if file_dir: + dir_depth = len(file_dir.split("/")) + if levels_up > dir_depth: + new_link = ( + f"{github_base}/{repo_name}/blob/main/{stripped_link}" + ) + return f"[{text}]({new_link})" + + # Inside docs tree: leave relative link unchanged for VitePress + return match.group(0) - # Handle absolute paths from root + # Handle absolute paths from root — redirect to GitHub if link.startswith("/"): - if link.startswith(f"{base_path}/"): - return match.group(0) # Link to file outside docs/ - point to GitHub stripped_link = link.lstrip("/") new_link = f"{github_base}/{repo_name}/blob/main/{stripped_link}" return f"[{text}]({new_link})" - # Handle simple filenames (same directory) - if "/" not in link: - stripped_link = link.replace(".md", "") - if file_dir: - new_link = f"{base_path}/{repo_name}/{file_dir}/{stripped_link}" - else: - new_link = f"{base_path}/{repo_name}/{stripped_link}" - return f"[{text}]({new_link})" - return match.group(0) # Apply transform to markdown links content = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", replace_link, content) - # Handle HTML media links - def replace_html_media_link(match): - attr_name = match.group(1) - link = match.group(2) - - if link.startswith(f"{base_path}/"): - return match.group(0) - if ".media/" in link: - media_part = link - while media_part.startswith("../"): - media_part = media_part[3:] - media_part = media_part.replace("./", "") - new_link = f"{base_path}/{repo_name}/{media_part}" - return f'{attr_name}="{new_link}"' - return match.group(0) - - content = re.sub( - r'(src|srcset)="([^"]*\.media/[^"]*)"', - replace_html_media_link, - content, - ) - return content @@ -295,63 +236,6 @@ def ensure_frontmatter(content: str) -> str: return content -def fix_broken_project_links( - content: str, - repo_name: str, - target_dir: str, - base_path: str = "/projects", - github_base: str = "https://github.com/gardenlinux", -) -> str: - """ - Fix links in /projects/ that point to non-existent files. - Replace with GitHub links. - - Args: - content: Markdown content - repo_name: Repository name - target_dir: Target directory path - base_path: Base path for projects - github_base: GitHub base URL - - Returns: - Content with fixed links - """ - target_path = Path(target_dir) - - def check_and_fix_link(match): - text = match.group(1) - link = match.group(2) - - # Only process /projects/{repo}/ links - if not link.startswith(f"{base_path}/{repo_name}/"): - return match.group(0) - - # Extract the path after /projects/{repo}/ - rel_path = link[len(f"{base_path}/{repo_name}/") :] - - potential_file = target_path / f"{rel_path}.md" - potential_index = target_path / rel_path / "index.md" - potential_dir = target_path / rel_path - - # If file exists, or directory exists with index.md, keep the link - if ( - potential_file.exists() - or potential_index.exists() - or ( - potential_dir.exists() - and potential_dir.is_dir() - and (potential_dir / "index.md").exists() - ) - ): - return match.group(0) - - github_link = f"{github_base}/{repo_name}/blob/main/{rel_path}" - return f"[{text}]({github_link})" - - content = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", check_and_fix_link, content) - return content - - def cleanup_github_markdown(content: str) -> str: """ Clean up GitHub release notes markdown for VitePress compatibility. diff --git a/tests/integration/test_aggregation.py b/tests/integration/test_aggregation.py index 84d6cc3..ef86440 100644 --- a/tests/integration/test_aggregation.py +++ b/tests/integration/test_aggregation.py @@ -46,7 +46,6 @@ def test_fetch_local_with_temp_dir(self, tmp_path): name="test-repo", url=f"file://{repo_path}", docs_path="docs", - target_path="projects/test-repo", ref="", ) @@ -75,7 +74,6 @@ def test_fetch_local_missing_docs_path(self, tmp_path, capsys): name="test-repo", url=f"file://{repo_path}", docs_path="docs", # Does not exist - target_path="projects/test-repo", ref="", ) @@ -105,7 +103,6 @@ def test_fetch_local_with_root_files_glob(self, tmp_path): name="test-repo", url=f"file://{repo_path}", docs_path="nonexistent-docs", # No standard docs; only root_files - target_path="projects/test-repo", ref="", root_files=["features/foo/*.md"], ) @@ -135,7 +132,6 @@ def test_fetch_local_resolves_relative_path(self, tmp_path): name="relative-repo", url="file://../mock-repo", docs_path="docs", - target_path="projects/relative-repo", ref="", ) @@ -179,7 +175,7 @@ def test_process_all_markdown(self, tmp_path): # Verify guide links were rewritten guide_content = (subdir / "guide.md").read_text() - assert "/projects/test-repo" in guide_content or "README" in guide_content + assert "README" in guide_content def test_process_markdown_with_frontmatter(self, tmp_path): """Test that frontmatter is properly handled.""" diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index a3640d6..27d0d22 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -6,11 +6,12 @@ import pytest from aggregation import load_config, save_config, RepoConfig +from aggregation.models import _DEFAULT_MEDIA_DIRECTORIES class TestLoadConfig: """Tests for load_config function.""" - + def test_load_valid_config(self, tmp_path): """Test loading valid configuration.""" config_data = { @@ -26,12 +27,12 @@ def test_load_valid_config(self, tmp_path): } config_file = tmp_path / "config.json" config_file.write_text(json.dumps(config_data)) - + repos = load_config(str(config_file)) assert len(repos) == 1 assert repos[0].name == "test-repo" assert repos[0].url == "https://github.com/test/repo" - + def test_load_multiple_repos(self, tmp_path): """Test loading config with multiple repositories.""" config_data = { @@ -53,12 +54,12 @@ def test_load_multiple_repos(self, tmp_path): } config_file = tmp_path / "config.json" config_file.write_text(json.dumps(config_data)) - + repos = load_config(str(config_file)) assert len(repos) == 2 assert repos[0].name == "repo1" assert repos[1].name == "repo2" - + def test_load_config_with_optional_fields(self, tmp_path): """Test loading config with optional fields.""" config_data = { @@ -71,39 +72,39 @@ def test_load_config_with_optional_fields(self, tmp_path): "ref": "main", "commit": "abc123", "root_files": ["README.md"], - "structure": {"old": "new"}, + "structure": "sphinx", } ] } config_file = tmp_path / "config.json" config_file.write_text(json.dumps(config_data)) - + repos = load_config(str(config_file)) assert repos[0].commit == "abc123" assert repos[0].root_files == ["README.md"] - assert repos[0].structure == {"old": "new"} - + assert repos[0].structure == "sphinx" + def test_load_invalid_json(self, tmp_path): """Test that invalid JSON causes exit.""" config_file = tmp_path / "config.json" config_file.write_text("{ invalid json") - + with pytest.raises(SystemExit): load_config(str(config_file)) - + def test_load_missing_repos_key(self, tmp_path): """Test that missing 'repos' key causes exit.""" config_data = {"other": "data"} config_file = tmp_path / "config.json" config_file.write_text(json.dumps(config_data)) - + with pytest.raises(SystemExit): load_config(str(config_file)) class TestSaveConfig: """Tests for save_config function.""" - + def test_save_single_repo(self, tmp_path): """Test saving configuration with single repository.""" repos = [ @@ -116,19 +117,19 @@ def test_save_single_repo(self, tmp_path): ) ] config_file = tmp_path / "config.json" - + save_config(str(config_file), repos) - + # Verify file was created and contains correct data assert config_file.exists() with open(config_file) as f: data = json.load(f) - + assert "repos" in data assert len(data["repos"]) == 1 assert data["repos"][0]["name"] == "test-repo" assert data["repos"][0]["ref"] == "main" - + def test_save_multiple_repos(self, tmp_path): """Test saving configuration with multiple repositories.""" repos = [ @@ -148,14 +149,14 @@ def test_save_multiple_repos(self, tmp_path): ), ] config_file = tmp_path / "config.json" - + save_config(str(config_file), repos) - + with open(config_file) as f: data = json.load(f) - + assert len(data["repos"]) == 2 - + def test_save_with_commit_lock(self, tmp_path): """Test saving configuration with commit field.""" repos = [ @@ -169,14 +170,14 @@ def test_save_with_commit_lock(self, tmp_path): ) ] config_file = tmp_path / "config.json" - + save_config(str(config_file), repos) - + with open(config_file) as f: data = json.load(f) - + assert data["repos"][0]["commit"] == "abc123" - + def test_save_omits_empty_optional_fields(self, tmp_path): """Test that empty optional fields are omitted.""" repos = [ @@ -189,16 +190,16 @@ def test_save_omits_empty_optional_fields(self, tmp_path): ) ] config_file = tmp_path / "config.json" - + save_config(str(config_file), repos) - + with open(config_file) as f: data = json.load(f) - + # Should not have empty optional fields assert "commit" not in data["repos"][0] assert "root_files" not in data["repos"][0] - + def test_round_trip(self, tmp_path): """Test that load/save round-trip preserves data.""" original_repos = [ @@ -213,11 +214,11 @@ def test_round_trip(self, tmp_path): ) ] config_file = tmp_path / "config.json" - + # Save and load save_config(str(config_file), original_repos) loaded_repos = load_config(str(config_file)) - + # Compare assert len(loaded_repos) == 1 assert loaded_repos[0].name == original_repos[0].name @@ -244,25 +245,24 @@ def test_load_config_rejects_invalid_scheme(self, tmp_path): with pytest.raises(SystemExit): load_config(str(config_file)) - def test_save_preserves_structure_dict(self, tmp_path): - """Test that a dict structure survives a save/load round trip.""" - structure_map = {"old_dir": "new_dir", "another": "different"} - repos = [ - RepoConfig( - name="test-repo", - url="https://github.com/test/repo", - docs_path="docs", - target_path="projects/test", - ref="main", - structure=structure_map, - ) - ] + def test_load_config_rejects_dict_structure(self, tmp_path): + """Test that a repo with a dict structure value causes SystemExit via from_dict().""" + config_data = { + "repos": [ + { + "name": "bad", + "url": "https://github.com/test/repo", + "target_path": "projects/bad", + "ref": "main", + "structure": {"tutorials": "tutorials"}, + } + ] + } config_file = tmp_path / "config.json" + config_file.write_text(json.dumps(config_data)) - save_config(str(config_file), repos) - loaded_repos = load_config(str(config_file)) - - assert loaded_repos[0].structure == structure_map + with pytest.raises(SystemExit): + load_config(str(config_file)) def test_save_omits_default_structure_flat(self, tmp_path): """Test that structure='flat' is not written to the JSON output.""" @@ -284,8 +284,8 @@ def test_save_omits_default_structure_flat(self, tmp_path): assert "structure" not in data["repos"][0] - def test_save_preserves_target_map_and_special_files(self, tmp_path): - """Test that target_map and special_files survive a save/load round trip.""" + def test_save_preserves_target_map(self, tmp_path): + """Test that target_map survives a save/load round trip.""" repos = [ RepoConfig( name="test-repo", @@ -294,7 +294,6 @@ def test_save_preserves_target_map_and_special_files(self, tmp_path): target_path="projects/test", ref="main", target_map={"src/api.md": "api/index.md"}, - special_files={"CHANGELOG.md": "reference/"}, ) ] config_file = tmp_path / "config.json" @@ -303,7 +302,6 @@ def test_save_preserves_target_map_and_special_files(self, tmp_path): loaded_repos = load_config(str(config_file)) assert loaded_repos[0].target_map == {"src/api.md": "api/index.md"} - assert loaded_repos[0].special_files == {"CHANGELOG.md": "reference/"} def test_save_local_repo_omits_empty_ref(self, tmp_path): """Test that local repos with ref='' do not emit a 'ref' key in JSON.""" @@ -324,3 +322,85 @@ def test_save_local_repo_omits_empty_ref(self, tmp_path): data = json.load(f) assert "ref" not in data["repos"][0] + + def test_save_omits_default_docs_path(self, tmp_path): + """Test that docs_path='docs' is not written to the JSON output.""" + repos = [ + RepoConfig( + name="test-repo", + url="https://github.com/test/repo", + docs_path="docs", + target_path="projects/test", + ref="main", + ) + ] + config_file = tmp_path / "config.json" + + save_config(str(config_file), repos) + + with open(config_file) as f: + data = json.load(f) + + assert "docs_path" not in data["repos"][0] + + def test_save_preserves_nondefault_docs_path(self, tmp_path): + """Test that a non-default docs_path is preserved in JSON output.""" + repos = [ + RepoConfig( + name="test-repo", + url="https://github.com/test/repo", + docs_path=".", + target_path="projects/test", + ref="main", + ) + ] + config_file = tmp_path / "config.json" + + save_config(str(config_file), repos) + + with open(config_file) as f: + data = json.load(f) + + assert data["repos"][0]["docs_path"] == "." + + def test_save_omits_default_media_directories(self, tmp_path): + """Test that default media_directories ['.media', 'assets'] is omitted.""" + repos = [ + RepoConfig( + name="test-repo", + url="https://github.com/test/repo", + docs_path="docs", + target_path="projects/test", + ref="main", + media_directories=list(_DEFAULT_MEDIA_DIRECTORIES), + ) + ] + config_file = tmp_path / "config.json" + + save_config(str(config_file), repos) + + with open(config_file) as f: + data = json.load(f) + + assert "media_directories" not in data["repos"][0] + + def test_save_preserves_nondefault_media_directories(self, tmp_path): + """Test that non-default media_directories are preserved in JSON output.""" + repos = [ + RepoConfig( + name="test-repo", + url="https://github.com/test/repo", + docs_path="docs", + target_path="projects/test", + ref="main", + media_directories=["images"], + ) + ] + config_file = tmp_path / "config.json" + + save_config(str(config_file), repos) + + with open(config_file) as f: + data = json.load(f) + + assert data["repos"][0]["media_directories"] == ["images"] diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 86bf281..6b61ccc 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -2,129 +2,172 @@ import pytest from aggregation import RepoConfig, AggregateResult +from aggregation.models import _DEFAULT_MEDIA_DIRECTORIES class TestRepoConfig: """Tests for RepoConfig dataclass.""" - + def test_from_dict_minimal(self): """Test creating RepoConfig from minimal valid dict.""" data = { "name": "test-repo", "url": "https://github.com/test/repo", "docs_path": "docs", - "target_path": "projects/test-repo", "ref": "main", } repo = RepoConfig.from_dict(data) assert repo.name == "test-repo" assert repo.url == "https://github.com/test/repo" assert repo.docs_path == "docs" - assert repo.target_path == "projects/test-repo" + assert repo.target_path == "" assert repo.ref == "main" assert repo.commit is None assert repo.root_files == [] assert repo.structure == "flat" - + + def test_from_dict_with_target_path(self): + """Test creating RepoConfig from dict including optional target_path.""" + data = { + "name": "test-repo", + "url": "https://github.com/test/repo", + "docs_path": "docs", + "target_path": "projects/test-repo", + "ref": "main", + } + repo = RepoConfig.from_dict(data) + assert repo.target_path == "projects/test-repo" + def test_from_dict_full(self): """Test creating RepoConfig from dict with all fields.""" data = { "name": "test-repo", "url": "https://github.com/test/repo", "docs_path": "docs", - "target_path": "projects/test-repo", "ref": "main", "commit": "abc123", "root_files": ["README.md", "LICENSE"], - "structure": {"old": "new"}, - "special_files": {"file.md": "special/"}, + "structure": "sphinx", "media_directories": [".media"], } repo = RepoConfig.from_dict(data) assert repo.commit == "abc123" assert repo.root_files == ["README.md", "LICENSE"] - assert repo.structure == {"old": "new"} - assert repo.special_files == {"file.md": "special/"} + assert repo.structure == "sphinx" assert repo.media_directories == [".media"] - + + def test_from_dict_defaults_docs_path(self): + """Test that from_dict defaults docs_path to 'docs' when key is absent.""" + data = { + "name": "test-repo", + "url": "https://github.com/test/repo", + "ref": "main", + } + repo = RepoConfig.from_dict(data) + assert repo.docs_path == "docs" + + def test_from_dict_defaults_media_directories(self): + """Test that from_dict defaults media_directories to the standard list when absent.""" + data = { + "name": "test-repo", + "url": "https://github.com/test/repo", + "ref": "main", + } + repo = RepoConfig.from_dict(data) + assert repo.media_directories == list(_DEFAULT_MEDIA_DIRECTORIES) + + def test_from_dict_explicit_empty_media_directories_preserved(self): + """Test that an explicit empty media_directories [] is preserved, not replaced by default.""" + data = { + "name": "test-repo", + "url": "https://github.com/test/repo", + "ref": "main", + "media_directories": [], + } + repo = RepoConfig.from_dict(data) + assert repo.media_directories == [] + + def test_from_dict_rejects_dict_structure(self): + """Test that a dict-typed structure value raises ValueError.""" + data = { + "name": "test-repo", + "url": "https://github.com/test/repo", + "ref": "main", + "structure": {"tutorials": "tutorials"}, + } + with pytest.raises(ValueError, match="'structure' must be a string"): + RepoConfig.from_dict(data) + def test_is_local_file_url(self): """Test is_local property with file:// URL.""" repo = RepoConfig( name="local", url="file://../gardenlinux", docs_path="docs", - target_path="projects/gardenlinux", ref="", ) assert repo.is_local is True assert repo.is_remote is False - + def test_is_remote_https_url(self): """Test is_remote property with https:// URL.""" repo = RepoConfig( name="remote", url="https://github.com/test/repo", docs_path="docs", - target_path="projects/test", ref="main", ) assert repo.is_remote is True assert repo.is_local is False - + def test_local_path_property(self): """Test local_path property strips file:// prefix.""" repo = RepoConfig( name="local", url="file://../gardenlinux", docs_path="docs", - target_path="projects/gardenlinux", ref="", ) assert repo.local_path == "../gardenlinux" - + def test_validate_local_without_ref(self): """Test that local repos don't require ref.""" repo = RepoConfig( name="local", url="file://../gardenlinux", docs_path="docs", - target_path="projects/gardenlinux", ref="", ) repo.validate() # Should not raise - + def test_validate_remote_requires_ref(self): """Test that remote repos must have ref.""" repo = RepoConfig( name="remote", url="https://github.com/test/repo", docs_path="docs", - target_path="projects/test", ref="", ) with pytest.raises(ValueError, match="must have 'ref' field"): repo.validate() - + def test_validate_invalid_url_scheme(self): """Test that invalid URL schemes are rejected.""" repo = RepoConfig( name="invalid", url="ftp://example.com/repo", docs_path="docs", - target_path="projects/test", ref="", ) with pytest.raises(ValueError, match="Invalid URL scheme"): repo.validate() - def test_from_dict_defaults_ref_to_main(self): """Test that from_dict defaults ref to 'main' when key is absent.""" data = { "name": "local-repo", "url": "file://../some-repo", "docs_path": "docs", - "target_path": "projects/some-repo", } repo = RepoConfig.from_dict(data) assert repo.ref == "main" @@ -135,7 +178,6 @@ def test_from_dict_explicit_empty_ref_falls_back_to_main(self): "name": "local-repo", "url": "file://../some-repo", "docs_path": "docs", - "target_path": "projects/some-repo", "ref": "", } repo = RepoConfig.from_dict(data) @@ -147,7 +189,6 @@ def test_validate_remote_with_ref_passes(self): name="remote", url="https://github.com/test/repo", docs_path="docs", - target_path="projects/test", ref="main", ) repo.validate() # Should not raise @@ -158,7 +199,6 @@ def test_local_path_with_absolute_url(self): name="local", url="file:///abs/path/to/repo", docs_path="docs", - target_path="projects/repo", ref="", ) assert repo.local_path == "/abs/path/to/repo" @@ -166,17 +206,17 @@ def test_local_path_with_absolute_url(self): class TestAggregateResult: """Tests for AggregateResult dataclass.""" - + def test_success_result(self): """Test creating success result.""" result = AggregateResult("test-repo", True, "abc123") assert result.repo_name == "test-repo" assert result.success is True assert result.resolved_commit == "abc123" - + def test_failure_result(self): """Test creating failure result.""" result = AggregateResult("test-repo", False, None) assert result.repo_name == "test-repo" assert result.success is False - assert result.resolved_commit is None \ No newline at end of file + assert result.resolved_commit is None diff --git a/tests/unit/test_structure.py b/tests/unit/test_structure.py index 596cbda..7c37a71 100644 --- a/tests/unit/test_structure.py +++ b/tests/unit/test_structure.py @@ -3,112 +3,210 @@ from pathlib import Path import pytest -from aggregation.structure import transform_directory_structure +from aggregation.structure import copy_targeted_docs, verify_internal_links -class TestTransformDirectoryStructure: - """Tests for transform_directory_structure function.""" +class TestCopyTargetedDocs: + """Tests for copy_targeted_docs function.""" - def test_dict_map_renames_directories(self, tmp_path): - """Directories are renamed according to the dict structure map.""" + def test_copies_file_with_github_target_path(self, tmp_path): + """Files with github_target_path are copied to the specified location.""" source = tmp_path / "source" - (source / "01_intro").mkdir(parents=True) - (source / "01_intro" / "index.md").write_text("# Intro") - (source / "02_guide").mkdir() - (source / "02_guide" / "guide.md").write_text("# Guide") - - target = tmp_path / "target" - - transform_directory_structure( - str(source), - str(target), - structure_map={"01_intro": "intro", "02_guide": "guide"}, + source.mkdir() + docs = tmp_path / "docs" + docs.mkdir() + + content = ( + "---\n" + "title: Example\n" + "github_target_path: docs/how-to/example.md\n" + "---\n\n# Example\n" ) + (source / "example.md").write_text(content) + + copy_targeted_docs(str(source), str(docs), "myrepo") - assert (target / "intro" / "index.md").exists() - assert (target / "guide" / "guide.md").exists() - # Original names should NOT appear in the target - assert not (target / "01_intro").exists() - assert not (target / "02_guide").exists() + assert (docs / "how-to" / "example.md").exists() - def test_flat_structure_copies_files(self, tmp_path): - """With structure_map='flat', all files are copied as-is.""" + def test_skips_file_without_github_target_path(self, tmp_path): + """Files without github_target_path are not copied.""" source = tmp_path / "source" source.mkdir() - (source / "readme.md").write_text("# README") - (source / "guide.md").write_text("# Guide") + docs = tmp_path / "docs" + docs.mkdir() + + (source / "untargeted.md").write_text("# No target\n") + + copy_targeted_docs(str(source), str(docs), "myrepo") + + # Nothing should be copied to docs/ + assert list(docs.rglob("*.md")) == [] + + def test_missing_source_dir_warns_gracefully(self, tmp_path, capsys): + """Missing source dir logs a warning and returns without error.""" + docs = tmp_path / "docs" + docs.mkdir() - target = tmp_path / "target" + copy_targeted_docs(str(tmp_path / "nonexistent"), str(docs), "myrepo") - transform_directory_structure(str(source), str(target), structure_map="flat") + captured = capsys.readouterr() + assert "Warning" in captured.out - assert (target / "readme.md").exists() - assert (target / "guide.md").exists() + def test_retargeted_media_colocated_with_target(self, tmp_path): + """Media next to a retargeted file lands at the target parent, not source parent. - def test_sphinx_structure_cleans_target(self, tmp_path): - """When structure_map='sphinx', a pre-existing target directory is wiped.""" + Concrete case: glrd overview/README.md → reference/supporting_tools/glrd.md + Media at overview/assets/ must end up at reference/supporting_tools/assets/. + """ source = tmp_path / "source" - source.mkdir() - (source / "new.md").write_text("# New") + (source / "overview" / "assets").mkdir(parents=True) + docs = tmp_path / "docs" + docs.mkdir() + + content = ( + "---\n" + "title: GLRD\n" + "github_target_path: docs/reference/supporting_tools/glrd.md\n" + "---\n\n![Overview](assets/x.png)\n" + ) + (source / "overview" / "README.md").write_text(content) + (source / "overview" / "assets" / "x.png").write_bytes(b"\x89PNG") + + copy_targeted_docs( + str(source), str(docs), "glrd", media_dirs=["assets"] + ) - target = tmp_path / "target" - target.mkdir() - stale = target / "stale.md" - stale.write_text("# Stale file from previous run") - assert stale.exists() + # Targeted file copied to correct location + assert (docs / "reference" / "supporting_tools" / "glrd.md").exists() + # Media colocated with retargeted file + assert (docs / "reference" / "supporting_tools" / "assets" / "x.png").exists() + # Media must NOT appear at the source-relative path + assert not (docs / "overview").exists() + + def test_identity_mapped_media_unchanged(self, tmp_path): + """Media next to an identity-mapped file stays at its source-relative location. + + Gardenlinux case: tutorials/cloud/first-boot-aws.md → tutorials/cloud/first-boot-aws.md + Media at tutorials/assets/img.png must still end up at tutorials/assets/img.png. + """ + source = tmp_path / "source" + (source / "tutorials" / "cloud").mkdir(parents=True) + (source / "tutorials" / "assets").mkdir(parents=True) + docs = tmp_path / "docs" + docs.mkdir() + + content = ( + "---\n" + "title: First Boot AWS\n" + "github_target_path: docs/tutorials/cloud/first-boot-aws.md\n" + "---\n\n![Img](../assets/img.png)\n" + ) + (source / "tutorials" / "cloud" / "first-boot-aws.md").write_text(content) + (source / "tutorials" / "assets" / "img.png").write_bytes(b"\x89PNG") - transform_directory_structure( - str(source), str(target), structure_map="sphinx" + copy_targeted_docs( + str(source), str(docs), "gardenlinux", media_dirs=["assets"] ) - assert not stale.exists(), "Stale file should have been removed by sphinx clean" - assert (target / "new.md").exists() + # Targeted file at its correct location + assert (docs / "tutorials" / "cloud" / "first-boot-aws.md").exists() + # Media at source-relative path (identity mapping: source parent "tutorials/cloud" + # maps to target parent "tutorials/cloud"; the assets dir parent is "tutorials", + # which has no file mapped from it, so fallback fires and preserves the path) + assert (docs / "tutorials" / "assets" / "img.png").exists() + - def test_special_files_placed_in_subdir(self, tmp_path): - """Files listed in special_files are copied to their configured subdirectory.""" +class TestVerifyInternalLinks: + """Tests for verify_internal_links function.""" + + def test_no_errors_when_no_shipped_files(self, tmp_path): + """Returns 0 when no files are shipped (no github_target_path).""" source = tmp_path / "source" source.mkdir() - (source / "CHANGELOG.md").write_text("# Changelog") - (source / "intro").mkdir() - (source / "intro" / "index.md").write_text("# Intro") + docs = tmp_path / "docs" + docs.mkdir() + + (source / "untargeted.md").write_text("[Link](other.md)\n") - target = tmp_path / "target" + errors = verify_internal_links(str(source), str(docs), "myrepo") + assert errors == 0 - transform_directory_structure( - str(source), - str(target), - structure_map={"intro": "intro"}, - special_files={"CHANGELOG.md": "reference/"}, + def test_no_errors_when_links_are_external(self, tmp_path): + """External links and anchors in shipped files do not trigger errors.""" + source = tmp_path / "source" + source.mkdir() + docs = tmp_path / "docs" + docs.mkdir() + + content = ( + "---\n" + "github_target_path: docs/reference/page.md\n" + "---\n\n" + "[External](https://example.com)\n" + "[Anchor](#section)\n" ) + (source / "page.md").write_text(content) - assert (target / "reference" / "CHANGELOG.md").exists() + errors = verify_internal_links(str(source), str(docs), "myrepo") + assert errors == 0 - def test_creates_target_if_missing(self, tmp_path): - """Target directory is created automatically if it does not exist.""" + def test_no_errors_when_linked_file_also_shipped(self, tmp_path): + """A shipped file linking to another shipped file produces no errors.""" source = tmp_path / "source" source.mkdir() - (source / "page.md").write_text("# Page") + docs = tmp_path / "docs" + docs.mkdir() - target = tmp_path / "nested" / "target" - assert not target.exists() - - transform_directory_structure(str(source), str(target), structure_map="flat") + content_a = ( + "---\n" + "github_target_path: docs/reference/a.md\n" + "---\n\n[B](b.md)\n" + ) + content_b = ( + "---\n" + "github_target_path: docs/reference/b.md\n" + "---\n\n# B\n" + ) + (source / "a.md").write_text(content_a) + (source / "b.md").write_text(content_b) - assert target.exists() - assert (target / "page.md").exists() + errors = verify_internal_links(str(source), str(docs), "myrepo") + assert errors == 0 - def test_underscore_prefixed_files_are_skipped_in_dict_mode(self, tmp_path): - """In dict-map mode, files/dirs starting with _ are skipped (not in structure_map).""" + def test_error_when_shipped_file_links_to_unshipped_file(self, tmp_path): + """A shipped file linking to an existing but unshipped file causes an error.""" source = tmp_path / "source" source.mkdir() - (source / "_private.md").write_text("# Private") - (source / "public.md").write_text("# Public") + docs = tmp_path / "docs" + docs.mkdir() + + content_a = ( + "---\n" + "github_target_path: docs/reference/a.md\n" + "---\n\n[Unshipped](unshipped.md)\n" + ) + # unshipped.md exists in source but has no github_target_path + content_u = "# Unshipped\n" + (source / "a.md").write_text(content_a) + (source / "unshipped.md").write_text(content_u) - target = tmp_path / "target" + errors = verify_internal_links(str(source), str(docs), "myrepo") + assert errors == 1 - # Use dict mode with an empty map: _private.md is not in map, not special, - # and starts with _, so the dict branch skips it. - transform_directory_structure(str(source), str(target), structure_map={}) + def test_no_error_for_missing_linked_file(self, tmp_path): + """Links to files that do not exist in the source tree are not flagged + (they may be VitePress-resolved or external paths).""" + source = tmp_path / "source" + source.mkdir() + docs = tmp_path / "docs" + docs.mkdir() + + content = ( + "---\n" + "github_target_path: docs/reference/page.md\n" + "---\n\n[Gone](does-not-exist.md)\n" + ) + (source / "page.md").write_text(content) - assert not (target / "_private.md").exists() - assert (target / "public.md").exists() + errors = verify_internal_links(str(source), str(docs), "myrepo") + assert errors == 0 diff --git a/tests/unit/test_transformer.py b/tests/unit/test_transformer.py index 028f7a4..7b0e653 100644 --- a/tests/unit/test_transformer.py +++ b/tests/unit/test_transformer.py @@ -8,85 +8,95 @@ quote_yaml_value, rewrite_links, ) -from aggregation.transformer import fix_broken_project_links class TestRewriteLinks: """Tests for rewrite_links function.""" - - def test_relative_links(self): - """Test that relative links are rewritten correctly.""" + + def test_dotslash_links_left_unchanged(self): + """./relative links are left unchanged for VitePress to resolve natively.""" content = "[Link](./other.md)" result = rewrite_links(content, "gardenlinux", "introduction/index.md") - assert "/projects/gardenlinux/introduction/other" in result - - def test_numbered_directory_links(self): - """Test that numbered directories in links are transformed.""" - content = "[Link](../01_developers/guide.md)" - result = rewrite_links(content, "gardenlinux", "introduction/index.md") - assert "developers/guide" in result - + assert result == content + + def test_dotdot_inside_docs_left_unchanged(self): + """../ links that stay inside the docs tree are left unchanged.""" + content = "[Link](../guide.md)" + result = rewrite_links(content, "gardenlinux", "subdir/index.md") + assert result == content + def test_preserve_external_links(self): """Test that external links are not modified.""" content = "[External](https://github.com/gardenlinux/gardenlinux)" result = rewrite_links(content, "gardenlinux", "") assert result == content - + def test_preserve_anchor_links(self): """Test that anchor links are preserved.""" content = "[Anchor](#section)" result = rewrite_links(content, "gardenlinux", "") assert result == content - - def test_media_links(self): - """Test that .media/ links are rewritten correctly.""" - content = "[Image](../.media/image.png)" - result = rewrite_links(content, "gardenlinux", "introduction/index.md") - assert "/projects/gardenlinux/.media/image.png" in result - - def test_relative_link_with_anchor(self): - """Test that relative links with anchors preserve the anchor.""" - content = "[Guide](./guide.md#section-one)" - result = rewrite_links(content, "gardenlinux", "docs/index.md") - assert "section-one" in result - assert "/projects/gardenlinux" in result def test_dotdot_outside_docs_falls_back_to_github(self): """Test that ../ links going outside docs/ fall back to GitHub.""" - # File is at the root (depth 1); two levels up would escape docs/ + # File is at depth 1 (subdir/index.md); two levels up escapes docs/ content = "[File](../../README.md)" result = rewrite_links(content, "gardenlinux", "subdir/index.md") assert "github.com/gardenlinux/gardenlinux/blob/main" in result + def test_absolute_path_redirects_to_github(self): + """Test that absolute paths redirect to GitHub.""" + content = "[File](/some/path/README.md)" + result = rewrite_links(content, "gardenlinux", "") + assert "github.com/gardenlinux/gardenlinux/blob/main/some/path/README.md" in result + + def test_bare_filename_left_unchanged(self): + """Bare filenames without path separators are left unchanged for VitePress.""" + content = "[Guide](guide.md)" + result = rewrite_links(content, "gardenlinux", "subdir/index.md") + assert result == content + + def test_media_relative_link_left_unchanged(self): + """.media/ relative links are left unchanged for VitePress to resolve.""" + content = "[Image](.media/image.png)" + result = rewrite_links(content, "gardenlinux", "explanation/index.md") + assert result == content + + def test_media_dotdot_link_left_unchanged_inside_docs(self): + """../.media/ that stays inside docs tree is left unchanged.""" + content = "[Image](../.media/image.png)" + result = rewrite_links(content, "gardenlinux", "explanation/subdir/index.md") + assert result == content + class TestQuoteYamlValue: """Tests for quote_yaml_value function.""" - + def test_quote_value_with_colon(self): """Test that YAML values with colons are quoted.""" value = "Getting Started: Creating Images" result = quote_yaml_value(value) assert '"' in result assert "Getting Started: Creating Images" in result - + def test_simple_value_not_quoted(self): """Test that simple YAML values are not quoted.""" value = "Simple Title" result = quote_yaml_value(value) assert result == "Simple Title" - + def test_already_quoted_value(self): """Test that already-quoted values are not double-quoted.""" value = '"Already Quoted"' result = quote_yaml_value(value) assert result == '"Already Quoted"' - + def test_value_with_hash(self): """Test that values with # are quoted.""" value = "Title #1" result = quote_yaml_value(value) assert '"' in result - + def test_value_with_special_chars(self): """Test various special characters that require quoting.""" special_chars = [":", "#", "@", "*", "&", "!"] @@ -98,25 +108,25 @@ def test_value_with_special_chars(self): class TestEnsureFrontmatter: """Tests for ensure_frontmatter function.""" - + def test_no_change_when_missing(self): """Test that content without frontmatter is returned unchanged.""" content = "# Test Title\n\nContent here." result = ensure_frontmatter(content) assert result == content - + def test_preserve_existing(self): """Test that existing frontmatter is preserved.""" content = "---\ntitle: Existing\n---\n\nContent" result = ensure_frontmatter(content) assert "title: Existing" in result - + def test_fix_colons(self): """Test that colons in existing frontmatter are quoted.""" content = "---\ntitle: Test: Example\n---\n\nContent" result = ensure_frontmatter(content) assert '"Test: Example"' in result - + def test_fix_multiple_fields(self): """Test that multiple frontmatter fields are fixed.""" content = "---\ntitle: Test: Example\nauthor: John Doe\ntags: tag1, tag2\n---\n\nContent" @@ -158,32 +168,3 @@ def test_malformed_frontmatter_returns_none_gracefully(self): fm, body = parse_frontmatter(content) assert fm is None assert body == content - - -class TestFixBrokenProjectLinks: - """Tests for fix_broken_project_links function.""" - - def test_existing_file_link_is_kept(self, tmp_path): - """A /projects/repo/ link whose target exists is left unchanged.""" - (tmp_path / "guide.md").write_text("# Guide") - content = "[Guide](/projects/myrepo/guide)" - result = fix_broken_project_links(content, "myrepo", str(tmp_path)) - assert result == content - - def test_missing_file_link_redirects_to_github(self, tmp_path): - """A /projects/repo/ link whose target is missing becomes a GitHub link.""" - content = "[Missing](/projects/myrepo/does-not-exist)" - result = fix_broken_project_links(content, "myrepo", str(tmp_path)) - assert result == "[Missing](https://github.com/gardenlinux/myrepo/blob/main/does-not-exist)" - - def test_external_link_is_not_touched(self, tmp_path): - """External links are left unchanged regardless of target existence.""" - content = "[External](https://example.com/page)" - result = fix_broken_project_links(content, "myrepo", str(tmp_path)) - assert result == content - - def test_other_project_link_is_not_touched(self, tmp_path): - """Links to other repos under /projects/ are not modified.""" - content = "[Other](/projects/other-repo/some-page)" - result = fix_broken_project_links(content, "myrepo", str(tmp_path)) - assert result == content \ No newline at end of file