From aa4ad3e4cc95cc9be8fb41bcc0b848a164ab0c49 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 7 Jul 2026 09:36:48 +0200 Subject: [PATCH 1/8] feat: fix maven repo gap Signed-off-by: Umberto Sgueglia --- services/apps/packages_worker/package.json | 2 + .../src/bin/maven-repo-url-backfill.ts | 59 ++++++++++++ .../src/maven/__tests__/normalize.test.ts | 31 +++++++ .../src/maven/backfillRepositoryUrl.ts | 76 ++++++++++++++++ .../apps/packages_worker/src/maven/extract.ts | 90 +++++++++++++++---- .../src/osspckgs/packages.ts | 77 ++++++++++++++++ 6 files changed, 316 insertions(+), 19 deletions(-) create mode 100644 services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts create mode 100644 services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index d500ab3301..81000d31bd 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -51,6 +51,8 @@ "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts", "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts", + "backfill:maven-repo-url": "SERVICE=maven tsx src/bin/maven-repo-url-backfill.ts", + "backfill:maven-repo-url:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-repo-url-backfill.ts", "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts", "import:maven-maintainers:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts", "import:sonatype-popularity": "SERVICE=maven tsx src/maven/scripts/importSonatypePopularityFromCsv.ts", diff --git a/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts b/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts new file mode 100644 index 0000000000..e2bfa1663a --- /dev/null +++ b/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts @@ -0,0 +1,59 @@ +import { getServiceLogger } from '@crowd/logging' + +import { getPackagesDb } from '../db' +import { backfillMavenRepositoryUrls } from '../maven/backfillRepositoryUrl' + +const log = getServiceLogger() + +let shuttingDown = false + +// Graceful stop: finish the in-flight batch, then exit. Safe to interrupt — every +// write is an idempotent UPDATE recomputed from declared_repository_url, so +// re-running simply reprocesses and skips rows that already match. +const shutdown = () => { + if (shuttingDown) return + shuttingDown = true + log.info('Shutting down maven repo-url backfill (stopping after the current batch)...') +} + +process.on('SIGINT', shutdown) +process.on('SIGTERM', shutdown) + +const DEFAULT_BATCH_SIZE = 5000 + +const main = async () => { + const dryRun = process.argv.includes('--dry-run') + const rawBatchSize = process.env.MAVEN_REPO_URL_BACKFILL_BATCH_SIZE + const parsedBatchSize = rawBatchSize === undefined ? DEFAULT_BATCH_SIZE : Number(rawBatchSize) + if (!Number.isInteger(parsedBatchSize) || parsedBatchSize <= 0) { + log.error( + { MAVEN_REPO_URL_BACKFILL_BATCH_SIZE: rawBatchSize }, + 'MAVEN_REPO_URL_BACKFILL_BATCH_SIZE must be a positive integer', + ) + process.exit(1) + } + const batchSize = parsedBatchSize + + log.info( + { dryRun, batchSize }, + 'maven repo-url backfill starting (recompute normalizeScmUrl from declared_repository_url, no POM fetch)...', + ) + + const qx = await getPackagesDb() + await qx.selectOne('SELECT 1') + log.info('Connected to packages-db.') + + const totals = await backfillMavenRepositoryUrls(qx, { + batchSize, + dryRun, + isShuttingDown: () => shuttingDown, + }) + + log.info({ ...totals, dryRun }, 'maven repo-url backfill complete') + process.exit(0) +} + +main().catch((err) => { + log.error({ err }, 'maven repo-url backfill fatal error') + process.exit(1) +}) diff --git a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts index f05a9beeb0..46fc921840 100644 --- a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts @@ -163,4 +163,35 @@ describe('normalizeScmUrl', () => { it('returns null for non-https result', () => { expect(normalizeScmUrl('svn://svn.apache.org/repos/commons-lang')).toBeNull() }) + + // Gap B — recover repository_url from inputs that were previously dropped + it('recovers scm:git: without a scheme', () => { + expect(normalizeScmUrl('scm:git:github.com/lum-ai/nxmlreader')).toBe( + 'https://github.com/lum-ai/nxmlreader', + ) + }) + + it('recovers a bare host/owner/repo without a scheme', () => { + expect(normalizeScmUrl('github.com/agiledigital/kamon-play-extensions')).toBe( + 'https://github.com/agiledigital/kamon-play-extensions', + ) + }) + + it('upgrades http to https and lower-cases the github path', () => { + expect(normalizeScmUrl('http://github.com/kevemueller/kTLSH/tree/master')).toBe( + 'https://github.com/kevemueller/ktlsh', + ) + }) + + // Gap C — reject non-repository URLs so they are never stored + it('returns null for website-only URLs', () => { + expect(normalizeScmUrl('https://meson.ai/')).toBeNull() + expect(normalizeScmUrl('http://source.android.com')).toBeNull() + }) + + it('returns null for placeholders and free-form text', () => { + expect(normalizeScmUrl('Private')).toBeNull() + expect(normalizeScmUrl('${scm-url}')).toBeNull() + expect(normalizeScmUrl('http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/foo')).toBeNull() + }) }) diff --git a/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts b/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts new file mode 100644 index 0000000000..cd96805675 --- /dev/null +++ b/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts @@ -0,0 +1,76 @@ +import { + listMavenPackagesForRepoUrlRecompute, + updateMavenRepositoryUrls, +} from '@crowd/data-access-layer' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceChildLogger } from '@crowd/logging' + +import { normalizeScmUrl } from './extract' + +const log = getServiceChildLogger('maven-repo-url-backfill') + +export type RepoUrlBackfillTotals = { + scanned: number + filled: number // Gap B: NULL → canonical value + cleared: number // Gap C: non-repo value → NULL + rewritten: number // non-canonical value → different canonical value + unchanged: number +} + +/** + * Recomputes `repository_url` for every Maven row directly from the stored + * `declared_repository_url`, applying the current `normalizeScmUrl`. No POMs are + * fetched — the raw SCM value is already in the DB. Fills recoverable NULLs + * (Gap B) and clears non-repository values (Gap C) via direct UPDATE. + * + * Idempotent and resumable: the id cursor is derived from the scan, so a + * re-run after an interrupt simply reprocesses from the start and skips rows + * that already match. + */ +export async function backfillMavenRepositoryUrls( + qx: QueryExecutor, + options: { batchSize: number; dryRun: boolean; isShuttingDown: () => boolean }, +): Promise { + const { batchSize, dryRun, isShuttingDown } = options + const totals: RepoUrlBackfillTotals = { + scanned: 0, + filled: 0, + cleared: 0, + rewritten: 0, + unchanged: 0, + } + + let afterId = 0 + for (;;) { + if (isShuttingDown()) { + log.info('Shutdown requested — stopping after the current batch.') + break + } + + const rows = await listMavenPackagesForRepoUrlRecompute(qx, { afterId, limit: batchSize }) + if (rows.length === 0) break + + const updates: { id: number; repositoryUrl: string | null }[] = [] + for (const row of rows) { + totals.scanned++ + const desired = normalizeScmUrl(row.declaredRepositoryUrl) + if (desired === row.repositoryUrl) { + totals.unchanged++ + continue + } + if (row.repositoryUrl === null) totals.filled++ + else if (desired === null) totals.cleared++ + else totals.rewritten++ + updates.push({ id: row.id, repositoryUrl: desired }) + } + + if (updates.length > 0 && !dryRun) { + await updateMavenRepositoryUrls(qx, updates) + } + + afterId = rows[rows.length - 1].id + log.info({ afterId, changes: updates.length, dryRun, ...totals }, 'Backfill progress') + } + + return totals +} diff --git a/services/apps/packages_worker/src/maven/extract.ts b/services/apps/packages_worker/src/maven/extract.ts index 2dad1d289f..f65f94942e 100644 --- a/services/apps/packages_worker/src/maven/extract.ts +++ b/services/apps/packages_worker/src/maven/extract.ts @@ -453,37 +453,89 @@ export async function extractArtifact(groupId: string, artifactId: string, versi // ─── SCM URL normalisation ─────────────────────────────────────────────────── /** - * Converts the raw SCM URL from a POM (declared_repository_url) into a clean - * HTTPS repository URL suitable for storage as repository_url. + * Known source-code-hosting hosts. A normalised repository_url is only produced + * when the URL resolves to one of these — anything else (homepages, doc sites, + * placeholders) yields null so it is never stored as a repository link. + * + * TODO(CM): host list pending product confirmation before rollout. + */ +const SCM_HOSTS = new Set([ + 'github.com', + 'gitlab.com', + 'bitbucket.org', + 'gitee.com', + 'codeberg.org', +]) + +/** Hosts whose owner/repo path is case-insensitive and should be lower-cased. */ +const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) + +/** + * Converts the raw SCM URL from a POM (declared_repository_url) into a clean, + * canonical `https:////` repository URL suitable for storage + * as repository_url. Returns null when the input does not resolve to a real + * repository on a known SCM host. * * Handles common Maven SCM URL forms: - * scm:git:git@github.com:owner/repo.git → https://github.com/owner/repo - * scm:git:https://github.com/owner/repo → https://github.com/owner/repo - * git://github.com/owner/repo.git → https://github.com/owner/repo - * https://github.com/owner/repo/tree/... → https://github.com/owner/repo + * scm:git:git@github.com:owner/repo.git → https://github.com/owner/repo + * scm:git:https://github.com/owner/repo → https://github.com/owner/repo + * scm:git:github.com/owner/repo → https://github.com/owner/repo + * github.com/owner/repo (no scheme) → https://github.com/owner/repo + * git://github.com/owner/repo.git → https://github.com/owner/repo + * http://github.com/owner/repo/tree/... → https://github.com/owner/repo + * + * Rejected (→ null): website-only URLs (https://meson.ai/), non-SCM hosts + * (svn://…, http://source.android.com), placeholders (Private, ${scm-url}). */ export function normalizeScmUrl(raw: string | null): string | null { if (!raw) return null - let url = raw.trim() + let s = raw.trim() + if (!s) return null - // Strip scm:git: or scm: prefix - url = url.replace(/^scm:git:/i, '').replace(/^scm:/i, '') + // Strip Maven scm:git: / scm: prefix + s = s.replace(/^scm:git:/i, '').replace(/^scm:/i, '') - // Convert SSH git@host:owner/repo → https://host/owner/repo - url = url.replace(/^git@([^:]+):(.+)$/, 'https://$1/$2') + // git+https://… → https://… + s = s.replace(/^git\+/, '') - // Convert git:// → https:// - url = url.replace(/^git:\/\//, 'https://') + // SCP form git@host:owner/repo → https://host/owner/repo + s = s.replace(/^git@([^:/]+):(.+)$/, 'https://$1/$2') - // Strip trailing .git - url = url.replace(/\.git$/, '') + // ssh://git@host/… → https://host/… + s = s.replace(/^ssh:\/\/git@([^/]+)\//, 'https://$1/') - // Strip /tree/... or /blob/... path suffixes (keep only host + owner + repo) - url = url.replace(/\/(tree|blob)(\/.*)?$/, '') + // git:// → https://, and upgrade http:// → https:// + s = s.replace(/^git:\/\//, 'https://').replace(/^http:\/\//, 'https://') - if (!url.startsWith('https://')) return null + // No scheme at all (e.g. "github.com/owner/repo") → assume https + if (!s.includes('://')) s = `https://${s}` + + let parsed: URL + try { + parsed = new URL(s) + } catch { + return null + } + + if (parsed.protocol !== 'https:') return null + + const host = parsed.hostname.toLowerCase().replace(/^www\./, '') + if (!SCM_HOSTS.has(host)) return null + + // Require at least owner + repo path segments + const segments = parsed.pathname.split('/').filter(Boolean) + if (segments.length < 2) return null + + let owner = segments[0] + let name = segments[1].replace(/\.git$/, '') + if (!owner || !name) return null + + if (CASE_INSENSITIVE_HOSTS.has(host)) { + owner = owner.toLowerCase() + name = name.toLowerCase() + } - return url.replace(/\/$/, '') + return `https://${host}/${owner}/${name}` } // ─── Private helpers ────────────────────────────────────────────────────────── diff --git a/services/libs/data-access-layer/src/osspckgs/packages.ts b/services/libs/data-access-layer/src/osspckgs/packages.ts index b9fe0a5ba7..83ca9152e9 100644 --- a/services/libs/data-access-layer/src/osspckgs/packages.ts +++ b/services/libs/data-access-layer/src/osspckgs/packages.ts @@ -118,6 +118,83 @@ export async function listMavenPackagesToSync( ) } +// ─── repository_url backfill ────────────────────────────────────────────────── + +export type MavenRepoUrlRow = { + id: number + declaredRepositoryUrl: string | null + repositoryUrl: string | null +} + +/** + * Keyset-paginated scan of Maven rows that carry a repository link (declared or + * canonical). Rows with neither are skipped — there is nothing to recompute. + * Used by the repository_url backfill to re-run the normalizer over stored data + * without re-fetching POMs from the registry. + */ +export async function listMavenPackagesForRepoUrlRecompute( + qx: QueryExecutor, + options: { afterId: number; limit: number }, +): Promise { + return qx.select( + ` + SELECT + id, + declared_repository_url AS "declaredRepositoryUrl", + repository_url AS "repositoryUrl" + FROM packages + WHERE ecosystem = 'maven' + AND id > $(afterId) + AND (declared_repository_url IS NOT NULL OR repository_url IS NOT NULL) + ORDER BY id ASC + LIMIT $(limit) + `, + { afterId: options.afterId, limit: options.limit }, + ) +} + +/** + * Applies a batch of recomputed repository_url values via direct UPDATE — the + * only way to clear a stale value, since the enrichment upsert COALESCEs and + * cannot write NULL. Splits clears (→ NULL) from sets to avoid NULLs inside a + * text[] array literal. + */ +export async function updateMavenRepositoryUrls( + qx: QueryExecutor, + updates: { id: number; repositoryUrl: string | null }[], +): Promise { + if (updates.length === 0) return + + const toClear = updates.filter((u) => u.repositoryUrl === null).map((u) => u.id) + const toSet = updates.filter( + (u): u is { id: number; repositoryUrl: string } => u.repositoryUrl !== null, + ) + + if (toClear.length > 0) { + await qx.result( + `UPDATE packages SET repository_url = NULL + WHERE id = ANY($(ids)::bigint[]) AND repository_url IS NOT NULL`, + { ids: toClear }, + ) + } + + if (toSet.length > 0) { + await qx.result( + ` + UPDATE packages p + SET repository_url = v.repository_url + FROM ( + SELECT unnest($(ids)::bigint[]) AS id, + unnest($(urls)::text[]) AS repository_url + ) v + WHERE p.id = v.id + AND p.repository_url IS DISTINCT FROM v.repository_url + `, + { ids: toSet.map((u) => u.id), urls: toSet.map((u) => u.repositoryUrl) }, + ) + } +} + // ─── packages touch ─────────────────────────────────────────────────────────── /** From 293a87b35bd580d960cf54c638507298fd9886e5 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 7 Jul 2026 10:25:00 +0200 Subject: [PATCH 2/8] fix: remove zombies Signed-off-by: Umberto Sgueglia --- .../src/maven/backfillRepositoryUrl.ts | 38 ++++++++++++++++++- .../src/maven/runMavenEnrichmentLoop.ts | 4 +- .../data-access-layer/src/osspckgs/repos.ts | 20 ++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts b/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts index cd96805675..45a28252cd 100644 --- a/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts +++ b/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts @@ -1,4 +1,5 @@ import { + deleteMavenPackageRepoLinks, listMavenPackagesForRepoUrlRecompute, updateMavenRepositoryUrls, } from '@crowd/data-access-layer' @@ -6,6 +7,7 @@ import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceChildLogger } from '@crowd/logging' import { normalizeScmUrl } from './extract' +import { writeRepoLink } from './runMavenEnrichmentLoop' const log = getServiceChildLogger('maven-repo-url-backfill') @@ -15,6 +17,8 @@ export type RepoUrlBackfillTotals = { cleared: number // Gap C: non-repo value → NULL rewritten: number // non-canonical value → different canonical value unchanged: number + linked: number // repos/package_repos link (re)written for a fill or rewrite + pruned: number // stale 'declared' link removed for a clear or rewrite } /** @@ -23,6 +27,15 @@ export type RepoUrlBackfillTotals = { * fetched — the raw SCM value is already in the DB. Fills recoverable NULLs * (Gap B) and clears non-repository values (Gap C) via direct UPDATE. * + * Link tables: package_repos is kept consistent with the recomputed + * repository_url. For rows that had a value and now change (rewrites) or are + * cleared, the stale source='declared' link is deleted; for rows that gain a + * canonical URL (fills and rewrites) the correct link is (re)written via the + * same `writeRepoLink` the enrichment loop uses. This matters because consumers + * such as security-contacts read the repo through repos ⋈ package_repos, not + * packages.repository_url. Note this is stricter than the incremental + * enrichment path, which is upsert-only and never prunes. + * * Idempotent and resumable: the id cursor is derived from the scan, so a * re-run after an interrupt simply reprocesses from the start and skips rows * that already match. @@ -38,6 +51,8 @@ export async function backfillMavenRepositoryUrls( cleared: 0, rewritten: 0, unchanged: 0, + linked: 0, + pruned: 0, } let afterId = 0 @@ -51,6 +66,10 @@ export async function backfillMavenRepositoryUrls( if (rows.length === 0) break const updates: { id: number; repositoryUrl: string | null }[] = [] + // Rows that had a link and now change/clear — their stale 'declared' link is pruned. + const pruneTargets: number[] = [] + // Rows that gained a canonical URL — their repo link is (re)written after the update. + const linkTargets: { id: number; repositoryUrl: string }[] = [] for (const row of rows) { totals.scanned++ const desired = normalizeScmUrl(row.declaredRepositoryUrl) @@ -62,10 +81,27 @@ export async function backfillMavenRepositoryUrls( else if (desired === null) totals.cleared++ else totals.rewritten++ updates.push({ id: row.id, repositoryUrl: desired }) + // A 'declared' link only exists when the row already had a value. + if (row.repositoryUrl !== null) pruneTargets.push(row.id) + if (desired !== null) linkTargets.push({ id: row.id, repositoryUrl: desired }) } if (updates.length > 0 && !dryRun) { - await updateMavenRepositoryUrls(qx, updates) + // Atomic per batch: the repository_url UPDATE, the stale-link prune, and the + // relink must commit together. Otherwise an interrupt between them leaves + // packages.repository_url updated but package_repos out of sync — and on a + // re-run the row is skipped (its repository_url already matches `desired`), + // so the inconsistency would never be repaired. On rollback the row stays + // unchanged and is reprocessed on the next run. + await qx.tx(async (t) => { + await updateMavenRepositoryUrls(t, updates) + await deleteMavenPackageRepoLinks(t, pruneTargets) + for (const target of linkTargets) { + await writeRepoLink(t, target.id, target.repositoryUrl) + } + }) + totals.pruned += pruneTargets.length + totals.linked += linkTargets.length } afterId = rows[rows.length - 1].id diff --git a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts index 7efdbd97e1..d9707a3aea 100644 --- a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts @@ -53,7 +53,7 @@ type PackageRow = MavenPackageToSync // ─── Helpers ────────────────────────────────────────────────────────────────── // prettier-ignore -async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl: string | null, changed: Set): Promise { +export async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl: string | null, changed?: Set): Promise { if (!repositoryUrl) return const parsed = parseRepoUrl(repositoryUrl) if (!parsed) return @@ -64,7 +64,7 @@ async function writeRepoLink(qx: QueryExecutor, packageId: number, repositoryUrl source: 'declared', confidence: 0.8, }) - repoChanged.forEach((f) => changed.add(f)) + repoChanged.forEach((f) => changed?.add(f)) } // Postgres deadlock (40P01) is transient: concurrent transactions upserting the same shared diff --git a/services/libs/data-access-layer/src/osspckgs/repos.ts b/services/libs/data-access-layer/src/osspckgs/repos.ts index 6c67f0e569..18bbb4e103 100644 --- a/services/libs/data-access-layer/src/osspckgs/repos.ts +++ b/services/libs/data-access-layer/src/osspckgs/repos.ts @@ -36,6 +36,26 @@ export async function upsertRepo(qx: QueryExecutor, item: IDbRepoUpsert): Promis return row.id as number } +/** + * Removes the `declared` repo links for the given packages. Used by the + * repository_url backfill to drop stale links when a package's canonical URL + * changes or is cleared, keeping package_repos consistent with + * packages.repository_url. Only touches source='declared' (the link Maven + * owns) — links from other enrichers (deps.dev, heuristic, manual) are left + * untouched. The shared `repos` rows are never deleted. + */ +export async function deleteMavenPackageRepoLinks( + qx: QueryExecutor, + packageIds: number[], +): Promise { + if (packageIds.length === 0) return + await qx.result( + `DELETE FROM package_repos + WHERE package_id = ANY($(packageIds)::bigint[]) AND source = 'declared'`, + { packageIds }, + ) +} + /** * Links a package to a repo with provenance metadata. * On conflict keeps the higher confidence value and refreshes verified_at. From bfda0b2e088a89a05b1d8403d639624ae7311328 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 7 Jul 2026 10:51:03 +0200 Subject: [PATCH 3/8] fix: critical onlhy Signed-off-by: Umberto Sgueglia --- .../src/bin/maven-repo-url-backfill.ts | 4 +++- .../src/maven/backfillRepositoryUrl.ts | 20 +++++++++++++++---- .../src/osspckgs/packages.ts | 6 +++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts b/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts index e2bfa1663a..d082aee5a3 100644 --- a/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts +++ b/services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts @@ -23,6 +23,7 @@ const DEFAULT_BATCH_SIZE = 5000 const main = async () => { const dryRun = process.argv.includes('--dry-run') + const criticalOnly = process.argv.includes('--critical-only') const rawBatchSize = process.env.MAVEN_REPO_URL_BACKFILL_BATCH_SIZE const parsedBatchSize = rawBatchSize === undefined ? DEFAULT_BATCH_SIZE : Number(rawBatchSize) if (!Number.isInteger(parsedBatchSize) || parsedBatchSize <= 0) { @@ -35,7 +36,7 @@ const main = async () => { const batchSize = parsedBatchSize log.info( - { dryRun, batchSize }, + { dryRun, criticalOnly, batchSize }, 'maven repo-url backfill starting (recompute normalizeScmUrl from declared_repository_url, no POM fetch)...', ) @@ -46,6 +47,7 @@ const main = async () => { const totals = await backfillMavenRepositoryUrls(qx, { batchSize, dryRun, + criticalOnly, isShuttingDown: () => shuttingDown, }) diff --git a/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts b/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts index 45a28252cd..f91c847559 100644 --- a/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts +++ b/services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts @@ -42,9 +42,14 @@ export type RepoUrlBackfillTotals = { */ export async function backfillMavenRepositoryUrls( qx: QueryExecutor, - options: { batchSize: number; dryRun: boolean; isShuttingDown: () => boolean }, + options: { + batchSize: number + dryRun: boolean + criticalOnly: boolean + isShuttingDown: () => boolean + }, ): Promise { - const { batchSize, dryRun, isShuttingDown } = options + const { batchSize, dryRun, criticalOnly, isShuttingDown } = options const totals: RepoUrlBackfillTotals = { scanned: 0, filled: 0, @@ -62,7 +67,11 @@ export async function backfillMavenRepositoryUrls( break } - const rows = await listMavenPackagesForRepoUrlRecompute(qx, { afterId, limit: batchSize }) + const rows = await listMavenPackagesForRepoUrlRecompute(qx, { + afterId, + limit: batchSize, + criticalOnly, + }) if (rows.length === 0) break const updates: { id: number; repositoryUrl: string | null }[] = [] @@ -105,7 +114,10 @@ export async function backfillMavenRepositoryUrls( } afterId = rows[rows.length - 1].id - log.info({ afterId, changes: updates.length, dryRun, ...totals }, 'Backfill progress') + log.info( + { afterId, changes: updates.length, dryRun, criticalOnly, ...totals }, + 'Backfill progress', + ) } return totals diff --git a/services/libs/data-access-layer/src/osspckgs/packages.ts b/services/libs/data-access-layer/src/osspckgs/packages.ts index 83ca9152e9..6a6bf42014 100644 --- a/services/libs/data-access-layer/src/osspckgs/packages.ts +++ b/services/libs/data-access-layer/src/osspckgs/packages.ts @@ -131,10 +131,13 @@ export type MavenRepoUrlRow = { * canonical). Rows with neither are skipped — there is nothing to recompute. * Used by the repository_url backfill to re-run the normalizer over stored data * without re-fetching POMs from the registry. + * + * `criticalOnly` restricts the scan to is_critical rows (index-backed by the + * partial index on is_critical) — used for a fast, consumer-facing first pass. */ export async function listMavenPackagesForRepoUrlRecompute( qx: QueryExecutor, - options: { afterId: number; limit: number }, + options: { afterId: number; limit: number; criticalOnly?: boolean }, ): Promise { return qx.select( ` @@ -144,6 +147,7 @@ export async function listMavenPackagesForRepoUrlRecompute( repository_url AS "repositoryUrl" FROM packages WHERE ecosystem = 'maven' + ${options.criticalOnly ? 'AND is_critical' : ''} AND id > $(afterId) AND (declared_repository_url IS NOT NULL OR repository_url IS NOT NULL) ORDER BY id ASC From 523fa7e0b4fc1ee439c8469f8788be5d36a3b108 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 7 Jul 2026 14:56:27 +0200 Subject: [PATCH 4/8] fix: add allowlist Signed-off-by: Umberto Sgueglia --- .../src/maven/__tests__/normalize.test.ts | 40 +++++++++++++++++++ .../apps/packages_worker/src/maven/extract.ts | 35 +++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts index 46fc921840..d050e4988c 100644 --- a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts @@ -194,4 +194,44 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('${scm-url}')).toBeNull() expect(normalizeScmUrl('http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/foo')).toBeNull() }) + + // SCP colon form: "host:owner/repo" where the colon is a path separator, not a port + it('recovers bare host:owner/repo SCP colon form', () => { + expect(normalizeScmUrl('github.com:japgolly/scalacss.git')).toBe( + 'https://github.com/japgolly/scalacss', + ) + }) + + it('recovers scheme://host:owner/repo SCP colon form', () => { + expect(normalizeScmUrl('https://github.com:networknt/light-4j.git')).toBe( + 'https://github.com/networknt/light-4j', + ) + }) + + it('recovers ssh://git@host:owner/repo SCP colon form', () => { + expect(normalizeScmUrl('ssh://git@github.com:apache/iotdb.git')).toBe( + 'https://github.com/apache/iotdb', + ) + expect(normalizeScmUrl('ssh://git@bitbucket.org:eci-elements/web-services.git')).toBe( + 'https://bitbucket.org/eci-elements/web-services', + ) + }) + + it('does not treat a numeric port as an SCP separator', () => { + expect(normalizeScmUrl('https://gitlab.com:443/foo/bar')).toBe('https://gitlab.com/foo/bar') + }) + + it('accepts allowlisted self-hosted GitLab/Gitea hosts', () => { + expect(normalizeScmUrl('https://git.neckar.it/neckarit/neckar-hub')).toBe( + 'https://git.neckar.it/neckarit/neckar-hub', + ) + expect(normalizeScmUrl('scm:git:https://gitlab.inria.fr/owner/repo.git')).toBe( + 'https://gitlab.inria.fr/owner/repo', + ) + }) + + it('still returns null for hosts not in the allowlist', () => { + expect(normalizeScmUrl('https://git.corp.adobe.com/team/project')).toBeNull() + expect(normalizeScmUrl('https://android.googlesource.com/platform/tools/base')).toBeNull() + }) }) diff --git a/services/apps/packages_worker/src/maven/extract.ts b/services/apps/packages_worker/src/maven/extract.ts index f65f94942e..bf966935a5 100644 --- a/services/apps/packages_worker/src/maven/extract.ts +++ b/services/apps/packages_worker/src/maven/extract.ts @@ -458,6 +458,18 @@ export async function extractArtifact(groupId: string, artifactId: string, versi * placeholders) yields null so it is never stored as a repository link. * * TODO(CM): host list pending product confirmation before rollout. + * + * Candidate additions found in Maven declared_repository_url (critical rows) but + * intentionally NOT added yet, because they need more than a host allowlist: + * - gitbox.apache.org / git.apache.org / git-wip-us.apache.org (~1.3k rows): + * paths are `/repos/asf/`, so the generic first-two-segments logic would + * collapse every Apache repo to `repos/asf`. Needs path-aware handling (skip + * the `/repos/asf/` prefix) or mapping to the github.com/apache mirror. + * - git.eclipse.org (~180 rows): Gerrit paths, likewise not owner/repo. + * - android.googlesource.com, ec.europa.eu (Bitbucket-Server /projects/x/repos/y): + * multi-segment paths, not owner/repo. + * - ambiguous `git.*` hosts (git.iem.at, git.i-novus.ru, git.oschina.net) and the + * internal git.corp.adobe.com: pending path/reachability confirmation. */ const SCM_HOSTS = new Set([ 'github.com', @@ -465,6 +477,13 @@ const SCM_HOSTS = new Set([ 'bitbucket.org', 'gitee.com', 'codeberg.org', + // Self-hosted GitLab / Gitea instances seen in Maven POMs with clean + // // paths (same shape as gitlab.com — handled by the generic logic). + 'gitlab.smartb.city', + 'gitlab.ow2.org', + 'gitlab.nuiton.org', + 'gitlab.inria.fr', + 'git.neckar.it', ]) /** Hosts whose owner/repo path is case-insensitive and should be lower-cased. */ @@ -483,6 +502,9 @@ const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) * github.com/owner/repo (no scheme) → https://github.com/owner/repo * git://github.com/owner/repo.git → https://github.com/owner/repo * http://github.com/owner/repo/tree/... → https://github.com/owner/repo + * github.com:owner/repo.git (SCP colon) → https://github.com/owner/repo + * https://github.com:owner/repo → https://github.com/owner/repo + * ssh://git@github.com:owner/repo.git → https://github.com/owner/repo * * Rejected (→ null): website-only URLs (https://meson.ai/), non-SCM hosts * (svn://…, http://source.android.com), placeholders (Private, ${scm-url}). @@ -501,14 +523,25 @@ export function normalizeScmUrl(raw: string | null): string | null { // SCP form git@host:owner/repo → https://host/owner/repo s = s.replace(/^git@([^:/]+):(.+)$/, 'https://$1/$2') + // ssh://git@host:owner/repo → https://host/owner/repo (SCP colon under ssh) + s = s.replace(/^ssh:\/\/git@([^:/]+):(?=\D)/, 'https://$1/') + // ssh://git@host/… → https://host/… s = s.replace(/^ssh:\/\/git@([^/]+)\//, 'https://$1/') + // scheme://host:owner/repo → scheme://host/owner/repo — the colon is an SCP path + // separator, not a port (guarded by \D so real numeric ports are left intact). + s = s.replace(/^(https?):\/\/([^:/]+):(?=\D)/, '$1://$2/') + // git:// → https://, and upgrade http:// → https:// s = s.replace(/^git:\/\//, 'https://').replace(/^http:\/\//, 'https://') // No scheme at all (e.g. "github.com/owner/repo") → assume https - if (!s.includes('://')) s = `https://${s}` + if (!s.includes('://')) { + // Bare SCP form "host:owner/repo" → "host/owner/repo" before assuming https. + s = s.replace(/^([^/:]+):(?=\D)/, '$1/') + s = `https://${s}` + } let parsed: URL try { From 4535615c3661d64b251b966dfc387c18e9ab05a0 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 7 Jul 2026 15:25:20 +0200 Subject: [PATCH 5/8] fix: add allowlist Signed-off-by: Umberto Sgueglia --- .../src/maven/__tests__/normalize.test.ts | 10 +++++++++- .../apps/packages_worker/src/maven/extract.ts | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts index d050e4988c..9f5f68ea6e 100644 --- a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts @@ -221,6 +221,12 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('https://gitlab.com:443/foo/bar')).toBe('https://gitlab.com/foo/bar') }) + it('recovers git://host:owner/repo SCP colon form', () => { + expect(normalizeScmUrl('git://github.com:appendium/objectlabkit.git')).toBe( + 'https://github.com/appendium/objectlabkit', + ) + }) + it('accepts allowlisted self-hosted GitLab/Gitea hosts', () => { expect(normalizeScmUrl('https://git.neckar.it/neckarit/neckar-hub')).toBe( 'https://git.neckar.it/neckarit/neckar-hub', @@ -228,10 +234,12 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('scm:git:https://gitlab.inria.fr/owner/repo.git')).toBe( 'https://gitlab.inria.fr/owner/repo', ) + expect(normalizeScmUrl('https://git.iem.at/owner/repo')).toBe('https://git.iem.at/owner/repo') }) - it('still returns null for hosts not in the allowlist', () => { + it('still returns null for internal or non-allowlisted hosts', () => { expect(normalizeScmUrl('https://git.corp.adobe.com/team/project')).toBeNull() + expect(normalizeScmUrl('https://gitlab.alibaba-inc.com/team/project')).toBeNull() expect(normalizeScmUrl('https://android.googlesource.com/platform/tools/base')).toBeNull() }) }) diff --git a/services/apps/packages_worker/src/maven/extract.ts b/services/apps/packages_worker/src/maven/extract.ts index bf966935a5..c089749b5d 100644 --- a/services/apps/packages_worker/src/maven/extract.ts +++ b/services/apps/packages_worker/src/maven/extract.ts @@ -479,11 +479,21 @@ const SCM_HOSTS = new Set([ 'codeberg.org', // Self-hosted GitLab / Gitea instances seen in Maven POMs with clean // // paths (same shape as gitlab.com — handled by the generic logic). + // Internal-only hosts (git.corp.adobe.com, gitlab.alibaba-inc.com) are excluded: + // their links are unreachable for consumers. The ≥2-segment owner/repo requirement + // acts as a safety net so a mis-classified host yields NULL, never a junk link. 'gitlab.smartb.city', 'gitlab.ow2.org', 'gitlab.nuiton.org', 'gitlab.inria.fr', 'git.neckar.it', + 'git.iem.at', + 'git.oschina.net', + 'git.i-novus.ru', + 'gitlab.protontech.ch', + 'git.catchpoint.net', + 'git.dorkbox.com', + 'git.adorsys.de', ]) /** Hosts whose owner/repo path is case-insensitive and should be lower-cased. */ @@ -529,13 +539,14 @@ export function normalizeScmUrl(raw: string | null): string | null { // ssh://git@host/… → https://host/… s = s.replace(/^ssh:\/\/git@([^/]+)\//, 'https://$1/') + // git:// → https://, and upgrade http:// → https:// — done before the SCP-colon + // rule below so that git://host:owner/repo is normalised too. + s = s.replace(/^git:\/\//, 'https://').replace(/^http:\/\//, 'https://') + // scheme://host:owner/repo → scheme://host/owner/repo — the colon is an SCP path // separator, not a port (guarded by \D so real numeric ports are left intact). s = s.replace(/^(https?):\/\/([^:/]+):(?=\D)/, '$1://$2/') - // git:// → https://, and upgrade http:// → https:// - s = s.replace(/^git:\/\//, 'https://').replace(/^http:\/\//, 'https://') - // No scheme at all (e.g. "github.com/owner/repo") → assume https if (!s.includes('://')) { // Bare SCP form "host:owner/repo" → "host/owner/repo" before assuming https. From b342b7c3291f8cb67599235eb697378cad766db4 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 8 Jul 2026 13:32:38 +0200 Subject: [PATCH 6/8] fix: add force to backfill Signed-off-by: Umberto Sgueglia --- .../packages_worker/src/bin/maven-backfill.ts | 18 +++- .../src/maven/__tests__/normalize.test.ts | 61 ++++++++++++- .../apps/packages_worker/src/maven/extract.ts | 89 ++++++++++++++++++- .../src/maven/runMavenEnrichmentLoop.ts | 61 +++++++++++++ .../src/osspckgs/packages.ts | 35 ++++++++ 5 files changed, 257 insertions(+), 7 deletions(-) diff --git a/services/apps/packages_worker/src/bin/maven-backfill.ts b/services/apps/packages_worker/src/bin/maven-backfill.ts index 1b0c557097..87c4d28f97 100644 --- a/services/apps/packages_worker/src/bin/maven-backfill.ts +++ b/services/apps/packages_worker/src/bin/maven-backfill.ts @@ -2,7 +2,10 @@ import { getServiceLogger } from '@crowd/logging' import { getMavenConfig } from '../config' import { getPackagesDb } from '../db' -import { runMavenCriticalBackfill } from '../maven/runMavenEnrichmentLoop' +import { + runMavenCriticalBackfill, + runMavenCriticalForceBackfill, +} from '../maven/runMavenEnrichmentLoop' const log = getServiceLogger() @@ -25,7 +28,14 @@ const main = async () => { process.env.MAVEN_FETCHER_BASE_URL_BACKFILL ?? 'https://maven-central.storage-download.googleapis.com/maven2' - log.info('maven backfill starting (one-shot, full extraction)...') + // --force: re-run POM extraction over EVERY critical row, ignoring the + // staleness window. Use to fully re-apply extraction changes (e.g. SCM + // interpolation) after the queue has already drained. The default path only + // picks rows due by refreshDays and cannot be coaxed into a full pass by + // setting refreshDays=0 (that reprocesses the first batch forever). + const force = process.argv.includes('--force') + + log.info({ force }, 'maven backfill starting (one-shot, full extraction)...') const config = getMavenConfig() log.info(config, 'Config loaded') @@ -34,7 +44,9 @@ const main = async () => { await qx.selectOne('SELECT 1') log.info('Connected to packages-db.') - const totals = await runMavenCriticalBackfill(qx, config, () => shuttingDown) + const totals = force + ? await runMavenCriticalForceBackfill(qx, config, () => shuttingDown) + : await runMavenCriticalBackfill(qx, config, () => shuttingDown) log.info({ ...totals }, 'maven backfill complete') process.exit(0) diff --git a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts index 9f5f68ea6e..bc779e48cf 100644 --- a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { normalizeScmUrl } from '../extract' +import { interpolateProperties, normalizeScmUrl } from '../extract' import { pickStableRelease } from '../metadata' import { isPrerelease, parseRepoUrl } from '../normalize' @@ -195,6 +195,12 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/foo')).toBeNull() }) + it('returns null when an unresolved placeholder is embedded in the repo path', () => { + // Would otherwise parse to github.com/owner/%7BartifactId%7D and slip through. + expect(normalizeScmUrl('https://github.com/owner/${artifactId}')).toBeNull() + expect(normalizeScmUrl('scm:git:https://github.com/${owner}/repo.git')).toBeNull() + }) + // SCP colon form: "host:owner/repo" where the colon is a path separator, not a port it('recovers bare host:owner/repo SCP colon form', () => { expect(normalizeScmUrl('github.com:japgolly/scalacss.git')).toBe( @@ -243,3 +249,56 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('https://android.googlesource.com/platform/tools/base')).toBeNull() }) }) + +describe('interpolateProperties', () => { + it('resolves a single ${...} placeholder from properties', () => { + expect( + interpolateProperties('${scm.github.url}', { + 'scm.github.url': 'https://github.com/owner/repo', + }), + ).toBe('https://github.com/owner/repo') + }) + + it('resolves multiple placeholders in one string', () => { + expect( + interpolateProperties('https://gitlab.com/${projectPath}', { + projectPath: 'group/project', + }), + ).toBe('https://gitlab.com/group/project') + }) + + it('resolves nested placeholders recursively', () => { + expect( + interpolateProperties('${scm.url}', { + 'scm.url': '${scm.base}/repo', + 'scm.base': 'https://github.com/owner', + }), + ).toBe('https://github.com/owner/repo') + }) + + it('resolves built-in project.* style placeholders', () => { + expect( + interpolateProperties('https://github.com/acme/${project.artifactId}', { + 'project.artifactId': 'my-lib', + }), + ).toBe('https://github.com/acme/my-lib') + }) + + it('leaves unresolved placeholders literal (missing property / method call)', () => { + expect(interpolateProperties('${scm.github.url}', {})).toBe('${scm.github.url}') + expect( + interpolateProperties('${pom.artifactId.substring(8)}', { 'pom.artifactId': 'foo' }), + ).toBe('${pom.artifactId.substring(8)}') + }) + + it('does not loop forever on self-referential placeholders', () => { + expect(interpolateProperties('${a}', { a: '${b}', b: '${a}' })).toContain('${') + }) + + it('composes with the normalizer end-to-end', () => { + const resolved = interpolateProperties('${scm.github.url}', { + 'scm.github.url': 'scm:git:https://github.com/Owner/Repo.git', + }) + expect(normalizeScmUrl(resolved)).toBe('https://github.com/owner/repo') + }) +}) diff --git a/services/apps/packages_worker/src/maven/extract.ts b/services/apps/packages_worker/src/maven/extract.ts index c089749b5d..8394c8dd97 100644 --- a/services/apps/packages_worker/src/maven/extract.ts +++ b/services/apps/packages_worker/src/maven/extract.ts @@ -47,6 +47,7 @@ interface PomData { developers?: { developer?: unknown } contributors?: { contributor?: unknown } parent?: { groupId?: unknown; artifactId?: unknown; version?: unknown } + properties?: unknown } interface PomPerson { @@ -262,6 +263,9 @@ interface ResolvedFields { developers: PomMaintainer[] contributors: PomMaintainer[] hops: number + // Merged across the resolved parent chain (child overrides parent), + // used to interpolate ${...} placeholders in the SCM URL. + properties: Record } // prettier-ignore @@ -273,9 +277,12 @@ async function resolveWithInheritance(groupId: string, artifactId: string, versi const scmUrl = extractStr(pom.scm?.url ?? pom.scm?.connection) const developers = extractPersons(pom.developers?.developer, 'author') const contributors = extractPersons(pom.contributors?.contributor, 'maintainer') + const properties = extractProperties(pom) const missingLicense = licenses.length === 0 - const missingScm = !scmUrl + // An unresolved ${...} placeholder counts as missing: the property that defines it + // may live in a parent POM, so we still need to walk the chain to collect it. + const missingScm = !scmUrl || scmUrl.includes('${') const missingDevelopers = developers.length === 0 || contributors.length === 0 const parent = extractParent(pom) @@ -306,6 +313,8 @@ async function resolveWithInheritance(groupId: string, artifactId: string, versi developers: developers.length > 0 ? developers : parentFields.developers, contributors: contributors.length > 0 ? contributors : parentFields.contributors, hops: parentFields.hops, + // Child properties override the parent's. + properties: { ...parentFields.properties, ...properties }, } } } @@ -319,6 +328,7 @@ async function resolveWithInheritance(groupId: string, artifactId: string, versi developers, contributors, hops: depth, + properties, } } @@ -355,7 +365,12 @@ export async function extractArtifactDirect(groupId: string, artifactId: string, } const licenses = extractLicenses(pom) - const scmUrl = extractStr(pom.scm?.url ?? pom.scm?.connection) + const rawScmUrl = extractStr(pom.scm?.url ?? pom.scm?.connection) + const props = { + ...extractProperties(pom), + ...builtinProjectProperties(groupId, artifactId, version), + } + const scmUrl = rawScmUrl ? interpolateProperties(rawScmUrl, props) : null const developers = extractPersons(pom.developers?.developer, 'author') const contributors = extractPersons(pom.contributors?.contributor, 'maintainer') @@ -414,6 +429,14 @@ export async function extractArtifact(groupId: string, artifactId: string, versi new Set(), baseUrl, ) + // Resolve ${...} placeholders in the SCM URL using the merged chain properties + // plus the leaf's built-in project.* values. Best-effort: unresolved placeholders + // stay literal and are rejected downstream by normalizeScmUrl. + const props = { + ...resolved.properties, + ...builtinProjectProperties(groupId, artifactId, version), + } + const scmUrl = resolved.scmUrl ? interpolateProperties(resolved.scmUrl, props) : null return { groupId, artifactId, @@ -422,7 +445,7 @@ export async function extractArtifact(groupId: string, artifactId: string, versi description: resolved.description, licenses: resolved.licenses, licensesRaw: resolved.licensesRaw, - scmUrl: resolved.scmUrl, + scmUrl, homepageUrl: resolved.homepageUrl, developers: resolved.developers, contributors: resolved.contributors, @@ -524,6 +547,12 @@ export function normalizeScmUrl(raw: string | null): string | null { let s = raw.trim() if (!s) return null + // A leftover ${...} means best-effort interpolation upstream could not resolve it. + // Reject outright: a placeholder embedded in the path (e.g. github.com/owner/${x}) + // otherwise survives URL parsing (percent-encoded to %7B…%7D) and would yield a junk + // repository_url instead of null. + if (s.includes('${')) return null + // Strip Maven scm:git: / scm: prefix s = s.replace(/^scm:git:/i, '').replace(/^scm:/i, '') @@ -612,6 +641,59 @@ function extractPersons(raw: unknown, role: 'author' | 'maintainer'): PomMaintai })) } +/** Flattens a POM's block into a string→string map (non-string values skipped). */ +function extractProperties(pom: PomData): Record { + const raw = pom.properties + if (!raw || typeof raw !== 'object') return {} + const out: Record = {} + for (const [key, value] of Object.entries(raw as Record)) { + if (typeof value === 'string') out[key] = value + else if (typeof value === 'number') out[key] = String(value) + } + return out +} + +/** Maven built-in project.* / pom.* properties for the leaf coordinates. */ +function builtinProjectProperties( + groupId: string, + artifactId: string, + version: string, +): Record { + return { + 'project.groupId': groupId, + 'project.artifactId': artifactId, + 'project.version': version, + 'pom.groupId': groupId, + 'pom.artifactId': artifactId, + 'pom.version': version, + groupId, + artifactId, + version, + } +} + +const MAX_INTERPOLATION_DEPTH = 10 + +/** + * Best-effort Maven property interpolation of ${...} placeholders. Resolves + * recursively (a property value may itself reference another) up to a depth cap + * to guard against cycles. Placeholders with no matching property (e.g. defined + * in a profile/settings, or method calls like ${x.substring(8)}) are left as-is + * so the SCM normaliser rejects them. + */ +export function interpolateProperties(value: string, props: Record): string { + let current = value + for (let i = 0; i < MAX_INTERPOLATION_DEPTH && current.includes('${'); i++) { + const next = current.replace(/\$\{([^{}]+)\}/g, (match, key) => { + const resolved = props[(key as string).trim()] + return resolved !== undefined ? resolved : match + }) + if (next === current) break + current = next + } + return current +} + function extractParent( pom: PomData, ): { groupId: string; artifactId: string; version: string } | null { @@ -634,5 +716,6 @@ function emptyFields(hops: number): ResolvedFields { developers: [], contributors: [], hops, + properties: {}, } } diff --git a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts index d9707a3aea..0a081956af 100644 --- a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts @@ -1,6 +1,7 @@ import { MavenPackageToSync, QueryExecutor, + listMavenCriticalPackagesById, listMavenPackagesToSync, logAuditFieldChange, replacePackageMaintainers, @@ -497,3 +498,63 @@ async function runPhase(qx: QueryExecutor, config: MavenConfig, isCritical: bool export async function runMavenCriticalBackfill(qx: QueryExecutor, config: MavenConfig, isShuttingDown: () => boolean): Promise { return runPhase(qx, config, true, isShuttingDown) } + +/** + * Force full-refresh backfill: re-runs POM extraction over EVERY critical Maven + * row, ignoring the staleness window. Unlike runMavenCriticalBackfill (which + * drains listMavenPackagesToSync by predicate and cannot terminate with + * refreshDays=0 — the freshly-synced top rows re-qualify every batch), this pages + * strictly by `id > afterId`, so each row is processed exactly once and the scan + * always terminates. Every page runs with forceFullExtraction=true. + * + * Trade-off vs. the normal path: rows are visited in id order, not + * dependent_count order, so if interrupted the most-depended-on packages are not + * guaranteed to be done first. Restart re-scans from id 0 (idempotent upserts). + */ +// prettier-ignore +export async function runMavenCriticalForceBackfill(qx: QueryExecutor, config: MavenConfig, isShuttingDown: () => boolean): Promise { + const total: BatchResult = { processed: 0, skipped: 0, error: 0, unchanged: 0 } + const startedAt = Date.now() + // Cursor kept as a string: id is a Postgres bigint, and Number() coercion would + // silently lose precision above 2^53, corrupting the cursor and skipping rows. + let afterId = '0' + let batchNum = 0 + + log.info('Force full-refresh started (all critical rows, keyset by id, ignoring refreshDays)') + + while (!isShuttingDown()) { + const page = await listMavenCriticalPackagesById(qx, { afterId, limit: config.batchSize }) + if (page.length === 0) break + + // Capture the cursor before processPackages reorders the page in place (it + // clusters by namespace for the parent-POM cache). Rows come back id-ordered, + // so the max id is the last element — kept as a string to preserve bigint precision. + afterId = page[page.length - 1].id + + const result = await processPackages(qx, config, page, true, true) + batchNum++ + total.processed += result.processed + total.skipped += result.skipped + total.error += result.error + total.unchanged += result.unchanged + + log.info( + { + batch: batchNum, + afterId, + totalProcessed: total.processed, + totalSkipped: total.skipped, + totalUnchanged: total.unchanged, + totalErrors: total.error, + elapsedSec: Math.round((Date.now() - startedAt) / 1000), + }, + 'Force batch done', + ) + } + + log.info( + { ...total, durationSec: Math.round((Date.now() - startedAt) / 1000) }, + 'Force full-refresh complete', + ) + return total +} diff --git a/services/libs/data-access-layer/src/osspckgs/packages.ts b/services/libs/data-access-layer/src/osspckgs/packages.ts index 6a6bf42014..8b63f8c5fc 100644 --- a/services/libs/data-access-layer/src/osspckgs/packages.ts +++ b/services/libs/data-access-layer/src/osspckgs/packages.ts @@ -118,6 +118,41 @@ export async function listMavenPackagesToSync( ) } +/** + * Keyset-paginated scan of every critical Maven package, independent of the + * staleness window. Unlike `listMavenPackagesToSync` — which drains by predicate + * (a row leaves the set once it is fresh) and therefore cannot terminate with + * refreshDays=0 — this pages strictly by `id > afterId`, so each row is visited + * exactly once and the scan always terminates. Used by the `--force` full-refresh + * backfill to re-run POM extraction over the entire critical set regardless of + * last_synced_at / ingestion_source. + */ +export async function listMavenCriticalPackagesById( + qx: QueryExecutor, + options: { afterId: string; limit: number }, +): Promise { + return qx.select( + ` + SELECT + p.id, + p.purl, + p.namespace, + p.name, + p.dependent_count AS "dependentPackagesCount", + p.dependent_repos_count AS "dependentReposCount", + p.latest_version AS "latestVersion" + FROM packages p + WHERE p.ecosystem = 'maven' + AND p.is_critical + AND p.namespace IS NOT NULL + AND p.id > $(afterId)::bigint + ORDER BY p.id ASC + LIMIT $(limit) + `, + { afterId: options.afterId, limit: options.limit }, + ) +} + // ─── repository_url backfill ────────────────────────────────────────────────── export type MavenRepoUrlRow = { From 970ad5e84f736e4d9bd56e2fc190e3af39fd8686 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 9 Jul 2026 10:51:49 +0200 Subject: [PATCH 7/8] fix: add apache normalize Signed-off-by: Umberto Sgueglia --- .../src/maven/__tests__/normalize.test.ts | 35 +++++++++++ .../apps/packages_worker/src/maven/extract.ts | 58 +++++++++++++++---- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts index bc779e48cf..4717808b5f 100644 --- a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts @@ -201,6 +201,41 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('scm:git:https://github.com/${owner}/repo.git')).toBeNull() }) + it('ignores an unresolved placeholder in a trailing suffix once owner/repo are valid', () => { + // ${project.scm.tag} lands in the /tree/ suffix, which is already ignored — + // only segments[0]/[1] (owner/repo) are inspected. + expect( + normalizeScmUrl('https://github.com/apache/httpcomponents-client/tree/${project.scm.tag}'), + ).toBe('https://github.com/apache/httpcomponents-client') + expect(normalizeScmUrl('https://github.com/apache/maven/tree/${project.scm.tag}')).toBe( + 'https://github.com/apache/maven', + ) + }) + + it('recovers Apache gitweb hosts (gitbox/git-wip-us/git.apache.org) in path form', () => { + expect(normalizeScmUrl('https://gitbox.apache.org/repos/asf/commons-lang.git')).toBe( + 'https://gitbox.apache.org/repos/asf/commons-lang', + ) + expect(normalizeScmUrl('https://git.apache.org/repos/asf/ant.git')).toBe( + 'https://git.apache.org/repos/asf/ant', + ) + }) + + it('recovers Apache gitweb hosts in classic ?p= query-string form', () => { + expect(normalizeScmUrl('https://gitbox.apache.org/repos/asf?p=commons-io.git')).toBe( + 'https://gitbox.apache.org/repos/asf/commons-io', + ) + expect(normalizeScmUrl('https://git-wip-us.apache.org/repos/asf?p=commons-math.git')).toBe( + 'https://git-wip-us.apache.org/repos/asf/commons-math', + ) + }) + + it('returns null for Apache gitweb hosts with no recoverable repo name', () => { + expect(normalizeScmUrl('https://gitbox.apache.org/')).toBeNull() + expect(normalizeScmUrl('https://gitbox.apache.org/repos/asf/')).toBeNull() + expect(normalizeScmUrl('https://gitbox.apache.org/some/other/path')).toBeNull() + }) + // SCP colon form: "host:owner/repo" where the colon is a path separator, not a port it('recovers bare host:owner/repo SCP colon form', () => { expect(normalizeScmUrl('github.com:japgolly/scalacss.git')).toBe( diff --git a/services/apps/packages_worker/src/maven/extract.ts b/services/apps/packages_worker/src/maven/extract.ts index 8394c8dd97..174f356ab9 100644 --- a/services/apps/packages_worker/src/maven/extract.ts +++ b/services/apps/packages_worker/src/maven/extract.ts @@ -484,15 +484,17 @@ export async function extractArtifact(groupId: string, artifactId: string, versi * * Candidate additions found in Maven declared_repository_url (critical rows) but * intentionally NOT added yet, because they need more than a host allowlist: - * - gitbox.apache.org / git.apache.org / git-wip-us.apache.org (~1.3k rows): - * paths are `/repos/asf/`, so the generic first-two-segments logic would - * collapse every Apache repo to `repos/asf`. Needs path-aware handling (skip - * the `/repos/asf/` prefix) or mapping to the github.com/apache mirror. - * - git.eclipse.org (~180 rows): Gerrit paths, likewise not owner/repo. + * - git.eclipse.org (~180 rows): Gerrit `/c/` paths, not owner/repo. + * - eclipse.gerrithub.io (~10 rows): Gerrit admin-UI paths (`/admin/repos/...`), + * not a clone URL shape. * - android.googlesource.com, ec.europa.eu (Bitbucket-Server /projects/x/repos/y): * multi-segment paths, not owner/repo. * - ambiguous `git.*` hosts (git.iem.at, git.i-novus.ru, git.oschina.net) and the * internal git.corp.adobe.com: pending path/reachability confirmation. + * + * gitbox.apache.org / git.apache.org / git-wip-us.apache.org are handled separately + * below (normalizeApacheGitwebUrl) rather than through this allowlist — their paths + * are `/repos/asf/` or `?p=` (classic gitweb query form), not owner/repo. */ const SCM_HOSTS = new Set([ 'github.com', @@ -522,6 +524,33 @@ const SCM_HOSTS = new Set([ /** Hosts whose owner/repo path is case-insensitive and should be lower-cased. */ const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) +/** + * Apache's gitweb-based hosts serve every repo under a fixed /repos/asf/ prefix, + * in two equivalent forms: /repos/asf/.git (path) or /repos/asf?p=.git + * (classic gitweb query-string). Neither is an owner/repo shape — the generic path + * logic would either reject the query form (no second segment) or collapse every + * repo to the literal segments "repos"/"asf" for the path form. Handled on the same + * host rather than mapped to a github.com/apache mirror, since that mapping isn't + * guaranteed to hold for every repo. + */ +const APACHE_GITWEB_HOSTS = new Set([ + 'gitbox.apache.org', + 'git.apache.org', + 'git-wip-us.apache.org', +]) + +function normalizeApacheGitwebUrl(host: string, pathname: string, search: string): string | null { + const queryRepo = new URLSearchParams(search).get('p') + const raw = + queryRepo ?? (pathname.startsWith('/repos/asf/') ? pathname.slice('/repos/asf/'.length) : null) + if (!raw) return null + + const name = raw.replace(/\.git$/, '').replace(/\/+$/, '') + if (!name || name.includes('/') || /\$\{|%7B/i.test(name)) return null + + return `https://${host}/repos/asf/${name}` +} + /** * Converts the raw SCM URL from a POM (declared_repository_url) into a clean, * canonical `https:////` repository URL suitable for storage @@ -547,12 +576,6 @@ export function normalizeScmUrl(raw: string | null): string | null { let s = raw.trim() if (!s) return null - // A leftover ${...} means best-effort interpolation upstream could not resolve it. - // Reject outright: a placeholder embedded in the path (e.g. github.com/owner/${x}) - // otherwise survives URL parsing (percent-encoded to %7B…%7D) and would yield a junk - // repository_url instead of null. - if (s.includes('${')) return null - // Strip Maven scm:git: / scm: prefix s = s.replace(/^scm:git:/i, '').replace(/^scm:/i, '') @@ -593,6 +616,11 @@ export function normalizeScmUrl(raw: string | null): string | null { if (parsed.protocol !== 'https:') return null const host = parsed.hostname.toLowerCase().replace(/^www\./, '') + + if (APACHE_GITWEB_HOSTS.has(host)) { + return normalizeApacheGitwebUrl(host, parsed.pathname, parsed.search) + } + if (!SCM_HOSTS.has(host)) return null // Require at least owner + repo path segments @@ -603,6 +631,14 @@ export function normalizeScmUrl(raw: string | null): string | null { let name = segments[1].replace(/\.git$/, '') if (!owner || !name) return null + // A leftover ${...} in owner/repo means best-effort interpolation upstream could + // not resolve it — reject rather than store a junk link. The URL parser + // percent-encodes { and } in the path, so check both forms. Placeholders in a + // trailing suffix (e.g. /tree/${project.scm.tag}) are irrelevant here since only + // segments[0]/[1] are inspected — that suffix is simply never read. + const hasPlaceholder = (seg: string) => /\$\{|%7B/i.test(seg) + if (hasPlaceholder(owner) || hasPlaceholder(name)) return null + if (CASE_INSENSITIVE_HOSTS.has(host)) { owner = owner.toLowerCase() name = name.toLowerCase() From f8c9e2f86671ffb70e4813b9e190b581086c3cd8 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 9 Jul 2026 11:36:48 +0200 Subject: [PATCH 8/8] fix: add eclipse normalize Signed-off-by: Umberto Sgueglia --- .../src/maven/__tests__/normalize.test.ts | 23 ++++++++++++++++ .../apps/packages_worker/src/maven/extract.ts | 26 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts index 4717808b5f..93bdc3360c 100644 --- a/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/normalize.test.ts @@ -236,6 +236,29 @@ describe('normalizeScmUrl', () => { expect(normalizeScmUrl('https://gitbox.apache.org/some/other/path')).toBeNull() }) + it('recovers git.eclipse.org cgit URLs, keeping the /c/ prefix and dropping trailing tree paths', () => { + expect(normalizeScmUrl('http://git.eclipse.org/c/eclipselink/javax.persistence.git')).toBe( + 'https://git.eclipse.org/c/eclipselink/javax.persistence', + ) + expect( + normalizeScmUrl('https://git.eclipse.org/c/eclipsescada/org.eclipse.scada.utils.git'), + ).toBe('https://git.eclipse.org/c/eclipsescada/org.eclipse.scada.utils') + expect( + normalizeScmUrl('http://git.eclipse.org/c/jetty/org.eclipse.jetty.project.git/tree'), + ).toBe('https://git.eclipse.org/c/jetty/org.eclipse.jetty.project') + expect( + normalizeScmUrl( + 'http://git.eclipse.org/c/jetty/org.eclipse.jetty.orbit.git/tree/jetty-orbit', + ), + ).toBe('https://git.eclipse.org/c/jetty/org.eclipse.jetty.orbit') + }) + + it('returns null for git.eclipse.org URLs with no recoverable repo path', () => { + expect(normalizeScmUrl('https://git.eclipse.org/')).toBeNull() + expect(normalizeScmUrl('https://git.eclipse.org/c/')).toBeNull() + expect(normalizeScmUrl('https://git.eclipse.org/c/onlyowner')).toBeNull() + }) + // SCP colon form: "host:owner/repo" where the colon is a path separator, not a port it('recovers bare host:owner/repo SCP colon form', () => { expect(normalizeScmUrl('github.com:japgolly/scalacss.git')).toBe( diff --git a/services/apps/packages_worker/src/maven/extract.ts b/services/apps/packages_worker/src/maven/extract.ts index 174f356ab9..1a530982c9 100644 --- a/services/apps/packages_worker/src/maven/extract.ts +++ b/services/apps/packages_worker/src/maven/extract.ts @@ -551,6 +551,28 @@ function normalizeApacheGitwebUrl(host: string, pathname: string, search: string return `https://${host}/repos/asf/${name}` } +/** + * Eclipse's cgit instance serves repos as /c//[.git][/tree/...], the + * same owner/repo shape as the generic hosts but under a fixed /c/ prefix that + * cgit requires for a working link (unlike GitHub-style hosts, a bare + * https://git.eclipse.org// does not resolve) — so the prefix is + * kept in the output rather than stripped like the generic path logic would. + */ +const ECLIPSE_CGIT_HOSTS = new Set(['git.eclipse.org']) + +function normalizeEclipseCgitUrl(host: string, pathname: string): string | null { + if (!pathname.startsWith('/c/')) return null + + const segments = pathname.slice('/c/'.length).split('/').filter(Boolean) + if (segments.length < 2) return null + + const owner = segments[0] + const name = segments[1].replace(/\.git$/, '') + if (!owner || !name || /\$\{|%7B/i.test(owner) || /\$\{|%7B/i.test(name)) return null + + return `https://${host}/c/${owner}/${name}` +} + /** * Converts the raw SCM URL from a POM (declared_repository_url) into a clean, * canonical `https:////` repository URL suitable for storage @@ -621,6 +643,10 @@ export function normalizeScmUrl(raw: string | null): string | null { return normalizeApacheGitwebUrl(host, parsed.pathname, parsed.search) } + if (ECLIPSE_CGIT_HOSTS.has(host)) { + return normalizeEclipseCgitUrl(host, parsed.pathname) + } + if (!SCM_HOSTS.has(host)) return null // Require at least owner + repo path segments