feat: fix maven repo gap (CM-1305)

services/apps/packages_worker/package.json

@@ -51,6 +51,8 @@
     "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
+    "backfill:maven-repo-url": "SERVICE=maven tsx src/bin/maven-repo-url-backfill.ts",
+    "backfill:maven-repo-url:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-repo-url-backfill.ts",
     "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
     "import:maven-maintainers:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
     "import:sonatype-popularity": "SERVICE=maven tsx src/maven/scripts/importSonatypePopularityFromCsv.ts",

services/apps/packages_worker/src/bin/maven-backfill.ts

@@ -2,7 +2,10 @@ import { getServiceLogger } from '@crowd/logging'
 
 import { getMavenConfig } from '../config'
 import { getPackagesDb } from '../db'
-import { runMavenCriticalBackfill } from '../maven/runMavenEnrichmentLoop'
+import {
+  runMavenCriticalBackfill,
+  runMavenCriticalForceBackfill,
+} from '../maven/runMavenEnrichmentLoop'
 
 const log = getServiceLogger()
 
@@ -25,7 +28,14 @@ const main = async () => {
     process.env.MAVEN_FETCHER_BASE_URL_BACKFILL ??
     'https://maven-central.storage-download.googleapis.com/maven2'
 
-  log.info('maven backfill starting (one-shot, full extraction)...')
+  // --force: re-run POM extraction over EVERY critical row, ignoring the
+  // staleness window. Use to fully re-apply extraction changes (e.g. SCM
+  // interpolation) after the queue has already drained. The default path only
+  // picks rows due by refreshDays and cannot be coaxed into a full pass by
+  // setting refreshDays=0 (that reprocesses the first batch forever).
+  const force = process.argv.includes('--force')
+
+  log.info({ force }, 'maven backfill starting (one-shot, full extraction)...')
 
   const config = getMavenConfig()
   log.info(config, 'Config loaded')
@@ -34,7 +44,9 @@ const main = async () => {
   await qx.selectOne('SELECT 1')
   log.info('Connected to packages-db.')
 
-  const totals = await runMavenCriticalBackfill(qx, config, () => shuttingDown)
+  const totals = force
+    ? await runMavenCriticalForceBackfill(qx, config, () => shuttingDown)
+    : await runMavenCriticalBackfill(qx, config, () => shuttingDown)
 
   log.info({ ...totals }, 'maven backfill complete')
   process.exit(0)

services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts

@@ -0,0 +1,61 @@
+import { getServiceLogger } from '@crowd/logging'
+
+import { getPackagesDb } from '../db'
+import { backfillMavenRepositoryUrls } from '../maven/backfillRepositoryUrl'
+
+const log = getServiceLogger()
+
+let shuttingDown = false
+
+// Graceful stop: finish the in-flight batch, then exit. Safe to interrupt — every
+// write is an idempotent UPDATE recomputed from declared_repository_url, so
+// re-running simply reprocesses and skips rows that already match.
+const shutdown = () => {
+  if (shuttingDown) return
+  shuttingDown = true
+  log.info('Shutting down maven repo-url backfill (stopping after the current batch)...')
+}
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+const DEFAULT_BATCH_SIZE = 5000
+
+const main = async () => {
+  const dryRun = process.argv.includes('--dry-run')
+  const criticalOnly = process.argv.includes('--critical-only')
+  const rawBatchSize = process.env.MAVEN_REPO_URL_BACKFILL_BATCH_SIZE
+  const parsedBatchSize = rawBatchSize === undefined ? DEFAULT_BATCH_SIZE : Number(rawBatchSize)
+  if (!Number.isInteger(parsedBatchSize) || parsedBatchSize <= 0) {
+    log.error(
+      { MAVEN_REPO_URL_BACKFILL_BATCH_SIZE: rawBatchSize },
+      'MAVEN_REPO_URL_BACKFILL_BATCH_SIZE must be a positive integer',
+    )
+    process.exit(1)
+  }
+  const batchSize = parsedBatchSize
+
+  log.info(
+    { dryRun, criticalOnly, batchSize },
+    'maven repo-url backfill starting (recompute normalizeScmUrl from declared_repository_url, no POM fetch)...',
+  )
+
+  const qx = await getPackagesDb()
+  await qx.selectOne('SELECT 1')
+  log.info('Connected to packages-db.')
+
+  const totals = await backfillMavenRepositoryUrls(qx, {
+    batchSize,
+    dryRun,
+    criticalOnly,
+    isShuttingDown: () => shuttingDown,
+  })
+
+  log.info({ ...totals, dryRun }, 'maven repo-url backfill complete')
+  process.exit(0)
+}
+
+main().catch((err) => {
+  log.error({ err }, 'maven repo-url backfill fatal error')
+  process.exit(1)
+})

services/apps/packages_worker/src/maven/__tests__/normalize.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest'
 
-import { normalizeScmUrl } from '../extract'
+import { interpolateProperties, normalizeScmUrl } from '../extract'
 import { pickStableRelease } from '../metadata'
 import { isPrerelease, parseRepoUrl } from '../normalize'
 
@@ -163,4 +163,200 @@ describe('normalizeScmUrl', () => {
   it('returns null for non-https result', () => {
     expect(normalizeScmUrl('svn://svn.apache.org/repos/commons-lang')).toBeNull()
   })
+
+  // Gap B — recover repository_url from inputs that were previously dropped
+  it('recovers scm:git: without a scheme', () => {
+    expect(normalizeScmUrl('scm:git:github.com/lum-ai/nxmlreader')).toBe(
+      'https://github.com/lum-ai/nxmlreader',
+    )
+  })
+
+  it('recovers a bare host/owner/repo without a scheme', () => {
+    expect(normalizeScmUrl('github.com/agiledigital/kamon-play-extensions')).toBe(
+      'https://github.com/agiledigital/kamon-play-extensions',
+    )
+  })
+
+  it('upgrades http to https and lower-cases the github path', () => {
+    expect(normalizeScmUrl('http://github.com/kevemueller/kTLSH/tree/master')).toBe(
+      'https://github.com/kevemueller/ktlsh',
+    )
+  })
+
+  // Gap C — reject non-repository URLs so they are never stored
+  it('returns null for website-only URLs', () => {
+    expect(normalizeScmUrl('https://meson.ai/')).toBeNull()
+    expect(normalizeScmUrl('http://source.android.com')).toBeNull()
+  })
+
+  it('returns null for placeholders and free-form text', () => {
+    expect(normalizeScmUrl('Private')).toBeNull()
+    expect(normalizeScmUrl('${scm-url}')).toBeNull()
+    expect(normalizeScmUrl('http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/foo')).toBeNull()
+  })
+
+  it('returns null when an unresolved placeholder is embedded in the repo path', () => {
+    // Would otherwise parse to github.com/owner/%7BartifactId%7D and slip through.
+    expect(normalizeScmUrl('https://github.com/owner/${artifactId}')).toBeNull()
+    expect(normalizeScmUrl('scm:git:https://github.com/${owner}/repo.git')).toBeNull()
+  })
+
+  it('ignores an unresolved placeholder in a trailing suffix once owner/repo are valid', () => {
+    // ${project.scm.tag} lands in the /tree/<ref> suffix, which is already ignored —
+    // only segments[0]/[1] (owner/repo) are inspected.
+    expect(
+      normalizeScmUrl('https://github.com/apache/httpcomponents-client/tree/${project.scm.tag}'),
+    ).toBe('https://github.com/apache/httpcomponents-client')
+    expect(normalizeScmUrl('https://github.com/apache/maven/tree/${project.scm.tag}')).toBe(
+      'https://github.com/apache/maven',
+    )
+  })
+
+  it('recovers Apache gitweb hosts (gitbox/git-wip-us/git.apache.org) in path form', () => {
+    expect(normalizeScmUrl('https://gitbox.apache.org/repos/asf/commons-lang.git')).toBe(
+      'https://gitbox.apache.org/repos/asf/commons-lang',
+    )
+    expect(normalizeScmUrl('https://git.apache.org/repos/asf/ant.git')).toBe(
+      'https://git.apache.org/repos/asf/ant',
+    )
+  })
+
+  it('recovers Apache gitweb hosts in classic ?p= query-string form', () => {
+    expect(normalizeScmUrl('https://gitbox.apache.org/repos/asf?p=commons-io.git')).toBe(
+      'https://gitbox.apache.org/repos/asf/commons-io',
+    )
+    expect(normalizeScmUrl('https://git-wip-us.apache.org/repos/asf?p=commons-math.git')).toBe(
+      'https://git-wip-us.apache.org/repos/asf/commons-math',
+    )
+  })
+
+  it('returns null for Apache gitweb hosts with no recoverable repo name', () => {
+    expect(normalizeScmUrl('https://gitbox.apache.org/')).toBeNull()
+    expect(normalizeScmUrl('https://gitbox.apache.org/repos/asf/')).toBeNull()
+    expect(normalizeScmUrl('https://gitbox.apache.org/some/other/path')).toBeNull()
+  })
+
+  it('recovers git.eclipse.org cgit URLs, keeping the /c/ prefix and dropping trailing tree paths', () => {
+    expect(normalizeScmUrl('http://git.eclipse.org/c/eclipselink/javax.persistence.git')).toBe(
+      'https://git.eclipse.org/c/eclipselink/javax.persistence',
+    )
+    expect(
+      normalizeScmUrl('https://git.eclipse.org/c/eclipsescada/org.eclipse.scada.utils.git'),
+    ).toBe('https://git.eclipse.org/c/eclipsescada/org.eclipse.scada.utils')
+    expect(
+      normalizeScmUrl('http://git.eclipse.org/c/jetty/org.eclipse.jetty.project.git/tree'),
+    ).toBe('https://git.eclipse.org/c/jetty/org.eclipse.jetty.project')
+    expect(
+      normalizeScmUrl(
+        'http://git.eclipse.org/c/jetty/org.eclipse.jetty.orbit.git/tree/jetty-orbit',
+      ),
+    ).toBe('https://git.eclipse.org/c/jetty/org.eclipse.jetty.orbit')
+  })
+
+  it('returns null for git.eclipse.org URLs with no recoverable repo path', () => {
+    expect(normalizeScmUrl('https://git.eclipse.org/')).toBeNull()
+    expect(normalizeScmUrl('https://git.eclipse.org/c/')).toBeNull()
+    expect(normalizeScmUrl('https://git.eclipse.org/c/onlyowner')).toBeNull()
+  })
+
+  // SCP colon form: "host:owner/repo" where the colon is a path separator, not a port
+  it('recovers bare host:owner/repo SCP colon form', () => {
+    expect(normalizeScmUrl('github.com:japgolly/scalacss.git')).toBe(
+      'https://github.com/japgolly/scalacss',
+    )
+  })
+
+  it('recovers scheme://host:owner/repo SCP colon form', () => {
+    expect(normalizeScmUrl('https://github.com:networknt/light-4j.git')).toBe(
+      'https://github.com/networknt/light-4j',
+    )
+  })
+
+  it('recovers ssh://git@host:owner/repo SCP colon form', () => {
+    expect(normalizeScmUrl('ssh://git@github.com:apache/iotdb.git')).toBe(
+      'https://github.com/apache/iotdb',
+    )
+    expect(normalizeScmUrl('ssh://git@bitbucket.org:eci-elements/web-services.git')).toBe(
+      'https://bitbucket.org/eci-elements/web-services',
+    )
+  })
+
+  it('does not treat a numeric port as an SCP separator', () => {
+    expect(normalizeScmUrl('https://gitlab.com:443/foo/bar')).toBe('https://gitlab.com/foo/bar')
+  })
+
+  it('recovers git://host:owner/repo SCP colon form', () => {
+    expect(normalizeScmUrl('git://github.com:appendium/objectlabkit.git')).toBe(
+      'https://github.com/appendium/objectlabkit',
+    )
+  })
+
+  it('accepts allowlisted self-hosted GitLab/Gitea hosts', () => {
+    expect(normalizeScmUrl('https://git.neckar.it/neckarit/neckar-hub')).toBe(
+      'https://git.neckar.it/neckarit/neckar-hub',
+    )
+    expect(normalizeScmUrl('scm:git:https://gitlab.inria.fr/owner/repo.git')).toBe(
+      'https://gitlab.inria.fr/owner/repo',
+    )
+    expect(normalizeScmUrl('https://git.iem.at/owner/repo')).toBe('https://git.iem.at/owner/repo')
+  })
+
+  it('still returns null for internal or non-allowlisted hosts', () => {
+    expect(normalizeScmUrl('https://git.corp.adobe.com/team/project')).toBeNull()
+    expect(normalizeScmUrl('https://gitlab.alibaba-inc.com/team/project')).toBeNull()
+    expect(normalizeScmUrl('https://android.googlesource.com/platform/tools/base')).toBeNull()
+  })
+})
+
+describe('interpolateProperties', () => {
+  it('resolves a single ${...} placeholder from properties', () => {
+    expect(
+      interpolateProperties('${scm.github.url}', {
+        'scm.github.url': 'https://github.com/owner/repo',
+      }),
+    ).toBe('https://github.com/owner/repo')
+  })
+
+  it('resolves multiple placeholders in one string', () => {
+    expect(
+      interpolateProperties('https://gitlab.com/${projectPath}', {
+        projectPath: 'group/project',
+      }),
+    ).toBe('https://gitlab.com/group/project')
+  })
+
+  it('resolves nested placeholders recursively', () => {
+    expect(
+      interpolateProperties('${scm.url}', {
+        'scm.url': '${scm.base}/repo',
+        'scm.base': 'https://github.com/owner',
+      }),
+    ).toBe('https://github.com/owner/repo')
+  })
+
+  it('resolves built-in project.* style placeholders', () => {
+    expect(
+      interpolateProperties('https://github.com/acme/${project.artifactId}', {
+        'project.artifactId': 'my-lib',
+      }),
+    ).toBe('https://github.com/acme/my-lib')
+  })
+
+  it('leaves unresolved placeholders literal (missing property / method call)', () => {
+    expect(interpolateProperties('${scm.github.url}', {})).toBe('${scm.github.url}')
+    expect(
+      interpolateProperties('${pom.artifactId.substring(8)}', { 'pom.artifactId': 'foo' }),
+    ).toBe('${pom.artifactId.substring(8)}')
+  })
+
+  it('does not loop forever on self-referential placeholders', () => {
+    expect(interpolateProperties('${a}', { a: '${b}', b: '${a}' })).toContain('${')
+  })
+
+  it('composes with the normalizer end-to-end', () => {
+    const resolved = interpolateProperties('${scm.github.url}', {
+      'scm.github.url': 'scm:git:https://github.com/Owner/Repo.git',
+    })
+    expect(normalizeScmUrl(resolved)).toBe('https://github.com/owner/repo')
+  })
 })