diff --git a/docs/assets/zotero-find-full-text.png b/docs/assets/zotero-find-full-text.png new file mode 100644 index 0000000..fe86586 Binary files /dev/null and b/docs/assets/zotero-find-full-text.png differ diff --git a/docs/how-to/export-cache-to-zotero.md b/docs/how-to/export-cache-to-zotero.md new file mode 100644 index 0000000..6f7628e --- /dev/null +++ b/docs/how-to/export-cache-to-zotero.md @@ -0,0 +1,114 @@ +# Export a Reference Cache to Zotero + +This workflow turns a checked-in public reference cache into a Zotero collection, +asks Zotero to find available PDFs, and then inventories the resulting attachments +without putting closed manuscripts into the project repository. + +The three stores remain separate: + +| Store | Purpose | Safe to commit? | +|---|---|---:| +| Project `references_cache` | Reproducible public validation evidence | Yes | +| Zotero library | User-managed references and PDF attachments | No | +| Private research cache | Extracted user-library text for agent context | No | + +Normal validation reads only the first store. + +## 1. Export metadata from the public cache + +Run the exporter from the `linkml-reference-validator` checkout. For dismech: + +```bash +uv run linkml-reference-validator cache export \ + --cache-dir /Users/cjm/repos/dismech/references_cache \ + --needs-full-text \ + --output /Users/cjm/Downloads/dismech-zotero.json +``` + +`--needs-full-text` is the default. It excludes cache entries that already have +full text, excludes non-publication identifiers, and deduplicates by normalized +DOI and then PMID. The CSL JSON output uses an explicit metadata allowlist: + +- title and authors; +- journal and year; +- DOI; or +- PMID and its public PubMed URL. + +It never contains cached article text, excerpts, provenance, PDFs, local paths, +or private-cache data. The command refuses to replace an existing output file; +use `--force` only after checking the destination. Use `--all` if you explicitly +want eligible references that already contain full text. + +## 2. Import the file into Zotero + +In Zotero: + +1. Choose **File → Import**. +2. Choose **A file**. +3. Select `dismech-zotero.json`. +4. Keep the imported records in a project-specific collection such as + **dismech**. + +CSL JSON is a standard format supported by Zotero. Importing the file adds only +bibliographic records; it does not copy anything from the validator's caches. + +## 3. Ask Zotero to find PDFs + +Select one or more top-level journal articles in the center item list, +right-click the selection, and choose **Find Full Text**. It is a built-in Zotero +command, not a plugin. + +![Zotero Find Full Text processing a batch of imported references](../assets/zotero-find-full-text.png) + +The command appears for an eligible parent item that has a DOI and does not +already have a file attachment. It may also be unavailable in a group library +where files cannot be added. If it is absent, first test a single article in +your personal library and confirm that its DOI field is populated. + +For a large import, start with a modest batch rather than all records at once. +Publishers can temporarily rate-limit repeated requests, and a later retry may +find additional files. Zotero reports **Full Text PDF** or **No file found** for +each attempted item. + +## 4. Inventory the enriched Zotero library + +Once Zotero finishes, scan the project cache again: + +```bash +uv run linkml-reference-validator cache enrich \ + --provider zotero \ + --cache-dir /Users/cjm/repos/dismech/references_cache \ + --dry-run +``` + +This read-only scan matches exact DOI, PMID, or PMCID identifiers and reports +which cached references now have usable Zotero text or PDF attachments. It does +not invoke **Find Full Text** and does not modify Zotero. + +After reviewing the matches, materialize them into the separate private research +cache: + +```bash +uv run linkml-reference-validator cache enrich \ + --provider zotero \ + --cache-dir /Users/cjm/repos/dismech/references_cache \ + --apply +``` + +Apply mode leaves `references_cache` unchanged. Private content goes to +`~/.cache/linkml-reference-validator/private` by default with owner-only +permissions. Agents may inspect it for context, but validation cannot use it as +evidence. + +## 5. Validate reproducibly + +Run validation normally. Results remain reproducible because the validator +reads the public project cache and accepts only public full-text provider results. +Neither the contents of Zotero nor the machine-specific private cache affect +validation. + +## See also + +- [Fetching Full Text and PDFs](fetch-full-text-and-pdfs.md) +- [Zotero: importing standardized formats](https://www.zotero.org/support/kb/importing_standardized_formats) +- [Zotero: adding files to a library](https://www.zotero.org/support/attaching_files) diff --git a/docs/how-to/fetch-full-text-and-pdfs.md b/docs/how-to/fetch-full-text-and-pdfs.md index a8b42c0..ef73eb0 100644 --- a/docs/how-to/fetch-full-text-and-pdfs.md +++ b/docs/how-to/fetch-full-text-and-pdfs.md @@ -46,9 +46,11 @@ your config YAML or on the `ReferenceValidationConfig` object): |-----|---------|-------------| | `fetch_full_text` | `true` | Attempt to obtain full text via the provider chain when a metadata source does not already return full text. | | `full_text_providers` | `[pmc, epmc_preprint, unpaywall, openalex]` | Ordered list of provider names to try until one yields usable full text. | +| `private_cache_dir` | `~/.cache/linkml-reference-validator/private` | Separate research cache for closed/user-library full text; never read by validation. | | `pdf_backend` | `pypdf` | Name of the PDF text-extraction backend. | | `download_pdfs` | `true` | If true, persist downloaded PDFs to the files cache directory. | | `full_text_providers_file` | `null` | Optional path to a YAML file defining custom full-text providers. | +| `zotero_base_url` | `http://localhost:23119/api/users/0` | Read-only Zotero local API library used by the opt-in `zotero` provider. | Two existing keys are also reused by the full-text machinery: @@ -71,6 +73,106 @@ pdf_backend: pypdf download_pdfs: true ``` +## Private manuscripts in Zotero + +The opt-in **`zotero`** provider can find non-open manuscripts that are already +in your personal Zotero library. It matches exact DOI, PMID, or PMCID identifiers; +it does not automatically accept fuzzy title matches. + +Zotero must be running, and its local API must be enabled in Zotero's advanced +settings. Inventory the reference cache before changing it: + +```bash +linkml-reference-validator cache enrich \ + --provider zotero \ + --cache-dir references_cache \ + --dry-run +``` + +The report distinguishes `found`, `not_found`, `already_full_text`, and `error`. +Dry-run is the default and never changes cache files. After reviewing exact +matches, apply usable text with: + +```bash +linkml-reference-validator cache enrich \ + --provider zotero \ + --cache-dir references_cache \ + --apply +``` + +Apply mode does **not** modify `references_cache`. It writes the enriched entry +to a separate research cache at `~/.cache/linkml-reference-validator/private` +by default. Agents may inspect that cache for background context, but ordinary +validation never reads it and never cites it as evidence. Validation therefore +has the same inputs locally, in CI, and on another contributor's machine. + +To keep the research cache in a private repository or another secured location: + +```bash +linkml-reference-validator cache enrich \ + --provider zotero \ + --cache-dir references_cache \ + --private-cache-dir /path/to/private-reference-cache \ + --apply +``` + +The same location can be configured for all commands: + +```yaml +cache_dir: references_cache +private_cache_dir: /path/to/private-reference-cache +``` + +Zotero-indexed text is preferred. If Zotero has not indexed an attachment, the +provider downloads it from Zotero's local PDF endpoint and sends it through the +normal PDF extractor. The source attachment key and `user_library` access type +are persisted, but the ephemeral localhost URL is not. + +!!! warning "Keep research caches private" + + Private cache entries contain extracted manuscript text and may contain a + copied PDF. Do not place this cache in a public repository or share it + unless you have permission to redistribute that content. + +Do not add `zotero` to `full_text_providers`. Even if it is configured there, +ordinary validation rejects its `user_library` result and continues to public +providers. Use `cache enrich` when you want to search Zotero. + +Support for reproducible validation backed by private manuscripts is a future, +separate feature. The proposed design is a trusted offline job that emits a +minimal signed excerpt attestation into the public cache. The validator would +verify the signature using a checked-in public key; neither agents nor CI would +have the signing key or permission to generate those files directly. + +## Export a cache to a Zotero collection + +Export references that still need full text as metadata-only CSL JSON: + +```bash +linkml-reference-validator cache export \ + --cache-dir references_cache \ + --format csl-json \ + --needs-full-text \ + --output project-zotero.json +``` + +The default `--needs-full-text` mode excludes cache entries that already contain +full text. It also excludes non-publication identifiers, deduplicates by +normalized DOI and then PMID, and exports only an explicit metadata allowlist: +title, authors, journal, year, DOI, PMID, and PubMed URL. Cached article text, +excerpts, provenance, PDFs, and local paths are never exported. Use `--all` only +when you deliberately want eligible records that already have full text. + +The exporter refuses to replace an existing output file unless `--force` is +given. Review the reported counts, then in Zotero choose **File → Import → A +file**, select the JSON file, and place the imported items in a project-specific +collection. Zotero supports CSL JSON as a standard import format. After import, +select the collection and run **Find Full Text**, then rerun `cache enrich`. + +See [Zotero's standardized-format import instructions](https://www.zotero.org/support/kb/importing_standardized_formats). +For the complete workflow with screenshots, see +[Export a Reference Cache to Zotero](export-cache-to-zotero.md). + ## Preprints Preprints are first-class references. They are increasingly cited for early diff --git a/docs/reference/cli.md b/docs/reference/cli.md index bc3ff24..923bcd7 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -761,6 +761,69 @@ linkml-reference-validator cache lookup PMID:16888623 --content linkml-reference-validator cache lookup PMID:16888623 --no-cache ``` +## cache export + +Export public bibliographic metadata to CSL JSON for Zotero import. + +```bash +linkml-reference-validator cache export --output project-zotero.json [OPTIONS] +``` + +**Options:** + +- `--output PATH, -o PATH` - Required destination file +- `--format TEXT` - Export format (default and currently supported: `csl-json`) +- `--needs-full-text` - Export only publication records needing full text (default) +- `--all` - Include eligible records that already contain full text +- `--cache-dir PATH, -c PATH` - Public reference cache directory +- `--force, -f` - Replace an existing output file +- `--config PATH` - Validation configuration file +- `--verbose, -v` - Enable detailed logging + +```bash +linkml-reference-validator cache export \ + --cache-dir references_cache \ + --needs-full-text \ + --output project-zotero.json +``` + +The output is a DOI/PMID-deduplicated metadata allowlist. It never includes +cached article content, excerpts, PDFs, local paths, or private-cache data. + +--- + +## cache enrich + +Inventory or enrich existing abstract-only cache entries through one full-text +provider. This is primarily intended for opt-in private-library providers such +as Zotero. + +```bash +linkml-reference-validator cache enrich [OPTIONS] +``` + +**Options:** + +- `--provider TEXT` - Registered provider name (default: `zotero`) +- `--cache-dir PATH, -c PATH` - Reference cache directory +- `--private-cache-dir PATH` - Separate private research-cache destination (default: `~/.cache/linkml-reference-validator/private`) +- `--dry-run` - Report matches without changing files (default) +- `--apply` - Materialize usable matches into the private research cache +- `--config PATH` - Validation configuration file +- `--verbose, -v` - Enable detailed logging + +```bash +# Safe inventory +linkml-reference-validator cache enrich --provider zotero --dry-run + +# Apply reviewed exact matches +linkml-reference-validator cache enrich --provider zotero --apply +``` + +The public source cache is never modified by this command, and validation never +reads the private destination. Private-library content may be copyrighted; keep +the research cache private. + --- ## Reference ID Formats diff --git a/docs/superpowers/plans/2026-08-05-private-library-full-text.md b/docs/superpowers/plans/2026-08-05-private-library-full-text.md new file mode 100644 index 0000000..85ff83e --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-private-library-full-text.md @@ -0,0 +1,190 @@ +# Private Library Full-Text Integration — Implementation Plan + +**Date:** 2026-08-05 +**Design:** [Private Library Full-Text Integration](../specs/2026-08-05-private-library-full-text-design.md) + +All implementation tasks follow test-driven development: add a failing test, +implement the smallest production change, then run the focused test and the full +`just test` suite. Network tests use realistic recorded/constructed API responses; +live tests remain opt-in. + +## Phase 1: Zotero local API minimum viable workflow + +### 1. Generalize full-text locations + +Add tests for materializing a local PDF path, rejecting a path over the size cap, +rejecting a non-PDF file with a `.pdf` suffix, and passing ephemeral request +headers without persisting them. + +Then: + +- add `local_path`, `headers`, `access_type`, and `source_item_id` to + `FullTextLocation`; +- teach `ReferenceFetcher._materialize()` to read a local file under the same size + and format-sniffing rules as HTTP content; +- teach `ContentAcquirer` to accept per-location headers; +- add private provenance fields to `ReferenceContent` and cache round-tripping; +- ensure secrets and localhost URLs are not written to frontmatter. + +### 2. Add a Zotero API client and exact identifier index + +Add contract tests using representative Zotero v3 JSON for: + +- local API availability detection; +- paginated top-level item loading; +- DOI normalization; +- PMID/PMCID extraction from structured fields and `extra`; +- incremental index refresh metadata; +- duplicate and ambiguous identifiers; +- child attachment selection; +- indexed-full-text success and PDF fallback. + +Then implement a small read-only client rather than coupling HTTP and matching +logic directly inside the provider. The default base URL is +`http://localhost:23119/api`; web mode remains disabled in this phase. + +### 3. Add `ZoteroFullTextProvider` + +Add provider tests showing: + +- no identifier returns `None`; +- an exact DOI/PMID hit returns Zotero indexed text when usable; +- short or absent indexed text returns the PDF file location; +- ambiguity returns no location and a diagnostic; +- connection errors remain retryable in the provider chain; +- provenance is `provider=zotero` and `access_type=user_library` with no OA claim. + +Register `zotero` for the explicit `cache enrich` workflow. Ordinary validation +must reject its `user_library` locations even if it is accidentally included in +the configured provider chain. + +### 4. Add `cache enrich` + +Add CLI tests first for dry-run, apply, misses, ambiguity, provider-unavailable, +and stable tabular output. Implement: + +```text +cache enrich --provider zotero --cache-dir PATH [--dry-run] +``` + +The command enumerates Markdown cache files, loads them through `ReferenceFetcher`, +and runs only the selected private provider even if `full_text_attempted` is +already true. Apply mode reuses the normal materialization path but writes an +owner-only private research cache outside the project by default. + +### 5. Positive local smoke test + +Run the dry-run command against a temporary cache entry whose DOI is known to +exist in the local Zotero library while Zotero is running. Verify: + +- exact DOI match; +- at least 500 extracted characters; +- cache provenance and checksum; +- no credential or private URL leakage; +- the source Zotero library remains unchanged. + +The checked-in test suite must not depend on the developer's personal library. + +## Phase 2: Zotero Web API + +### 6. Add authenticated read-only web mode + +Test and implement user/group library configuration, pagination, `since`-based +index refresh, backoff/rate-limit headers, attachment download authentication, +and Zotero storage/WebDAV absence behavior. + +Credentials come from `ZOTERO_API_KEY` or a credential callback. Add explicit +configuration for user/group library ID; never accept API keys in URLs. + +### 7. Operational documentation + +Document how to enable the local API, create a read-only Zotero key, scan before +applying, protect private caches, and keep private content out of validation. + +## Phase 3: Paperpile local sync + +### 8. Build a generic local library index + +Extract identifier normalization and match reporting from the Zotero client into +a source-neutral index. Add BibTeX ingestion with tests for DOI, PMID, title, +year, author, duplicate keys, and malformed entries. Add a cacheable PDF probe +that reads metadata/first-page text only once per file and stores its checksum. + +### 9. Add `PaperpileFullTextProvider` for local files + +Test and implement configuration for: + +- a local Paperpile/Google Drive PDF root; +- a Paperpile automatic BibTeX export file or HTTPS download link; +- an optional manual Paperpile JSON export. + +Only exact DOI/PMID matches apply automatically. Title/year/author candidates are +reported by `cache enrich --dry-run` and require explicit user confirmation in a +future task. + +### 10. Measure matching quality + +Run dry-run evaluation on a representative private cache and record precision, +coverage, ambiguity, and unmatched reasons separately. Do not weaken matching +thresholds to improve coverage. Use findings to decide whether guarded metadata +matching should ever be enabled for apply mode. + +## Phase 4: Paperpile via Google Drive + +### 11. Add a read-only Drive adapter + +Test with Google Drive API fixtures, then implement OAuth read-only listing and +download, incremental change tracking, native Google shortcut handling, and +streamed size limits. Reuse the Paperpile BibTeX identity index from Phase 3. + +This phase must not mutate Drive or Paperpile. Tokens use the system credential +store and are excluded from logs and config serialization. + +## Phase 5: Privacy hardening and signed attestations + +### 12. Separate private content from shareable cache metadata + +Completed in the first implementation: private content uses an owner-only +research cache outside the project by default. Validation reads only the public +cache, and its ordinary provider chain rejects explicitly non-open locations. + +### 13. Specify and implement offline signed excerpt attestations + +Define a canonical, versioned `excerpts.yaml` schema and bind each record to its +reference ID, claim hash, normalized excerpt hash, private source checksum, +location, result, generator version, timestamp, and key ID. A trusted offline +job reads the private cache and signs with Ed25519. Only the verification key is +checked in; agents and CI cannot write trusted attestations. Add tests for valid, +tampered, wrong-reference, wrong-claim, unknown-key, expired-key, and unsigned +artifacts before enabling this optional validation mode. + +## Phase 6: Browse a cache in Zotero + +### 14. Export cache metadata for Zotero import + +Implemented `cache export --format csl-json` with deterministic DOI/PMID +deduplication and missing-full-text filtering. It exports an explicit +bibliographic metadata allowlist and always excludes cached text, excerpts, +provenance, PDFs, private-cache data, and local paths. The command refuses to +overwrite an output file without `--force`. + +### 15. Consider an opt-in Zotero collection sync + +Only after export is stable, design an idempotent write integration that creates +or updates a dedicated Zotero collection through the documented write API. It +must require explicit write authorization and a dry-run diff; it must never +alter unrelated Zotero items or attach private files implicitly. + +## Definition of done for the minimal first task + +- `cache enrich --provider zotero --dry-run` inventories an existing reference + cache without modifying it. +- Exact DOI/PMID matches identify PDF attachments in the supported Zotero local + API. +- Apply mode extracts usable text into the private research cache without changing the + public source cache. +- Normal validation ignores the private research cache and rejects private + provider results. +- Misses and ambiguities are visible and safe. +- No private API, direct SQLite dependency, secret persistence, or library write. +- Focused tests, doctests, mypy, Ruff, and the full `just test` suite pass. diff --git a/docs/superpowers/specs/2026-08-05-private-library-full-text-design.md b/docs/superpowers/specs/2026-08-05-private-library-full-text-design.md new file mode 100644 index 0000000..f46e3f8 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-private-library-full-text-design.md @@ -0,0 +1,309 @@ +# Private Library Full-Text Integration — Design + +**Date:** 2026-08-05 +**Status:** Partially implemented; validation isolation revised +**Approach:** Keep user-authorized manuscript libraries in a separate research +workflow, outside the reproducible public validation path. + +## Problem + +The validator can obtain open full text from PMC, Europe PMC, Unpaywall, and +OpenAlex, but many references are available only through a user's lawful +subscription or personal manuscript library. The user may already have the PDF +in Zotero or Paperpile even when every open-access provider misses. + +The first useful workflow is: + +> Given an existing reference cache, report which abstract-only references have +> an exact-identity PDF match in the user's Zotero library, then optionally enrich +> those cache entries from the matched PDFs. + +Earlier mentions of "Zenodo library" meant **Zotero**. This design concerns +Zotero and Paperpile personal libraries; existing Zenodo repository support is +unrelated. + +## Goals + +- Search user-authorized libraries in an explicit cache-enrichment workflow. +- Match references conservatively, preferring exact DOI, PMID, or PMCID matches. +- Reuse the existing PDF acquisition, extraction, minimum-text, and cache logic. +- Record that content came from a private/user library without claiming it is + open access. +- Keep credentials out of configuration files and logs. +- Provide a dry-run inventory command before changing cached references. +- Support Zotero first, then Paperpile without depending on undocumented APIs. +- Keep validation deterministic: it reads only the public project cache and + accepts evidence only from public providers. + +## Non-goals + +- Bypassing publisher authentication, DRM, or paywalls. +- Redistributing private PDFs or derived full text. +- Writing to Zotero, Paperpile, Google Drive, or Zenodo. +- Depending on Zotero's private SQLite schema for the production integration. +- Depending on reverse-engineered Paperpile web endpoints. +- Using fuzzy-only matches automatically. + +## Existing architecture fit + +The current pipeline already separates metadata resolution from full-text +location: + +```text +reference ID + -> metadata source + -> ReferenceIdentifiers + -> ordered FullTextProvider chain + -> acquire PDF/HTML/XML + -> extract text + -> save content and provenance in the reference cache +``` + +Private libraries run beside, not inside, the validation pipeline: + +```text +Zotero/Paperpile -> cache enrich -> private research cache -> agent context only + +public reference cache -> public provider chain -> validation evidence +``` + +The existing `FullTextProvider.locate()` interface is nearly sufficient. Two +small generalizations are needed: + +1. `ReferenceIdentifiers` needs matching metadata (`title`, `year`, and + optionally first author) for diagnostic candidates and future guarded + fallback matching. +2. `FullTextLocation` needs either a local path or request headers. Zotero's + local API can return text or a localhost file URL, but Zotero Web API and + Google Drive downloads require authenticated headers. A local synced + Paperpile PDF should not be routed through `requests`. + +The preferred model is: + +```python +@dataclass +class FullTextLocation: + url: Optional[str] = None + local_path: Optional[Path] = None + text: Optional[str] = None + headers: dict[str, str] = field(default_factory=dict, repr=False) + format_hint: Optional[str] = None + provider: str = "" + access_type: Optional[str] = None # open | user_library | institutional + source_item_id: Optional[str] = None +``` + +At most one of `url`, `local_path`, and `text` should be populated. Header +values must never be serialized into the cache. + +## Identity matching policy + +Wrong-paper matches are more harmful than misses. Matching therefore has +explicit confidence levels: + +| Match | Automatic use | Notes | +|---|---:|---| +| Normalized DOI equality | Yes | Strip DOI URL/prefix, whitespace, and case | +| PMID or PMCID equality | Yes | Use structured library fields or a parsed explicit identifier | +| DOI equality found in PDF metadata/text | Yes, after identifier validation | Useful for an unindexed local folder | +| Title + year + first author | Report only initially | May become opt-in after measuring false positives | +| Title similarity alone | No | Diagnostic candidate only | + +Ambiguous exact matches are not failures: choose a usable PDF only if all exact +matches identify the same parent work; otherwise report `ambiguous` and make no +cache change. + +## Zotero design + +### Supported access modes + +1. **Local API (first implementation).** Zotero exposes a read-only API at + `http://localhost:23119/api/`. It uses the local database and requires no + authentication for reads. The Zotero desktop application must be running and + local API access enabled. +2. **Web API (second implementation).** Private libraries require a read-only API + key. The key is read from an environment variable or secret store. File + attachments can be downloaded from `/items//file`. + +The supported APIs are preferable to reading `zotero.sqlite`: direct database +access couples the validator to private schema details, can see inconsistent +state, and complicates linked-file path resolution. + +### Lookup algorithm + +Zotero's API quick search does not provide reliable field-specific DOI search. +The provider should build a lightweight local index of top-level item JSON: + +```text +normalized DOI -> Zotero parent item key(s) +PMID -> Zotero parent item key(s) +PMCID -> Zotero parent item key(s) +``` + +For the local API the index can be loaded once per process. For the Web API it +must be paginated and persisted using Zotero library versions and `since` so +later refreshes are incremental. + +For a matched parent item: + +1. Request `/items//children`. +2. Select PDF attachments deterministically: primary attachment first, then the + newest non-supplement attachment. +3. Prefer `/items//fulltext`, because Zotero may already have + indexed the PDF. +4. If indexed text is absent or too short, download `/items//file` + and use the existing PDF extractor. +5. Return provider `zotero`, access type `user_library`, and the attachment key. + +### Observed local feasibility + +On the development machine, `/Users/cjm/Zotero/zotero.sqlite` exists and contains +283 PDF attachments. The supported local API was not reachable during this +design pass because Zotero was not running. The sole reference in this checkout's +`references_cache` (`PMID:23456789`, DOI `10.1002/cncr.27976`) has no exact DOI +match in that local library. This confirms there is enough local test data for a +provider, but a different cached reference is needed for a real positive smoke +test. + +## Paperpile design + +No documented public Paperpile library API was found. The supported integration +should compose two official export/sync mechanisms: + +- A continuously updated BibTeX export, available through a download link, + Google Drive, or GitHub, supplies DOI and citation metadata. +- Google Drive sync supplies PDFs and supplementary files in the user's + Paperpile folder. + +Paperpile also offers a manual JSON export containing attachment metadata, but +the export excludes the PDF bytes and is not documented as a continuously +synced API. It is useful as an optional richer index, not as the primary +automation contract. + +### Phased access modes + +1. **Local synced folder.** Point the validator at a read-only local Paperpile + Google Drive folder plus a synced BibTeX file. This needs no Google OAuth and + is easy to test. +2. **Google Drive API.** Use OAuth read-only access to find and download files for + environments without Drive for desktop. Match them against the synced BibTeX + index. Tokens stay in the platform keychain/secret store, never YAML. +3. **Paperpile API only if officially documented.** Do not build on observed + browser traffic or private endpoints. + +### Paperpile matching + +Build an index from BibTeX DOI/PMID/title/year fields. Associate PDF files using, +in order: + +1. an attachment path present in an explicitly supplied Paperpile JSON export; +2. DOI found in the PDF metadata or first pages; +3. a deterministic Paperpile filename/title match, confirmed by year and author. + +Only exact identifiers should auto-enrich in the first release. Metadata-based +matches appear in the dry-run report for user review. + +## Browsing a cache as a library + +Cache-to-Zotero export is a useful companion feature. The safest first version +should generate a standard CSL JSON or RIS file for manual import into a new +Zotero collection. Each cache entry can +export title, authors, year, DOI/PMID, content type, provider, and cache path. +Private PDFs and extracted full text should be excluded by default; the Zotero +provider already links an enriched entry back to its source attachment key. + +A later `cache sync-zotero` command could create/update a dedicated collection +through Zotero's write API, but that mutates the user's library and needs explicit +authorization, stable cache-to-item identifiers, idempotency, and conflict tests. + +## Cache and rights policy + +Private source content must not be labeled with an OA status. Add these persisted +provenance fields: + +| Field | Example | Purpose | +|---|---|---| +| `full_text_provider` | `zotero` | Existing provider provenance | +| `full_text_access_type` | `user_library` | Separates lawful private access from OA | +| `full_text_source_item_id` | Zotero attachment key | Auditable source identity | +| `full_text_source_checksum` | SHA-256 | Detects replacement without exposing credentials | + +Because the Markdown cache contains extracted text, private content is stored in +a separate research cache at `~/.cache/linkml-reference-validator/private` by +default, with owner-only directory and file permissions. Users may explicitly +point it at a private repository. Validation never reads this cache. A private +provider result is also rejected by the ordinary provider chain, which then +continues to public providers. + +## Future signed excerpt attestations + +Private-backed validation, if enabled later, must be reproducible without giving +the manuscript or a signing credential to an agent or CI. A trusted offline job +would be the only writer of an artifact such as +`CACHEDIR/PMID_12345678/excerpts.yaml`. It would: + +1. read the private research cache and the claims to validate; +2. select only the minimal supporting or contradicting excerpt; +3. record the reference ID, claim hash, normalized excerpt hash, source-file + checksum, location, outcome, tool version, timestamp, and signing-key ID; +4. sign a canonical serialization with an offline Ed25519 private key; and +5. copy the attestation—not the PDF or full extracted text—into the public cache. + +The repository contains only the public verification key. The validator accepts +an attestation only when its signature is valid and all bound identifiers and +hashes match the current claim. Editing or moving fields invalidates the +signature. Agents are forbidden from writing these artifacts directly and, more +importantly, cannot create a valid one because they never receive the private +key. Unsigned or invalid artifacts are ignored by default and become hard errors +when a future private-attestation mode is explicitly requested. + +The exact excerpt length and redistribution policy need review before this mode +is implemented. Until then, signed attestations are design-only and all +validation evidence comes from the public cache. + +## CLI shape + +The first command is an inventory-first operation: + +```bash +linkml-reference-validator cache enrich \ + --provider zotero \ + --cache-dir references_cache \ + --dry-run +``` + +Output is one row per cached reference: + +```text +REFERENCE RESULT MATCH ATTACHMENT +PMID:12345678 found doi zotero:ABCD1234 +DOI:10.example/x not_found - - +DOI:10.example/y ambiguous title/year - +``` + +Without `--dry-run`, exact matches are written to the separate private research cache +through the same materialization path used by other providers. The public source +cache is never modified. The command exits nonzero only for +configuration/provider errors, not ordinary library misses. + +## Security and operational rules + +- Read-only by default; no library or Drive writes. +- Tokens only through environment variables or a credential provider. +- Redact authorization headers and signed URLs from logs and cache frontmatter. +- Never serialize a localhost/session URL as a durable source URL. +- Apply existing download size limits to local files as well as HTTP downloads. +- Verify `%PDF-` magic bytes and compute SHA-256 before extraction. +- Preserve provider errors as retryable; a normal library miss is definitive for + that index version. +- Do not fall through to a lower-confidence private match automatically. + +## Sources + +- [Zotero Web API v3 and local API](https://www.zotero.org/support/dev/web_api/v3/basics) +- [Zotero attachment file download](https://www.zotero.org/support/dev/web_api/v3/file_upload) +- [Zotero full-text content API](https://www.zotero.org/support/dev/web_api/v3/fulltext_content) +- [Paperpile automatic BibTeX sync](https://paperpile.com/h/sync-bibtex-files/) +- [Paperpile library data export](https://paperpile.com/h/export-library-data/) +- [Paperpile Google Drive sync](https://paperpile.com/h/sync-google-drive/) +- [Zenodo REST API](https://developers.zenodo.org/) diff --git a/mkdocs.yml b/mkdocs.yml index 5f82008..e05f7d7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,6 +42,7 @@ nav: - Validating Reference Titles: how-to/validate-titles.md - Using Local Files and URLs: how-to/use-local-files-and-urls.md - Fetching Full Text and PDFs: how-to/fetch-full-text-and-pdfs.md + - Exporting a Cache to Zotero: how-to/export-cache-to-zotero.md - Adding a New Reference Source: how-to/add-reference-source.md - Skipping Unsupported References: how-to/skip-unsupported-references.md - Repairing Validation Errors: how-to/repair-validation-errors.md diff --git a/src/linkml_reference_validator/cli/cache.py b/src/linkml_reference_validator/cli/cache.py index b179b4e..7f8e5df 100644 --- a/src/linkml_reference_validator/cli/cache.py +++ b/src/linkml_reference_validator/cli/cache.py @@ -1,11 +1,19 @@ """Cache subcommands for linkml-reference-validator.""" +import json import logging +from pathlib import Path +from typing import Optional import typer from typing_extensions import Annotated from linkml_reference_validator.etl.reference_fetcher import ReferenceFetcher +from linkml_reference_validator.etl.csl_export import ( + export_identity, + reference_to_csl_json, +) +from linkml_reference_validator.etl.fulltext.base import FullTextProviderRegistry from .shared import ( CacheDirOption, VerboseOption, @@ -16,6 +24,7 @@ ) logger = logging.getLogger(__name__) +MAX_CONSECUTIVE_PROVIDER_ERRORS = 3 # Option for showing file content ContentOption = Annotated[ @@ -41,6 +50,67 @@ no_args_is_help=True, ) +ProviderOption = Annotated[ + str, + typer.Option( + "--provider", + help="Registered full-text provider to use (for example: zotero)", + ), +] + +DryRunOption = Annotated[ + bool, + typer.Option( + "--dry-run/--apply", + help="Report matches without changing cache files, or apply usable matches", + ), +] + +PrivateCacheDirOption = Annotated[ + Optional[Path], + typer.Option( + "--private-cache-dir", + help=( + "Destination for private full text. Defaults outside the project to " + "~/.cache/linkml-reference-validator/private" + ), + ), +] + +ExportOutputOption = Annotated[ + Path, + typer.Option( + "--output", + "-o", + help="Destination CSL JSON file for Zotero import", + ), +] + +ExportFormatOption = Annotated[ + str, + typer.Option( + "--format", + help="Bibliographic export format (currently: csl-json)", + ), +] + +NeedsFullTextOption = Annotated[ + bool, + typer.Option( + "--needs-full-text/--all", + help="Export only records needing full text, or all publication records", + ), +] + +ExportForceOption = Annotated[ + bool, + typer.Option( + "--force", + "-f", + help="Replace the output file if it already exists", + ), +] + @cache_app.command(name="reference") def reference_command( @@ -141,3 +211,163 @@ def lookup_command( typer.echo(cache_path.read_text(encoding="utf-8")) else: typer.echo(str(cache_path.absolute())) + + +@cache_app.command(name="enrich") +def enrich_command( + provider: ProviderOption = "zotero", + config_file: ConfigFileOption = None, + cache_dir: CacheDirOption = None, + private_cache_dir: PrivateCacheDirOption = None, + dry_run: DryRunOption = True, + verbose: VerboseOption = False, +): + """Inventory or enrich cached references from one full-text provider. + + Dry-run is the default and never changes cache files. Use ``--apply`` only + after reviewing matches; private full text may be copyrighted and should not + be committed or shared. + + Examples: + + linkml-reference-validator cache enrich --provider zotero --dry-run + + linkml-reference-validator cache enrich --provider zotero --apply + """ + setup_logging(verbose) + + if FullTextProviderRegistry.get(provider) is None: + typer.echo(f"Unknown full-text provider: {provider}", err=True) + raise typer.Exit(2) + + config = load_validation_config(config_file) + if cache_dir: + config.cache_dir = cache_dir + if private_cache_dir: + config.private_cache_dir = private_cache_dir + fetcher = ReferenceFetcher(config) + + found = 0 + applied = 0 + errors = 0 + scanned = 0 + consecutive_errors = 0 + for reference in fetcher.iter_cached_references(): + scanned += 1 + if not fetcher.needs_full_text(reference): + typer.echo(f"{reference.reference_id}\talready_full_text\t-") + continue + try: # provider is an external-system boundary + location = fetcher.locate_full_text(reference, provider) + except Exception as exc: + errors += 1 + consecutive_errors += 1 + typer.echo(f"{reference.reference_id}\terror\t{exc}") + if consecutive_errors >= MAX_CONSECUTIVE_PROVIDER_ERRORS: + typer.echo( + "Stopping after " + f"{MAX_CONSECUTIVE_PROVIDER_ERRORS} consecutive provider errors; " + "check that the provider is available.", + err=True, + ) + break + continue + + consecutive_errors = 0 + + if location is None: + typer.echo(f"{reference.reference_id}\tnot_found\t-") + continue + + found += 1 + source = f"{location.provider or provider}:{location.source_item_id or '-'}" + if dry_run: + typer.echo(f"{reference.reference_id}\tfound\t{source}") + continue + + if fetcher.apply_full_text_location( + reference, location, provider, private=True + ): + applied += 1 + typer.echo(f"{reference.reference_id}\tapplied\t{source}") + else: + typer.echo(f"{reference.reference_id}\tunusable\t{source}") + + typer.echo(f"Scanned: {scanned}") + typer.echo(f"Found: {found}") + if not dry_run: + typer.echo(f"Applied: {applied}") + typer.echo(f"Private cache: {config.private_cache_dir.expanduser()}") + if errors: + typer.echo(f"Errors: {errors}", err=True) + raise typer.Exit(1) + + +@cache_app.command(name="export") +def export_command( + output: ExportOutputOption, + export_format: ExportFormatOption = "csl-json", + needs_full_text: NeedsFullTextOption = True, + config_file: ConfigFileOption = None, + cache_dir: CacheDirOption = None, + force: ExportForceOption = False, + verbose: VerboseOption = False, +): + """Export public bibliographic metadata for import into Zotero. + + The export is an allowlisted CSL JSON projection. It never contains cached + article text, excerpts, PDFs, local paths, or private-cache data. By default, + only DOI/PMID records that still need full text are included. + """ + setup_logging(verbose) + + if export_format != "csl-json": + typer.echo(f"Unsupported export format: {export_format}", err=True) + raise typer.Exit(2) + if output.exists() and not force: + typer.echo( + f"Output already exists: {output}. Use --force to replace it.", + err=True, + ) + raise typer.Exit(2) + + config = load_validation_config(config_file) + if cache_dir: + config.cache_dir = cache_dir + fetcher = ReferenceFetcher(config) + records: list[dict[str, object]] = [] + seen: set[str] = set() + duplicates = 0 + skipped_full_text = 0 + skipped_identifier = 0 + scanned = 0 + for reference in fetcher.iter_cached_references(): + scanned += 1 + if needs_full_text and not fetcher.needs_full_text(reference): + skipped_full_text += 1 + continue + identity = export_identity(reference) + if identity is None: + skipped_identifier += 1 + continue + if identity in seen: + duplicates += 1 + continue + record = reference_to_csl_json(reference) + if record is None: + skipped_identifier += 1 + continue + seen.add(identity) + records.append(record) + + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(records, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + typer.echo(f"Scanned: {scanned}") + typer.echo(f"Exported: {len(records)}") + typer.echo(f"Duplicates: {duplicates}") + typer.echo(f"Skipped with full text: {skipped_full_text}") + typer.echo(f"Skipped without DOI/PMID: {skipped_identifier}") + typer.echo(f"Output: {output}") diff --git a/src/linkml_reference_validator/etl/acquire.py b/src/linkml_reference_validator/etl/acquire.py index 32ee2af..4a8e889 100644 --- a/src/linkml_reference_validator/etl/acquire.py +++ b/src/linkml_reference_validator/etl/acquire.py @@ -2,7 +2,10 @@ import logging import time +from pathlib import Path from typing import Optional +from urllib.parse import urljoin, urlparse +from urllib.request import url2pathname import requests # type: ignore @@ -105,6 +108,15 @@ def fetch_bytes( Returns ``(None, content_type)`` on non-200 responses or when the size cap is exceeded. """ + return self._fetch_bytes(url, config, redirects_remaining=5) + + def _fetch_bytes( + self, + url: str, + config: ReferenceValidationConfig, + redirects_remaining: int, + ) -> tuple[Optional[bytes], Optional[str]]: + """Download bytes while handling bounded HTTP and Zotero file redirects.""" time.sleep(config.rate_limit_delay) headers = { @@ -112,7 +124,40 @@ def fetch_bytes( } # ``with`` guarantees the streamed connection is released on every path, # including the early return when the size cap is exceeded mid-stream. - with requests.get(url, headers=headers, timeout=60, stream=True) as response: + manual_redirects = self._is_loopback_url(url) + with requests.get( + url, + headers=headers, + timeout=60, + stream=True, + allow_redirects=not manual_redirects, + ) as response: + if response.status_code in {301, 302, 303, 307, 308}: + location = response.headers.get("location") + if not location or redirects_remaining == 0: + logger.warning(f"Download redirect failed for {url}") + return None, None + + resolved_location = urljoin(url, location) + parsed_location = urlparse(resolved_location) + if parsed_location.scheme == "file": + if not self._is_loopback_url(url): + logger.warning( + f"Refusing non-local redirect to a file URI from {url}" + ) + return None, None + return self._read_local_file(resolved_location, config) + + if parsed_location.scheme in {"http", "https"}: + return self._fetch_bytes( + resolved_location, + config, + redirects_remaining=redirects_remaining - 1, + ) + + logger.warning(f"Unsupported redirect scheme for {url}") + return None, None + if response.status_code != 200: logger.warning(f"Download failed for {url} - status {response.status_code}") return None, None @@ -132,3 +177,37 @@ def fetch_bytes( return None, content_type return bytes(chunks), content_type + + @staticmethod + def _is_loopback_url(url: str) -> bool: + """Return True only for an HTTP endpoint on the local machine.""" + parsed = urlparse(url) + return parsed.scheme == "http" and parsed.hostname in { + "localhost", + "127.0.0.1", + "::1", + } + + @staticmethod + def _read_local_file( + file_url: str, config: ReferenceValidationConfig + ) -> tuple[Optional[bytes], Optional[str]]: + """Read a local-API-authorized file URI under the normal size cap.""" + parsed = urlparse(file_url) + if parsed.netloc not in {"", "localhost"}: + logger.warning("Refusing a non-local file URI") + return None, None + + path = Path(url2pathname(parsed.path)) + content_type = "application/pdf" if path.suffix.lower() == ".pdf" else None + if not path.is_file(): + logger.warning(f"Local redirected file does not exist: {path}") + return None, content_type + + max_size = config.max_supplementary_file_size + if max_size and path.stat().st_size > max_size: + logger.warning( + f"Local redirected file exceeded size cap ({max_size} bytes); skipping" + ) + return None, content_type + return path.read_bytes(), content_type diff --git a/src/linkml_reference_validator/etl/csl_export.py b/src/linkml_reference_validator/etl/csl_export.py new file mode 100644 index 0000000..feb8e60 --- /dev/null +++ b/src/linkml_reference_validator/etl/csl_export.py @@ -0,0 +1,73 @@ +"""Convert public reference-cache metadata to Zotero-importable CSL JSON.""" + +import re +from typing import Optional + +from linkml_reference_validator.etl.identifiers import normalize_doi +from linkml_reference_validator.models import ReferenceContent + + +def reference_pmid(reference: ReferenceContent) -> Optional[str]: + """Return a numeric PMID when the reference ID is a PubMed identifier. + + Examples: + >>> reference_pmid(ReferenceContent(reference_id="PMID:12345")) + '12345' + >>> reference_pmid(ReferenceContent(reference_id="NCIT:C123")) is None + True + """ + prefix, separator, identifier = reference.reference_id.partition(":") + if separator and prefix.upper() == "PMID" and identifier.isdigit(): + return identifier + return None + + +def export_identity(reference: ReferenceContent) -> Optional[str]: + """Return the DOI-first identity used to deduplicate exported records.""" + doi = normalize_doi(reference.doi) + if doi: + return f"doi:{doi}" + pmid = reference_pmid(reference) + if pmid: + return f"pmid:{pmid}" + return None + + +def reference_to_csl_json(reference: ReferenceContent) -> Optional[dict[str, object]]: + """Map one publication to an allowlisted CSL JSON metadata record. + + Cached content, provenance, local paths, and attachments are deliberately + unavailable to this function and therefore cannot enter its output. + + Examples: + >>> reference_to_csl_json(ReferenceContent( + ... reference_id="PMID:123", title="A paper", year="2024" + ... )) + {'id': 'PMID:123', 'type': 'article-journal', 'title': 'A paper', 'issued': {'date-parts': [[2024]]}, 'PMID': '123', 'URL': 'https://pubmed.ncbi.nlm.nih.gov/123/'} + """ + doi = normalize_doi(reference.doi) + pmid = reference_pmid(reference) + if doi is None and pmid is None: + return None + + record: dict[str, object] = { + "id": reference.reference_id, + "type": "article-journal", + } + if reference.title: + record["title"] = reference.title + if reference.authors: + record["author"] = [{"literal": author} for author in reference.authors] + if reference.journal: + record["container-title"] = reference.journal + if reference.year: + match = re.match(r"\s*(\d{4})", reference.year) + if match: + record["issued"] = {"date-parts": [[int(match.group(1))]]} + if doi: + record["DOI"] = doi + if pmid: + record["PMID"] = pmid + if pmid and not doi: + record["URL"] = f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/" + return record diff --git a/src/linkml_reference_validator/etl/fulltext/__init__.py b/src/linkml_reference_validator/etl/fulltext/__init__.py index 5ff3b57..f309ef9 100644 --- a/src/linkml_reference_validator/etl/fulltext/__init__.py +++ b/src/linkml_reference_validator/etl/fulltext/__init__.py @@ -10,6 +10,7 @@ from linkml_reference_validator.etl.fulltext.epmc_preprint import EuropePMCPreprintProvider from linkml_reference_validator.etl.fulltext.unpaywall import UnpaywallProvider from linkml_reference_validator.etl.fulltext.openalex import OpenAlexProvider +from linkml_reference_validator.etl.fulltext.zotero import ZoteroFullTextProvider __all__ = [ "FullTextProvider", @@ -18,4 +19,5 @@ "EuropePMCPreprintProvider", "UnpaywallProvider", "OpenAlexProvider", + "ZoteroFullTextProvider", ] diff --git a/src/linkml_reference_validator/etl/fulltext/zotero.py b/src/linkml_reference_validator/etl/fulltext/zotero.py new file mode 100644 index 0000000..110d600 --- /dev/null +++ b/src/linkml_reference_validator/etl/fulltext/zotero.py @@ -0,0 +1,252 @@ +"""Locate user-authorized full text through Zotero's read-only API. + +The first implementation targets Zotero's supported local API. It builds a +process-local exact identifier index, finds PDF children of an unambiguous +parent item, and prefers text Zotero has already indexed before returning the +attachment's file endpoint for normal PDF extraction. +""" + +import re +from collections import defaultdict +from typing import Optional + +import requests # type: ignore + +from linkml_reference_validator.etl.fulltext.base import ( + FullTextProvider, + FullTextProviderRegistry, +) +from linkml_reference_validator.etl.identifiers import normalize_doi +from linkml_reference_validator.models import ( + FullTextLocation, + ReferenceIdentifiers, + ReferenceValidationConfig, +) + +MIN_ZOTERO_INDEXED_TEXT_CHARS = 500 +_EXTRA_IDENTIFIER_PATTERNS = { + "pmid": re.compile(r"^PMID\s*:\s*(\d+)\s*$", re.IGNORECASE | re.MULTILINE), + "pmcid": re.compile( + r"^PMCID\s*:\s*(PMC\d+)\s*$", re.IGNORECASE | re.MULTILINE + ), +} + + +class ZoteroAPIError(RuntimeError): + """Raised when Zotero's API cannot complete a read operation.""" + + +class ZoteroClient: + """Small read-only client for the Zotero v3 item API.""" + + def __init__( + self, + base_url: str, + session: Optional[requests.Session] = None, + page_size: int = 100, + ): + """Initialize the client with a library base URL and optional session.""" + self.base_url = base_url.rstrip("/") + self._session = session or requests.Session() + self._page_size = page_size + self._identifier_index: Optional[dict[tuple[str, str], set[str]]] = None + + def find_parent_keys(self, ids: ReferenceIdentifiers) -> set[str]: + """Return unambiguous parent candidates matching supplied identifiers. + + Identifier types absent from Zotero metadata are ignored. When two + supplied identifier types match different Zotero records, the empty set + is returned instead of choosing either record. + """ + index = self._get_identifier_index() + requested = { + "doi": normalize_doi(ids.doi), + "pmid": ids.pmid.strip() if ids.pmid else None, + "pmcid": ids.pmcid.strip().upper() if ids.pmcid else None, + } + matched_sets = [ + index[(kind, value)] + for kind, value in requested.items() + if value and index.get((kind, value)) + ] + if not matched_sets: + return set() + candidates = set(matched_sets[0]) + for matches in matched_sets[1:]: + candidates.intersection_update(matches) + return candidates + + def pdf_attachments(self, parent_key: str) -> list[dict]: + """Return PDF attachment items belonging to a parent item.""" + items = self._get_json(f"items/{parent_key}/children") + if not isinstance(items, list): + raise ZoteroAPIError("Zotero children response was not a list") + return [ + item + for item in items + if isinstance(item, dict) + and isinstance(item.get("data"), dict) + and item["data"].get("itemType") == "attachment" + and item["data"].get("contentType") == "application/pdf" + ] + + def indexed_full_text(self, attachment_key: str) -> Optional[str]: + """Return Zotero-indexed attachment text, or None when unavailable.""" + response = self._session.get( + f"{self.base_url}/items/{attachment_key}/fulltext", + headers={"Zotero-API-Version": "3"}, + timeout=30, + ) + if response.status_code == 404: + return None + self._require_success(response.status_code, "full-text lookup") + data = response.json() + if not isinstance(data, dict): + raise ZoteroAPIError("Zotero full-text response was not an object") + content = data.get("content") + return content if isinstance(content, str) and content.strip() else None + + def attachment_file_url(self, attachment_key: str) -> str: + """Return the supported Zotero attachment download endpoint.""" + return f"{self.base_url}/items/{attachment_key}/file" + + def _get_identifier_index(self) -> dict[tuple[str, str], set[str]]: + """Load and cache exact identifiers for top-level Zotero items.""" + if self._identifier_index is not None: + return self._identifier_index + + items = self._get_paginated_items("items/top") + + index: dict[tuple[str, str], set[str]] = defaultdict(set) + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("data"), dict): + continue + data = item["data"] + key = data.get("key") or item.get("key") + if not isinstance(key, str): + continue + + doi = normalize_doi(data.get("DOI")) + if doi: + index[("doi", doi)].add(key) + + extra = data.get("extra", "") + if isinstance(extra, str): + for kind, pattern in _EXTRA_IDENTIFIER_PATTERNS.items(): + match = pattern.search(extra) + if match: + value = match.group(1) + if kind == "pmcid": + value = value.upper() + index[(kind, value)].add(key) + + self._identifier_index = dict(index) + return self._identifier_index + + def _get_paginated_items(self, path: str) -> list[object]: + """GET every page from a Zotero collection endpoint.""" + items: list[object] = [] + start = 0 + while True: + response = self._session.get( + f"{self.base_url}/{path.lstrip('/')}", + headers={"Zotero-API-Version": "3"}, + params={"limit": self._page_size, "start": start}, + timeout=30, + ) + self._require_success(response.status_code, path) + page = response.json() + if not isinstance(page, list): + raise ZoteroAPIError("Zotero items response was not a list") + items.extend(page) + + total_header = response.headers.get("Total-Results") + total = int(total_header) if total_header is not None else None + start += len(page) + if not page or (total is not None and start >= total): + break + if total is None and len(page) < self._page_size: + break + return items + + def _get_json(self, path: str) -> object: + """GET a Zotero JSON resource and require a successful response.""" + response = self._session.get( + f"{self.base_url}/{path.lstrip('/')}", + headers={"Zotero-API-Version": "3"}, + timeout=30, + ) + self._require_success(response.status_code, path) + return response.json() + + @staticmethod + def _require_success(status_code: int, operation: str) -> None: + """Raise a descriptive error for a non-successful Zotero response.""" + if status_code != 200: + raise ZoteroAPIError( + f"Zotero API returned {status_code} during {operation}" + ) + + +@FullTextProviderRegistry.register +class ZoteroFullTextProvider(FullTextProvider): + """Find full text in a user's Zotero library by exact identifier.""" + + def __init__(self, client: Optional[ZoteroClient] = None): + """Initialize with an optional client for testing or custom embedding.""" + self._client = client + self._client_is_injected = client is not None + self._client_base_url = client.base_url if client is not None else None + + @classmethod + def name(cls) -> str: + """Return the provider-chain name.""" + return "zotero" + + def locate( + self, ids: ReferenceIdentifiers, config: ReferenceValidationConfig + ) -> Optional[FullTextLocation]: + """Return indexed text or a PDF endpoint for an exact Zotero match.""" + if self._client is None or ( + not self._client_is_injected + and self._client_base_url != config.zotero_base_url.rstrip("/") + ): + self._client = ZoteroClient(config.zotero_base_url) + self._client_base_url = config.zotero_base_url.rstrip("/") + client = self._client + parent_keys = client.find_parent_keys(ids) + if len(parent_keys) != 1: + return None + + parent_key = next(iter(parent_keys)) + attachments = client.pdf_attachments(parent_key) + if not attachments: + return None + + keyed_attachments = [] + for attachment in attachments: + data = attachment["data"] + attachment_key = data.get("key") or attachment.get("key") + if isinstance(attachment_key, str): + keyed_attachments.append((attachment_key, attachment)) + if not keyed_attachments: + return None + attachment_key, _ = min(keyed_attachments, key=lambda pair: pair[0]) + + text = client.indexed_full_text(attachment_key) + if text and len(text.strip()) >= MIN_ZOTERO_INDEXED_TEXT_CHARS: + return FullTextLocation( + text=text, + format_hint="text", + provider=self.name(), + access_type="user_library", + source_item_id=attachment_key, + ) + + return FullTextLocation( + url=client.attachment_file_url(attachment_key), + format_hint="pdf", + provider=self.name(), + access_type="user_library", + source_item_id=attachment_key, + ) diff --git a/src/linkml_reference_validator/etl/identifiers.py b/src/linkml_reference_validator/etl/identifiers.py index 0157bf6..167ff7d 100644 --- a/src/linkml_reference_validator/etl/identifiers.py +++ b/src/linkml_reference_validator/etl/identifiers.py @@ -14,6 +14,25 @@ logger = logging.getLogger(__name__) +def normalize_doi(value: Optional[str]) -> Optional[str]: + """Return a lowercase bare DOI suitable for exact identity matching. + + Examples: + >>> normalize_doi(" https://doi.org/10.1000/Example ") + '10.1000/example' + >>> normalize_doi("doi:10.1000/ABC") + '10.1000/abc' + >>> normalize_doi(None) is None + True + """ + if not value: + return None + normalized = value.strip().lower() + normalized = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", normalized) + normalized = re.sub(r"^doi\s*:\s*", "", normalized) + return normalized or None + + def _split_reference_id(reference_id: str) -> tuple[Optional[str], Optional[str]]: """Split a reference id into (prefix, identifier). diff --git a/src/linkml_reference_validator/etl/reference_fetcher.py b/src/linkml_reference_validator/etl/reference_fetcher.py index ce4a2c3..6805527 100644 --- a/src/linkml_reference_validator/etl/reference_fetcher.py +++ b/src/linkml_reference_validator/etl/reference_fetcher.py @@ -6,6 +6,7 @@ import logging import re +from collections.abc import Iterator from pathlib import Path from typing import Any, Optional @@ -146,27 +147,27 @@ def fetch( source = source_class() content = source.fetch(identifier, self.config) - if content and self.config.fetch_full_text and self._needs_full_text(content): + if content and self.config.fetch_full_text and self.needs_full_text(content): content = self._enrich_with_full_text(content) if content: self._cache[normalized_reference_id] = content - self._save_to_disk(content) + self._save_by_access(content) return content - def _needs_full_text(self, content: ReferenceContent) -> bool: + def needs_full_text(self, content: ReferenceContent) -> bool: """Return True if the content lacks full text and the chain should run. Examples: >>> config = ReferenceValidationConfig() >>> fetcher = ReferenceFetcher(config) >>> from linkml_reference_validator.models import ReferenceContent - >>> fetcher._needs_full_text( + >>> fetcher.needs_full_text( ... ReferenceContent(reference_id="DOI:1", content_type="abstract_only") ... ) True - >>> fetcher._needs_full_text( + >>> fetcher.needs_full_text( ... ReferenceContent(reference_id="DOI:1", content_type="full_text_xml") ... ) False @@ -184,7 +185,7 @@ def _maybe_retry_full_text(self, content: ReferenceContent) -> ReferenceContent: """ if ( not self.config.fetch_full_text - or not self._needs_full_text(content) + or not self.needs_full_text(content) or content.full_text_attempted ): return content @@ -193,16 +194,30 @@ def _maybe_retry_full_text(self, content: ReferenceContent) -> ReferenceContent: content = self._enrich_with_full_text(content) after = (content.content, content.content_type, content.full_text_attempted) if after != before: - self._save_to_disk(content) + self._save_by_access(content) return content + def _save_by_access(self, content: ReferenceContent) -> None: + """Persist content according to its access provenance. + + Ordinary enrichment rejects private locations before this point. The + private branch remains a defensive safeguard for callers handling an + explicitly private ``ReferenceContent`` outside validation. + """ + self._save_to_disk( + content, + private=content.full_text_access_type not in (None, "open"), + ) + def _enrich_with_full_text(self, content: ReferenceContent) -> ReferenceContent: - """Walk the provider chain; merge the first usable full text into content. + """Merge the first usable public full text from the provider chain. If no provider yields usable full text but the chain was consulted without a transient error, mark ``full_text_attempted`` so the record is not re-queried on every later run. A provider/download error leaves the flag unset so a - subsequent run retries (PR #48 review #1). + subsequent run retries (PR #48 review #1). Locations with an explicit + non-open access type are ignored: private-library material is available to + the separate cache-enrichment workflow, never to ordinary validation. """ ids = build_identifiers(content) abstract = content.content @@ -224,22 +239,21 @@ def _enrich_with_full_text(self, content: ReferenceContent) -> ReferenceContent: if location is None: continue - text, fmt, pdf_bytes, error = self._materialize(location) - if error: - had_error = True - if not text or len(text.strip()) < MIN_FULL_TEXT_CHARS: + if location.access_type not in (None, "open"): + logger.info( + "Ignoring non-public full text from provider '%s' for %s", + provider_name, + content.reference_id, + ) continue - content.content = f"{abstract}\n\n{text}" if abstract else text - content.content_type = _FORMAT_TO_CONTENT_TYPE.get(fmt or "text", "full_text") - content.full_text_provider = location.provider or provider_name - content.full_text_url = location.url - content.oa_status = location.oa_status - content.license = location.license - content.full_text_attempted = True - if pdf_bytes is not None and self.config.download_pdfs: - content.local_pdf_path = self._save_pdf(content.reference_id, pdf_bytes) - return content + applied, error = self._apply_full_text_location( + content, abstract, location, provider_name + ) + if error: + had_error = True + if applied: + return content # No usable full text: only record a definitive attempt if nothing went wrong, # so a transient failure stays retryable on the next run. @@ -247,6 +261,83 @@ def _enrich_with_full_text(self, content: ReferenceContent) -> ReferenceContent: content.full_text_attempted = True return content + def locate_full_text( + self, content: ReferenceContent, provider_name: str + ) -> Optional[FullTextLocation]: + """Locate full text for cached content using exactly one provider. + + This is the inventory primitive used by ``cache enrich --dry-run``. It + deliberately ignores ``full_text_attempted`` because a newly configured + private library may contain a manuscript that public providers missed. + + Raises: + ValueError: If ``provider_name`` is not registered. + """ + provider = FullTextProviderRegistry.get(provider_name) + if provider is None: + raise ValueError(f"Unknown full-text provider: {provider_name}") + return provider.locate(build_identifiers(content), self.config) + + def apply_full_text_location( + self, + content: ReferenceContent, + location: FullTextLocation, + provider_name: str, + private: bool = False, + ) -> bool: + """Materialize one located resource, update content, and persist it.""" + private = private or location.access_type not in (None, "open") + abstract = content.content + applied, _ = self._apply_full_text_location( + content, abstract, location, provider_name, private=private + ) + if applied: + if not private: + normalized_id = self.normalize_reference_id(content.reference_id) + self._cache[normalized_id] = content + self._save_to_disk(content, private=private) + return applied + + def iter_cached_references(self) -> Iterator[ReferenceContent]: + """Yield modern Markdown cache entries in deterministic path order.""" + for cache_path in sorted(self.config.get_cache_dir().glob("*.md")): + content_text = cache_path.read_text(encoding="utf-8") + reference = self._load_markdown_format(content_text, cache_path.stem) + if reference is not None: + yield reference + + def _apply_full_text_location( + self, + content: ReferenceContent, + abstract: Optional[str], + location: FullTextLocation, + provider_name: str, + private: bool = False, + ) -> tuple[bool, bool]: + """Apply one location and return ``(applied, transient_error)``.""" + text, fmt, pdf_bytes, error = self._materialize(location) + if not text or len(text.strip()) < MIN_FULL_TEXT_CHARS: + return False, error + + content.content = f"{abstract}\n\n{text}" if abstract else text + content.content_type = _FORMAT_TO_CONTENT_TYPE.get(fmt or "text", "full_text") + content.full_text_provider = location.provider or provider_name + # Non-public endpoints are not durable provenance and may contain + # session-specific access information. + content.full_text_url = ( + None if location.access_type not in (None, "open") else location.url + ) + content.oa_status = location.oa_status + content.license = location.license + content.full_text_access_type = location.access_type + content.full_text_source_item_id = location.source_item_id + content.full_text_attempted = True + if pdf_bytes is not None and self.config.download_pdfs: + content.local_pdf_path = self._save_pdf( + content.reference_id, pdf_bytes, private=private + ) + return True, error + def _materialize( self, location: FullTextLocation ) -> tuple[Optional[str], Optional[str], Optional[bytes], bool]: @@ -293,14 +384,23 @@ def _materialize( pdf_bytes = data if fmt == "pdf" else None return text, fmt, pdf_bytes, False - def _save_pdf(self, reference_id: str, data: bytes) -> str: + def _save_pdf( + self, reference_id: str, data: bytes, private: bool = False + ) -> str: """Persist a downloaded PDF and return its path relative to the cache dir.""" safe_id = ( reference_id.replace(":", "_").replace("/", "_").replace("?", "_").replace("=", "_") ) - files_dir = self.config.get_files_cache_dir() + files_dir = ( + self.config.get_private_files_cache_dir() + if private + else self.config.get_files_cache_dir() + ) pdf_path = files_dir / f"{safe_id}.pdf" pdf_path.write_bytes(data) + if private: + pdf_path.chmod(0o600) + return str(pdf_path.relative_to(self.config.get_private_cache_dir())) return str(pdf_path.relative_to(self.config.cache_dir)) def _parse_reference_id(self, reference_id: str) -> tuple[str, str]: @@ -410,8 +510,17 @@ def get_cache_path(self, reference_id: str) -> Path: >>> path.name 'url_https___example.com_book_chapter1.md' """ - safe_id = reference_id.replace(":", "_").replace("/", "_").replace("?", "_").replace("=", "_") - cache_dir = self.config.get_cache_dir() + return self._cache_path(reference_id, self.config.get_cache_dir()) + + @staticmethod + def _cache_path(reference_id: str, cache_dir: Path) -> Path: + """Return a cache path under an already selected cache directory.""" + safe_id = ( + reference_id.replace(":", "_") + .replace("/", "_") + .replace("?", "_") + .replace("=", "_") + ) return cache_dir / f"{safe_id}.md" def _quote_yaml_value(self, value: str) -> str: @@ -464,13 +573,20 @@ def _quote_yaml_value(self, value: str) -> str: return value - def _save_to_disk(self, reference: ReferenceContent) -> None: + def _save_to_disk( + self, reference: ReferenceContent, private: bool = False + ) -> None: """Save reference content to disk cache as markdown with YAML frontmatter. Args: reference: Reference content to save """ - cache_path = self.get_cache_path(reference.reference_id) + cache_path = self._cache_path( + reference.reference_id, + self.config.get_private_cache_dir() + if private + else self.config.get_cache_dir(), + ) lines = [] lines.append("---") @@ -514,6 +630,16 @@ def _save_to_disk(self, reference: ReferenceContent) -> None: lines.append(f"license: {self._quote_yaml_value(reference.license)}") if reference.local_pdf_path: lines.append(f"local_pdf_path: {self._quote_yaml_value(reference.local_pdf_path)}") + if reference.full_text_access_type: + lines.append( + "full_text_access_type: " + f"{self._quote_yaml_value(reference.full_text_access_type)}" + ) + if reference.full_text_source_item_id: + lines.append( + "full_text_source_item_id: " + f"{self._quote_yaml_value(reference.full_text_source_item_id)}" + ) if reference.metadata and "extra_fields_captured" in reference.metadata: extra_fields = reference.metadata.get("extra_fields_captured") if isinstance(extra_fields, list): @@ -567,12 +693,16 @@ def _save_to_disk(self, reference: ReferenceContent) -> None: lines.append(reference.content) cache_path.write_text("\n".join(lines), encoding="utf-8") + if private: + cache_path.chmod(0o600) logger.info(f"Cached {reference.reference_id} to {cache_path}") def _load_from_disk(self, reference_id: str) -> Optional[ReferenceContent]: - """Load reference content from disk cache. + """Load reference content from the public validation cache. Supports both new markdown format with YAML frontmatter and legacy text format. + Private research caches are intentionally excluded so validation results are + reproducible across local machines and CI. Args: reference_id: Reference identifier @@ -671,6 +801,8 @@ def _load_markdown_format( oa_status=frontmatter.get("oa_status"), license=frontmatter.get("license"), local_pdf_path=frontmatter.get("local_pdf_path"), + full_text_access_type=frontmatter.get("full_text_access_type"), + full_text_source_item_id=frontmatter.get("full_text_source_item_id"), is_preprint=frontmatter.get("is_preprint"), peer_review_status=frontmatter.get("peer_review_status"), full_text_attempted=bool(frontmatter.get("full_text_attempted", False)), diff --git a/src/linkml_reference_validator/models.py b/src/linkml_reference_validator/models.py index 682247e..808cf45 100644 --- a/src/linkml_reference_validator/models.py +++ b/src/linkml_reference_validator/models.py @@ -363,6 +363,18 @@ class ReferenceValidationConfig(BaseModel): default=Path("references_cache"), description="Directory for caching downloaded references", ) + private_cache_dir: Path = Field( + default_factory=lambda: Path.home() + / ".cache" + / "linkml-reference-validator" + / "private", + description=( + "Separate private research cache for user-library full text. It is " + "never read by ordinary validation and defaults " + "outside the current project so closed manuscripts are not added " + "to a checked-in public reference cache." + ), + ) reference_base_dir: Optional[Path] = Field( default=None, description="Base directory for resolving relative file: references. If None, uses CWD.", @@ -482,6 +494,13 @@ class ReferenceValidationConfig(BaseModel): default=None, description="Optional path to a YAML file defining custom full-text providers.", ) + zotero_base_url: str = Field( + default="http://localhost:23119/api/users/0", + description=( + "Base URL for the read-only Zotero local API library. " + "Zotero must be running with local API access enabled." + ), + ) def get_cache_dir(self) -> Path: """Create and return the cache directory. @@ -512,6 +531,24 @@ def get_files_cache_dir(self) -> Path: files_dir.mkdir(parents=True, exist_ok=True) return files_dir + def get_private_cache_dir(self) -> Path: + """Create and return the owner-only private research-cache directory.""" + private_dir = self.private_cache_dir.expanduser() + private_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + mode = private_dir.stat().st_mode & 0o777 + if mode & 0o077: + private_dir.chmod(mode & 0o700) + return private_dir + + def get_private_files_cache_dir(self) -> Path: + """Create and return the owner-only private binary-files directory.""" + files_dir = self.get_private_cache_dir() / "files" + files_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + mode = files_dir.stat().st_mode & 0o777 + if mode & 0o077: + files_dir.chmod(mode & 0o700) + return files_dir + @dataclass class JSONAPISourceConfig: @@ -663,6 +700,8 @@ class FullTextLocation: license: Optional[str] = None provider: str = "" version: Optional[str] = None # "publishedVersion" | "acceptedVersion" | ... + access_type: Optional[str] = None # "open" | "user_library" | "institutional" + source_item_id: Optional[str] = None @dataclass @@ -719,6 +758,8 @@ class ReferenceContent: oa_status: Optional[str] = None license: Optional[str] = None local_pdf_path: Optional[str] = None + full_text_access_type: Optional[str] = None + full_text_source_item_id: Optional[str] = None # Preprint / peer-review status, surfaced so downstream KBs can apply policies # such as "a preprint may not be the sole support for a claim". Left as None # when the publication type is unknown; only asserted when positively detected diff --git a/tests/test_acquire.py b/tests/test_acquire.py index 8d896da..87e3583 100644 --- a/tests/test_acquire.py +++ b/tests/test_acquire.py @@ -69,6 +69,7 @@ def test_fetch_bytes_returns_content_and_type(mock_get, tmp_path): data, ctype = ContentAcquirer().fetch_bytes("https://x/y.pdf", config) assert data == b"%PDF-" assert ctype == "application/pdf" + assert mock_get.call_args.kwargs["allow_redirects"] is True @patch("linkml_reference_validator.etl.acquire.requests.get") @@ -114,3 +115,110 @@ def test_fetch_bytes_non_200_returns_none(mock_get, tmp_path): config = ReferenceValidationConfig(cache_dir=tmp_path / "cache", rate_limit_delay=0.0) data, ctype = ContentAcquirer().fetch_bytes("https://x/missing.pdf", config) assert data is None + + +@patch("linkml_reference_validator.etl.acquire.requests.get") +def test_fetch_bytes_follows_zotero_file_redirect(mock_get, tmp_path): + """A Zotero local-API redirect to an authorized local file is readable.""" + pdf_path = tmp_path / "paper with spaces.pdf" + pdf_path.write_bytes(b"%PDF-1.7\nlocal zotero attachment") + mock_get.return_value = _cm_response( + status_code=302, + headers={"location": pdf_path.as_uri()}, + ) + config = ReferenceValidationConfig(cache_dir=tmp_path / "cache", rate_limit_delay=0.0) + + data, content_type = ContentAcquirer().fetch_bytes( + "http://localhost:23119/api/users/0/items/ABC/file", config + ) + + assert data == b"%PDF-1.7\nlocal zotero attachment" + assert content_type == "application/pdf" + assert mock_get.call_args.kwargs["allow_redirects"] is False + + +@patch("linkml_reference_validator.etl.acquire.requests.get") +def test_fetch_bytes_applies_size_cap_to_zotero_local_file(mock_get, tmp_path): + """Local-library files obey the same size limit as network downloads.""" + pdf_path = tmp_path / "large.pdf" + pdf_path.write_bytes(b"%PDF-" + b"x" * 20) + mock_get.return_value = _cm_response( + status_code=302, + headers={"location": pdf_path.as_uri()}, + ) + config = ReferenceValidationConfig( + cache_dir=tmp_path / "cache", + rate_limit_delay=0.0, + max_supplementary_file_size=10, + ) + + data, content_type = ContentAcquirer().fetch_bytes( + "http://localhost:23119/api/users/0/items/ABC/file", config + ) + + assert data is None + assert content_type == "application/pdf" + + +@patch("linkml_reference_validator.etl.acquire.requests.get") +def test_fetch_bytes_refuses_remote_redirect_to_local_file(mock_get, tmp_path): + """An arbitrary remote server cannot cause a local file to be read.""" + pdf_path = tmp_path / "private.pdf" + pdf_path.write_bytes(b"%PDF-private") + mock_get.return_value = _cm_response( + status_code=302, + headers={"location": pdf_path.as_uri()}, + ) + config = ReferenceValidationConfig(cache_dir=tmp_path / "cache", rate_limit_delay=0.0) + + data, content_type = ContentAcquirer().fetch_bytes( + "https://untrusted.example/redirect", config + ) + + assert data is None + assert content_type is None + + +@patch("linkml_reference_validator.etl.acquire.requests.get") +def test_fetch_bytes_delegates_ordinary_http_redirects_to_requests(mock_get, tmp_path): + """Requests retains its redirect budget, relative URL, and cookie handling.""" + downloaded = _cm_response( + status_code=200, + headers={"content-type": "application/pdf"}, + ) + downloaded.iter_content.return_value = [b"%PDF-content"] + mock_get.return_value = downloaded + config = ReferenceValidationConfig(cache_dir=tmp_path / "cache", rate_limit_delay=0.0) + + data, content_type = ContentAcquirer().fetch_bytes( + "https://publisher.example/paper", config + ) + + assert data == b"%PDF-content" + assert content_type == "application/pdf" + assert mock_get.call_count == 1 + assert mock_get.call_args.kwargs["allow_redirects"] is True + + +@patch("linkml_reference_validator.etl.acquire.requests.get") +def test_fetch_bytes_resolves_relative_zotero_redirect(mock_get, tmp_path): + """Relative local-API redirects are resolved without leaving loopback.""" + downloaded = _cm_response( + status_code=200, + headers={"content-type": "application/pdf"}, + ) + downloaded.iter_content.return_value = [b"%PDF-content"] + mock_get.side_effect = [ + _cm_response(status_code=302, headers={"location": "/download/PDF1"}), + downloaded, + ] + config = ReferenceValidationConfig(cache_dir=tmp_path / "cache", rate_limit_delay=0.0) + + data, content_type = ContentAcquirer().fetch_bytes( + "http://localhost:23119/api/users/0/items/PDF1/file", config + ) + + assert data == b"%PDF-content" + assert content_type == "application/pdf" + assert mock_get.call_args_list[0].kwargs["allow_redirects"] is False + assert mock_get.call_args_list[1].args[0] == "http://localhost:23119/download/PDF1" diff --git a/tests/test_cli_cache_enrich.py b/tests/test_cli_cache_enrich.py new file mode 100644 index 0000000..f781e72 --- /dev/null +++ b/tests/test_cli_cache_enrich.py @@ -0,0 +1,227 @@ +"""Tests for inventorying and enriching a cache from one full-text provider.""" + +from typer.testing import CliRunner + +from linkml_reference_validator.cli import app +from linkml_reference_validator.etl.fulltext.base import ( + FullTextProvider, + FullTextProviderRegistry, +) +from linkml_reference_validator.models import FullTextLocation + + +class _PrivateHitProvider(FullTextProvider): + """Test provider that finds one DOI without mocking validator internals.""" + + @classmethod + def name(cls) -> str: + """Return the provider name used by the CLI tests.""" + return "private_hit" + + def locate(self, ids, config): + """Return full text only for the cache fixture DOI.""" + if ids.doi != "10.1000/hit": + return None + return FullTextLocation( + text="private manuscript body " * 30, + format_hint="text", + provider=self.name(), + access_type="user_library", + source_item_id="ATTACHMENT1", + ) + + +class _UnavailableProvider(FullTextProvider): + """Provider representing a Zotero API that is entirely unavailable.""" + + calls = 0 + + @classmethod + def name(cls) -> str: + """Return the provider name used by the fail-fast test.""" + return "unavailable_provider" + + def locate(self, ids, config): + """Raise the same connection failure for every attempted reference.""" + type(self).calls += 1 + raise ConnectionError("connection refused") + + +def _write_cached_reference(cache_dir, reference_id: str, doi: str) -> None: + """Write a minimal valid Markdown cache entry.""" + safe_id = reference_id.replace(":", "_").replace("/", "_") + (cache_dir / f"{safe_id}.md").write_text( + "---\n" + f"reference_id: {reference_id}\n" + f"doi: {doi}\n" + "content_type: abstract_only\n" + "full_text_attempted: true\n" + "---\n\n" + "abstract text\n", + encoding="utf-8", + ) + + +def _write_full_text_reference(cache_dir, reference_id: str, doi: str) -> None: + """Write a cache entry that must not be enriched a second time.""" + safe_id = reference_id.replace(":", "_").replace("/", "_") + (cache_dir / f"{safe_id}.md").write_text( + "---\n" + f"reference_id: {reference_id}\n" + f"doi: {doi}\n" + "content_type: full_text_pdf\n" + "full_text_attempted: true\n" + "---\n\n" + "existing full text\n", + encoding="utf-8", + ) + + +def test_cache_enrich_dry_run_reports_hit_without_mutating_cache(tmp_path): + """Dry-run inventories exact provider hits and leaves cache bytes unchanged.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + _write_cached_reference(cache_dir, "DOI:10.1000/hit", "10.1000/hit") + _write_cached_reference(cache_dir, "DOI:10.1000/miss", "10.1000/miss") + hit_path = cache_dir / "DOI_10.1000_hit.md" + private_cache_dir = tmp_path / "private-cache" + original = hit_path.read_bytes() + FullTextProviderRegistry.register(_PrivateHitProvider) + + result = CliRunner().invoke( + app, + [ + "cache", + "enrich", + "--provider", + "private_hit", + "--cache-dir", + str(cache_dir), + "--private-cache-dir", + str(private_cache_dir), + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert "DOI:10.1000/hit\tfound\tprivate_hit:ATTACHMENT1" in result.output + assert "DOI:10.1000/miss\tnot_found\t-" in result.output + assert "Found: 1" in result.output + assert hit_path.read_bytes() == original + assert not private_cache_dir.exists() + + +def test_cache_enrich_apply_persists_private_text_and_provenance(tmp_path): + """Apply writes a private research cache and leaves the public source unchanged.""" + cache_dir = tmp_path / "cache" + private_cache_dir = tmp_path / "private-cache" + cache_dir.mkdir() + _write_cached_reference(cache_dir, "DOI:10.1000/hit", "10.1000/hit") + public_path = cache_dir / "DOI_10.1000_hit.md" + original = public_path.read_bytes() + FullTextProviderRegistry.register(_PrivateHitProvider) + + result = CliRunner().invoke( + app, + [ + "cache", + "enrich", + "--provider", + "private_hit", + "--cache-dir", + str(cache_dir), + "--private-cache-dir", + str(private_cache_dir), + "--apply", + ], + ) + + assert result.exit_code == 0, result.output + assert public_path.read_bytes() == original + private_path = private_cache_dir / "DOI_10.1000_hit.md" + cached = private_path.read_text(encoding="utf-8") + assert "content_type: full_text" in cached + assert "full_text_provider: private_hit" in cached + assert "full_text_access_type: user_library" in cached + assert "full_text_source_item_id: ATTACHMENT1" in cached + assert "private manuscript body" in cached + assert private_path.stat().st_mode & 0o777 == 0o600 + assert private_cache_dir.stat().st_mode & 0o777 == 0o700 + assert f"Private cache: {private_cache_dir}" in result.output + + +def test_cache_enrich_rejects_unknown_provider(tmp_path): + """An unknown provider is a configuration error, not a cache-wide miss.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + + result = CliRunner().invoke( + app, + [ + "cache", + "enrich", + "--provider", + "does_not_exist", + "--cache-dir", + str(cache_dir), + ], + ) + + assert result.exit_code == 2 + assert "Unknown full-text provider" in result.output + + +def test_cache_enrich_skips_references_that_already_have_full_text(tmp_path): + """Existing full text is never duplicated or replaced by a library scan.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + _write_full_text_reference(cache_dir, "DOI:10.1000/hit", "10.1000/hit") + path = cache_dir / "DOI_10.1000_hit.md" + original = path.read_bytes() + FullTextProviderRegistry.register(_PrivateHitProvider) + + result = CliRunner().invoke( + app, + [ + "cache", + "enrich", + "--provider", + "private_hit", + "--cache-dir", + str(cache_dir), + "--apply", + ], + ) + + assert result.exit_code == 0, result.output + assert "DOI:10.1000/hit\talready_full_text\t-" in result.output + assert path.read_bytes() == original + + +def test_cache_enrich_fails_fast_after_repeated_provider_errors(tmp_path): + """A dead Zotero service does not produce one error for every cache entry.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + for number in range(5): + _write_cached_reference( + cache_dir, f"DOI:10.1000/{number}", f"10.1000/{number}" + ) + _UnavailableProvider.calls = 0 + FullTextProviderRegistry.register(_UnavailableProvider) + + result = CliRunner().invoke( + app, + [ + "cache", + "enrich", + "--provider", + "unavailable_provider", + "--cache-dir", + str(cache_dir), + ], + ) + + assert result.exit_code == 1 + assert _UnavailableProvider.calls == 3 + assert "Stopping after 3 consecutive provider errors" in result.output + assert "Scanned: 3" in result.output diff --git a/tests/test_cli_cache_export.py b/tests/test_cli_cache_export.py new file mode 100644 index 0000000..3a42794 --- /dev/null +++ b/tests/test_cli_cache_export.py @@ -0,0 +1,246 @@ +"""Tests for exporting public cache metadata for Zotero import.""" + +import json + +from typer.testing import CliRunner + +from linkml_reference_validator.cli import app +from linkml_reference_validator.etl.csl_export import reference_to_csl_json +from linkml_reference_validator.models import ReferenceContent + + +def _write_reference( + cache_dir, + filename: str, + *, + reference_id: str, + title: str, + content_type: str = "abstract_only", + doi: str | None = None, + authors: list[str] | None = None, + journal: str | None = None, + year: str | None = None, +) -> None: + """Write one representative Markdown reference-cache entry.""" + fields = [ + "---", + f'reference_id: "{reference_id}"', + f'title: "{title}"', + f"content_type: {content_type}", + ] + if doi: + fields.append(f'doi: "{doi}"') + if authors: + fields.append("authors:") + fields.extend(f'- "{author}"' for author in authors) + if journal: + fields.append(f'journal: "{journal}"') + if year: + fields.append(f"year: '{year}'") + fields.extend( + [ + 'local_pdf_path: "files/private.pdf"', + "---", + "", + "PRIVATE OR CACHED CONTENT MUST NEVER BE EXPORTED", + ] + ) + (cache_dir / filename).write_text("\n".join(fields), encoding="utf-8") + + +def test_cache_export_writes_deduplicated_metadata_only_csl_json(tmp_path): + """Default export emits missing-full-text DOI/PMID metadata without content.""" + cache_dir = tmp_path / "references_cache" + cache_dir.mkdir() + _write_reference( + cache_dir, + "A.md", + reference_id="DOI:10.1000/Example", + title="Example article", + doi="https://doi.org/10.1000/Example", + authors=["Ada Lovelace", "Grace Hopper"], + journal="Journal of Tests", + year="2024", + ) + _write_reference( + cache_dir, + "B.md", + reference_id="PMID:999", + title="Duplicate record", + doi="10.1000/example", + ) + _write_reference( + cache_dir, + "C.md", + reference_id="PMID:12345", + title="PubMed-only article", + authors=["Single Name"], + year="2020 May", + ) + _write_reference( + cache_dir, + "D.md", + reference_id="DOI:10.1000/already-full", + title="Already full text", + doi="10.1000/already-full", + content_type="full_text_pdf", + ) + _write_reference( + cache_dir, + "E.md", + reference_id="NCIT:C123", + title="Not a publication", + ) + output = tmp_path / "dismech-zotero.json" + + result = CliRunner().invoke( + app, + [ + "cache", + "export", + "--cache-dir", + str(cache_dir), + "--format", + "csl-json", + "--needs-full-text", + "--output", + str(output), + ], + ) + + assert result.exit_code == 0, result.output + records = json.loads(output.read_text(encoding="utf-8")) + assert records == [ + { + "id": "DOI:10.1000/Example", + "type": "article-journal", + "title": "Example article", + "author": [{"literal": "Ada Lovelace"}, {"literal": "Grace Hopper"}], + "container-title": "Journal of Tests", + "issued": {"date-parts": [[2024]]}, + "DOI": "10.1000/example", + }, + { + "id": "PMID:12345", + "type": "article-journal", + "title": "PubMed-only article", + "author": [{"literal": "Single Name"}], + "issued": {"date-parts": [[2020]]}, + "PMID": "12345", + "URL": "https://pubmed.ncbi.nlm.nih.gov/12345/", + }, + ] + exported_text = output.read_text(encoding="utf-8") + assert "PRIVATE OR CACHED CONTENT" not in exported_text + assert "local_pdf_path" not in exported_text + assert "Scanned: 5" in result.output + assert "Exported: 2" in result.output + assert "Duplicates: 1" in result.output + assert "Skipped with full text: 1" in result.output + assert "Skipped without DOI/PMID: 1" in result.output + + +def test_cache_export_all_includes_records_that_already_have_full_text(tmp_path): + """The explicit --all mode exports an otherwise eligible full-text record.""" + cache_dir = tmp_path / "references_cache" + cache_dir.mkdir() + _write_reference( + cache_dir, + "full.md", + reference_id="DOI:10.1000/full", + title="Full text record", + doi="10.1000/full", + content_type="full_text_pdf", + ) + output = tmp_path / "all.json" + + result = CliRunner().invoke( + app, + [ + "cache", + "export", + "--cache-dir", + str(cache_dir), + "--all", + "--output", + str(output), + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(output.read_text(encoding="utf-8"))[0]["DOI"] == "10.1000/full" + + +def test_csl_record_includes_doi_and_pmid_when_both_are_known(): + """Zotero receives every useful exact identifier for a publication.""" + record = reference_to_csl_json( + ReferenceContent(reference_id="PMID:123", doi="doi:10.1000/BOTH") + ) + + assert record is not None + assert record["DOI"] == "10.1000/both" + assert record["PMID"] == "123" + assert "URL" not in record + + +def test_cache_export_refuses_to_overwrite_without_force(tmp_path): + """Export does not silently replace an existing Zotero import file.""" + cache_dir = tmp_path / "references_cache" + cache_dir.mkdir() + output = tmp_path / "existing.json" + output.write_text("keep me", encoding="utf-8") + + result = CliRunner().invoke( + app, + [ + "cache", + "export", + "--cache-dir", + str(cache_dir), + "--output", + str(output), + ], + ) + + assert result.exit_code == 2 + assert "already exists" in result.output + assert output.read_text(encoding="utf-8") == "keep me" + + forced = CliRunner().invoke( + app, + [ + "cache", + "export", + "--cache-dir", + str(cache_dir), + "--output", + str(output), + "--force", + ], + ) + + assert forced.exit_code == 0, forced.output + assert json.loads(output.read_text(encoding="utf-8")) == [] + + +def test_cache_export_rejects_unknown_format(tmp_path): + """Only the documented CSL JSON format is accepted initially.""" + cache_dir = tmp_path / "references_cache" + cache_dir.mkdir() + + result = CliRunner().invoke( + app, + [ + "cache", + "export", + "--cache-dir", + str(cache_dir), + "--format", + "ris", + "--output", + str(tmp_path / "output.ris"), + ], + ) + + assert result.exit_code == 2 + assert "Unsupported export format" in result.output diff --git a/tests/test_models.py b/tests/test_models.py index 2ed0d48..c9268ef 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -17,6 +17,19 @@ def test_config_defaults(): assert config.cache_dir == Path("references_cache") assert config.rate_limit_delay == 0.5 assert config.literal_bracket_patterns == [] + assert config.private_cache_dir.is_absolute() + assert config.private_cache_dir.name == "private" + + +def test_private_cache_does_not_broaden_existing_owner_permissions(tmp_path): + """Accessors restrict shared bits without adding owner permissions.""" + private_dir = tmp_path / "private" + private_dir.mkdir(mode=0o500) + private_dir.chmod(0o500) + config = ReferenceValidationConfig(private_cache_dir=private_dir) + + assert config.get_private_cache_dir() == private_dir + assert private_dir.stat().st_mode & 0o777 == 0o500 def test_config_custom_values(): diff --git a/tests/test_reference_fetcher.py b/tests/test_reference_fetcher.py index d0a7e37..1151733 100644 --- a/tests/test_reference_fetcher.py +++ b/tests/test_reference_fetcher.py @@ -120,6 +120,199 @@ def test_load_from_disk_not_found(fetcher): assert result is None +def test_private_cache_does_not_overlay_public_validation_cache(tmp_path): + """Validation reads only the public cache when a private entry also exists.""" + public_dir = tmp_path / "public" + private_dir = tmp_path / "private" + public_fetcher = ReferenceFetcher( + ReferenceValidationConfig( + cache_dir=public_dir, + private_cache_dir=private_dir, + fetch_full_text=False, + ) + ) + public_fetcher._save_to_disk( + ReferenceContent( + reference_id="DOI:10.1000/hit", + doi="10.1000/hit", + content="public abstract", + content_type="abstract_only", + ) + ) + public_fetcher._save_to_disk( + ReferenceContent( + reference_id="DOI:10.1000/hit", + doi="10.1000/hit", + content="closed full text", + content_type="full_text", + full_text_access_type="user_library", + ), + private=True, + ) + + loaded = public_fetcher._load_from_disk("DOI:10.1000/hit") + + assert loaded is not None + assert loaded.content == "public abstract" + assert loaded.full_text_access_type is None + + +def test_normal_fetch_never_writes_user_library_text_to_public_cache(tmp_path): + """Validation skips private evidence and continues to a public provider.""" + + class _PrivateProvider(FullTextProvider): + @classmethod + def name(cls): + """Return the test provider name.""" + return "private_fetch" + + def locate(self, ids, config): + """Return a private manuscript for the fixture DOI.""" + return FullTextLocation( + text="closed manuscript text " * 30, + format_hint="text", + provider=self.name(), + access_type="user_library", + source_item_id="PRIVATE1", + ) + + class _PublicProvider(FullTextProvider): + @classmethod + def name(cls): + """Return the test provider name.""" + return "public_fetch" + + def locate(self, ids, config): + """Return public evidence for the fixture DOI.""" + return FullTextLocation( + text="public full text " * 30, + format_hint="text", + provider=self.name(), + access_type="open", + ) + + public_dir = tmp_path / "public" + private_dir = tmp_path / "private" + config = ReferenceValidationConfig( + cache_dir=public_dir, + private_cache_dir=private_dir, + rate_limit_delay=0.0, + full_text_providers=["private_fetch", "public_fetch"], + ) + FullTextProviderRegistry.register(_PrivateProvider) + FullTextProviderRegistry.register(_PublicProvider) + fetcher = ReferenceFetcher(config) + metadata = ReferenceContent( + reference_id="DOI:10.1000/private", + doi="10.1000/private", + content="public abstract", + content_type="abstract_only", + ) + + with patch( + "linkml_reference_validator.etl.reference_fetcher.ReferenceSourceRegistry.get_source" + ) as get_source: + get_source.return_value.return_value.fetch.return_value = metadata + result = fetcher.fetch("DOI:10.1000/private") + + assert result is not None + assert result.content == "public abstract\n\n" + "public full text " * 30 + assert "closed manuscript text" not in result.content + assert result.full_text_access_type == "open" + public_path = public_dir / "DOI_10.1000_private.md" + assert public_path.exists() + assert "public full text" in public_path.read_text(encoding="utf-8") + private_path = private_dir / "DOI_10.1000_private.md" + assert not private_path.exists() + + +def test_applying_private_enrichment_does_not_enter_validation_memory_cache(tmp_path): + """A fetcher reused after private enrichment still returns public evidence.""" + public_dir = tmp_path / "public" + private_dir = tmp_path / "private" + fetcher = ReferenceFetcher( + ReferenceValidationConfig( + cache_dir=public_dir, + private_cache_dir=private_dir, + fetch_full_text=False, + ) + ) + public = ReferenceContent( + reference_id="PMID:123", + content="public abstract", + content_type="abstract_only", + ) + fetcher._save_to_disk(public) + + applied = fetcher.apply_full_text_location( + public, + FullTextLocation( + text="closed manuscript text " * 30, + format_hint="text", + provider="zotero", + access_type="user_library", + source_item_id="PRIVATE1", + ), + "zotero", + private=True, + ) + + assert applied is True + loaded = fetcher.fetch("PMID:123") + assert loaded is not None + assert loaded.content == "public abstract" + assert loaded.full_text_access_type is None + + +def test_private_location_forces_private_persistence_without_flag(tmp_path): + """Private provenance is a safety floor even when a caller omits ``private``.""" + public_dir = tmp_path / "public" + private_dir = tmp_path / "private" + fetcher = ReferenceFetcher( + ReferenceValidationConfig( + cache_dir=public_dir, + private_cache_dir=private_dir, + fetch_full_text=False, + ) + ) + content = ReferenceContent( + reference_id="PMID:123", content="abstract", content_type="abstract_only" + ) + + applied = fetcher.apply_full_text_location( + content, + FullTextLocation( + text="closed manuscript text " * 30, + format_hint="text", + access_type="user_library", + ), + "zotero", + ) + + assert applied is True + assert not (public_dir / "PMID_123.md").exists() + assert (private_dir / "PMID_123.md").exists() + + +def test_iter_cached_references_streams_in_sorted_order(tmp_path): + """Large caches are yielded one record at a time in deterministic order.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + for filename, reference_id in (("B.md", "PMID:2"), ("A.md", "PMID:1")): + (cache_dir / filename).write_text( + f"---\nreference_id: {reference_id}\ncontent_type: abstract_only\n---\n", + encoding="utf-8", + ) + fetcher = ReferenceFetcher( + ReferenceValidationConfig(cache_dir=cache_dir, fetch_full_text=False) + ) + + references = fetcher.iter_cached_references() + + assert iter(references) is references + assert [reference.reference_id for reference in references] == ["PMID:1", "PMID:2"] + + def test_save_and_load_preprint_metadata(fetcher, tmp_path): """Preprint status round-trips through the disk cache frontmatter.""" ref = ReferenceContent( @@ -841,7 +1034,7 @@ def test_enrich_skips_when_already_full_text(tmp_path): content = ReferenceContent( reference_id="PMID:1", content="lots of full text", content_type="full_text_xml" ) - assert fetcher._needs_full_text(content) is False + assert fetcher.needs_full_text(content) is False def test_enrich_downloads_and_extracts_pdf(tmp_path): @@ -902,6 +1095,8 @@ def test_provenance_round_trips_through_cache(tmp_path): oa_status="gold", license="cc-by", local_pdf_path="files/DOI_10.1_x.pdf", + full_text_access_type="user_library", + full_text_source_item_id="ATTACHMENT1", ) fetcher._save_to_disk(content) loaded = fetcher._load_from_disk("DOI:10.1/x") @@ -912,6 +1107,8 @@ def test_provenance_round_trips_through_cache(tmp_path): assert loaded.oa_status == "gold" assert loaded.license == "cc-by" assert loaded.local_pdf_path == "files/DOI_10.1_x.pdf" + assert loaded.full_text_access_type == "user_library" + assert loaded.full_text_source_item_id == "ATTACHMENT1" def test_fetcher_registers_custom_full_text_providers(tmp_path): diff --git a/tests/test_zotero_provider.py b/tests/test_zotero_provider.py new file mode 100644 index 0000000..7c8d14b --- /dev/null +++ b/tests/test_zotero_provider.py @@ -0,0 +1,277 @@ +"""Tests for Zotero private-library full-text lookup.""" + +from unittest.mock import MagicMock + +import pytest + +from linkml_reference_validator.models import ( + ReferenceIdentifiers, + ReferenceValidationConfig, +) + + +def _json_response( + data: object, status_code: int = 200, headers: dict[str, str] | None = None +) -> MagicMock: + """Return a response double carrying a realistic Zotero JSON payload.""" + response = MagicMock() + response.status_code = status_code + response.headers = headers or {} + response.json.return_value = data + return response + + +def _zotero_item( + key: str, + *, + doi: str | None = None, + extra: str = "", + item_type: str = "journalArticle", + content_type: str | None = None, + filename: str | None = None, +) -> dict: + """Build the subset of a Zotero v3 item used by the provider contract.""" + data = {"key": key, "itemType": item_type, "extra": extra} + if doi is not None: + data["DOI"] = doi + if content_type is not None: + data["contentType"] = content_type + if filename is not None: + data["filename"] = filename + return {"key": key, "data": data} + + +def test_normalize_doi_handles_common_zotero_forms(): + """DOIs compare independently of URL form, prefix, whitespace, or case.""" + from linkml_reference_validator.etl.fulltext.zotero import normalize_doi + + assert normalize_doi(" https://doi.org/10.1000/ABC ") == "10.1000/abc" + assert normalize_doi("doi:10.1000/ABC") == "10.1000/abc" + assert normalize_doi("") is None + + +def test_client_indexes_exact_doi_pmid_and_pmcid(): + """The client builds an exact identifier index from Zotero item JSON.""" + from linkml_reference_validator.etl.fulltext.zotero import ZoteroClient + + session = MagicMock() + session.get.return_value = _json_response( + [ + _zotero_item( + "PARENT1", + doi="https://doi.org/10.1000/ABC", + extra="PMID: 12345678\nPMCID: PMC7654321", + ), + _zotero_item("PARENT2", doi="10.2000/other"), + ] + ) + client = ZoteroClient("http://localhost:23119/api/users/0", session=session) + + assert client.find_parent_keys(ReferenceIdentifiers(doi="10.1000/abc")) == { + "PARENT1" + } + assert client.find_parent_keys(ReferenceIdentifiers(pmid="12345678")) == { + "PARENT1" + } + assert client.find_parent_keys(ReferenceIdentifiers(pmcid="PMC7654321")) == { + "PARENT1" + } + + +def test_client_paginates_top_level_items(): + """Identifier indexing covers every Zotero page, not only the first one.""" + from linkml_reference_validator.etl.fulltext.zotero import ZoteroClient + + session = MagicMock() + session.get.side_effect = [ + _json_response( + [_zotero_item("PARENT1", doi="10.1000/first")], + headers={"Total-Results": "2"}, + ), + _json_response( + [_zotero_item("PARENT2", doi="10.1000/second")], + headers={"Total-Results": "2"}, + ), + ] + client = ZoteroClient( + "http://localhost:23119/api/users/0", session=session, page_size=1 + ) + + assert client.find_parent_keys(ReferenceIdentifiers(doi="10.1000/second")) == { + "PARENT2" + } + assert [call.kwargs["params"] for call in session.get.call_args_list] == [ + {"limit": 1, "start": 0}, + {"limit": 1, "start": 1}, + ] + + +def test_client_requires_all_supplied_identifiers_to_name_same_parent(): + """Conflicting exact identifiers are ambiguous rather than silently merged.""" + from linkml_reference_validator.etl.fulltext.zotero import ZoteroClient + + session = MagicMock() + session.get.return_value = _json_response( + [ + _zotero_item("DOI_PARENT", doi="10.1000/a"), + _zotero_item("PMID_PARENT", extra="PMID: 123"), + ] + ) + client = ZoteroClient("http://localhost:23119/api/users/0", session=session) + + assert client.find_parent_keys( + ReferenceIdentifiers(doi="10.1000/a", pmid="123") + ) == set() + + +def test_provider_prefers_zotero_indexed_full_text(): + """Usable Zotero-indexed text avoids downloading and reparsing the PDF.""" + from linkml_reference_validator.etl.fulltext.zotero import ( + ZoteroClient, + ZoteroFullTextProvider, + ) + + session = MagicMock() + session.get.side_effect = [ + _json_response([_zotero_item("PARENT", doi="10.1000/a")]), + _json_response( + [ + _zotero_item( + "PDF1", + item_type="attachment", + content_type="application/pdf", + filename="article.pdf", + ) + ] + ), + _json_response({"content": "indexed full text " * 40, "indexedPages": 8}), + ] + client = ZoteroClient("http://localhost:23119/api/users/0", session=session) + provider = ZoteroFullTextProvider(client=client) + + location = provider.locate( + ReferenceIdentifiers(doi="10.1000/a"), + ReferenceValidationConfig(rate_limit_delay=0.0), + ) + + assert location is not None + assert location.text == "indexed full text " * 40 + assert location.format_hint == "text" + assert location.provider == "zotero" + assert location.access_type == "user_library" + assert location.source_item_id == "PDF1" + assert location.oa_status is None + + +def test_provider_falls_back_to_pdf_file_endpoint(): + """An unindexed attachment is returned as a PDF URL for normal extraction.""" + from linkml_reference_validator.etl.fulltext.zotero import ( + ZoteroClient, + ZoteroFullTextProvider, + ) + + session = MagicMock() + session.get.side_effect = [ + _json_response([_zotero_item("PARENT", extra="PMID: 123")]), + _json_response( + [ + _zotero_item( + "PDF1", + item_type="attachment", + content_type="application/pdf", + filename="article.pdf", + ) + ] + ), + _json_response({}, status_code=404), + ] + client = ZoteroClient("http://localhost:23119/api/users/0", session=session) + provider = ZoteroFullTextProvider(client=client) + + location = provider.locate( + ReferenceIdentifiers(pmid="123"), + ReferenceValidationConfig(rate_limit_delay=0.0), + ) + + assert location is not None + assert location.url == "http://localhost:23119/api/users/0/items/PDF1/file" + assert location.format_hint == "pdf" + assert location.access_type == "user_library" + assert location.source_item_id == "PDF1" + + +def test_provider_returns_none_for_ambiguous_doi(): + """Duplicate DOI records do not choose an arbitrary parent item.""" + from linkml_reference_validator.etl.fulltext.zotero import ( + ZoteroClient, + ZoteroFullTextProvider, + ) + + session = MagicMock() + session.get.return_value = _json_response( + [ + _zotero_item("PARENT1", doi="10.1000/a"), + _zotero_item("PARENT2", doi="10.1000/a"), + ] + ) + client = ZoteroClient("http://localhost:23119/api/users/0", session=session) + + assert ZoteroFullTextProvider(client=client).locate( + ReferenceIdentifiers(doi="10.1000/a"), + ReferenceValidationConfig(rate_limit_delay=0.0), + ) is None + + +def test_provider_reuses_default_client_between_references(monkeypatch): + """Scanning a cache builds the Zotero library index only once.""" + from linkml_reference_validator.etl.fulltext import zotero + + client = MagicMock() + client.find_parent_keys.return_value = set() + constructor = MagicMock(return_value=client) + monkeypatch.setattr(zotero, "ZoteroClient", constructor) + provider = zotero.ZoteroFullTextProvider() + config = ReferenceValidationConfig( + zotero_base_url="http://localhost:23119/api/users/0" + ) + + provider.locate(ReferenceIdentifiers(doi="10.1000/a"), config) + provider.locate(ReferenceIdentifiers(doi="10.1000/b"), config) + + constructor.assert_called_once_with(config.zotero_base_url) + + +def test_provider_selects_pdf_attachment_deterministically(): + """Multiple PDFs use a stable key-based choice independent of API order.""" + from linkml_reference_validator.etl.fulltext.zotero import ZoteroFullTextProvider + + client = MagicMock() + client.find_parent_keys.return_value = {"PARENT"} + client.pdf_attachments.return_value = [ + _zotero_item("PDF_Z", item_type="attachment", content_type="application/pdf"), + _zotero_item("PDF_A", item_type="attachment", content_type="application/pdf"), + ] + client.indexed_full_text.return_value = None + client.attachment_file_url.side_effect = lambda key: f"file:{key}" + + location = ZoteroFullTextProvider(client=client).locate( + ReferenceIdentifiers(doi="10.1000/a"), ReferenceValidationConfig() + ) + + assert location is not None + assert location.source_item_id == "PDF_A" + + +def test_client_surfaces_disabled_local_api(): + """A disabled Zotero local API is an external error, not a library miss.""" + from linkml_reference_validator.etl.fulltext.zotero import ( + ZoteroAPIError, + ZoteroClient, + ) + + session = MagicMock() + session.get.return_value = _json_response({}, status_code=403) + client = ZoteroClient("http://localhost:23119/api/users/0", session=session) + + with pytest.raises(ZoteroAPIError, match="403"): + client.find_parent_keys(ReferenceIdentifiers(doi="10.1000/a"))