diff --git a/CHANGELOG.md b/CHANGELOG.md index 71049304..7d7c8559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - `*Options` TypedDict types and a `Context` type alias in `pyld.options` for JSON-LD API option dicts (typing and documentation). - `pyld.SqliteCacheRequestsDocumentLoader`: a SQLite-backed HTTP cache document loader using `requests-cache`. +- `pyld.FileDocumentLoader`: a document loader for local `file:` URLs with optional root confinement. ### Fixed - If value objects contain array values for `@type` during expansion, an error is now raised. Fixes [expand#ter54](https://w3c.github.io/json-ld-api/tests/expand-manifest.html#ter54) and [toRdf#ter54](https://w3c.github.io/json-ld-api/tests/toRdf-manifest.html#ter54). diff --git a/docs/examples/data/person.jsonld b/docs/examples/data/person.jsonld new file mode 100644 index 00000000..63dc7217 --- /dev/null +++ b/docs/examples/data/person.jsonld @@ -0,0 +1,6 @@ +{ + "@context": { + "name": "http://schema.org/name" + }, + "name": "Ada Lovelace" +} diff --git a/docs/examples/document_loaders/file_basic.py b/docs/examples/document_loaders/file_basic.py new file mode 100644 index 00000000..7121c762 --- /dev/null +++ b/docs/examples/document_loaders/file_basic.py @@ -0,0 +1,13 @@ +import json +from pathlib import Path + +from pyld import FileDocumentLoader, jsonld + +person = Path(__file__).resolve().parent.parent / 'data' / 'person.jsonld' + +loader = FileDocumentLoader() +result = jsonld.expand( + person.as_uri(), + options={'documentLoader': loader}, +) +print(json.dumps(result, indent=2)) diff --git a/docs/examples/document_loaders/file_root.py b/docs/examples/document_loaders/file_root.py new file mode 100644 index 00000000..743f7222 --- /dev/null +++ b/docs/examples/document_loaders/file_root.py @@ -0,0 +1,14 @@ +import json +from pathlib import Path + +from pyld import FileDocumentLoader, jsonld + +data_dir = Path(__file__).resolve().parent.parent / 'data' +person = data_dir / 'person.jsonld' + +loader = FileDocumentLoader(root=data_dir) +result = jsonld.expand( + person.as_uri(), + options={'documentLoader': loader}, +) +print(json.dumps(result, indent=2)) diff --git a/docs/reference/document-loaders/file.md b/docs/reference/document-loaders/file.md new file mode 100644 index 00000000..b4f023b3 --- /dev/null +++ b/docs/reference/document-loaders/file.md @@ -0,0 +1,29 @@ +--- +hide: [toc] +--- +# :material-file-outline: `FileDocumentLoader` + +::: pyld.FileDocumentLoader + options: + show_root_heading: false + show_bases: false + heading_level: 3 + members: false + +{{ example('document_loaders/file_basic.py', output_syntax='json') }} + +## Content Types + +The content type is chosen based on the file extension as follows: + +{{ file_content_types_table() }} + +Unsupported extensions raise `JsonLdError` with code `loading document failed`. + +## Root Confinement + +Pass `root` to refuse paths that resolve outside a directory. Resolved paths +and symlink targets are checked, so `..` traversal and symlink escapes are +rejected: + +{{ example('document_loaders/file_root.py', output_syntax='json') }} diff --git a/docs/reference/document-loaders/frozen.md b/docs/reference/document-loaders/frozen.md index c22012c6..3ba6a358 100644 --- a/docs/reference/document-loaders/frozen.md +++ b/docs/reference/document-loaders/frozen.md @@ -1,8 +1,11 @@ # :material-snowflake: `FrozenDocumentLoader` -`FrozenDocumentLoader` serves only URLs in an allowlist and refuses all other -document loads. It is intended for air-gapped runs, reproducible builds, and -deployments that must avoid remote context fetching. +::: pyld.FrozenDocumentLoader + options: + show_root_heading: false + show_bases: false + heading_level: 3 + members: false With no arguments, the loader serves the curated `BUNDLED_CONTEXTS` mapping: diff --git a/docs/reference/document-loaders/index.md b/docs/reference/document-loaders/index.md index 8e6607a9..383597c5 100644 --- a/docs/reference/document-loaders/index.md +++ b/docs/reference/document-loaders/index.md @@ -32,6 +32,12 @@ class-based loaders for common cases and supports custom subclasses of Serve only documents from an allowlist for air-gapped or reproducible runs. +- [:material-file-outline:{ .lg .middle } `FileDocumentLoader`](file.md) + + --- + + Read local JSON-LD documents from `file:` URLs. + - [:material-code-braces:{ .lg .middle } __Custom Document Loaders__](custom.md) --- @@ -42,7 +48,9 @@ class-based loaders for common cases and supports custom subclasses of ## Default Document Loader -The default document loader is selected at import time. PyLD uses -`RequestsDocumentLoader` if `requests` is available, falls back to -`AioHttpDocumentLoader` if `aiohttp` is available, and otherwise installs a -dummy loader that raises when invoked. +The default document loader is selected at import time, in this order: + +1. [`RequestsDocumentLoader`](requests.md) if `requests` is available +2. [`AioHttpDocumentLoader`](aiohttp.md) if `aiohttp` is available + +If neither is installed, document loading raises. diff --git a/docs_macros.py b/docs_macros.py index 1a3c3643..fd066217 100644 --- a/docs_macros.py +++ b/docs_macros.py @@ -236,6 +236,22 @@ def bundled_contexts_table(): rows.append(f'| `{url}` | `{Path(path).name}` |') return '\n'.join(rows) + @env.macro + def file_content_types_table(): + from pyld.documentloader.file import CONTENT_TYPES + + by_type: dict[str, list[str]] = {} + for extension, content_type in CONTENT_TYPES.items(): + by_type.setdefault(content_type, []).append(f'`{extension}`') + + rows = [ + '| Extension | Content type |', + '| --- | --- |', + ] + for content_type, extensions in by_type.items(): + rows.append(f'| {", ".join(extensions)} | `{content_type}` |') + return '\n'.join(rows) + @env.macro def skipped_tests_table(): from runtests import TEST_TYPES diff --git a/lib/pyld/__init__.py b/lib/pyld/__init__.py index ae05cb95..c107b47c 100644 --- a/lib/pyld/__init__.py +++ b/lib/pyld/__init__.py @@ -4,6 +4,7 @@ from .context_resolver import ContextResolver from .documentloader.aiohttp import AioHttpDocumentLoader from .documentloader.base import DocumentLoader, RemoteDocument +from .documentloader.file import FileDocumentLoader from .documentloader.frozen import BUNDLED_CONTEXTS, FrozenDocumentLoader from .documentloader.requests import RequestsDocumentLoader from .documentloader.requests_sqlite_cache import SqliteCacheRequestsDocumentLoader @@ -13,6 +14,7 @@ 'BUNDLED_CONTEXTS', 'ContextResolver', 'DocumentLoader', + 'FileDocumentLoader', 'FrozenDocumentLoader', 'RequestsDocumentLoader', 'RemoteDocument', diff --git a/lib/pyld/documentloader/file.py b/lib/pyld/documentloader/file.py new file mode 100644 index 00000000..d47a766a --- /dev/null +++ b/lib/pyld/documentloader/file.py @@ -0,0 +1,107 @@ +""" +Local filesystem document loader for ``file:`` URLs. + +.. module:: jsonld.documentloader.file + :synopsis: FileDocumentLoader for local JSON-LD documents +""" + +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import url2pathname + +from pyld.documentloader.base import DocumentLoader, RemoteDocument +from pyld.jsonld import JsonLdError + +CONTENT_TYPES = { + '.jsonld': 'application/ld+json', + '.json': 'application/json', + '.html': 'text/html', + '.htm': 'text/html', + '.xhtml': 'application/xhtml+xml', +} + + +class FileDocumentLoader(DocumentLoader): + """Document loader that reads local files for `file:` URLs. + + Accepts `file:` URLs, scheme-less absolute paths, and `pathlib.Path` + instances. Any other scheme raises `JsonLdError`. When `root` is set, + only paths that resolve under that directory are served. + + :param root: optional directory that confines readable paths; when set, + paths that resolve outside this directory are refused. + """ + + def __init__(self, root: str | Path | None = None) -> None: + self.root = Path(root).resolve() if root is not None else None + + def __call__( + self, url: str | Path, options: dict | None = None + ) -> RemoteDocument: + """Retrieve the JSON-LD document at `url`. + + :param url: a `file:` URL, scheme-less absolute path, or + `pathlib.Path`. + :param options: loader options (unused; accepted for interface parity). + :return: a `RemoteDocument`. + """ + if options is None: + options = {} + + fragment = '' + if isinstance(url, Path): + path = url.resolve() + else: + parts = urlparse(url) + if parts.scheme == 'file': + path_text = parts.path + elif parts.scheme == '': + path_text = url + else: + raise JsonLdError( + 'URL could not be dereferenced; only "file" URLs are ' + 'supported.', + 'jsonld.InvalidUrl', + {'url': url}, + code='loading document failed', + ) + path = Path(url2pathname(path_text)).resolve() + fragment = parts.fragment + + if self.root is not None and not path.is_relative_to(self.root): + raise JsonLdError( + 'URL could not be dereferenced; path is outside the ' + 'configured root.', + 'jsonld.LoadDocumentError', + {'url': url, 'root': str(self.root)}, + code='loading document failed', + ) + + content_type = CONTENT_TYPES.get(path.suffix.lower()) + if content_type is None: + raise JsonLdError( + 'URL could not be dereferenced; unsupported file extension.', + 'jsonld.LoadDocumentError', + {'url': url, 'suffix': path.suffix}, + code='loading document failed', + ) + + try: + document = path.read_text(encoding='utf-8') + except OSError as cause: + raise JsonLdError( + 'Could not retrieve a JSON-LD document from the URL.', + 'jsonld.LoadDocumentError', + {'url': url}, + code='loading document failed', + ) from cause + + document_url = path.as_uri() + if fragment: + document_url = f'{document_url}#{fragment}' + return { + 'contentType': content_type, + 'contextUrl': None, + 'documentUrl': document_url, + 'document': document, + } diff --git a/lib/pyld/documentloader/frozen/__init__.py b/lib/pyld/documentloader/frozen/__init__.py index 9fcb51fe..b924c116 100644 --- a/lib/pyld/documentloader/frozen/__init__.py +++ b/lib/pyld/documentloader/frozen/__init__.py @@ -2,13 +2,13 @@ Frozen JSON-LD document loader. A document loader that serves *only* the URLs in its ``documents`` allowlist -and refuses everything else with :class:`pyld.jsonld.JsonLdError`. Suitable for -secure / air-gapped / privacy-sensitive deployments and for honoring the -guidance in the W3C *JSON-LD Best Practices* note that clients SHOULD attempt -to use a locally cached version of contexts (§ Cache JSON-LD Contexts, +and refuses everything else with ``JsonLdError``. Suitable for secure / +air-gapped / privacy-sensitive deployments and for honoring the guidance in +the W3C *JSON-LD Best Practices* note that clients SHOULD attempt to use a +locally cached version of contexts (§ Cache JSON-LD Contexts, https://w3c.github.io/json-ld-bp/#cache-json-ld-contexts). -This module also defines :data:`BUNDLED_CONTEXTS`, a curated mapping of +This module also defines ``BUNDLED_CONTEXTS``, a curated mapping of high-traffic public W3C / W3ID JSON-LD context URLs to vendored on-disk copies shipped with the package. See ``scripts/download_contexts.py`` for how the files in ``bundled/`` are refreshed. @@ -45,15 +45,18 @@ class FrozenDocumentLoader(DocumentLoader): """Document loader that serves only a sealed allowlist of URLs. ``documents`` maps each allowed URL to either a parsed JSON-LD ``dict`` or - a :class:`pathlib.Path` pointing to a JSON file on disk. Path entries are - read and parsed lazily on first request, then cached in place so subsequent + a ``pathlib.Path`` pointing to a JSON file on disk. Path entries are read + and parsed lazily on first request, then cached in place so subsequent calls skip the file read. Any URL not present in the mapping raises - :class:`pyld.jsonld.JsonLdError` with code ``'loading document failed'``. + ``JsonLdError`` with code ``'loading document failed'``. With no arguments, a ``FrozenDocumentLoader`` serves the curated - :data:`BUNDLED_CONTEXTS` set. To extend rather than replace the bundle:: + ``BUNDLED_CONTEXTS`` set. To extend rather than replace the bundle:: FrozenDocumentLoader(documents=dict(BUNDLED_CONTEXTS, **extras)) + + :param documents: allowlist mapping each URL to a parsed JSON-LD ``dict`` + or a ``pathlib.Path`` to a JSON file; defaults to ``BUNDLED_CONTEXTS``. """ def __init__(self, documents: Mapping[str, dict | Path] | None = None) -> None: diff --git a/tests/test_file_document_loader.py b/tests/test_file_document_loader.py new file mode 100644 index 00000000..d0c21836 --- /dev/null +++ b/tests/test_file_document_loader.py @@ -0,0 +1,223 @@ +"""Tests for FileDocumentLoader.""" + +import json +from pathlib import Path +from urllib.request import pathname2url + +import pytest + +from pyld import FileDocumentLoader, jsonld +from pyld.jsonld import JsonLdError + +_PERSON = { + '@context': {'name': 'http://schema.org/name'}, + 'name': 'Ada Lovelace', +} + + +def _file_url(path: Path, fragment: str | None = None) -> str: + url = path.resolve().as_uri() + if fragment: + return f'{url}#{fragment}' + return url + + +def test_loads_jsonld_file(tmp_path): + """A .jsonld file is returned as raw text with the ld+json content type.""" + path = tmp_path / 'person.jsonld' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + + result = FileDocumentLoader()(_file_url(path), {}) + + assert result['contentType'] == 'application/ld+json' + assert result['contextUrl'] is None + assert result['documentUrl'] == _file_url(path) + assert json.loads(result['document']) == _PERSON + + +def test_loads_json_file(tmp_path): + """A .json file is returned with the application/json content type.""" + path = tmp_path / 'person.json' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + + result = FileDocumentLoader()(_file_url(path), {}) + + assert result['contentType'] == 'application/json' + assert json.loads(result['document']) == _PERSON + + +def test_scheme_less_path_is_accepted(tmp_path): + """A scheme-less absolute path is treated as a file URL.""" + path = tmp_path / 'person.jsonld' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + + result = FileDocumentLoader()(str(path.resolve()), {}) + + assert json.loads(result['document']) == _PERSON + assert result['documentUrl'].startswith('file:') + + +def test_pathlib_path_is_accepted(tmp_path): + """A pathlib.Path is accepted and reported as a file URL.""" + path = tmp_path / 'person.jsonld' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + + result = FileDocumentLoader()(path, {}) + + assert json.loads(result['document']) == _PERSON + assert result['documentUrl'] == path.resolve().as_uri() + + +def test_percent_encoded_path_with_space(tmp_path): + """Percent-encoded spaces in a file URL resolve to the on-disk path.""" + path = tmp_path / 'my person.jsonld' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + encoded = path.resolve().as_uri() + assert '%20' in encoded + + result = FileDocumentLoader()(encoded, {}) + + assert json.loads(result['document']) == _PERSON + + +def test_relative_context_resolves_against_document_url(tmp_path): + """Relative @context references resolve against the file document URL.""" + context = {'@context': {'name': 'http://schema.org/name'}} + (tmp_path / 'context.jsonld').write_text(json.dumps(context), encoding='utf-8') + doc = {'@context': 'context.jsonld', 'name': 'Ada Lovelace'} + doc_path = tmp_path / 'person.jsonld' + doc_path.write_text(json.dumps(doc), encoding='utf-8') + + loader = FileDocumentLoader() + expanded = jsonld.expand( + _file_url(doc_path), + options={'documentLoader': loader}, + ) + + assert expanded == [{'http://schema.org/name': [{'@value': 'Ada Lovelace'}]}] + + +def test_html_file_extracts_script(tmp_path): + """An HTML file with a JSON-LD script is extractable via expand.""" + html = ( + '' + '' + ) + path = tmp_path / 'person.html' + path.write_text(html, encoding='utf-8') + + result = FileDocumentLoader()(_file_url(path), {}) + assert result['contentType'] == 'text/html' + + expanded = jsonld.expand( + _file_url(path), + options={'documentLoader': FileDocumentLoader()}, + ) + assert expanded == [{'http://schema.org/name': [{'@value': 'Ada Lovelace'}]}] + + +def test_fragment_selects_html_script(tmp_path): + """A fragment id selects the matching script element from an HTML file.""" + html = ( + '' + '' + '' + ) + path = tmp_path / 'person.html' + path.write_text(html, encoding='utf-8') + + expanded = jsonld.expand( + _file_url(path, fragment='second'), + options={'documentLoader': FileDocumentLoader()}, + ) + assert expanded == [{'http://schema.org/name': [{'@value': 'Second'}]}] + + +def test_missing_file_raises_load_document_error(tmp_path): + """A missing file raises JsonLdError with code loading document failed.""" + missing = tmp_path / 'missing.jsonld' + with pytest.raises(JsonLdError) as exc: + FileDocumentLoader()(_file_url(missing), {}) + assert exc.value.code == 'loading document failed' + + +def test_directory_raises_load_document_error(tmp_path): + """A directory path raises JsonLdError with code loading document failed.""" + with pytest.raises(JsonLdError) as exc: + FileDocumentLoader()(_file_url(tmp_path), {}) + assert exc.value.code == 'loading document failed' + + +def test_unknown_extension_is_refused(tmp_path): + """An unsupported file extension is refused.""" + path = tmp_path / 'person.txt' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + + with pytest.raises(JsonLdError) as exc: + FileDocumentLoader()(_file_url(path), {}) + assert exc.value.code == 'loading document failed' + + +def test_http_url_is_refused(): + """HTTP URLs are refused by the file-only loader.""" + with pytest.raises(JsonLdError) as exc: + FileDocumentLoader()('https://example.com/person.jsonld', {}) + assert exc.value.code == 'loading document failed' + assert exc.value.type == 'jsonld.InvalidUrl' + + +def test_root_allows_file_inside_root(tmp_path): + """A file under the configured root is loadable.""" + allowed = tmp_path / 'allowed' + allowed.mkdir() + path = allowed / 'person.jsonld' + path.write_text(json.dumps(_PERSON), encoding='utf-8') + + result = FileDocumentLoader(root=allowed)(_file_url(path), {}) + assert json.loads(result['document']) == _PERSON + + +def test_root_rejects_parent_traversal(tmp_path): + """Paths that escape the configured root via .. are refused.""" + root = tmp_path / 'root' + root.mkdir() + outside = tmp_path / 'outside.jsonld' + outside.write_text(json.dumps(_PERSON), encoding='utf-8') + + # Build a file URL whose path contains a .. segment that resolves outside. + escape = root / 'nested' / '..' / '..' / 'outside.jsonld' + url = 'file://' + pathname2url(str(escape)) + + with pytest.raises(JsonLdError) as exc: + FileDocumentLoader(root=root)(url, {}) + assert exc.value.code == 'loading document failed' + + +def test_root_rejects_symlink_escape(tmp_path): + """A symlink that points outside the configured root is refused.""" + root = tmp_path / 'root' + root.mkdir() + outside = tmp_path / 'secret.jsonld' + outside.write_text(json.dumps(_PERSON), encoding='utf-8') + link = root / 'escape.jsonld' + link.symlink_to(outside) + + with pytest.raises(JsonLdError) as exc: + FileDocumentLoader(root=root)(_file_url(link), {}) + assert exc.value.code == 'loading document failed'