diff --git a/.context/plan.md b/.context/plan.md index cebd2695..856b806e 100644 --- a/.context/plan.md +++ b/.context/plan.md @@ -18,4 +18,6 @@ +- 2026-07-10 - Added `get_available_hed_versions()` with two-tier (time + ETag) caching for GitHub schema-version listing; addressed all Copilot review rounds - PR #1351 (merged). + diff --git a/docs/user_guide.md b/docs/user_guide.md index 16146767..207b8834 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -327,6 +327,146 @@ The five available checks are: Each method returns a list of issue dictionaries and updates `sv.summary` (a `ComplianceSummary` instance) with what was checked. +## Schema caching + +```{index} schema caching, cache directory, offline use, GitHub rate limits, HED_GITHUB_TOKEN +``` + +The official HED schemas live in the [hed-standard/hed-schemas](https://github.com/hed-standard/hed-schemas) GitHub repository. HEDTools caches schema data locally so that repeated validation runs — and repeated program starts — don't re-download the same content from GitHub every time. This section explains what gets cached, where, and how to control it. + +### The cache directory + +By default, schemas are cached in `~/.hedtools/hed_cache/`. You can check or change this location: + +```python +from hed.schema import get_cache_directory, set_cache_directory + +# Check the current cache location +print(get_cache_directory()) + +# Use a custom location instead (e.g. a writable path in a container image) +set_cache_directory("/data/hed_cache") +``` + +A few common schema versions also ship *inside* the hedtools package itself (the "installed cache"), so a fresh install can validate against those versions with no network access at all, before anything has been downloaded. + +### How `load_schema_version()` uses the cache + +```{index} load_schema_version; caching +``` + +When you call `load_schema_version("8.4.0")`, HEDTools looks for that version in the local cache first (including its `prerelease/` subfolder). If it's not there, it downloads just that one version's XML content from GitHub and stores it in the cache directory — a one-time cost per version, per machine: + +```python +from hed.schema import load_schema_version + +schema = load_schema_version("8.4.0") # downloaded and cached on first use +schema = load_schema_version("8.4.0") # reused from disk on every call after that +``` + +Nothing here is fetched again once a version is cached, since released schemas are immutable — the same version number never changes content. + +### Listing available versions without downloading + +```{index} get_available_hed_versions, get_hed_versions, version picker +``` + +There are two different functions for listing versions, and it's easy to reach for the wrong one: + +| Function | Where it looks | Network calls | +| ------------------------------ | ----------------------------------------- | ----------------------------------------- | +| `get_hed_versions()` | Local cache only (installed + downloaded) | None | +| `get_available_hed_versions()` | GitHub, live | Small listing requests, no schema content | + +`get_available_hed_versions()` is the right choice for something like a version-picker dropdown — it tells you everything currently published on GitHub without downloading any of it: + +```python +from hed.schema import get_available_hed_versions, load_schema_version + +# Standard schema versions only, newest first +versions = get_available_hed_versions() +# ['8.4.0', '8.3.0', '8.2.0', ...] + +# Everything: standard schema plus every library, keyed by library name +# (the standard schema is under the None key) +all_versions = get_available_hed_versions(library_name="all") +# {None: ['8.4.0', ...], 'score': ['2.1.0', '1.0.0'], 'lang': ['1.1.0']} + +# Include prerelease (in-development) versions +versions_with_prerelease = get_available_hed_versions(check_prerelease=True) + +# Once the user picks one, load it — this is the step that actually downloads content +schema = load_schema_version(versions[0]) +``` + +If GitHub can't be reached (offline, rate-limited, etc.), `get_available_hed_versions()` returns an empty list or dict rather than raising, so it's safe to call from a user-facing listing without wrapping it in a try/except. + +### How repeated calls stay cheap + +```{index} caching; rate limits, ETag, ETag-conditional +``` + +`get_available_hed_versions()` is designed to be called often — e.g. every time a web page loads — without adding up to a lot of GitHub traffic. It keeps its own small on-disk listing cache (separate from the downloaded schema content) and checks it in increasingly cheap tiers before ever making a real request: + +1. **Recently checked** — if a given piece of information was checked within the last `cache_time_threshold` seconds (60 by default), it's reused with no network call at all. +2. **Confirmed unchanged** — otherwise, a conditional request is made using a stored ETag. If GitHub confirms nothing changed (a 304 response), the previous result is reused. +3. **Directory-level gate** — before checking individual folders, one cheap check asks whether the `standard_schema/` directory (or the `library_schemas/` directory, depending on what you asked for) has had any new commits at all since the last full check. Since hed-schemas typically only changes every few days or weeks, this lets an entire crawl be skipped in the common case where nothing has changed anywhere relevant. + +For most uses the defaults are fine, but two parameters are available for tuning: + +```python +# Force an immediate, confirmed-fresh answer (e.g. right after publishing a release) +get_available_hed_versions(force_refresh=True) + +# Widen the gap between the cheap directory-level "did anything change?" checks, +# independent of how fresh content is once a real change is found. Useful for a +# long-running, unauthenticated service - see "Authenticating with GitHub" below. +get_available_hed_versions(repo_check_interval=3600) +``` + +### Authenticating with GitHub + +```{index} GITHUB_TOKEN, HED_GITHUB_TOKEN, authentication, rate limits +``` + +GitHub's API allows 60 requests per hour per IP address for unauthenticated callers, versus 5,000 per hour for authenticated ones. Authentication also makes conditional (ETag) requests free of charge against that limit — for unauthenticated callers, even a confirmed-unchanged response still counts against the 60/hour budget. + +If your use of HEDTools makes frequent GitHub calls — a web service checking for new versions, a CI pipeline, a container that restarts often — set a GitHub personal access token (no special scopes needed; it only needs to read a public repository) as an environment variable: + +```bash +export HED_GITHUB_TOKEN=ghp_your_token_here +# GITHUB_TOKEN is also recognized (checked second), which many CI systems set automatically +``` + +HEDTools picks this up automatically; no code changes are needed. + +### Working offline / pre-populating the cache + +```{index} cache_xml_versions, offline, pre-populating cache +``` + +To prepare an environment that won't have network access later (an offline workstation, a Docker image built once and deployed many times), pre-download everything with `cache_xml_versions()`: + +```python +from hed.schema import cache_xml_versions, set_cache_directory + +set_cache_directory("/opt/hed_cache") # optional: a specific location to ship or mount +cache_xml_versions() # downloads every discovered version's full content +``` + +This is a much heavier operation than `get_available_hed_versions()` — it downloads every version it finds, not just a listing — so it's meant to be run once (e.g. during image build or setup), not on a request-handling hot path. It's throttled independently (won't re-run within 30 minutes of its last successful run in the same cache folder) to avoid accidental repeated use. + +### Clearing the cache + +There's no dedicated CLI command for this; the cache is just a regular directory, so you can remove it directly and it will be rebuilt automatically the next time it's needed: + +```python +import shutil +from hed.schema import get_cache_directory + +shutil.rmtree(get_cache_directory(), ignore_errors=True) +``` + ## Jupyter notebooks ```{index} Jupyter notebooks, examples, workflows @@ -860,6 +1000,8 @@ from hed.schema import get_cache_directory print(get_cache_directory()) ``` +See the [Schema caching](#schema-caching) chapter for more on how the cache works, working offline, and avoiding GitHub rate limits. + #### Validation errors **Problem:** Unexpected validation issues diff --git a/hed/schema/hed_cache.py b/hed/schema/hed_cache.py index f6314e06..5b292094 100644 --- a/hed/schema/hed_cache.py +++ b/hed/schema/hed_cache.py @@ -5,6 +5,7 @@ import shutil import os import time +import math import json from hashlib import sha1 @@ -40,6 +41,18 @@ DEFAULT_HED_LIST_VERSIONS_URL = "https://api.github.com/repos/hed-standard/hed-schemas/contents/standard_schema" LIBRARY_HED_URL = "https://api.github.com/repos/hed-standard/hed-schemas/contents/library_schemas" LIBRARY_DATA_URL = "https://raw.githubusercontent.com/hed-standard/hed-schemas/main/library_data.json" +# Cheap, path-scoped endpoints used to detect whether the standard-schema directory or the +# library-schemas directory has changed since the last full crawl of that directory, without +# touching any of the per-version URLs below. Each is scoped (via GitHub's commits-list +# `?path=` filter) to just the one top-level directory a caller actually needs, so a commit +# elsewhere in the repo (docs, CI config, README, etc.) never forces a recrawl here. See +# _get_last_commit_sha() and its use in get_available_hed_versions(). +STANDARD_SCHEMA_HEAD_URL = ( + "https://api.github.com/repos/hed-standard/hed-schemas/commits?path=standard_schema&per_page=1" +) +LIBRARY_SCHEMAS_HEAD_URL = ( + "https://api.github.com/repos/hed-standard/hed-schemas/commits?path=library_schemas&per_page=1" +) DEFAULT_URL_LIST = (DEFAULT_HED_LIST_VERSIONS_URL,) DEFAULT_LIBRARY_URL_LIST = (LIBRARY_HED_URL,) @@ -52,6 +65,12 @@ AVAILABLE_VERSIONS_CACHE_FILENAME = "available_versions_cache.json" AVAILABLE_VERSIONS_TIME_THRESHOLD = 60 +# Keys inside available_versions_cache.json (alongside the per-URL entries) recording the +# last commit SHA affecting each top-level directory, as of the last time that directory's +# per-folder cache entries were actually crawled. +_STANDARD_HEAD_CACHE_KEY = "_standard_schema_head_sha_at_last_crawl" +_LIBRARY_HEAD_CACHE_KEY = "_library_schemas_head_sha_at_last_crawl" + HED_CACHE_DIRECTORY = os.path.join(Path.home(), ".hedtools/hed_cache/") # This is the schemas included in the hedtools package. @@ -279,6 +298,50 @@ def cache_xml_versions( return 0 +def _get_last_commit_sha(url, etag_cache, force_refresh=False, cache_time_threshold=0): + """Best-effort fetch of the latest commit SHA affecting one top-level directory. + + Used by get_available_hed_versions() as a single, cheap gate ahead of the much more + expensive per-folder crawl of that same directory (standard schema, or library listing plus + every library's own hedxml/prerelease folders): if this SHA is unchanged since the last full + crawl of that directory, nothing under it could have changed, so every per-folder URL under + it can be treated as still fresh without individually re-checking - let alone re-fetching - + any of them. Scoping the check to one directory, via GitHub's commits-list `?path=` filter, + means an unrelated commit elsewhere in the repo (docs, CI config, README, etc.) never forces + a recrawl here. hed-schemas changes on the order of days or weeks, so in steady state this + turns what would otherwise be a recurring multi-request burst into a single request per + directory actually needed. + + This participates in the exact same two-tier caching (time-based, then ETag-conditional) + that every other URL here does, via the same etag_cache dict and _get_json_with_etag() - + it is not a separate caching mechanism, just one more URL in the same cache. + + Parameters: + url (str): STANDARD_SCHEMA_HEAD_URL or LIBRARY_SCHEMAS_HEAD_URL. + etag_cache (dict or None): Same per-URL cache dict get_available_hed_versions() uses. + force_refresh (bool): Passed through to _get_json_with_etag(). + cache_time_threshold (int): Passed through to _get_json_with_etag(). + + Returns: + str or None: The latest commit SHA touching this directory, or None if it couldn't be + determined - network error, rate limit, an empty commit history, or a + response shaped differently than expected. Callers must treat None as + "unknown", not "unchanged": every caller of this function falls back to the + pre-existing per-folder behavior whenever this returns None, so a problem + with this one extra endpoint can never produce a stale answer - at worst, it + costs the same (already-optimized) per-folder crawl that would have run + without this gate at all. + """ + try: + loaded_json = _get_json_with_etag(url, etag_cache, force_refresh, cache_time_threshold) + if not loaded_json: + return None + sha = loaded_json[0].get("sha") + return sha if isinstance(sha, str) else None + except Exception: + return None + + def get_available_hed_versions( hed_base_urls=DEFAULT_URL_LIST, hed_library_urls=DEFAULT_LIBRARY_URL_LIST, @@ -288,6 +351,7 @@ def get_available_hed_versions( cache_folder=None, force_refresh=False, cache_time_threshold=AVAILABLE_VERSIONS_TIME_THRESHOLD, + repo_check_interval=None, ) -> Union[list, dict]: """List HED schema versions available on GitHub, without downloading or caching their content. @@ -332,6 +396,20 @@ def get_available_hed_versions( Default is 60 seconds - short enough that new releases show up quickly, long enough that a caller polling this in a tight loop doesn't generate a request per call. + repo_check_interval (int or None): How long, in seconds, to go between the cheap + top-level-directory gate checks described in Notes, before + even asking GitHub whether that directory's latest commit + changed. None (the default) reuses cache_time_threshold, so + behavior is unchanged unless this is set explicitly. A + longer interval directly helps an unauthenticated caller: + GitHub's 60-requests/hour anonymous cap is charged even for + a confirmed-unchanged (304) response, unlike for + authenticated requests, where conditional 304s are free. + Since hed-schemas typically only changes every few days or + weeks, an unauthenticated long-running caller (e.g. a web + service polling this periodically) can safely set this much + larger - minutes to hours - without meaningfully delaying + when a real change is noticed. Returns: Union[list, dict]: List of version numbers, or {library_name: [versions]} if @@ -399,6 +477,29 @@ def get_available_hed_versions( cache_xml_versions() downloads. - force_refresh=True skips layer 1 above but still uses layer 2 (the conditional GET), so it stays cheap when nothing has actually changed. + - Before crawling either the standard schema or the library folders, one extra cheap + check runs first, scoped to just that one top-level directory: the latest commit SHA + affecting standard_schema/ (if the result could include it) or library_schemas/ (if + the result could include library data) - see _get_last_commit_sha() - compared + against the SHA that was current the last time that directory was actually fully + crawled. hed-schemas changes on the order of days or weeks, so if the SHA still + matches, every per-folder URL under that directory is known to still be correct + without checking - or fetching - any of them individually; the "couple dozen + requests" figure above is the cold-start/actual-change case, not the (far more common) + steady-state case. Scoping each gate to just the directory it protects means a commit + elsewhere in the repo (docs, CI config, README, etc.) never forces either directory to + be recrawled. If a gate check itself fails for any reason, it's treated as "unknown" + and that directory falls back to the per-URL behavior described above, unaffected by + the gate ever having existed. When the gate confirms a directory is unchanged, that + directory's per-folder URLs are treated as fresh indefinitely - but only the ones that + were actually fetched successfully; see the "Note" on _get_json_with_etag() for why a + per-URL failure is never shielded from retry by this, no matter how long the gate goes + without seeing a change. + - The gate checks use repo_check_interval (defaulting to cache_time_threshold) rather + than cache_time_threshold directly, so the (already very cheap) gate checks can be + made even less frequent without affecting how fresh a genuinely-changed directory's + own content is once a change is actually detected. This matters most for + unauthenticated callers - see repo_check_interval above. """ if isinstance(hed_base_urls, str): hed_base_urls = [hed_base_urls] @@ -418,12 +519,33 @@ def get_available_hed_versions( # all - a plain standard-schema request (library_name=None) never looks at it. needs_libraries = library_name is not None + if repo_check_interval is None: + repo_check_interval = cache_time_threshold + all_hed_versions = {} if needs_standard: + # A single, cheap check ahead of the standard-schema crawl below, scoped to just that + # directory: if standard_schema/ hasn't changed since the last time this crawl actually + # ran, nothing any of its per-folder URLs would return could have changed either, so + # every one of them can be treated as still fresh - regardless of how much wall-clock + # time has passed - without individually re-checking or re-fetching any of them. A None + # result (network error, rate limit, unexpected response) means "unknown", so these are + # simply left equal to the caller's own values, i.e. exactly the pre-existing behavior. + standard_head_sha = _get_last_commit_sha( + STANDARD_SCHEMA_HEAD_URL, url_cache, force_refresh, repo_check_interval + ) + standard_folder_force_refresh = force_refresh + standard_folder_cache_time_threshold = cache_time_threshold + if standard_head_sha is not None and standard_head_sha == url_cache.get(_STANDARD_HEAD_CACHE_KEY): + standard_folder_force_refresh = False + standard_folder_cache_time_threshold = math.inf + if standard_head_sha is not None: + url_cache[_STANDARD_HEAD_CACHE_KEY] = standard_head_sha + for hed_base_url in hed_base_urls: try: new_hed_versions = _get_hed_xml_versions_one_library( - hed_base_url, url_cache, force_refresh, cache_time_threshold + hed_base_url, url_cache, standard_folder_force_refresh, standard_folder_cache_time_threshold ) _merge_in_versions(all_hed_versions, new_hed_versions) except Exception: @@ -436,6 +558,18 @@ def get_available_hed_versions( # harmless to the caller as a plain connection failure. continue if needs_libraries: + # Same idea as the standard-schema gate above, scoped to library_schemas/ instead - + # kept as an entirely separate gate (rather than one whole-repo check) so a commit + # under, say, standard_schema/ or docs/ never forces a library recrawl, and vice versa. + library_head_sha = _get_last_commit_sha(LIBRARY_SCHEMAS_HEAD_URL, url_cache, force_refresh, repo_check_interval) + library_folder_force_refresh = force_refresh + library_folder_cache_time_threshold = cache_time_threshold + if library_head_sha is not None and library_head_sha == url_cache.get(_LIBRARY_HEAD_CACHE_KEY): + library_folder_force_refresh = False + library_folder_cache_time_threshold = math.inf + if library_head_sha is not None: + url_cache[_LIBRARY_HEAD_CACHE_KEY] = library_head_sha + # "all" means no filter (list every library); a specific name restricts the # helper to just that one library's folder, instead of listing and fetching # every library found under hed_library_urls. @@ -447,8 +581,8 @@ def get_available_hed_versions( library_name=library_filter, skip_folders=skip_folders, etag_cache=url_cache, - force_refresh=force_refresh, - cache_time_threshold=cache_time_threshold, + force_refresh=library_folder_force_refresh, + cache_time_threshold=library_folder_cache_time_threshold, ) if library_filter is not None and new_hed_versions: # When filtered to one library, _get_hed_xml_versions_from_url_all_libraries() @@ -558,7 +692,10 @@ def _get_json_with_etag(url, etag_cache, force_refresh=False, cache_time_thresho force_refresh (bool): Skip tier 1 (the time-based shortcut) even if the cached entry is still within cache_time_threshold. Tier 2's conditional GET is still used, so this remains cheap when nothing has changed. - cache_time_threshold (int): How long, in seconds, tier 1 applies for. + cache_time_threshold (int): How long, in seconds, tier 1 applies for. For a *recorded + failure* specifically, this is capped at + AVAILABLE_VERSIONS_TIME_THRESHOLD regardless of the value + passed in - see the note below. Returns: The parsed JSON body for this URL - freshly fetched, or reused from etag_cache. @@ -566,6 +703,25 @@ def _get_json_with_etag(url, etag_cache, force_refresh=False, cache_time_thresho Raises: urllib.error.URLError: If the URL could not be reached (including a recent, still-fresh failure recorded by an earlier call - see etag_cache above). + + Note: + A successful result and a recorded failure are not equally safe to hold onto for a long + cache_time_threshold. get_available_hed_versions() has two independent directory-level + gates - one for standard_schema/, one for library_schemas/ - and each deliberately passes + a very large (or infinite) threshold here once it's confirmed its own directory hasn't + changed, so that already-fetched, successful per-folder entries under that directory can + be reused indefinitely without individually re-checking each one. This applies equally to + both gates, since they funnel through this same function. But a *failure* entry (body is + None) isn't "known good data" that staying unchanged makes safe to keep reusing - it's + the absence of data, typically from a transient rate-limit or network error unrelated to + whether the directory's content has actually changed. Honoring a large threshold for a + failure would mean a transient error picked up during one crawl keeps being silently + skipped (see the broad except clauses in _get_hed_xml_versions_one_library() and + _get_hed_xml_versions_from_url_all_libraries()) for as long as the directory happens not + to change - potentially days or weeks - even though the underlying problem may have + resolved within minutes. To prevent that, for either gate, a recorded failure is always + retried within AVAILABLE_VERSIONS_TIME_THRESHOLD seconds, no matter how large a + cache_time_threshold was passed in for this call. """ cached_entry = etag_cache.get(url) if etag_cache is not None else None if not isinstance(cached_entry, dict): @@ -575,8 +731,13 @@ def _get_json_with_etag(url, etag_cache, force_refresh=False, cache_time_thresho cached_entry = None if cached_entry and not force_refresh: + effective_threshold = cache_time_threshold + if cached_entry.get("body") is None: + # See the "Note" above: never let a recorded failure be shielded from retry by a + # large threshold that was only ever meant to apply to confirmed-good data. + effective_threshold = min(cache_time_threshold, AVAILABLE_VERSIONS_TIME_THRESHOLD) age = time.time() - cached_entry.get("timestamp", 0) - if age <= cache_time_threshold: + if age <= effective_threshold: if cached_entry.get("body") is None: raise URLError(f"A recent attempt to reach {url} failed; not retrying yet") return cached_entry["body"] diff --git a/tests/schema/test_hed_schema_io.py b/tests/schema/test_hed_schema_io.py index 839152b0..1f7711ce 100644 --- a/tests/schema/test_hed_schema_io.py +++ b/tests/schema/test_hed_schema_io.py @@ -9,7 +9,9 @@ import os import json +import math import tempfile +from urllib.error import URLError from semantic_version import Version from hed.errors import HedExceptions from hed.schema import HedKey @@ -240,9 +242,15 @@ def test_get_available_hed_versions_skips_unneeded_urls(self): f"library_name='score' should never touch standard-schema URLs, but cached: {cached_urls}", ) library_base = hed_cache.LIBRARY_HED_URL + # Besides the actual per-library content URLs, the cache also legitimately holds + # two library_schemas/-gate entries that aren't library-content URLs at all: the + # gate's own commits-endpoint URL, and the plain-string SHA marker recorded under + # it (see _LIBRARY_HEAD_CACHE_KEY) - neither is scoped to a specific library, so + # both are expected regardless of which library_name was requested. + non_library_keys = {library_base, hed_cache.LIBRARY_SCHEMAS_HEAD_URL, hed_cache._LIBRARY_HEAD_CACHE_KEY} for url in cached_urls: - if url == library_base: - continue # the one unavoidable "list all libraries" call + if url in non_library_keys: + continue # the one unavoidable "list all libraries" call, or a gate entry # Trailing slash matters here - without it, a hypothetical library named # e.g. "scoreboard" would incorrectly pass a plain startswith(".../score"). self.assertTrue( @@ -295,6 +303,248 @@ def test_get_available_hed_versions_tolerates_corrupted_cache_entry(self): self.skipTest("GitHub unreachable or rate-limited in this environment") _assert_valid_sorted_versions(self, versions) + def test_get_last_commit_sha_returns_a_sha_for_standard_and_library_paths(self): + """_get_last_commit_sha() should return a real commit SHA for each scoped gate URL.""" + for url in (hed_cache.STANDARD_SCHEMA_HEAD_URL, hed_cache.LIBRARY_SCHEMAS_HEAD_URL): + sha = hed_cache._get_last_commit_sha(url, {}) + if sha is None: + self.skipTest("GitHub unreachable or rate-limited in this environment") + self.assertIsInstance(sha, str) + self.assertTrue(sha, f"expected a non-empty SHA for {url}") + + def test_get_last_commit_sha_degrades_gracefully_on_bad_path(self): + """_get_last_commit_sha() should return None, not raise, for a path with no commit history. + + Points at a real repo (hed-standard/hed-schemas) but a path that never existed, rather + than mocking a bad response, per this repo's testing policy. GitHub's commits-list + endpoint returns an empty array (not a 404) for such a path. + """ + bad_url = ( + "https://api.github.com/repos/hed-standard/hed-schemas/commits" + "?path=this-path-does-not-exist-abc123&per_page=1" + ) + sha = hed_cache._get_last_commit_sha(bad_url, {}) + self.assertIsNone(sha) + + def test_get_json_with_etag_retries_failed_entry_despite_large_threshold(self): + """A recorded failure should be retried well before a very large cache_time_threshold + would otherwise allow, even though a successful entry is legitimately allowed to be + reused for that long. + + Regression test for a gap flagged in review: get_available_hed_versions()'s + directory-level gate passes math.inf as the per-folder cache_time_threshold once it + confirms a directory is unchanged, so that already-fetched, successful entries can be + reused indefinitely. But a *failed* per-URL entry isn't "known good data" - honoring + that same infinite threshold for it would mean a transient rate-limit/network error + gets silently, permanently skipped until the gate happens to see a real change (days or + weeks later). Uses a real, guaranteed-unreachable address (no mocks, per this repo's + testing policy) so the "a retry actually happened" assertion reflects a genuine second + network attempt, not a mocked one - and works regardless of whether GitHub itself is + reachable in this environment. + """ + unreachable_url = "http://127.0.0.1:1/this-will-never-respond" + etag_cache = {} + + with self.assertRaises(URLError): + hed_cache._get_json_with_etag(unreachable_url, etag_cache, force_refresh=False, cache_time_threshold=0) + + self.assertIn(unreachable_url, etag_cache) + first_failure_timestamp = etag_cache[unreachable_url]["timestamp"] + + # Simulate the failure having been recorded a while ago - well past + # AVAILABLE_VERSIONS_TIME_THRESHOLD, but still comfortably inside a "the gate confirmed + # nothing changed" infinite threshold. + etag_cache[unreachable_url]["timestamp"] = first_failure_timestamp - ( + hed_cache.AVAILABLE_VERSIONS_TIME_THRESHOLD + 30 + ) + + with self.assertRaises(URLError): + hed_cache._get_json_with_etag( + unreachable_url, etag_cache, force_refresh=False, cache_time_threshold=math.inf + ) + + # If the large threshold had been honored for this failure, tier 1 would have re-raised + # immediately from the stale cached entry without touching the network again, and the + # timestamp would still be the artificially backdated one. Seeing a fresh timestamp + # proves a real retry happened instead. + self.assertGreater(etag_cache[unreachable_url]["timestamp"], first_failure_timestamp) + + def test_get_available_hed_versions_skips_crawl_when_standard_schema_unchanged(self): + """A repeat call should skip the standard-schema crawl once its scoped gate confirms + nothing changed under standard_schema/ on GitHub - even when force_refresh=True forces + the gate itself to revalidate. + + Verified by checking that every per-folder URL's stored timestamp is untouched by the + second call, while relying on real network calls throughout (no mocking, per this + repo's testing policy) - this assumes hed-schemas genuinely doesn't change mid-test, + which is exactly the (overwhelmingly common) scenario this gate exists for. + + The gate itself is best-effort: _get_last_commit_sha() can return None (commits endpoint + unreachable or rate-limited independently of the rest of GitHub's API), in which case + get_available_hed_versions() falls back to a normal per-folder crawl and can still return + a non-empty, correct version list without ever recording the gate SHA. That's correct + behavior, not something this test is meant to catch - so skip rather than fail when the + gate key never got written. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(hed_cache, "HED_CACHE_DIRECTORY", tmp_dir): + first = hed_cache.get_available_hed_versions() + if not first: + self.skipTest("GitHub unreachable or rate-limited in this environment") + + cache_filename = os.path.join(tmp_dir, hed_cache.AVAILABLE_VERSIONS_CACHE_FILENAME) + with open(cache_filename) as f: + cache_after_first = json.load(f) + if hed_cache._STANDARD_HEAD_CACHE_KEY not in cache_after_first: + self.skipTest( + "standard-schema gate SHA unavailable this run (commits endpoint unreachable or " + "rate-limited); get_available_hed_versions() correctly fell back to a per-folder crawl" + ) + gate_keys = {hed_cache._STANDARD_HEAD_CACHE_KEY, hed_cache._LIBRARY_HEAD_CACHE_KEY} + folder_urls = [ + url + for url in cache_after_first + if url not in gate_keys and url != hed_cache.STANDARD_SCHEMA_HEAD_URL + ] + self.assertTrue(folder_urls, "expected at least one per-folder URL cached") + folder_timestamps_before = {url: cache_after_first[url]["timestamp"] for url in folder_urls} + + second = hed_cache.get_available_hed_versions(force_refresh=True) + self.assertEqual(second, first, "result should be unchanged since nothing on GitHub changed") + + with open(cache_filename) as f: + cache_after_second = json.load(f) + for url in folder_urls: + self.assertEqual( + cache_after_second[url]["timestamp"], + folder_timestamps_before[url], + f"{url} should not have been re-checked once the standard-schema gate confirmed " + "nothing had changed", + ) + + def test_get_available_hed_versions_recrawls_when_standard_schema_head_sha_stale(self): + """A stored standard-schema gate SHA that doesn't match the real one should trigger a + normal crawl of standard_schema/. + + Hand-writing a deliberately wrong SHA into the cache file (mimicking what a real repo + change would look like on the next call) - without mocking any network response - and + confirming get_available_hed_versions() still returns real, complete data rather than + incorrectly trusting the stale marker, and that the marker gets corrected afterward. + + Refreshing the marker is itself best-effort: _get_last_commit_sha() can return None + (commits endpoint unreachable or rate-limited independently of the rest of GitHub's + API), in which case get_available_hed_versions() falls back to a normal per-folder crawl + and still returns valid data, but deliberately leaves the existing marker untouched + rather than overwriting it with nothing useful. That's correct behavior, not something + this test is meant to catch - so skip rather than fail when the marker wasn't refreshed. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(hed_cache, "HED_CACHE_DIRECTORY", tmp_dir): + cache_filename = os.path.join(tmp_dir, hed_cache.AVAILABLE_VERSIONS_CACHE_FILENAME) + with open(cache_filename, "w") as f: + json.dump({hed_cache._STANDARD_HEAD_CACHE_KEY: "not-a-real-sha"}, f) + + versions = hed_cache.get_available_hed_versions() + if not versions: + self.skipTest("GitHub unreachable or rate-limited in this environment") + _assert_valid_sorted_versions(self, versions) + + with open(cache_filename) as f: + cache_after = json.load(f) + if cache_after[hed_cache._STANDARD_HEAD_CACHE_KEY] == "not-a-real-sha": + self.skipTest( + "standard-schema gate SHA unavailable this run (commits endpoint unreachable or " + "rate-limited); get_available_hed_versions() correctly fell back to a per-folder crawl " + "without refreshing the marker" + ) + + def test_get_available_hed_versions_gates_are_scoped_independently(self): + """A stale library gate marker should not force a standard-schema recrawl, and vice + versa - confirming the two gates (standard_schema/ and library_schemas/) are + independent rather than one shared whole-repo check. + + Both gate SHAs are best-effort: _get_last_commit_sha() can return None (its commits + endpoint unreachable or rate-limited independently of the rest of GitHub's API), in + which case get_available_hed_versions() falls back to a normal per-folder crawl for that + directory and never records the corresponding gate key - correct behavior, but not one + this test (which specifically manipulates the library gate key) can exercise. Skip + rather than fail when either key never got written. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(hed_cache, "HED_CACHE_DIRECTORY", tmp_dir): + first = hed_cache.get_available_hed_versions(library_name="all") + if not first: + self.skipTest("GitHub unreachable or rate-limited in this environment") + + cache_filename = os.path.join(tmp_dir, hed_cache.AVAILABLE_VERSIONS_CACHE_FILENAME) + with open(cache_filename) as f: + cache_after_first = json.load(f) + if ( + hed_cache._STANDARD_HEAD_CACHE_KEY not in cache_after_first + or hed_cache._LIBRARY_HEAD_CACHE_KEY not in cache_after_first + ): + self.skipTest( + "a gate SHA was unavailable this run (commits endpoint unreachable or rate-limited); " + "get_available_hed_versions() correctly fell back to a per-folder crawl" + ) + + standard_folder_urls = [ + url for url in cache_after_first if url.startswith(hed_cache.DEFAULT_HED_LIST_VERSIONS_URL) + ] + self.assertTrue(standard_folder_urls, "expected at least one standard-schema folder URL cached") + standard_timestamps_before = {url: cache_after_first[url]["timestamp"] for url in standard_folder_urls} + + # Corrupt only the library gate marker - the standard-schema gate marker is left + # untouched, so its crawl should still be skipped even though force_refresh=True + # forces both gates to revalidate against GitHub. + cache_after_first[hed_cache._LIBRARY_HEAD_CACHE_KEY] = "not-a-real-sha" + with open(cache_filename, "w") as f: + json.dump(cache_after_first, f) + + hed_cache.get_available_hed_versions(library_name="all", force_refresh=True) + + with open(cache_filename) as f: + cache_after_second = json.load(f) + for url in standard_folder_urls: + self.assertEqual( + cache_after_second[url]["timestamp"], + standard_timestamps_before[url], + f"{url} should not have been re-checked - only the library gate was made stale", + ) + + def test_get_available_hed_versions_repo_check_interval_limits_gate_frequency(self): + """A larger repo_check_interval should reduce how often the gate URL itself is hit, + independently of cache_time_threshold for the per-folder content URLs. + + Verified by checking the on-disk cache after two rapid calls: with a large + repo_check_interval, the gate URL's own cache entry should not be re-checked on the + second call (tier-1 reuse). This can't assert anything about GitHub's actual request + count without mocking, which this repo's testing policy avoids - only the client-side + effect of not even attempting a network call is checkable here. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(hed_cache, "HED_CACHE_DIRECTORY", tmp_dir): + first = hed_cache.get_available_hed_versions(repo_check_interval=3600) + if not first: + self.skipTest("GitHub unreachable or rate-limited in this environment") + + cache_filename = os.path.join(tmp_dir, hed_cache.AVAILABLE_VERSIONS_CACHE_FILENAME) + with open(cache_filename) as f: + cache_after_first = json.load(f) + gate_entry = cache_after_first.get(hed_cache.STANDARD_SCHEMA_HEAD_URL) + self.assertIsInstance(gate_entry, dict, "expected the gate URL itself to be cached") + gate_timestamp_before = gate_entry["timestamp"] + + hed_cache.get_available_hed_versions(repo_check_interval=3600) + + with open(cache_filename) as f: + cache_after_second = json.load(f) + self.assertEqual( + cache_after_second[hed_cache.STANDARD_SCHEMA_HEAD_URL]["timestamp"], + gate_timestamp_before, + "a large repo_check_interval should have skipped re-checking the gate itself", + ) + def test_load_schema_version_default_no_standard_raises(self): """Test that empty version with only library schemas raises HedFileError.""" with tempfile.TemporaryDirectory() as tmp_dir: