diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8c5de1ae..c73c51d5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -5,6 +5,7 @@ ## Session summaries and reports **When you fix bugs, implement features, or investigate issues, always create a summary report in `.status/` directory.** Name it descriptively (e.g., `.status/fix_missing_extra_sections.md`, `.status/investigation_cache_bug.md`, `.status/implement_feature_x.md`). The report should document: + - What problem was being solved - What approach was taken - What code changes were made and why diff --git a/hed/schema/hed_cache.py b/hed/schema/hed_cache.py index 5b292094..d4fe4c0b 100644 --- a/hed/schema/hed_cache.py +++ b/hed/schema/hed_cache.py @@ -137,8 +137,9 @@ def get_hed_versions(local_hed_directory=None, library_name=None, check_prerelea hed_files += os.listdir(hed_dir) except FileNotFoundError: pass - if not hed_files: + if not any(version_pattern.match(f) for f in hed_files): cache_local_versions(local_hed_directory) + hed_files = [] for hed_dir in local_directories: try: hed_files += os.listdir(hed_dir) @@ -166,15 +167,16 @@ def get_hed_versions(local_hed_directory=None, library_name=None, check_prerelea def get_hed_version_path(xml_version, library_name=None, local_hed_directory=None) -> Union[str, None]: """Get the HED XML file path for a given version. - Searches the local cache first. If the version is not found and local_hed_directory - is the default HED cache, the cache is refreshed from GitHub before a second lookup. + Searches the local cache first (including the bundled schemas that are always present). + If the version is not found and local_hed_directory is the default HED cache, downloads + only the single requested file from GitHub — never the entire catalog. No network call is made for custom directories. Parameters: xml_version (str): The version string to look up. library_name (str or None): Optional schema library name. local_hed_directory (str or None): Path to local HED directory. Defaults to HED_CACHE_DIRECTORY. - Passing a custom path disables the automatic GitHub refresh. + Passing a custom path disables the automatic GitHub download. Returns: Union[str, None]: The path to the requested HED XML file, or None. @@ -187,13 +189,12 @@ def get_hed_version_path(xml_version, library_name=None, local_hed_directory=Non if result: return result - # Version not found locally — try refreshing cache from GitHub (default cache only). - # cache_xml_versions() returns -1 on failure (network error, lock contention, rate limit). - # In that case the second lookup will return None, which the caller treats as "version not found". + # Version not found locally — download only this specific version from GitHub. + # Never bulk-download the entire catalog; that is cache_xml_versions()'s job. if not xml_version or local_hed_directory != HED_CACHE_DIRECTORY: return None - cache_xml_versions() + _download_schema_version(xml_version, library_name, local_hed_directory) return _find_hed_version_path(xml_version, library_name, local_hed_directory) @@ -272,6 +273,10 @@ def cache_xml_versions( if not cache_folder: cache_folder = HED_CACHE_DIRECTORY + # Always seed the cache with bundled schemas first so the cache is usable even if the + # subsequent GitHub download fails (network error, rate limit, etc.). + cache_local_versions(cache_folder) + try: with CacheLock(cache_folder): if isinstance(hed_base_urls, str): @@ -508,9 +513,39 @@ def get_available_hed_versions( if not cache_folder: cache_folder = HED_CACHE_DIRECTORY + # Resolve repo_check_interval early so the manifest fast path can honour it. + # The manifest IS the "has anything changed?" gate for the whole repo, so it should + # be throttled by repo_check_interval (the interval for those repo-level checks), + # not by the shorter cache_time_threshold (which governs individual folder listings). + if repo_check_interval is None: + repo_check_interval = cache_time_threshold + url_cache = _read_available_versions_cache(cache_folder) cache_before = json.dumps(url_cache, sort_keys=True) + # Fast path: read the repo-level manifest in a single fetch from the raw/CDN host (not subject + # to the GitHub REST API rate limit) instead of crawling the API directory listings. Only used + # for the canonical hed-schemas URLs; any custom/forked URL set falls through to the crawl. Any + # failure (unreachable, malformed, or an unrecognized manifest format) also falls through. + # The manifest is used as the "has the repo changed?" check: its per-file sha fields are the + # hashes we compare against locally cached files, and repo_check_interval controls how often + # we even ask GitHub whether the manifest itself has changed. + if ( + list(hed_base_urls) == list(DEFAULT_URL_LIST) + and list(hed_library_urls) == list(DEFAULT_LIBRARY_URL_LIST) + and tuple(skip_folders) == tuple(DEFAULT_SKIP_FOLDERS) + ): + from hed.schema import schema_version_manifest as _manifest + + try: + manifest_json = _get_json_with_etag(_manifest.MANIFEST_URL, url_cache, force_refresh, repo_check_interval) + if _manifest.is_supported(manifest_json): + if json.dumps(url_cache, sort_keys=True) != cache_before: + _write_available_versions_cache(cache_folder, url_cache) + return _manifest.available_versions(manifest_json, library_name, check_prerelease) + except Exception: + pass # fall through to the REST API crawl below + # Only fetch the standard-schema URLs when the result could actually include them # (library_name is None or "all") - a request for one specific library has no use for # this data, and fetching it anyway would be pure wasted requests. @@ -519,9 +554,6 @@ def get_available_hed_versions( # all - a plain standard-schema request (library_name=None) never looks at it. needs_libraries = library_name is not None - if repo_check_interval is None: - repo_check_interval = cache_time_threshold - all_hed_versions = {} if needs_standard: # A single, cheap check ahead of the standard-schema crawl below, scoped to just that @@ -1020,6 +1052,63 @@ def _safe_move_tmp_to_folder(temp_hed_xml_file, dest_filename): return dest_filename +def _download_schema_version(xml_version, library_name, cache_folder): + """Download a single specific schema version from GitHub. + + Fetches the directory listing for only the relevant library (2-3 small API calls), then + downloads only the one requested XML file. Uses SHA comparison so a file whose content + has not changed on GitHub is never re-downloaded. + + This is the on-demand complement to cache_xml_versions() (which bulk-downloads every + version). Call this when you need one specific version that is not in the local cache; + call cache_xml_versions() only when you want to pre-populate the entire catalog (e.g. + for an offline/air-gapped environment). + + Parameters: + xml_version (str): The version string to download. + library_name (str or None): The library name, None for the standard schema. + cache_folder (str): The folder to save the downloaded file in. + + Returns: + str or None: The local file path on success, None if the version was not found on + GitHub or the download failed. + """ + # Fast path: resolve this version from the manifest (single raw/CDN fetch, no REST API), then + # download just the one file. Falls back to the REST API resolution below on any failure, + # including a malformed manifest that passes is_supported() but raises inside find_version_info + # or _cache_hed_version (unexpected types, missing keys, etc.). + from hed.schema import schema_version_manifest as _manifest + + try: + manifest_json = _get_json_with_etag(_manifest.MANIFEST_URL, None) + if _manifest.is_supported(manifest_json): + version_info = _manifest.find_version_info(manifest_json, xml_version, library_name) + if version_info is not None: + return _cache_hed_version(xml_version, library_name, version_info, cache_folder) + except Exception: + pass # fall through to REST API resolution below + + if library_name is None: + url = DEFAULT_HED_LIST_VERSIONS_URL + else: + url = f"{LIBRARY_HED_URL}/{library_name}" + + try: + available = _get_hed_xml_versions_one_library(url) + except Exception: + return None + + versions_for_library = available.get(library_name) + if not versions_for_library: + return None + + version_info = versions_for_library.get(xml_version) + if version_info is None: + return None + + return _cache_hed_version(xml_version, library_name, version_info, cache_folder) + + def _cache_hed_version(version, library_name, version_info, cache_folder): """Cache the given HED version""" sha_hash, download_url, prerelease = version_info diff --git a/hed/schema/schema_validation/hed_id_validator.py b/hed/schema/schema_validation/hed_id_validator.py index 74eca95d..6ceb41fe 100644 --- a/hed/schema/schema_validation/hed_id_validator.py +++ b/hed/schema/schema_validation/hed_id_validator.py @@ -1,9 +1,8 @@ """Validator for HED ID consistency across schema versions.""" -from hed.schema.hed_cache import get_library_data +from hed.schema.hed_cache import get_library_data, get_available_hed_versions, get_hed_versions from semantic_version import Version from hed.schema.hed_schema_io import load_schema_version -from hed.schema.hed_cache import get_hed_versions from hed.schema.hed_schema_constants import HedKey from hed.errors.error_types import SchemaAttributeErrors from hed.errors.error_reporter import ErrorHandler @@ -52,7 +51,13 @@ def __init__(self, hed_schema): @staticmethod def _get_previous_version(version, library): current_version = Version(version) - all_schema_versions = get_hed_versions(library_name=library, check_prerelease=False) + # Use the manifest-based listing as the authoritative version source so that a version + # released after the local cache was last populated is not silently skipped. Fall back + # to the local cache when the network is unavailable. + library_name = library if library else None + all_schema_versions = get_available_hed_versions(library_name=library_name) or get_hed_versions( + library_name=library, check_prerelease=False + ) for old_version in all_schema_versions: if Version(old_version) < current_version: prev_version = old_version diff --git a/hed/schema/schema_version_manifest.py b/hed/schema/schema_version_manifest.py new file mode 100644 index 00000000..74a3cc4b --- /dev/null +++ b/hed/schema/schema_version_manifest.py @@ -0,0 +1,148 @@ +"""Repo-level manifest fast path for HED schema version discovery. + +hed-schemas publishes a single ``schema_versions.json`` at its repository root listing every schema +version (released / prerelease / deprecated) for the standard schema and each library, with each +XML file's git blob SHA. hedtools can read that one file from the raw/CDN host instead of crawling +the GitHub REST API directory listings (which are metered - 60 requests/hour unauthenticated, with +even conditional 304s counting against that cap). + +This module is deliberately I/O-light and has no dependency on the rest of ``hed.schema`` (it never +imports :mod:`hed.schema.hed_cache`), so hed_cache can use it as a drop-in fast path while keeping +the REST API crawl as a fallback. The network fetch itself is performed by the caller (hed_cache) +via its existing ETag-aware helper; the functions here only parse an already-loaded manifest dict. + +Deprecated versions are recorded in the manifest so that the static HED schema browser can list and +display them directly (no hedtools call involved). hedtools' own compute paths do not operate on +deprecated schemas, so neither the listing nor the load/download helpers here surface them. + +Manifest shape (``manifest_format_version`` 1):: + + { + "manifest_format_version": 1, + "generated": "", + "repo_commit": "", + "libraries": { + "": {"released": [{"version", "file", "sha", "date"}], "prerelease": [...], "deprecated": [...]}, + "score": {...}, + ... + } + } + +The standard schema is keyed by the empty string ``""`` in the manifest; internally hedtools uses +``None`` for the standard schema, so that mapping is applied here. +""" + +from __future__ import annotations + +from semantic_version import Version + +# The manifest is served as raw file content (CDN-backed), not through the REST API, so it is not +# subject to the GitHub REST API rate limit. +RAW_CONTENT_BASE = "https://raw.githubusercontent.com/hed-standard/hed-schemas" +MANIFEST_FILE = "schema_versions.json" +MANIFEST_URL = f"{RAW_CONTENT_BASE}/main/{MANIFEST_FILE}" + +# Only this on-disk format is understood; a manifest advertising anything else is ignored so a +# future format change can never be silently misread (the caller falls back to the REST API crawl). +SUPPORTED_MANIFEST_FORMAT_VERSION = 1 + +_STANDARD_MANIFEST_KEY = "" # the standard schema's key inside manifest["libraries"] + + +def is_supported(manifest) -> bool: + """Return True if ``manifest`` is a dict in a format version this module understands. + + Parameters: + manifest: The parsed manifest (any type - non-dicts and unknown formats return False). + + Returns: + bool: True only for a dict whose ``manifest_format_version`` matches the supported version + and whose ``libraries`` value is a dict (the shape all downstream helpers require). + """ + return ( + isinstance(manifest, dict) + and manifest.get("manifest_format_version") == SUPPORTED_MANIFEST_FORMAT_VERSION + and isinstance(manifest.get("libraries"), dict) + ) + + +def raw_url_for(file_path: str, ref: str) -> str: + """Build the raw/CDN URL for a repo-relative file path at a given git ref (commit/branch/tag).""" + return f"{RAW_CONTENT_BASE}/{ref}/{file_path}" + + +def _sort_versions(versions): + """Sort version strings newest-first using semantic-versioning precedence.""" + return sorted(versions, key=Version, reverse=True) + + +def _versions_for_key(manifest, manifest_key, check_prerelease): + """Collect (and sort, newest-first) the versions for one manifest library key.""" + categories = manifest.get("libraries", {}).get(manifest_key, {}) + versions = [entry["version"] for entry in categories.get("released", [])] + if check_prerelease: + versions += [entry["version"] for entry in categories.get("prerelease", [])] + # ``deprecated`` is intentionally never included, matching the default skip_folders behavior of + # the REST API crawl (which skips the ``deprecated`` folder). + return _sort_versions(versions) + + +def available_versions(manifest, library_name=None, check_prerelease=False): + """Return versions from ``manifest`` in the exact shape of get_available_hed_versions(). + + Parameters: + manifest (dict): A supported manifest (see :func:`is_supported`). + library_name (str or None): None for the standard schema only; "all" for every library as a + dict; or a specific library name. + check_prerelease (bool): If True, include prerelease versions. + + Returns: + Union[list, dict]: A list of version strings, or - for "all" - a dict + {library_name_or_None: [versions]} with the standard schema under None. + """ + libraries = manifest.get("libraries", {}) + if library_name == "all": + result = {} + for manifest_key in libraries: + internal_key = None if manifest_key == _STANDARD_MANIFEST_KEY else manifest_key + versions = _versions_for_key(manifest, manifest_key, check_prerelease) + if versions: + result[internal_key] = versions + return result + + manifest_key = _STANDARD_MANIFEST_KEY if library_name is None else library_name + return _versions_for_key(manifest, manifest_key, check_prerelease) + + +def find_version_info(manifest, xml_version, library_name, ref=None): + """Locate one version in ``manifest`` and return its download info, or None if absent. + + Parameters: + manifest (dict): A supported manifest. + xml_version (str): The version string to find. + library_name (str or None): The library name, None for the standard schema. + ref (str or None): Git ref for the raw URL. Defaults to the manifest's ``repo_commit`` (so + the fetched bytes are immutable and match the recorded SHA), then "main". + + Returns: + tuple or None: ``(sha, download_url, prerelease)`` matching the ``version_info`` shape that + hed_cache._cache_hed_version() consumes, or None if the version is not + listed as a released or prerelease version. + + Notes: + - Only ``released`` and ``prerelease`` versions are searched; ``deprecated`` versions are + intentionally NOT loadable through this path. Deprecated schemas exist only so the static + HED schema browser can display them (reading them straight from the manifest); hedtools' + compute paths (validation, etc.) do not operate on deprecated schemas. This also keeps + behavior consistent with the REST API fallback, whose crawl skips the ``deprecated`` + folder. + """ + manifest_key = _STANDARD_MANIFEST_KEY if library_name is None else library_name + categories = manifest.get("libraries", {}).get(manifest_key, {}) + if ref is None: + ref = manifest.get("repo_commit") or "main" + for category, is_prerelease in (("released", False), ("prerelease", True)): + for entry in categories.get(category, []): + if entry["version"] == xml_version: + return (entry["sha"], raw_url_for(entry["file"], ref), is_prerelease) + return None diff --git a/spec_tests/hed-tests b/spec_tests/hed-tests index 9844ab8e..d2fc8f6f 160000 --- a/spec_tests/hed-tests +++ b/spec_tests/hed-tests @@ -1 +1 @@ -Subproject commit 9844ab8eaec3c2dafe211e88ee8407a28b1d76ac +Subproject commit d2fc8f6f90ae3ac73a73d9e6c2105833b13334ad diff --git a/spec_tests/test_hed_cache.py b/spec_tests/test_hed_cache.py index 9734b796..6ef57bbf 100644 --- a/spec_tests/test_hed_cache.py +++ b/spec_tests/test_hed_cache.py @@ -122,7 +122,14 @@ def test_get_hed_version_path_no_auto_refresh_for_custom_directory(self): shutil.rmtree(empty_dir) def test_get_hed_version_path_auto_refresh_downloads_missing_version(self): - """get_hed_version_path automatically downloads from GitHub when a version is not cached locally.""" + """get_hed_version_path downloads only the single requested version from GitHub. + + Uses a testlib version, which is not bundled with hedtools, so the download must come from + GitHub rather than the local bundle. testlib lives in a normal ``hedxml/`` folder, so this + exercises the real on-demand download path (manifest fast path and REST API fallback alike) + without relying on deprecated versions. Only the listing for the relevant library and the + one XML file are fetched - not the entire catalog. + """ # Use a fresh cache directory so the version is definitely not present fresh_cache = os.path.join(os.path.dirname(self.hed_cache_dir), "schema_cache_auto_refresh/") if os.path.exists(fresh_cache): @@ -131,10 +138,17 @@ def test_get_hed_version_path_auto_refresh_downloads_missing_version(self): saved = hed_cache.HED_CACHE_DIRECTORY try: hed_cache.HED_CACHE_DIRECTORY = fresh_cache - # 8.0.0 is a released version that should be downloadable from GitHub - result = hed_cache.get_hed_version_path("8.0.0") - self.assertIsNotNone(result, "Auto-refresh should download 8.0.0 from GitHub") + # testlib 2.0.0 is a released library version that is NOT bundled - it must come from GitHub. + result = hed_cache.get_hed_version_path("2.0.0", library_name="testlib") + self.assertIsNotNone(result, "get_hed_version_path should download testlib 2.0.0 from GitHub on demand") self.assertTrue(os.path.exists(result)) + # Only the one requested file should have been downloaded (plus any bundled schemas the + # cache seeds); no full-catalog download. + xml_files = [f for f in os.listdir(fresh_cache) if hed_cache.version_pattern.match(f)] + self.assertTrue( + any("testlib" in f and "2.0.0" in f for f in xml_files), + f"HED_testlib_2.0.0.xml must be in the cache after on-demand download; found: {xml_files}", + ) finally: hed_cache.HED_CACHE_DIRECTORY = saved shutil.rmtree(fresh_cache, ignore_errors=True) diff --git a/tests/schema/test_hed_schema_io.py b/tests/schema/test_hed_schema_io.py index a2cd9df7..f8c5e760 100644 --- a/tests/schema/test_hed_schema_io.py +++ b/tests/schema/test_hed_schema_io.py @@ -132,6 +132,139 @@ def test_get_hed_versions_includes_bundled_score(self): self.assertIsInstance(versions, list) self.assertTrue(versions, "score library schemas must always be present in the bundled schema_data") + def test_get_hed_versions_falls_back_when_dir_has_only_non_xml_files(self): + """get_hed_versions() must trigger the bundled-schema fallback even when the cache directory + exists and is non-empty — provided none of the files match the HED XML pattern. + + This is the scenario that occurs when cache_xml_versions() fails after acquiring its lock: + the directory is created and cache_lock.lock is written, but no XML schemas are downloaded. + Before the fix, get_hed_versions() treated any non-empty directory as "already populated" + and skipped the fallback, returning []. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + # Plant only a lock file — no XML schema files. + with open(os.path.join(tmp_dir, "cache_lock.lock"), "w") as f: + f.write("") + + standard_versions = hed_cache.get_hed_versions(tmp_dir) + self.assertIsInstance(standard_versions, list) + self.assertTrue( + standard_versions, "bundled standard schemas must be found even with only a lock file present" + ) + + score_versions = hed_cache.get_hed_versions(tmp_dir, library_name="score") + self.assertIsInstance(score_versions, list) + self.assertTrue(score_versions, "bundled score schemas must be found even with only a lock file present") + + def test_cache_xml_versions_seeds_bundled_schemas_on_github_failure(self): + """cache_xml_versions() must seed the cache with bundled schemas even when the GitHub + download fails (network error, rate limit, etc.). + + cache_local_versions() is called unconditionally before the CacheLock context, so the + cache always contains at least the bundled released schemas. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + hed_cache, "_get_hed_xml_versions_one_library", side_effect=URLError("simulated failure") + ): + result = hed_cache.cache_xml_versions(cache_folder=tmp_dir) + + # GitHub download failed — cache_xml_versions returns -1. + self.assertEqual(result, -1) + + # Bundled schemas must still be present despite the failure. + cached_files = os.listdir(tmp_dir) + xml_files = [f for f in cached_files if hed_cache.version_pattern.match(f)] + self.assertTrue(xml_files, "bundled schemas must be seeded into the cache even when GitHub download fails") + + standard = hed_cache.get_hed_versions(tmp_dir) + self.assertTrue(standard, "standard schemas must be in cache after failed cache_xml_versions") + score = hed_cache.get_hed_versions(tmp_dir, library_name="score") + self.assertTrue(score, "score schemas must be in cache after failed cache_xml_versions") + + def test_get_hed_version_path_downloads_only_requested_version(self): + """get_hed_version_path must download only the single requested version, never the full catalog. + + The old implementation fell back to cache_xml_versions() (bulk download of every schema). + The new implementation calls _download_schema_version() which fetches the listing for + only the relevant library and downloads only the one file. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + # Patch _download_schema_version so we can verify it's called with the right args, + # and patch cache_xml_versions to fail the test if it's called at all. + downloaded_calls = [] + + def fake_download(xml_version, library_name, cache_folder): + downloaded_calls.append((xml_version, library_name)) + return None # simulates GitHub failure; get_hed_version_path returns None + + with patch.object(hed_cache, "_download_schema_version", side_effect=fake_download): + with patch.object(hed_cache, "cache_xml_versions") as mock_bulk: + with patch.object(hed_cache, "HED_CACHE_DIRECTORY", tmp_dir): + # Request a version not in the bundle. + hed_cache.get_hed_version_path("7.1.1") + + mock_bulk.assert_not_called() + self.assertEqual( + downloaded_calls, + [("7.1.1", None)], + "Only the requested version should be downloaded, not the full catalog", + ) + + def test_deprecated_library_folder_is_not_fetched(self): + """cache_xml_versions must never fetch schemas from a library folder named 'deprecated'. + + DEFAULT_SKIP_FOLDERS = ("deprecated",) causes _get_hed_xml_versions_from_url_all_libraries + to skip any top-level library subfolder with that name. The test injects a fake GitHub + listing that includes both 'score' and 'deprecated' library folders, then confirms only + score-related URLs are ever requested. + """ + fake_library_listing = [ + {"type": "dir", "name": "score"}, + {"type": "dir", "name": "deprecated"}, + {"type": "dir", "name": "lang"}, + ] + fetched_library_names = [] + + def fake_get_json(url, *args, **kwargs): + if url == hed_cache.LIBRARY_HED_URL: + return fake_library_listing + # Record which library sub-URLs are actually requested. + for lib in ("score", "deprecated", "lang"): + if f"/{lib}" in url: + fetched_library_names.append(lib) + return [] + + with patch.object(hed_cache, "_get_json_with_etag", side_effect=fake_get_json): + hed_cache._get_hed_xml_versions_from_url_all_libraries(hed_cache.LIBRARY_HED_URL) + + self.assertNotIn("deprecated", fetched_library_names, "deprecated library folder must never be fetched") + self.assertIn("score", fetched_library_names, "score library should be fetched") + self.assertIn("lang", fetched_library_names, "lang library should be fetched") + + def test_deprecated_subfolder_within_hedxml_is_skipped(self): + """_get_hed_xml_versions_one_folder must silently skip directory entries. + + A deprecated/ subfolder inside hedxml/ (or any other dir entry in the API response) + is type='dir'. The function skips all dir entries, so no deprecated schema stored + inside a subdirectory can ever be downloaded or cached. + """ + fake_hedxml_listing = [ + {"type": "file", "name": "HED8.4.0.xml", "sha": "abc", "download_url": ""}, + {"type": "dir", "name": "deprecated"}, + ] + + with patch.object(hed_cache, "_get_json_with_etag", return_value=fake_hedxml_listing): + result = hed_cache._get_hed_xml_versions_one_folder("https://fake-url/hedxml") + + # Standard schema should be found. + self.assertIn(None, result) + self.assertIn("8.4.0", result[None]) + # 'deprecated' must not appear as a library key or a version. + self.assertNotIn("deprecated", result) + for lib_versions in result.values(): + self.assertNotIn("deprecated", lib_versions) + def test_get_available_hed_versions_lists_without_downloading(self): """get_available_hed_versions() should list real GitHub versions without caching any content.""" with tempfile.TemporaryDirectory() as tmp_dir: @@ -266,7 +399,16 @@ def test_get_available_hed_versions_skips_unneeded_urls(self): # gate's own commits-endpoint URL, and the plain-string SHA marker recorded under # it (see _LIBRARY_HEAD_CACHE_KEY) - neither is scoped to a specific library, so # both are expected regardless of which library_name was requested. - non_library_keys = {library_base, hed_cache.LIBRARY_SCHEMAS_HEAD_URL, hed_cache._LIBRARY_HEAD_CACHE_KEY} + # The manifest fast path (schema_version_manifest) fetches a single global manifest + # file before doing any per-library crawl, so its URL is also library-neutral. + from hed.schema import schema_version_manifest as _manifest + + non_library_keys = { + library_base, + hed_cache.LIBRARY_SCHEMAS_HEAD_URL, + hed_cache._LIBRARY_HEAD_CACHE_KEY, + _manifest.MANIFEST_URL, + } for url in cached_urls: if url in non_library_keys: continue # the one unavoidable "list all libraries" call, or a gate entry @@ -545,15 +687,19 @@ def test_get_available_hed_versions_gates_are_scoped_independently(self): ) def test_get_available_hed_versions_repo_check_interval_limits_gate_frequency(self): - """A larger repo_check_interval should reduce how often the gate URL itself is hit, - independently of cache_time_threshold for the per-folder content URLs. - - Verified by checking the on-disk cache after two rapid calls: with a large - repo_check_interval, the gate URL's own cache entry should not be re-checked on the - second call (tier-1 reuse). This can't assert anything about GitHub's actual request - count without mocking, which this repo's testing policy avoids - only the client-side - effect of not even attempting a network call is checkable here. + """A larger repo_check_interval should reduce how often the "has the repo changed?" gate + is re-fetched, independently of cache_time_threshold for per-folder content URLs. + + With the manifest fast path, the manifest URL IS that gate: its per-file sha values are + the hashes we compare against locally cached files, and repo_check_interval controls how + often we ask GitHub whether the manifest itself has changed. With a large interval, a + rapid second call must reuse the cached manifest without making a new request (tier-1 + reuse). This can't assert actual HTTP request counts without mocking (which this repo's + testing policy avoids); we verify the client-side effect - the cached timestamp for the + gate URL is unchanged - instead. """ + from hed.schema import schema_version_manifest as _manifest + with tempfile.TemporaryDirectory() as tmp_dir: with patch.object(hed_cache, "HED_CACHE_DIRECTORY", tmp_dir): first = hed_cache.get_available_hed_versions(repo_check_interval=3600) @@ -563,8 +709,14 @@ def test_get_available_hed_versions_repo_check_interval_limits_gate_frequency(se cache_filename = os.path.join(tmp_dir, hed_cache.AVAILABLE_VERSIONS_CACHE_FILENAME) with open(cache_filename) as f: cache_after_first = json.load(f) - gate_entry = cache_after_first.get(hed_cache.STANDARD_SCHEMA_HEAD_URL) - self.assertIsInstance(gate_entry, dict, "expected the gate URL itself to be cached") + + # Determine which gate was used. Normally the manifest fast path fires and the + # manifest URL is the gate; STANDARD_SCHEMA_HEAD_URL is only present when the + # manifest was unavailable and the REST API crawl ran instead. + manifest_entry = cache_after_first.get(_manifest.MANIFEST_URL) + gate_url = _manifest.MANIFEST_URL if manifest_entry is not None else hed_cache.STANDARD_SCHEMA_HEAD_URL + gate_entry = cache_after_first.get(gate_url) + self.assertIsInstance(gate_entry, dict, f"expected the gate URL to be cached: {gate_url}") gate_timestamp_before = gate_entry["timestamp"] hed_cache.get_available_hed_versions(repo_check_interval=3600) @@ -572,9 +724,9 @@ def test_get_available_hed_versions_repo_check_interval_limits_gate_frequency(se with open(cache_filename) as f: cache_after_second = json.load(f) self.assertEqual( - cache_after_second[hed_cache.STANDARD_SCHEMA_HEAD_URL]["timestamp"], + cache_after_second[gate_url]["timestamp"], gate_timestamp_before, - "a large repo_check_interval should have skipped re-checking the gate itself", + f"a large repo_check_interval should have skipped re-checking the gate: {gate_url}", ) def test_load_schema_version_default_no_standard_raises(self): diff --git a/tests/schema/test_schema_validator_hed_id.py b/tests/schema/test_schema_validator_hed_id.py index fbd38190..0adcefac 100644 --- a/tests/schema/test_schema_validator_hed_id.py +++ b/tests/schema/test_schema_validator_hed_id.py @@ -40,6 +40,11 @@ def test_get_previous_version(self): self.assertEqual(HedIDValidator._get_previous_version("8.2.0", ""), "8.1.0") self.assertEqual(HedIDValidator._get_previous_version("8.0.0", ""), None) self.assertEqual(HedIDValidator._get_previous_version("3.0.0", "testlib"), "testlib_2.2.0") + # Regression guard: the previous version must come from the full (manifest) version list, + # not just whatever happens to be in the local cache. This reaches deeper into testlib's + # history than the case above, so it fails if the list is ever truncated/incomplete again + # (the bug that surfaced on a fresh CI runner). testlib releases: 1.0.2, 2.0.0, 2.1.0, ... + self.assertEqual(HedIDValidator._get_previous_version("2.0.0", "testlib"), "testlib_1.0.2") def test_verify_tag_id(self): event_entry = self.hed_schema84.tags["Event"] diff --git a/tests/schema/test_schema_version_manifest.py b/tests/schema/test_schema_version_manifest.py new file mode 100644 index 00000000..fe0bf66a --- /dev/null +++ b/tests/schema/test_schema_version_manifest.py @@ -0,0 +1,173 @@ +"""Unit tests for hed.schema.schema_version_manifest (manifest parsing fast path). + +These are pure parsing tests over an in-memory manifest fixture - no network, no real schema files. +The live fetch path (manifest -> hed_cache.get_available_hed_versions / _download_schema_version) is +exercised by the existing schema-cache spec tests. +""" + +import unittest + +from hed.schema import schema_version_manifest as manifest + + +SAMPLE_MANIFEST = { + "manifest_format_version": 1, + "generated": "2026-07-16T00:00:00+00:00", + "repo_commit": "abc123def456", + "libraries": { + "": { + "released": [ + {"version": "8.4.0", "file": "standard_schema/hedxml/HED8.4.0.xml", "sha": "sha840", "date": "d"}, + {"version": "8.3.0", "file": "standard_schema/hedxml/HED8.3.0.xml", "sha": "sha830", "date": "d"}, + ], + "prerelease": [ + {"version": "8.5.0", "file": "standard_schema/prerelease/HED8.5.0.xml", "sha": "sha850", "date": "d"}, + ], + "deprecated": [ + # 8.3.5 is deliberately "mid-range" (between released 8.3.0 and 8.4.0) so the tests + # cover a deprecated version that falls inside the released range, not just below it. + { + "version": "8.3.5", + "file": "standard_schema/hedxml/deprecated/HED8.3.5.xml", + "sha": "sha835", + "date": "d", + }, + { + "version": "7.2.0", + "file": "standard_schema/hedxml/deprecated/HED7.2.0.xml", + "sha": "sha720", + "date": "d", + }, + ], + }, + "score": { + "released": [ + { + "version": "2.1.0", + "file": "library_schemas/score/hedxml/HED_score_2.1.0.xml", + "sha": "shasc", + "date": "d", + }, + ], + "prerelease": [], + "deprecated": [ + # A deprecated *library* version, to confirm deprecated exclusion is not standard-only. + { + "version": "2.0.0", + "file": "library_schemas/score/hedxml/deprecated/HED_score_2.0.0.xml", + "sha": "shascd", + "date": "d", + }, + ], + }, + "mouse": { # prerelease only - no released versions + "released": [], + "prerelease": [ + { + "version": "1.0.0", + "file": "library_schemas/mouse/prerelease/HED_mouse_1.0.0.xml", + "sha": "shamo", + "date": "d", + }, + ], + "deprecated": [], + }, + }, +} + + +class TestIsSupported(unittest.TestCase): + def test_supported(self): + self.assertTrue(manifest.is_supported(SAMPLE_MANIFEST)) + + def test_unsupported_format_version(self): + self.assertFalse(manifest.is_supported({"manifest_format_version": 2, "libraries": {}})) + + def test_non_dict(self): + self.assertFalse(manifest.is_supported(None)) + self.assertFalse(manifest.is_supported([])) + self.assertFalse(manifest.is_supported("not a manifest")) + + +class TestAvailableVersions(unittest.TestCase): + def test_standard_default_excludes_prerelease_and_deprecated(self): + # Also proves the mid-range deprecated version (8.3.5) is excluded from the listing. + self.assertEqual(manifest.available_versions(SAMPLE_MANIFEST), ["8.4.0", "8.3.0"]) + + def test_standard_with_prerelease_newest_first(self): + self.assertEqual( + manifest.available_versions(SAMPLE_MANIFEST, None, check_prerelease=True), + ["8.5.0", "8.4.0", "8.3.0"], + ) + + def test_specific_library(self): + # score has a deprecated 2.0.0; it must not appear in the released listing. + self.assertEqual(manifest.available_versions(SAMPLE_MANIFEST, "score"), ["2.1.0"]) + + def test_all_maps_standard_to_none_and_omits_empty(self): + result = manifest.available_versions(SAMPLE_MANIFEST, "all") + self.assertEqual(result[None], ["8.4.0", "8.3.0"]) + self.assertEqual(result["score"], ["2.1.0"]) + # mouse has no released versions and check_prerelease is False -> omitted entirely. + self.assertNotIn("mouse", result) + + def test_all_with_prerelease_includes_prerelease_only_library(self): + result = manifest.available_versions(SAMPLE_MANIFEST, "all", check_prerelease=True) + self.assertEqual(result["mouse"], ["1.0.0"]) + self.assertIn(None, result) + + def test_unknown_library_returns_empty(self): + self.assertEqual(manifest.available_versions(SAMPLE_MANIFEST, "nosuch"), []) + + +class TestFindVersionInfo(unittest.TestCase): + def test_released_pins_to_repo_commit(self): + info = manifest.find_version_info(SAMPLE_MANIFEST, "8.4.0", None) + self.assertEqual( + info, + ( + "sha840", + "https://raw.githubusercontent.com/hed-standard/hed-schemas/" + "abc123def456/standard_schema/hedxml/HED8.4.0.xml", + False, + ), + ) + + def test_prerelease_flag_true(self): + info = manifest.find_version_info(SAMPLE_MANIFEST, "8.5.0", None) + self.assertIsNotNone(info) + self.assertTrue(info[2]) + + def test_library_version(self): + info = manifest.find_version_info(SAMPLE_MANIFEST, "2.1.0", "score") + self.assertEqual(info[0], "shasc") + self.assertTrue(info[1].endswith("library_schemas/score/hedxml/HED_score_2.1.0.xml")) + self.assertFalse(info[2]) + + def test_ref_override(self): + info = manifest.find_version_info(SAMPLE_MANIFEST, "8.4.0", None, ref="main") + self.assertIn("/main/", info[1]) + + def test_missing_version_returns_none(self): + self.assertIsNone(manifest.find_version_info(SAMPLE_MANIFEST, "9.9.9", None)) + self.assertIsNone(manifest.find_version_info(SAMPLE_MANIFEST, "1.0.0", "nosuch")) + + def test_deprecated_versions_are_not_loadable(self): + # Deprecated schemas are display-only (for the static schema browser); the download/load + # path must never surface them - for the standard schema or any library, and regardless of + # where the version falls in the range (below the released range, or in the middle of it). + self.assertIsNone(manifest.find_version_info(SAMPLE_MANIFEST, "8.3.5", None)) # mid-range standard + self.assertIsNone(manifest.find_version_info(SAMPLE_MANIFEST, "7.2.0", None)) # old standard + self.assertIsNone(manifest.find_version_info(SAMPLE_MANIFEST, "2.0.0", "score")) # deprecated library + + +class TestRawUrl(unittest.TestCase): + def test_raw_url_for(self): + self.assertEqual( + manifest.raw_url_for("standard_schema/hedxml/HED8.4.0.xml", "deadbeef"), + "https://raw.githubusercontent.com/hed-standard/hed-schemas/deadbeef/standard_schema/hedxml/HED8.4.0.xml", + ) + + +if __name__ == "__main__": + unittest.main()