Cache rework to better use hed-schemas version manifest

.github/copilot-instructions.md

@@ -5,6 +5,7 @@
 ## Session summaries and reports
 
 **When you fix bugs, implement features, or investigate issues, always create a summary report in `.status/` directory.** Name it descriptively (e.g., `.status/fix_missing_extra_sections.md`, `.status/investigation_cache_bug.md`, `.status/implement_feature_x.md`). The report should document:
+
 - What problem was being solved
 - What approach was taken
 - What code changes were made and why

hed/schema/hed_cache.py

@@ -137,8 +137,9 @@ def get_hed_versions(local_hed_directory=None, library_name=None, check_prerelea
             hed_files += os.listdir(hed_dir)
         except FileNotFoundError:
             pass
-    if not hed_files:
+    if not any(version_pattern.match(f) for f in hed_files):
         cache_local_versions(local_hed_directory)
+        hed_files = []
         for hed_dir in local_directories:
             try:
                 hed_files += os.listdir(hed_dir)
@@ -166,15 +167,16 @@ def get_hed_versions(local_hed_directory=None, library_name=None, check_prerelea
 def get_hed_version_path(xml_version, library_name=None, local_hed_directory=None) -> Union[str, None]:
     """Get the HED XML file path for a given version.
 
-    Searches the local cache first. If the version is not found and local_hed_directory
-    is the default HED cache, the cache is refreshed from GitHub before a second lookup.
+    Searches the local cache first (including the bundled schemas that are always present).
+    If the version is not found and local_hed_directory is the default HED cache, downloads
+    only the single requested file from GitHub — never the entire catalog.
     No network call is made for custom directories.
 
     Parameters:
         xml_version (str): The version string to look up.
         library_name (str or None): Optional schema library name.
         local_hed_directory (str or None): Path to local HED directory. Defaults to HED_CACHE_DIRECTORY.
-            Passing a custom path disables the automatic GitHub refresh.
+            Passing a custom path disables the automatic GitHub download.
 
     Returns:
         Union[str, None]: The path to the requested HED XML file, or None.
@@ -187,13 +189,12 @@ def get_hed_version_path(xml_version, library_name=None, local_hed_directory=Non
     if result:
         return result
 
-    # Version not found locally — try refreshing cache from GitHub (default cache only).
-    # cache_xml_versions() returns -1 on failure (network error, lock contention, rate limit).
-    # In that case the second lookup will return None, which the caller treats as "version not found".
+    # Version not found locally — download only this specific version from GitHub.
+    # Never bulk-download the entire catalog; that is cache_xml_versions()'s job.
     if not xml_version or local_hed_directory != HED_CACHE_DIRECTORY:
         return None
 
-    cache_xml_versions()
+    _download_schema_version(xml_version, library_name, local_hed_directory)
     return _find_hed_version_path(xml_version, library_name, local_hed_directory)
 
 
@@ -272,6 +273,10 @@ def cache_xml_versions(
     if not cache_folder:
         cache_folder = HED_CACHE_DIRECTORY
 
+    # Always seed the cache with bundled schemas first so the cache is usable even if the
+    # subsequent GitHub download fails (network error, rate limit, etc.).
+    cache_local_versions(cache_folder)
+
     try:
         with CacheLock(cache_folder):
             if isinstance(hed_base_urls, str):
@@ -508,9 +513,39 @@ def get_available_hed_versions(
     if not cache_folder:
         cache_folder = HED_CACHE_DIRECTORY
 
+    # Resolve repo_check_interval early so the manifest fast path can honour it.
+    # The manifest IS the "has anything changed?" gate for the whole repo, so it should
+    # be throttled by repo_check_interval (the interval for those repo-level checks),
+    # not by the shorter cache_time_threshold (which governs individual folder listings).
+    if repo_check_interval is None:
+        repo_check_interval = cache_time_threshold
+
     url_cache = _read_available_versions_cache(cache_folder)
     cache_before = json.dumps(url_cache, sort_keys=True)
 
+    # Fast path: read the repo-level manifest in a single fetch from the raw/CDN host (not subject
+    # to the GitHub REST API rate limit) instead of crawling the API directory listings. Only used
+    # for the canonical hed-schemas URLs; any custom/forked URL set falls through to the crawl. Any
+    # failure (unreachable, malformed, or an unrecognized manifest format) also falls through.
+    # The manifest is used as the "has the repo changed?" check: its per-file sha fields are the
+    # hashes we compare against locally cached files, and repo_check_interval controls how often
+    # we even ask GitHub whether the manifest itself has changed.
+    if (
+        list(hed_base_urls) == list(DEFAULT_URL_LIST)
+        and list(hed_library_urls) == list(DEFAULT_LIBRARY_URL_LIST)
+        and tuple(skip_folders) == tuple(DEFAULT_SKIP_FOLDERS)
+    ):
+        from hed.schema import schema_version_manifest as _manifest
+
+        try:
+            manifest_json = _get_json_with_etag(_manifest.MANIFEST_URL, url_cache, force_refresh, repo_check_interval)
+            if _manifest.is_supported(manifest_json):
+                if json.dumps(url_cache, sort_keys=True) != cache_before:
+                    _write_available_versions_cache(cache_folder, url_cache)
+                return _manifest.available_versions(manifest_json, library_name, check_prerelease)
+        except Exception:
+            pass  # fall through to the REST API crawl below
+
     # Only fetch the standard-schema URLs when the result could actually include them
     # (library_name is None or "all") - a request for one specific library has no use for
     # this data, and fetching it anyway would be pure wasted requests.
@@ -519,9 +554,6 @@ def get_available_hed_versions(
     # all - a plain standard-schema request (library_name=None) never looks at it.
     needs_libraries = library_name is not None
 
-    if repo_check_interval is None:
-        repo_check_interval = cache_time_threshold
-
     all_hed_versions = {}
     if needs_standard:
         # A single, cheap check ahead of the standard-schema crawl below, scoped to just that
@@ -1020,6 +1052,63 @@ def _safe_move_tmp_to_folder(temp_hed_xml_file, dest_filename):
     return dest_filename
 
 
+def _download_schema_version(xml_version, library_name, cache_folder):
+    """Download a single specific schema version from GitHub.
+
+    Fetches the directory listing for only the relevant library (2-3 small API calls), then
+    downloads only the one requested XML file.  Uses SHA comparison so a file whose content
+    has not changed on GitHub is never re-downloaded.
+
+    This is the on-demand complement to cache_xml_versions() (which bulk-downloads every
+    version). Call this when you need one specific version that is not in the local cache;
+    call cache_xml_versions() only when you want to pre-populate the entire catalog (e.g.
+    for an offline/air-gapped environment).
+
+    Parameters:
+        xml_version (str): The version string to download.
+        library_name (str or None): The library name, None for the standard schema.
+        cache_folder (str): The folder to save the downloaded file in.
+
+    Returns:
+        str or None: The local file path on success, None if the version was not found on
+                     GitHub or the download failed.
+    """
+    # Fast path: resolve this version from the manifest (single raw/CDN fetch, no REST API), then
+    # download just the one file. Falls back to the REST API resolution below on any failure,
+    # including a malformed manifest that passes is_supported() but raises inside find_version_info
+    # or _cache_hed_version (unexpected types, missing keys, etc.).
+    from hed.schema import schema_version_manifest as _manifest
+
+    try:
+        manifest_json = _get_json_with_etag(_manifest.MANIFEST_URL, None)
+        if _manifest.is_supported(manifest_json):
+            version_info = _manifest.find_version_info(manifest_json, xml_version, library_name)
+            if version_info is not None:
+                return _cache_hed_version(xml_version, library_name, version_info, cache_folder)
+    except Exception:
+        pass  # fall through to REST API resolution below
+
+    if library_name is None:
+        url = DEFAULT_HED_LIST_VERSIONS_URL
+    else:
+        url = f"{LIBRARY_HED_URL}/{library_name}"
+
+    try:
+        available = _get_hed_xml_versions_one_library(url)
+    except Exception:
+        return None
+
+    versions_for_library = available.get(library_name)
+    if not versions_for_library:
+        return None
+
+    version_info = versions_for_library.get(xml_version)
+    if version_info is None:
+        return None
+
+    return _cache_hed_version(xml_version, library_name, version_info, cache_folder)
+
+
 def _cache_hed_version(version, library_name, version_info, cache_folder):
     """Cache the given HED version"""
     sha_hash, download_url, prerelease = version_info

hed/schema/schema_validation/hed_id_validator.py

@@ -1,9 +1,8 @@
 """Validator for HED ID consistency across schema versions."""
 
-from hed.schema.hed_cache import get_library_data
+from hed.schema.hed_cache import get_library_data, get_available_hed_versions, get_hed_versions
 from semantic_version import Version
 from hed.schema.hed_schema_io import load_schema_version
-from hed.schema.hed_cache import get_hed_versions
 from hed.schema.hed_schema_constants import HedKey
 from hed.errors.error_types import SchemaAttributeErrors
 from hed.errors.error_reporter import ErrorHandler
@@ -52,7 +51,13 @@ def __init__(self, hed_schema):
     @staticmethod
     def _get_previous_version(version, library):
         current_version = Version(version)
-        all_schema_versions = get_hed_versions(library_name=library, check_prerelease=False)
+        # Use the manifest-based listing as the authoritative version source so that a version
+        # released after the local cache was last populated is not silently skipped.  Fall back
+        # to the local cache when the network is unavailable.
+        library_name = library if library else None
+        all_schema_versions = get_available_hed_versions(library_name=library_name) or get_hed_versions(
+            library_name=library, check_prerelease=False
+        )
         for old_version in all_schema_versions:
             if Version(old_version) < current_version:
                 prev_version = old_version

hed/schema/schema_version_manifest.py

@@ -0,0 +1,148 @@
+"""Repo-level manifest fast path for HED schema version discovery.
+
+hed-schemas publishes a single ``schema_versions.json`` at its repository root listing every schema
+version (released / prerelease / deprecated) for the standard schema and each library, with each
+XML file's git blob SHA. hedtools can read that one file from the raw/CDN host instead of crawling
+the GitHub REST API directory listings (which are metered - 60 requests/hour unauthenticated, with
+even conditional 304s counting against that cap).
+
+This module is deliberately I/O-light and has no dependency on the rest of ``hed.schema`` (it never
+imports :mod:`hed.schema.hed_cache`), so hed_cache can use it as a drop-in fast path while keeping
+the REST API crawl as a fallback. The network fetch itself is performed by the caller (hed_cache)
+via its existing ETag-aware helper; the functions here only parse an already-loaded manifest dict.
+
+Deprecated versions are recorded in the manifest so that the static HED schema browser can list and
+display them directly (no hedtools call involved). hedtools' own compute paths do not operate on
+deprecated schemas, so neither the listing nor the load/download helpers here surface them.
+
+Manifest shape (``manifest_format_version`` 1)::
+
+    {
+      "manifest_format_version": 1,
+      "generated": "<iso timestamp>",
+      "repo_commit": "<hed-schemas HEAD sha>",
+      "libraries": {
+        "":      {"released": [{"version", "file", "sha", "date"}], "prerelease": [...], "deprecated": [...]},
+        "score": {...},
+        ...
+      }
+    }
+
+The standard schema is keyed by the empty string ``""`` in the manifest; internally hedtools uses
+``None`` for the standard schema, so that mapping is applied here.
+"""
+
+from __future__ import annotations
+
+from semantic_version import Version
+
+# The manifest is served as raw file content (CDN-backed), not through the REST API, so it is not
+# subject to the GitHub REST API rate limit.
+RAW_CONTENT_BASE = "https://raw.githubusercontent.com/hed-standard/hed-schemas"
+MANIFEST_FILE = "schema_versions.json"
+MANIFEST_URL = f"{RAW_CONTENT_BASE}/main/{MANIFEST_FILE}"
+
+# Only this on-disk format is understood; a manifest advertising anything else is ignored so a
+# future format change can never be silently misread (the caller falls back to the REST API crawl).
+SUPPORTED_MANIFEST_FORMAT_VERSION = 1
+
+_STANDARD_MANIFEST_KEY = ""  # the standard schema's key inside manifest["libraries"]
+
+
+def is_supported(manifest) -> bool:
+    """Return True if ``manifest`` is a dict in a format version this module understands.
+
+    Parameters:
+        manifest: The parsed manifest (any type - non-dicts and unknown formats return False).
+
+    Returns:
+        bool: True only for a dict whose ``manifest_format_version`` matches the supported version
+              and whose ``libraries`` value is a dict (the shape all downstream helpers require).
+    """
+    return (
+        isinstance(manifest, dict)
+        and manifest.get("manifest_format_version") == SUPPORTED_MANIFEST_FORMAT_VERSION
+        and isinstance(manifest.get("libraries"), dict)
+    )
+
+
+def raw_url_for(file_path: str, ref: str) -> str:
+    """Build the raw/CDN URL for a repo-relative file path at a given git ref (commit/branch/tag)."""
+    return f"{RAW_CONTENT_BASE}/{ref}/{file_path}"
+
+
+def _sort_versions(versions):
+    """Sort version strings newest-first using semantic-versioning precedence."""
+    return sorted(versions, key=Version, reverse=True)
+
+
+def _versions_for_key(manifest, manifest_key, check_prerelease):
+    """Collect (and sort, newest-first) the versions for one manifest library key."""
+    categories = manifest.get("libraries", {}).get(manifest_key, {})
+    versions = [entry["version"] for entry in categories.get("released", [])]
+    if check_prerelease:
+        versions += [entry["version"] for entry in categories.get("prerelease", [])]
+    # ``deprecated`` is intentionally never included, matching the default skip_folders behavior of
+    # the REST API crawl (which skips the ``deprecated`` folder).
+    return _sort_versions(versions)
+
+
+def available_versions(manifest, library_name=None, check_prerelease=False):
+    """Return versions from ``manifest`` in the exact shape of get_available_hed_versions().
+
+    Parameters:
+        manifest (dict): A supported manifest (see :func:`is_supported`).
+        library_name (str or None): None for the standard schema only; "all" for every library as a
+                                    dict; or a specific library name.
+        check_prerelease (bool): If True, include prerelease versions.
+
+    Returns:
+        Union[list, dict]: A list of version strings, or - for "all" - a dict
+                           {library_name_or_None: [versions]} with the standard schema under None.
+    """
+    libraries = manifest.get("libraries", {})
+    if library_name == "all":
+        result = {}
+        for manifest_key in libraries:
+            internal_key = None if manifest_key == _STANDARD_MANIFEST_KEY else manifest_key
+            versions = _versions_for_key(manifest, manifest_key, check_prerelease)
+            if versions:
+                result[internal_key] = versions
+        return result
+
+    manifest_key = _STANDARD_MANIFEST_KEY if library_name is None else library_name
+    return _versions_for_key(manifest, manifest_key, check_prerelease)
+
+
+def find_version_info(manifest, xml_version, library_name, ref=None):
+    """Locate one version in ``manifest`` and return its download info, or None if absent.
+
+    Parameters:
+        manifest (dict): A supported manifest.
+        xml_version (str): The version string to find.
+        library_name (str or None): The library name, None for the standard schema.
+        ref (str or None): Git ref for the raw URL. Defaults to the manifest's ``repo_commit`` (so
+                           the fetched bytes are immutable and match the recorded SHA), then "main".
+
+    Returns:
+        tuple or None: ``(sha, download_url, prerelease)`` matching the ``version_info`` shape that
+                       hed_cache._cache_hed_version() consumes, or None if the version is not
+                       listed as a released or prerelease version.
+
+    Notes:
+        - Only ``released`` and ``prerelease`` versions are searched; ``deprecated`` versions are
+          intentionally NOT loadable through this path. Deprecated schemas exist only so the static
+          HED schema browser can display them (reading them straight from the manifest); hedtools'
+          compute paths (validation, etc.) do not operate on deprecated schemas. This also keeps
+          behavior consistent with the REST API fallback, whose crawl skips the ``deprecated``
+          folder.
+    """
+    manifest_key = _STANDARD_MANIFEST_KEY if library_name is None else library_name
+    categories = manifest.get("libraries", {}).get(manifest_key, {})
+    if ref is None:
+        ref = manifest.get("repo_commit") or "main"
+    for category, is_prerelease in (("released", False), ("prerelease", True)):
+        for entry in categories.get(category, []):
+            if entry["version"] == xml_version:
+                return (entry["sha"], raw_url_for(entry["file"], ref), is_prerelease)
+    return None