diff --git a/catalog-info.yaml b/catalog-info.yaml index 3a61eb47e..43b41c1a8 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -156,7 +156,7 @@ spec: build_branches: true build_pull_request_forks: false cancel_deleted_branch_builds: true - filter_condition: build.branch == "main" || build.branch == "9.4" || build.branch == "9.3" || build.branch == "8.19" || build.branch == "7.17" + filter_condition: build.branch == "main" || build.branch == "9.5" || build.branch == "9.4" || build.branch == "9.3" || build.branch == "8.19" filter_enabled: true publish_blocked_as_pending: true publish_commit_status: false @@ -164,10 +164,6 @@ spec: trigger_mode: code repository: elastic/ml-cpp schedules: - Daily 7_17: - branch: '7.17' - cronline: 30 04 * * * - message: Daily SNAPSHOT build for 7.17 Daily 8_19: branch: '8.19' cronline: 30 03 * * * @@ -180,6 +176,10 @@ spec: branch: '9.4' cronline: 30 01 * * * message: Daily SNAPSHOT build for 9.4 + Daily 9.5: + branch: '9.5' + cronline: 30 05 * * * + message: Daily SNAPSHOT build for 9.5 Daily main: branch: main cronline: 30 00 * * * diff --git a/dev-tools/bump_main_minor_freeze.sh b/dev-tools/bump_main_minor_freeze.sh index 9886cdd2b..c8d51c70a 100755 --- a/dev-tools/bump_main_minor_freeze.sh +++ b/dev-tools/bump_main_minor_freeze.sh @@ -31,6 +31,7 @@ source "${SCRIPT_DIR}/version_bump_lib.sh" PYTHON="${PYTHON:-python3}" VALIDATION_PY="${SCRIPT_DIR}/version_bump_validation.py" UPDATE_BACKPORTRC_PY="${SCRIPT_DIR}/update_backportrc.py" +UPDATE_CATALOG_PY="${SCRIPT_DIR}/update_catalog_snapshot.py" CREATE_PR_SH="${SCRIPT_DIR}/create_github_pull_request.sh" : "${NEW_VERSION:?NEW_VERSION must be set}" @@ -43,6 +44,7 @@ TARGET_BRANCH="main" GRADLE_PROPS="gradle.properties" BACKPORTRC=".backportrc.json" +CATALOG_INFO="catalog-info.yaml" if [ "$DRY_RUN" = "true" ]; then echo "=== DRY RUN MODE — will not push or create PR ===" @@ -115,14 +117,22 @@ then exit 1 fi +# Register the newly cut release branch in the daily snapshot build pipeline. +if ! "$PYTHON" "$UPDATE_CATALOG_PY" \ + --path "$CATALOG_INFO" \ + --branch "$BRANCH" +then + exit 1 +fi + if git diff-index --quiet HEAD --; then - echo "main already at ${MAIN_NEW_VERSION} and .backportrc.json is up to date — nothing to do" + echo "main already at ${MAIN_NEW_VERSION}, .backportrc.json and catalog-info.yaml up to date — nothing to do" version_bump_set_buildkite_meta "ml_cpp_main_bump_needed" "false" exit 0 fi configure_git -git add "$GRADLE_PROPS" "$BACKPORTRC" +git add "$GRADLE_PROPS" "$BACKPORTRC" "$CATALOG_INFO" git commit -m "[ML] Bump version to ${MAIN_NEW_VERSION} (minor freeze)" if [ "$DRY_RUN" = "true" ]; then @@ -145,6 +155,7 @@ Automated minor feature-freeze bump for \`${TARGET_BRANCH}\`. | **Release branch** | \`${BRANCH}\` @ \`${NEW_VERSION}\` | | **elasticsearchVersion on main** | \`${current_version}\` → \`${MAIN_NEW_VERSION}\` | | **.backportrc.json** | Adds \`${BRANCH}\`; maps \`v${MAIN_NEW_VERSION}\` → \`main\` | +| **catalog-info.yaml** | Adds \`${BRANCH}\` to the snapshot pipeline filter and a daily snapshot schedule | When merging is enabled (\`VERSION_BUMP_NO_MERGE\` not true): **auto-merge** if \`VERSION_BUMP_MERGE_AUTO=true\`. EOF diff --git a/dev-tools/unittest/test_update_catalog_snapshot.py b/dev-tools/unittest/test_update_catalog_snapshot.py new file mode 100644 index 000000000..6ae93042a --- /dev/null +++ b/dev-tools/unittest/test_update_catalog_snapshot.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Pytest tests for dev-tools/update_catalog_snapshot.py. + +Verifies a newly cut release branch is registered in the snapshot build pipeline +(filter_condition + daily schedule) with a minimal, comment-preserving diff, that +the operation is idempotent, and that the real catalog-info.yaml stays valid YAML. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_DEV_TOOLS = Path(__file__).resolve().parents[1] +if str(_DEV_TOOLS) not in sys.path: + sys.path.insert(0, str(_DEV_TOOLS)) + +import update_catalog_snapshot as ucs # noqa: E402 + +_REPO_ROOT = _DEV_TOOLS.parent +_CATALOG = _REPO_ROOT / "catalog-info.yaml" + +# A branch that will never actually ship, so real-catalog tests stay stable and +# never flip to a no-op (changed == False) once a plausible version is released. +_SENTINEL_BRANCH = "99.0" + +# Minimal fixture mirroring the snapshot pipeline block followed by another +# section, so scoping (only the snapshot block is edited) is exercised. +_FIXTURE = """\ +# Declare the snapshot build pipeline +--- +apiVersion: "backstage.io/v1alpha1" +kind: "Resource" +metadata: + name: "ml-cpp-snapshot-builds" +spec: + implementation: + spec: + provider_settings: + build_branches: true + filter_condition: build.branch == "main" || build.branch == "9.4" || build.branch == "7.17" + filter_enabled: true + schedules: + Daily 7_17: + branch: '7.17' + cronline: 30 04 * * * + message: Daily SNAPSHOT build for 7.17 + Daily 9.4: + branch: '9.4' + cronline: 30 01 * * * + message: Daily SNAPSHOT build for 9.4 + Daily main: + branch: main + cronline: 30 00 * * * + message: Daily SNAPSHOT build for main + skip_intermediate_builds: true + +# Declare the staging build pipeline +--- +metadata: + name: "ml-cpp-staging-builds" +spec: + implementation: + spec: + provider_settings: + filter_condition: 'build.branch == "main"' +""" + + +def test_adds_branch_to_filter_after_main() -> None: + new_text, changed = ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5") + assert changed is True + filter_line = next( + line for line in new_text.splitlines() if "filter_condition:" in line and "build_branches" not in line + ) + # 9.5 must appear immediately after main and before 9.4 (newest-first). + assert 'build.branch == "main" || build.branch == "9.5" || build.branch == "9.4"' in filter_line + + +def test_adds_schedule_before_main_with_next_free_hour() -> None: + new_text, _ = ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5") + lines = new_text.splitlines() + # New schedule inserted just before "Daily main:". + keys = [ln.strip().rstrip(":") for ln in lines if ln.strip().startswith("Daily ")] + assert keys == ["Daily 7_17", "Daily 9.4", "Daily 9.5", "Daily main"] + # Next free half-past hour is 05 (max existing is 04). + assert "cronline: 30 05 * * *" in new_text + assert "message: Daily SNAPSHOT build for 9.5" in new_text + assert "branch: '9.5'" in new_text + + +def test_does_not_touch_staging_section() -> None: + new_text, _ = ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5") + staging = new_text.split("# Declare the staging build pipeline", 1)[1] + assert "9.5" not in staging + + +def test_idempotent() -> None: + once, changed1 = ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5") + assert changed1 is True + twice, changed2 = ucs.add_release_branch_to_snapshot(once, "9.5") + assert changed2 is False + assert twice == once + + +def test_partial_state_filter_only_is_completed() -> None: + # Branch already in filter but no schedule yet: the schedule must still be added. + text, _ = ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5") + # Remove the schedule block we just added, keep the filter entry. + without_schedule = text.replace( + " Daily 9.5:\n" + " branch: '9.5'\n" + " cronline: 30 05 * * *\n" + " message: Daily SNAPSHOT build for 9.5\n", + "", + ) + completed, changed = ucs.add_release_branch_to_snapshot(without_schedule, "9.5") + assert changed is True + assert "Daily 9.5:" in completed + + +def test_partial_state_schedule_only_is_completed() -> None: + # Inverse of the above: branch already scheduled but missing from the filter + # (e.g. a hand edit). The filter entry must still be added. + text, _ = ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5") + # Remove only the filter entry we just added, keep the schedule block. + without_filter = text.replace(' || build.branch == "9.5"', "") + assert 'build.branch == "9.5"' not in without_filter + assert "Daily 9.5:" in without_filter + + completed, changed = ucs.add_release_branch_to_snapshot(without_filter, "9.5") + assert changed is True + assert 'build.branch == "9.5"' in completed + # The schedule must not be duplicated when it was already present. + assert completed.count("Daily 9.5:") == 1 + + +def test_fallback_appends_after_last_entry_when_no_daily_main() -> None: + # A schedules block without a "Daily main" entry: the new schedule must be + # appended after the last existing entry, before the next sibling key + # (skip_intermediate_builds), not prepended after "schedules:". + fixture = """\ +# Declare the snapshot build pipeline +--- +metadata: + name: "ml-cpp-snapshot-builds" +spec: + implementation: + spec: + provider_settings: + filter_condition: build.branch == "main" || build.branch == "9.4" + schedules: + Daily 9.4: + branch: '9.4' + cronline: 30 01 * * * + message: Daily SNAPSHOT build for 9.4 + skip_intermediate_builds: true + +# Declare the staging build pipeline +--- +metadata: + name: "other" +""" + new_text, changed = ucs.add_release_branch_to_snapshot(fixture, "9.5") + assert changed is True + lines = new_text.splitlines() + keys = [ln.strip().rstrip(":") for ln in lines if ln.strip().startswith("Daily ")] + assert keys == ["Daily 9.4", "Daily 9.5"] + # New block sits between the last schedule entry and the sibling key. + msg_idx = next(i for i, ln in enumerate(lines) if "Daily SNAPSHOT build for 9.5" in ln) + sib_idx = next(i for i, ln in enumerate(lines) if "skip_intermediate_builds:" in ln) + assert msg_idx < sib_idx + yaml = pytest.importorskip("yaml") + docs = list(yaml.safe_load_all(new_text)) + snap = next( + d for d in docs if d and d.get("metadata", {}).get("name") == "ml-cpp-snapshot-builds" + ) + assert snap["spec"]["implementation"]["spec"]["schedules"]["Daily 9.5"]["branch"] == "9.5" + + +def test_rejects_non_semver_branch() -> None: + with pytest.raises(ValueError, match="MAJOR.MINOR"): + ucs.add_release_branch_to_snapshot(_FIXTURE, "main") + with pytest.raises(ValueError, match="MAJOR.MINOR"): + ucs.add_release_branch_to_snapshot(_FIXTURE, "9.5.0") + + +def test_missing_snapshot_anchor_raises() -> None: + with pytest.raises(ValueError, match="anchor not found"): + ucs.add_release_branch_to_snapshot("no snapshot pipeline here\n", "9.5") + + +@pytest.mark.skipif(not _CATALOG.is_file(), reason="catalog-info.yaml not found") +def test_real_catalog_edit_is_valid_yaml_and_unique_crons() -> None: + yaml = pytest.importorskip("yaml") + text = _CATALOG.read_text(encoding="utf-8") + new_text, changed = ucs.add_release_branch_to_snapshot(text, _SENTINEL_BRANCH) + assert changed is True + + docs = list(yaml.safe_load_all(new_text)) + snap = next( + d for d in docs if d and d.get("metadata", {}).get("name") == "ml-cpp-snapshot-builds" + ) + spec = snap["spec"]["implementation"]["spec"] + assert f'build.branch == "{_SENTINEL_BRANCH}"' in spec["provider_settings"]["filter_condition"] + assert spec["schedules"][f"Daily {_SENTINEL_BRANCH}"]["branch"] == _SENTINEL_BRANCH + + crons = [s["cronline"] for s in spec["schedules"].values()] + assert len(crons) == len(set(crons)), f"cronline collision: {crons}" + + +@pytest.mark.skipif(not _CATALOG.is_file(), reason="catalog-info.yaml not found") +def test_real_catalog_only_snapshot_block_changes() -> None: + text = _CATALOG.read_text(encoding="utf-8") + new_text, _ = ucs.add_release_branch_to_snapshot(text, _SENTINEL_BRANCH) + # The staging section (and everything after it) must be untouched. + marker = "# Declare the staging build pipeline" + assert text.split(marker, 1)[1] == new_text.split(marker, 1)[1] diff --git a/dev-tools/update_catalog_snapshot.py b/dev-tools/update_catalog_snapshot.py new file mode 100644 index 000000000..51b5d9fcf --- /dev/null +++ b/dev-tools/update_catalog_snapshot.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +"""Register a newly cut release branch in the ml-cpp snapshot build pipeline. + +When a minor feature freeze cuts a new MAJOR.MINOR release branch, the daily +snapshot build pipeline defined in ``catalog-info.yaml`` must be told about it in +two places: + + * ``filter_condition`` -- otherwise scheduled/triggered builds for the branch + are blocked. + * ``schedules`` -- a new ``Daily `` cron entry so the branch gets a + daily SNAPSHOT build. + +The edit is deliberately a scoped, comment-preserving text transform (rather than +a YAML round-trip) so the rest of ``catalog-info.yaml`` -- comments, ordering and +formatting -- is left byte-for-byte identical. Only the snapshot pipeline block is +touched; the staging pipeline already matches any MAJOR.MINOR via a regex filter +and has no schedules, so it needs no change. + +The operation is idempotent: re-running for a branch that is already registered +makes no change (important because the centralized version bump may re-trigger a +completed freeze). +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import Tuple + +# Anchors the snapshot pipeline document. Unique within catalog-info.yaml. +_SNAPSHOT_ANCHOR = 'name: "ml-cpp-snapshot-builds"' +# Section boundaries are the human-authored "# Declare the ... pipeline" comments. +_SECTION_END_RE = re.compile(r"^# Declare ", re.MULTILINE) +_FILTER_RE = re.compile(r"^(?P[ \t]*)filter_condition:[ \t]*(?P.*)$", re.MULTILINE) +_SCHEDULES_RE = re.compile(r"^(?P[ \t]*)schedules:[ \t]*$", re.MULTILINE) +_CRON_HOUR_RE = re.compile(r"cronline:[ \t]*30[ \t]+(\d{1,2})[ \t]") +_BRANCH_RE = re.compile(r"^[0-9]+\.[0-9]+$") + + +def _snapshot_bounds(text: str) -> Tuple[int, int]: + start = text.find(_SNAPSHOT_ANCHOR) + if start == -1: + raise ValueError(f"snapshot pipeline anchor not found ({_SNAPSHOT_ANCHOR!r})") + end_match = _SECTION_END_RE.search(text, start) + end = end_match.start() if end_match else len(text) + return start, end + + +def _update_filter_condition(section: str, branch: str) -> Tuple[str, bool]: + m = _FILTER_RE.search(section) + if m is None: + raise ValueError("filter_condition not found in snapshot pipeline") + val = m.group("val") + token = f'build.branch == "{branch}"' + if token in val: + return section, False + main_token = 'build.branch == "main"' + if main_token in val: + # Insert immediately after main to keep newest-first ordering. + pos = val.index(main_token) + len(main_token) + new_val = f"{val[:pos]} || {token}{val[pos:]}" + else: + new_val = f"{val} || {token}" + new_line = f"{m.group('indent')}filter_condition: {new_val}" + return section[: m.start()] + new_line + section[m.end() :], True + + +def _next_cron_hour(section: str) -> int: + hours = [int(h) for h in _CRON_HOUR_RE.findall(section)] + next_hour = (max(hours) + 1) if hours else 0 + if next_hour > 23: + raise ValueError( + "no free daily 'half past' snapshot slot: hours 0-23 are all taken; " + "prune EOL branch schedules in catalog-info.yaml before adding more" + ) + return next_hour + + +def _add_schedule(section: str, branch: str) -> Tuple[str, bool]: + m = _SCHEDULES_RE.search(section) + if m is None: + raise ValueError("schedules block not found in snapshot pipeline") + # Already scheduled? (accept single/double quoted or bare branch value) + if re.search(rf"^[ \t]*branch:[ \t]*['\"]?{re.escape(branch)}['\"]?[ \t]*$", section, re.MULTILINE): + return section, False + + schedules_indent = m.group("indent") + key_indent = schedules_indent + " " + field_indent = key_indent + " " + cron_hour = _next_cron_hour(section) + block = ( + f"{key_indent}Daily {branch}:\n" + f"{field_indent}branch: '{branch}'\n" + f"{field_indent}cronline: 30 {cron_hour:02d} * * *\n" + f"{field_indent}message: Daily SNAPSHOT build for {branch}\n" + ) + + # Insert before the "Daily main:" entry so main stays last. + main_entry = re.compile(rf"^{re.escape(key_indent)}Daily main:[ \t]*$", re.MULTILINE) + main_m = main_entry.search(section, m.end()) + if main_m is not None: + insert_at = main_m.start() + return section[:insert_at] + block + section[insert_at:], True + + # Fallback (no "Daily main" entry): append after the last schedule entry, i.e. + # just before the first following line indented at or below the "schedules:" key + # (a sibling mapping key) or the end of the section. + schedules_col = len(schedules_indent) + insert_at = len(section) + for sibling in re.finditer(r"^(?P[ \t]*)\S", section[m.end() :], re.MULTILINE): + if len(sibling.group("indent")) <= schedules_col: + insert_at = m.end() + sibling.start() + break + return section[:insert_at] + block + section[insert_at:], True + + +def add_release_branch_to_snapshot(text: str, branch: str) -> Tuple[str, bool]: + """Add ``branch`` to the snapshot pipeline filter and schedules. Idempotent. + + Returns (new_text, changed). + """ + if not _BRANCH_RE.match(branch): + raise ValueError(f"branch must be MAJOR.MINOR (e.g. 9.5), got {branch!r}") + + start, end = _snapshot_bounds(text) + section = text[start:end] + + section, filter_changed = _update_filter_condition(section, branch) + section, schedule_changed = _add_schedule(section, branch) + + changed = filter_changed or schedule_changed + if not changed: + return text, False + return text[:start] + section + text[end:], True + + +def _cmd_update(args: argparse.Namespace) -> int: + path = Path(args.path) + if not path.is_file(): + print(f"ERROR: {path} not found", file=sys.stderr) + return 1 + + text = path.read_text(encoding="utf-8") + try: + new_text, changed = add_release_branch_to_snapshot(text, args.branch) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + + if not changed: + print(f"OK: {path} already schedules snapshot builds for branch {args.branch}") + return 0 + + path.write_text(new_text, encoding="utf-8") + print(f"Updated {path}: added branch {args.branch} to snapshot filter and schedules") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Register a release branch in the ml-cpp snapshot build pipeline" + ) + parser.add_argument("--path", default="catalog-info.yaml", help="Path to catalog-info.yaml") + parser.add_argument("--branch", required=True, help="Release branch (MAJOR.MINOR, e.g. 9.5)") + args = parser.parse_args() + return _cmd_update(args) + + +if __name__ == "__main__": + sys.exit(main())