Gerg/clone-bosh-releases.py

## clone-bosh-releases.py
#!/usr/bin/env python3
"""
Clone (or update existing clones) for every BOSH release listed in
cf-deployment.yml, checked out at the exact tagged version, with submodules
initialized recursively.

Runs all releases in parallel with a live Rich progress display.

Usage:
    ./clone-bosh-releases.py [cf-deployment.yml] [dest-dir]

Arguments:
    cf-deployment.yml  Path to the manifest
                       (default: repo root/cf-deployment.yml)
    dest-dir           Directory to clone releases into
                       (default: repo root/releases/)

Dependencies:
    pip install rich

Requirements:
    - Python 3.10+  (uses union type syntax: X | Y)
    - git           (must be on PATH)

Behavior:
    - Each release is cloned from GitHub using the URL embedded in the
      bosh.io release URL.
    - The exact version tag is checked out (tries v<version> then <version>).
    - Submodules are initialized recursively if a .gitmodules file is present.
    - Re-running is idempotent: existing clones are fetched and updated rather
      than re-cloned, and repos already at the correct commit are left alone.
    - All releases are processed in parallel (default: 8 workers).

NOTE: This script was generated by claude-4.6-sonnet via Cursor. Relinquish
your soul to the machines at your own peril.
"""

import re
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from pathlib import Path
from threading import Lock

try:
    from rich.console import Console
    from rich.live import Live
    from rich.table import Table
    from rich.text import Text
    RICH = True
except ImportError:
    RICH = False

MAX_WORKERS = 8


if not RICH:
    _STRIP_MARKUP = re.compile(r'\[/?[^\]]+\]')

    _builtin_print = print

    class Console:  # noqa: F811
        def print(self, msg="", **_):
            _builtin_print(_STRIP_MARKUP.sub("", str(msg)))

    class Live:  # noqa: F811
        def __init__(self, *_, **__):
            pass

        def __enter__(self):
            return self

        def __exit__(self, *_):
            pass

        def update(self, _):
            pass

    class Table:  # noqa: F811
        pass

    class Text:  # noqa: F811
        def __init__(self, text, **_):
            self.plain = text


STATUS_COLORS = {
    "pending":    "grey50",
    "cloning":    "cyan",
    "fetching":   "cyan",
    "checkout":   "yellow",
    "submodules": "yellow",
    "done":       "green",
    "warning":    "dark_orange",
    "error":      "red",
}


@dataclass
class Release:
    name: str
    slug: str
    version: str
    status: str = "pending"
    detail: str = ""


def parse_releases(manifest_path: Path) -> list[Release]:
    text = manifest_path.read_text()
    m = re.search(r'^releases:\n(.*)', text, re.MULTILINE | re.DOTALL)
    if not m:
        sys.exit("ERROR: could not find 'releases:' section")

    releases = []
    for stanza in re.split(r'\n(?=- name:)', m.group(1)):
        name_m = re.search(r'- name:\s+(\S+)', stanza)
        url_m = re.search(
            r'url:\s+https://bosh\.io/d/github\.com/([^?]+)\?', stanza
        )
        version_m = re.search(r'version:\s+(\S+)', stanza)
        if name_m and url_m and version_m:
            releases.append(Release(
                name=name_m.group(1),
                slug=url_m.group(1).rstrip('/'),
                version=version_m.group(1),
            ))
    return releases


def build_table(releases: list[Release]) -> Table:
    table = Table(box=None, pad_edge=False, show_header=True, expand=True)
    table.add_column("Release", style="bold", min_width=28)
    table.add_column("Version", min_width=12)
    table.add_column("Status", min_width=12)
    table.add_column("Detail", style="grey70")

    for r in releases:
        color = STATUS_COLORS.get(r.status, "white")
        table.add_row(
            r.name, r.version, Text(r.status, style=color), r.detail
        )
    return table


def run(
    args: list[str], cwd: Path | None = None
) -> subprocess.CompletedProcess:
    return subprocess.run(args, cwd=cwd, capture_output=True, text=True)


def resolve_tag(dest: Path, version: str) -> str | None:
    """Try v<version> before <version> to handle both tag conventions."""
    for candidate in (f"v{version}", version):
        r = run(
            ["git", "rev-parse", "--verify", "--quiet",
             f"refs/tags/{candidate}"],
            cwd=dest,
        )
        if r.returncode == 0:
            return candidate
    return None


def tag_commit(dest: Path, tag: str) -> str | None:
    """Dereference annotated tags to their underlying commit SHA."""
    r = run(["git", "rev-parse", f"refs/tags/{tag}^{{}}"], cwd=dest)
    return r.stdout.strip() if r.returncode == 0 else None


def process_release(
    release: Release,
    dest_dir: Path,
    lock: Lock,
    live: Live,
    releases: list[Release],
) -> None:
    dest = dest_dir / release.name
    clone_url = f"https://github.com/{release.slug}.git"

    def update(status: str, detail: str = "") -> None:
        with lock:
            release.status = status
            release.detail = detail
            if RICH:
                live.update(build_table(releases))
            else:
                suffix = f" ({detail})" if detail else ""
                print(f"[{status.upper():>10}] {release.name}{suffix}")

    try:
        if not (dest / ".git").exists():
            update("cloning")
            r = run(
                ["git", "clone", "--filter=blob:none", clone_url, str(dest)]
            )
            if r.returncode != 0:
                stderr = r.stderr.strip()
                msg = stderr.splitlines()[-1] if stderr else "clone failed"
                update("error", msg)
                return
        else:
            update("fetching")
            r = run([
                "git", "-C", str(dest),
                "fetch", "--tags", "--force", "--quiet", "origin",
            ])
            if r.returncode != 0:
                stderr = r.stderr.strip()
                msg = stderr.splitlines()[-1] if stderr else "fetch failed"
                update("error", msg)
                return

        tag = resolve_tag(dest, release.version)
        if tag is None:
            update("warning", f"no tag for {release.version}")
            return

        current = run(
            ["git", "-C", str(dest), "rev-parse", "HEAD"]
        ).stdout.strip()
        target = tag_commit(dest, tag)
        if target is None:
            update("error", f"could not resolve commit for tag {tag}")
            return

        if current != target:
            update("checkout", tag)
            r = run(["git", "-C", str(dest), "checkout", "--quiet", tag])
            if r.returncode != 0:
                update("error", "checkout failed")
                return

        if (dest / ".gitmodules").exists():
            update("submodules")
            r = run([
                "git", "-C", str(dest), "submodule", "update",
                "--init", "--recursive", "--filter=blob:none",
            ])
            if r.returncode != 0:
                update("error", "submodule update failed")
                return

        update("done", tag)

    except Exception as exc:
        update("error", str(exc))


def main() -> None:
    script_dir = Path(__file__).parent.resolve()
    repo_root = script_dir.parent

    manifest_path = (
        Path(sys.argv[1]) if len(sys.argv) > 1
        else repo_root / "cf-deployment.yml"
    )
    dest_dir = (
        Path(sys.argv[2]) if len(sys.argv) > 2
        else repo_root / "releases"
    )

    if not manifest_path.exists():
        sys.exit(f"ERROR: manifest not found: {manifest_path}")

    dest_dir.mkdir(parents=True, exist_ok=True)

    releases = parse_releases(manifest_path)
    if not releases:
        sys.exit(f"ERROR: no releases parsed from {manifest_path}")

    console = Console()
    console.print("[bold]cf-deployment BOSH release cloner[/bold]")
    console.print(f"Manifest : {manifest_path}")
    console.print(f"Dest     : {dest_dir}")
    console.print(
        f"Releases : {len(releases)}  |  Workers: {MAX_WORKERS}\n"
    )

    lock: Lock = Lock()

    initial = build_table(releases) if RICH else None
    try:
        with Live(initial, console=console, refresh_per_second=8) as live:
            with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
                futures = {
                    pool.submit(
                        process_release, r, dest_dir, lock, live, releases
                    ): r
                    for r in releases
                }
                try:
                    for _ in as_completed(futures):
                        pass  # updates are pushed from within process_release
                except KeyboardInterrupt:
                    for f in futures:
                        f.cancel()
                    pool.shutdown(wait=False, cancel_futures=True)
                    raise
    except KeyboardInterrupt:
        console.print("\n[yellow]Interrupted.[/yellow]")
        sys.exit(130)

    done = sum(1 for r in releases if r.status == "done")
    warnings = sum(1 for r in releases if r.status == "warning")
    errors = sum(1 for r in releases if r.status == "error")

    console.print()
    if errors:
        console.print(
            f"[red]Completed with errors:[/red] "
            f"{done} done, {warnings} warnings, {errors} errors"
        )
        for r in releases:
            if r.status == "error":
                console.print(f"  [red]✗[/red] {r.name}: {r.detail}")
        sys.exit(1)
    elif warnings:
        console.print(
            f"[dark_orange]Done:[/dark_orange] "
            f"{done} done, {warnings} warnings"
        )
        for r in releases:
            if r.status == "warning":
                console.print(
                    f"  [dark_orange]⚠[/dark_orange] {r.name}: {r.detail}"
                )
    else:
        console.print(
            f"[green]All {done} releases cloned successfully.[/green]"
        )


if __name__ == "__main__":
    main()

# MIT License
#
# Copyright (c) 2026 Greg Cobb
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
	#!/usr/bin/env python3
	"""
	Clone (or update existing clones) for every BOSH release listed in
	cf-deployment.yml, checked out at the exact tagged version, with submodules
	initialized recursively.

	Runs all releases in parallel with a live Rich progress display.

	Usage:
	./clone-bosh-releases.py [cf-deployment.yml] [dest-dir]

	Arguments:
	cf-deployment.yml Path to the manifest
	(default: repo root/cf-deployment.yml)
	dest-dir Directory to clone releases into
	(default: repo root/releases/)

	Dependencies:
	pip install rich

	Requirements:
	- Python 3.10+ (uses union type syntax: X \| Y)
	- git (must be on PATH)

	Behavior:
	- Each release is cloned from GitHub using the URL embedded in the
	bosh.io release URL.
	- The exact version tag is checked out (tries v<version> then <version>).
	- Submodules are initialized recursively if a .gitmodules file is present.
	- Re-running is idempotent: existing clones are fetched and updated rather
	than re-cloned, and repos already at the correct commit are left alone.
	- All releases are processed in parallel (default: 8 workers).

	NOTE: This script was generated by claude-4.6-sonnet via Cursor. Relinquish
	your soul to the machines at your own peril.
	"""

	import re
	import subprocess
	import sys
	from concurrent.futures import ThreadPoolExecutor, as_completed
	from dataclasses import dataclass
	from pathlib import Path
	from threading import Lock

	try:
	from rich.console import Console
	from rich.live import Live
	from rich.table import Table
	from rich.text import Text
	RICH = True
	except ImportError:
	RICH = False

	MAX_WORKERS = 8


	if not RICH:
	_STRIP_MARKUP = re.compile(r'\[/?[^\]]+\]')

	_builtin_print = print

	class Console: # noqa: F811
	def print(self, msg="", **_):
	_builtin_print(_STRIP_MARKUP.sub("", str(msg)))

	class Live: # noqa: F811
	def __init__(self, _, *__):
	pass

	def __enter__(self):
	return self

	def __exit__(self, *_):
	pass

	def update(self, _):
	pass

	class Table: # noqa: F811
	pass

	class Text: # noqa: F811
	def __init__(self, text, **_):
	self.plain = text


	STATUS_COLORS = {
	"pending": "grey50",
	"cloning": "cyan",
	"fetching": "cyan",
	"checkout": "yellow",
	"submodules": "yellow",
	"done": "green",
	"warning": "dark_orange",
	"error": "red",
	}


	@dataclass
	class Release:
	name: str
	slug: str
	version: str
	status: str = "pending"
	detail: str = ""


	def parse_releases(manifest_path: Path) -> list[Release]:
	text = manifest_path.read_text()
	m = re.search(r'^releases:\n(.*)', text, re.MULTILINE \| re.DOTALL)
	if not m:
	sys.exit("ERROR: could not find 'releases:' section")

	releases = []
	for stanza in re.split(r'\n(?=- name:)', m.group(1)):
	name_m = re.search(r'- name:\s+(\S+)', stanza)
	url_m = re.search(
	r'url:\s+https://bosh\.io/d/github\.com/([^?]+)\?', stanza
	)
	version_m = re.search(r'version:\s+(\S+)', stanza)
	if name_m and url_m and version_m:
	releases.append(Release(
	name=name_m.group(1),
	slug=url_m.group(1).rstrip('/'),
	version=version_m.group(1),
	))
	return releases


	def build_table(releases: list[Release]) -> Table:
	table = Table(box=None, pad_edge=False, show_header=True, expand=True)
	table.add_column("Release", style="bold", min_width=28)
	table.add_column("Version", min_width=12)
	table.add_column("Status", min_width=12)
	table.add_column("Detail", style="grey70")

	for r in releases:
	color = STATUS_COLORS.get(r.status, "white")
	table.add_row(
	r.name, r.version, Text(r.status, style=color), r.detail
	)
	return table


	def run(
	args: list[str], cwd: Path \| None = None
	) -> subprocess.CompletedProcess:
	return subprocess.run(args, cwd=cwd, capture_output=True, text=True)


	def resolve_tag(dest: Path, version: str) -> str \| None:
	"""Try v<version> before <version> to handle both tag conventions."""
	for candidate in (f"v{version}", version):
	r = run(
	["git", "rev-parse", "--verify", "--quiet",
	f"refs/tags/{candidate}"],
	cwd=dest,
	)
	if r.returncode == 0:
	return candidate
	return None


	def tag_commit(dest: Path, tag: str) -> str \| None:
	"""Dereference annotated tags to their underlying commit SHA."""
	r = run(["git", "rev-parse", f"refs/tags/{tag}^{{}}"], cwd=dest)
	return r.stdout.strip() if r.returncode == 0 else None


	def process_release(
	release: Release,
	dest_dir: Path,
	lock: Lock,
	live: Live,
	releases: list[Release],
	) -> None:
	dest = dest_dir / release.name
	clone_url = f"https://github.com/{release.slug}.git"

	def update(status: str, detail: str = "") -> None:
	with lock:
	release.status = status
	release.detail = detail
	if RICH:
	live.update(build_table(releases))
	else:
	suffix = f" ({detail})" if detail else ""
	print(f"[{status.upper():>10}] {release.name}{suffix}")

	try:
	if not (dest / ".git").exists():
	update("cloning")
	r = run(
	["git", "clone", "--filter=blob:none", clone_url, str(dest)]
	)
	if r.returncode != 0:
	stderr = r.stderr.strip()
	msg = stderr.splitlines()[-1] if stderr else "clone failed"
	update("error", msg)
	return
	else:
	update("fetching")
	r = run([
	"git", "-C", str(dest),
	"fetch", "--tags", "--force", "--quiet", "origin",
	])
	if r.returncode != 0:
	stderr = r.stderr.strip()
	msg = stderr.splitlines()[-1] if stderr else "fetch failed"
	update("error", msg)
	return

	tag = resolve_tag(dest, release.version)
	if tag is None:
	update("warning", f"no tag for {release.version}")
	return

	current = run(
	["git", "-C", str(dest), "rev-parse", "HEAD"]
	).stdout.strip()
	target = tag_commit(dest, tag)
	if target is None:
	update("error", f"could not resolve commit for tag {tag}")
	return

	if current != target:
	update("checkout", tag)
	r = run(["git", "-C", str(dest), "checkout", "--quiet", tag])
	if r.returncode != 0:
	update("error", "checkout failed")
	return

	if (dest / ".gitmodules").exists():
	update("submodules")
	r = run([
	"git", "-C", str(dest), "submodule", "update",
	"--init", "--recursive", "--filter=blob:none",
	])
	if r.returncode != 0:
	update("error", "submodule update failed")
	return

	update("done", tag)

	except Exception as exc:
	update("error", str(exc))


	def main() -> None:
	script_dir = Path(__file__).parent.resolve()
	repo_root = script_dir.parent

	manifest_path = (
	Path(sys.argv[1]) if len(sys.argv) > 1
	else repo_root / "cf-deployment.yml"
	)
	dest_dir = (
	Path(sys.argv[2]) if len(sys.argv) > 2
	else repo_root / "releases"
	)

	if not manifest_path.exists():
	sys.exit(f"ERROR: manifest not found: {manifest_path}")

	dest_dir.mkdir(parents=True, exist_ok=True)

	releases = parse_releases(manifest_path)
	if not releases:
	sys.exit(f"ERROR: no releases parsed from {manifest_path}")

	console = Console()
	console.print("[bold]cf-deployment BOSH release cloner[/bold]")
	console.print(f"Manifest : {manifest_path}")
	console.print(f"Dest : {dest_dir}")
	console.print(
	f"Releases : {len(releases)} \| Workers: {MAX_WORKERS}\n"
	)

	lock: Lock = Lock()

	initial = build_table(releases) if RICH else None
	try:
	with Live(initial, console=console, refresh_per_second=8) as live:
	with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
	futures = {
	pool.submit(
	process_release, r, dest_dir, lock, live, releases
	): r
	for r in releases
	}
	try:
	for _ in as_completed(futures):
	pass # updates are pushed from within process_release
	except KeyboardInterrupt:
	for f in futures:
	f.cancel()
	pool.shutdown(wait=False, cancel_futures=True)
	raise
	except KeyboardInterrupt:
	console.print("\n[yellow]Interrupted.[/yellow]")
	sys.exit(130)

	done = sum(1 for r in releases if r.status == "done")
	warnings = sum(1 for r in releases if r.status == "warning")
	errors = sum(1 for r in releases if r.status == "error")

	console.print()
	if errors:
	console.print(
	f"[red]Completed with errors:[/red] "
	f"{done} done, {warnings} warnings, {errors} errors"
	)
	for r in releases:
	if r.status == "error":
	console.print(f" [red]✗[/red] {r.name}: {r.detail}")
	sys.exit(1)
	elif warnings:
	console.print(
	f"[dark_orange]Done:[/dark_orange] "
	f"{done} done, {warnings} warnings"
	)
	for r in releases:
	if r.status == "warning":
	console.print(
	f" [dark_orange]⚠[/dark_orange] {r.name}: {r.detail}"
	)
	else:
	console.print(
	f"[green]All {done} releases cloned successfully.[/green]"
	)


	if __name__ == "__main__":
	main()

	# MIT License
	#
	# Copyright (c) 2026 Greg Cobb
	#
	# Permission is hereby granted, free of charge, to any person obtaining a
	# copy of this software and associated documentation files (the "Software"),
	# to deal in the Software without restriction, including without limitation
	# the rights to use, copy, modify, merge, publish, distribute, sublicense,
	# and/or sell copies of the Software, and to permit persons to whom the
	# Software is furnished to do so, subject to the following conditions:
	#
	# The above copyright notice and this permission notice shall be included in
	# all copies or substantial portions of the Software.
	#
	# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
	# DEALINGS IN THE SOFTWARE.
No results found