Skip to content

Instantly share code, notes, and snippets.

@davidfstr
Created August 2, 2025 11:45
Show Gist options
  • Select an option

  • Save davidfstr/84e6deb243ec3a091d7cdfbe8318566d to your computer and use it in GitHub Desktop.

Select an option

Save davidfstr/84e6deb243ec3a091d7cdfbe8318566d to your computer and use it in GitHub Desktop.
download_universal2_wheels - Claude 4 AI draft (BEFORE) and human draft (AFTER)
#!/usr/bin/env python3
"""
Script to create universal2 wheels for ALL dependencies from poetry.lock
using uPip, so Poetry can install from a local wheelhouse only.
Uses 'poetry export' to properly handle platform markers and get only
the packages that would actually be installed on this platform.
"""
import subprocess
import sys
import tempfile
from pathlib import Path
# Check for packaging library (should be available since Poetry uses it)
try:
from packaging.requirements import Requirement
PACKAGING_AVAILABLE = True
except ImportError:
print("WARNING: packaging library not available. Marker evaluation will be disabled.")
PACKAGING_AVAILABLE = False
def install_poetry_export_plugin():
"""Install the poetry-plugin-export if not already installed."""
try:
# Check if export plugin is available
result = subprocess.run(['poetry', 'export', '--help'],
capture_output=True, text=True, timeout=10)
if result.returncode == 0:
print("βœ… Poetry export plugin already available")
return True
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
pass
print("πŸ“¦ Installing poetry-plugin-export...")
try:
result = subprocess.run(['poetry', 'self', 'add', 'poetry-plugin-export'],
capture_output=True, text=True, timeout=60)
if result.returncode == 0:
print("βœ… Successfully installed poetry-plugin-export")
return True
else:
print(f"❌ Failed to install poetry-plugin-export:")
print(f" stdout: {result.stdout}")
print(f" stderr: {result.stderr}")
return False
except subprocess.TimeoutExpired:
print("⏰ Timeout installing poetry-plugin-export")
return False
def export_requirements():
"""Export current dependencies to requirements.txt format using poetry export."""
print("πŸ“‹ Exporting dependencies with poetry export...")
try:
# Export only main dependencies (not dev dependencies) for current platform
result = subprocess.run([
'poetry', 'export',
'--format', 'requirements.txt',
'--without-hashes', # Exclude hashes for cleaner parsing
'--only', 'main' # Only main dependencies, not dev dependencies
], capture_output=True, text=True, timeout=30)
if result.returncode != 0:
print(f"❌ Poetry export failed:")
print(f" stdout: {result.stdout}")
print(f" stderr: {result.stderr}")
return None
requirements_content = result.stdout.strip()
if not requirements_content:
print("⚠️ Warning: poetry export returned empty content")
return None
print(f"βœ… Successfully exported {len(requirements_content.splitlines())} requirements")
return requirements_content
except subprocess.TimeoutExpired:
print("⏰ Timeout during poetry export")
return None
def parse_requirements(requirements_content):
"""Parse requirements.txt content to extract package names and versions that apply to current environment."""
if not PACKAGING_AVAILABLE:
print("WARNING: Falling back to simple parsing without marker evaluation")
return parse_requirements_simple(requirements_content)
packages = []
for line in requirements_content.splitlines():
line = line.strip()
# Skip empty lines and comments
if not line or line.startswith('#'):
continue
try:
# Parse the requirement (handles markers automatically)
req = Requirement(line)
# Check if the marker applies to current environment
if req.marker is None or req.marker.evaluate():
# Extract name and version from specifier
if req.specifier and len(req.specifier) == 1:
spec = list(req.specifier)[0]
if spec.operator == '==':
packages.append((req.name, spec.version))
else:
print(f"⚠️ Skipping {req.name}: non-exact version specifier {spec}")
else:
print(f"⚠️ Skipping {req.name}: complex version specifier {req.specifier}")
else:
print(f"πŸ“„ Skipping {req.name}: marker doesn't apply to current environment")
except Exception as e:
print(f"⚠️ Failed to parse requirement line: {line}")
print(f" Error: {e}")
return packages
def parse_requirements_simple(requirements_content):
"""Fallback simple parser without marker evaluation."""
packages = []
for line in requirements_content.splitlines():
line = line.strip()
# Skip empty lines and comments
if not line or line.startswith('#'):
continue
# Skip lines with markers (since we can't evaluate them)
if ';' in line:
print(f"πŸ“„ Skipping {line.split('==')[0] if '==' in line else line}: has markers (can't evaluate)")
continue
# Parse package==version format
if '==' in line:
name, version = line.split('==', 1)
packages.append((name.strip(), version.strip()))
else:
print(f"⚠️ Skipping complex requirement: {line}")
return packages
def fix_upip_binary():
"""Fix the uPip binary import bug."""
try:
upip_path = subprocess.check_output(['which', 'uPip'], text=True).strip()
with open(upip_path, 'r') as f:
content = f.read()
# Fix the import statement
fixed_content = content.replace(
'from uPip.cli import processInput',
'from universalPip.cli import processInput'
)
if content != fixed_content:
with open(upip_path, 'w') as f:
f.write(fixed_content)
print(f"Fixed uPip binary at {upip_path}")
else:
print("uPip binary already fixed or doesn't need fixing")
except subprocess.CalledProcessError:
print("WARNING: uPip binary not found, skipping fix")
def create_universal2_wheel(package_name, version, destination):
"""Create a universal2 wheel for a specific package."""
print(f"Creating universal2 wheel for {package_name} {version}...")
# Count wheels before uPip runs
wheelhouse = Path(destination)
wheels_before = set(wheelhouse.glob("*.whl"))
try:
cmd = ['uPip', '--makeU', package_name, '--version', version, '--destination', destination]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
# Count wheels after uPip runs
wheels_after = set(wheelhouse.glob("*.whl"))
new_wheels = wheels_after - wheels_before
if result.returncode == 0:
if new_wheels:
# Check if any new wheels are universal2
universal2_wheels = [w for w in new_wheels if "universal2" in w.name]
if universal2_wheels:
print(f"βœ… Success: {package_name} {version} (created {len(universal2_wheels)} universal2 wheels)")
return True
else:
print(f"πŸ“¦ Success: {package_name} {version} (created {len(new_wheels)} wheels, but no universal2)")
return True
else:
# uPip succeeded but created no wheels - likely pure Python
if "Any wheel found" in result.stdout or "none-any" in result.stdout:
print(f"πŸ“„ Success: {package_name} {version} (pure Python, no wheel needed)")
else:
print(f"πŸ€” Success: {package_name} {version} (no wheels created, reason unclear)")
return True
else:
# Check for the specific IndexError that indicates uPip's closest distribution bug
if "IndexError: list index out of range" in result.stderr:
print(f"⚠️ Skipped: {package_name} {version} (uPip closest distribution bug)")
print(f" This is likely a pure Python package that doesn't need universal2 wheels")
return True # Treat as success since these are usually pure Python packages
else:
print(f"⚠️ Failed: {package_name} {version}")
print(f" stdout: {result.stdout}")
print(f" stderr: {result.stderr}")
return False
except subprocess.TimeoutExpired:
print(f"⏰ Timeout: {package_name} {version}")
return False
except Exception as e:
print(f"❌ Error: {package_name} {version}: {e}")
return False
def main():
"""Main function to create universal2 wheels for all dependencies."""
# Ensure we're in the right directory
if not Path("poetry.lock").exists():
print("ERROR: Must run from directory containing poetry.lock")
sys.exit(1)
# Install poetry export plugin if needed
if not install_poetry_export_plugin():
print("ERROR: Failed to install poetry-plugin-export")
sys.exit(1)
# Create destination directory
wheelhouse = Path(".uwheels")
wheelhouse.mkdir(exist_ok=True)
# Fix uPip binary
fix_upip_binary()
# Export requirements using poetry export
print("πŸ” Exporting platform-specific dependencies...")
requirements_content = export_requirements()
if not requirements_content:
print("ERROR: Failed to export requirements from poetry")
sys.exit(1)
# Parse requirements.txt format
packages = parse_requirements(requirements_content)
print(f"πŸ“¦ Found {len(packages)} platform-appropriate packages")
# Show what we're going to build
print("πŸ“‹ Packages to build universal2 wheels for:")
for name, version in packages[:10]: # Show first 10
print(f" - {name} {version}")
if len(packages) > 10:
print(f" ... and {len(packages) - 10} more")
# Create universal2 wheels for all packages
success_count = 0
failed_packages = []
pure_python_count = 0
universal2_created_count = 0
for package_name, version in packages:
# Count wheels before processing this package
wheels_before = set(wheelhouse.glob("*universal2.whl"))
success = create_universal2_wheel(package_name, version, str(wheelhouse))
if success:
success_count += 1
# Count wheels after processing this package
wheels_after = set(wheelhouse.glob("*universal2.whl"))
new_universal2_wheels = wheels_after - wheels_before
if new_universal2_wheels:
universal2_created_count += 1
else:
pure_python_count += 1
else:
failed_packages.append((package_name, version))
print(f"\nπŸ“Š Results:")
print(f" βœ… Total successful: {success_count}/{len(packages)}")
print(f" πŸ”₯ Created universal2 wheels: {universal2_created_count}")
print(f" πŸ“„ Pure Python (no wheel needed): {pure_python_count}")
print(f" ❌ Failed: {len(failed_packages)}")
if failed_packages:
print(f"\n❌ Failed packages:")
for name, version in failed_packages[:20]: # Show first 20 failures
print(f" - {name} {version}")
if len(failed_packages) > 20:
print(f" ... and {len(failed_packages) - 20} more failures")
# Clean up non-universal2 wheels
print(f"\n🧹 Cleaning up non-universal2 wheels...")
non_universal2_wheels = list(wheelhouse.glob("*.whl"))
non_universal2_wheels = [w for w in non_universal2_wheels if "universal2" not in w.name]
for wheel in non_universal2_wheels:
wheel.unlink()
print(f" Removed: {wheel.name}")
# Show final wheelhouse contents
universal2_wheels = list(wheelhouse.glob("*universal2.whl"))
print(f"\nπŸ“¦ Final wheelhouse contains {len(universal2_wheels)} universal2 wheels:")
for wheel in sorted(universal2_wheels)[:15]: # Show first 15
print(f" {wheel.name}")
if len(universal2_wheels) > 15:
print(f" ... and {len(universal2_wheels) - 15} more")
if len(universal2_wheels) == 0:
print("⚠️ WARNING: No universal2 wheels were created!")
# Don't exit with error - some packages might not have wheels or might be pure Python
print("πŸ’‘ This might be expected if all dependencies are pure Python packages")
print(f"\nβœ… Created universal2 wheelhouse with {len(universal2_wheels)} packages")
print(f"πŸ’‘ Note: Pure Python packages don't need universal2 wheels and will be downloaded normally")
if __name__ == "__main__":
main()
#!/bin/bash
# Downloads universal2 and pure Python wheels for all dependencies to .uwheels
# Create/clean output directories
mkdir -p .uwheels
rm -rf .uwheels/*
mkdir .uwheels/forge
# Convert poetry.lock to requirements-local.txt
poetry export --format requirements.txt --all-groups --without-hashes > requirements.txt
cat requirements.txt | python setup/localize_requirements.py > requirements-local.txt
BEST_UNIVERSAL2_PLATFORM=$(python -c "import packaging.tags; print([t.platform for t in packaging.tags.sys_tags() if 'universal2' in t.platform][0])")
BEST_X86_64_PLATFORM=$(echo "$BEST_UNIVERSAL2_PLATFORM" | sed 's/universal2/x86_64/')
BEST_ARM64_PLATFORM=$(echo "$BEST_UNIVERSAL2_PLATFORM" | sed 's/universal2/arm64/')
# For each requirement, download the best universal2 or pure Python wheel
for REQUIREMENT in $(cat requirements-local.txt); do
# Download best universal2 or pure Python wheel, if available
echo "Downloading: ${REQUIREMENT}"
pip download --only-binary=:all: --dest .uwheels --no-deps "$REQUIREMENT" --platform "$BEST_UNIVERSAL2_PLATFORM" > /dev/null 2>&1
if [ $? -ne 0 ]; then
REQUIREMENT_DONE=0
mkdir .uwheels/forge/$REQUIREMENT
# Download best x86_64 wheel, or fallback to source package if not available
pip download --only-binary=:all: --dest .uwheels/forge/$REQUIREMENT --no-deps "$REQUIREMENT" --platform "$BEST_X86_64_PLATFORM" > /dev/null 2>&1
if [ $? -ne 0 ]; then
pip download --no-binary=:all: --dest .uwheels --no-deps "$REQUIREMENT" > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "*** No wheel available for $REQUIREMENT on $BEST_X86_64_PLATFORM. No source package either."
exit 1
fi
REQUIREMENT_DONE=1
fi
if [ $REQUIREMENT_DONE -eq 0 ]; then
# Download best arm64 wheel
pip download --only-binary=:all: --dest .uwheels/forge/$REQUIREMENT --no-deps "$REQUIREMENT" --platform "$BEST_ARM64_PLATFORM" > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "*** No wheel available for $REQUIREMENT on $BEST_ARM64_PLATFORM."
exit 1
fi
# Merge the x86_64 and arm64 wheels into a universal2 wheel
delocate-merge -w .uwheels .uwheels/forge/$REQUIREMENT/*.whl
if [ $? -ne 0 ]; then
echo "*** Could not merge wheels for $REQUIREMENT."
exit 1
fi
fi
rm -rf .uwheels/forge/$REQUIREMENT
fi
done
# Clean up
rmdir .uwheels/forge
rm requirements.txt requirements-local.txt
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment