Created
August 2, 2025 11:45
-
-
Save davidfstr/84e6deb243ec3a091d7cdfbe8318566d to your computer and use it in GitHub Desktop.
download_universal2_wheels - Claude 4 AI draft (BEFORE) and human draft (AFTER)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """ | |
| Script to create universal2 wheels for ALL dependencies from poetry.lock | |
| using uPip, so Poetry can install from a local wheelhouse only. | |
| Uses 'poetry export' to properly handle platform markers and get only | |
| the packages that would actually be installed on this platform. | |
| """ | |
| import subprocess | |
| import sys | |
| import tempfile | |
| from pathlib import Path | |
| # Check for packaging library (should be available since Poetry uses it) | |
| try: | |
| from packaging.requirements import Requirement | |
| PACKAGING_AVAILABLE = True | |
| except ImportError: | |
| print("WARNING: packaging library not available. Marker evaluation will be disabled.") | |
| PACKAGING_AVAILABLE = False | |
| def install_poetry_export_plugin(): | |
| """Install the poetry-plugin-export if not already installed.""" | |
| try: | |
| # Check if export plugin is available | |
| result = subprocess.run(['poetry', 'export', '--help'], | |
| capture_output=True, text=True, timeout=10) | |
| if result.returncode == 0: | |
| print("β Poetry export plugin already available") | |
| return True | |
| except (subprocess.CalledProcessError, subprocess.TimeoutExpired): | |
| pass | |
| print("π¦ Installing poetry-plugin-export...") | |
| try: | |
| result = subprocess.run(['poetry', 'self', 'add', 'poetry-plugin-export'], | |
| capture_output=True, text=True, timeout=60) | |
| if result.returncode == 0: | |
| print("β Successfully installed poetry-plugin-export") | |
| return True | |
| else: | |
| print(f"β Failed to install poetry-plugin-export:") | |
| print(f" stdout: {result.stdout}") | |
| print(f" stderr: {result.stderr}") | |
| return False | |
| except subprocess.TimeoutExpired: | |
| print("β° Timeout installing poetry-plugin-export") | |
| return False | |
| def export_requirements(): | |
| """Export current dependencies to requirements.txt format using poetry export.""" | |
| print("π Exporting dependencies with poetry export...") | |
| try: | |
| # Export only main dependencies (not dev dependencies) for current platform | |
| result = subprocess.run([ | |
| 'poetry', 'export', | |
| '--format', 'requirements.txt', | |
| '--without-hashes', # Exclude hashes for cleaner parsing | |
| '--only', 'main' # Only main dependencies, not dev dependencies | |
| ], capture_output=True, text=True, timeout=30) | |
| if result.returncode != 0: | |
| print(f"β Poetry export failed:") | |
| print(f" stdout: {result.stdout}") | |
| print(f" stderr: {result.stderr}") | |
| return None | |
| requirements_content = result.stdout.strip() | |
| if not requirements_content: | |
| print("β οΈ Warning: poetry export returned empty content") | |
| return None | |
| print(f"β Successfully exported {len(requirements_content.splitlines())} requirements") | |
| return requirements_content | |
| except subprocess.TimeoutExpired: | |
| print("β° Timeout during poetry export") | |
| return None | |
| def parse_requirements(requirements_content): | |
| """Parse requirements.txt content to extract package names and versions that apply to current environment.""" | |
| if not PACKAGING_AVAILABLE: | |
| print("WARNING: Falling back to simple parsing without marker evaluation") | |
| return parse_requirements_simple(requirements_content) | |
| packages = [] | |
| for line in requirements_content.splitlines(): | |
| line = line.strip() | |
| # Skip empty lines and comments | |
| if not line or line.startswith('#'): | |
| continue | |
| try: | |
| # Parse the requirement (handles markers automatically) | |
| req = Requirement(line) | |
| # Check if the marker applies to current environment | |
| if req.marker is None or req.marker.evaluate(): | |
| # Extract name and version from specifier | |
| if req.specifier and len(req.specifier) == 1: | |
| spec = list(req.specifier)[0] | |
| if spec.operator == '==': | |
| packages.append((req.name, spec.version)) | |
| else: | |
| print(f"β οΈ Skipping {req.name}: non-exact version specifier {spec}") | |
| else: | |
| print(f"β οΈ Skipping {req.name}: complex version specifier {req.specifier}") | |
| else: | |
| print(f"π Skipping {req.name}: marker doesn't apply to current environment") | |
| except Exception as e: | |
| print(f"β οΈ Failed to parse requirement line: {line}") | |
| print(f" Error: {e}") | |
| return packages | |
| def parse_requirements_simple(requirements_content): | |
| """Fallback simple parser without marker evaluation.""" | |
| packages = [] | |
| for line in requirements_content.splitlines(): | |
| line = line.strip() | |
| # Skip empty lines and comments | |
| if not line or line.startswith('#'): | |
| continue | |
| # Skip lines with markers (since we can't evaluate them) | |
| if ';' in line: | |
| print(f"π Skipping {line.split('==')[0] if '==' in line else line}: has markers (can't evaluate)") | |
| continue | |
| # Parse package==version format | |
| if '==' in line: | |
| name, version = line.split('==', 1) | |
| packages.append((name.strip(), version.strip())) | |
| else: | |
| print(f"β οΈ Skipping complex requirement: {line}") | |
| return packages | |
| def fix_upip_binary(): | |
| """Fix the uPip binary import bug.""" | |
| try: | |
| upip_path = subprocess.check_output(['which', 'uPip'], text=True).strip() | |
| with open(upip_path, 'r') as f: | |
| content = f.read() | |
| # Fix the import statement | |
| fixed_content = content.replace( | |
| 'from uPip.cli import processInput', | |
| 'from universalPip.cli import processInput' | |
| ) | |
| if content != fixed_content: | |
| with open(upip_path, 'w') as f: | |
| f.write(fixed_content) | |
| print(f"Fixed uPip binary at {upip_path}") | |
| else: | |
| print("uPip binary already fixed or doesn't need fixing") | |
| except subprocess.CalledProcessError: | |
| print("WARNING: uPip binary not found, skipping fix") | |
| def create_universal2_wheel(package_name, version, destination): | |
| """Create a universal2 wheel for a specific package.""" | |
| print(f"Creating universal2 wheel for {package_name} {version}...") | |
| # Count wheels before uPip runs | |
| wheelhouse = Path(destination) | |
| wheels_before = set(wheelhouse.glob("*.whl")) | |
| try: | |
| cmd = ['uPip', '--makeU', package_name, '--version', version, '--destination', destination] | |
| result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) | |
| # Count wheels after uPip runs | |
| wheels_after = set(wheelhouse.glob("*.whl")) | |
| new_wheels = wheels_after - wheels_before | |
| if result.returncode == 0: | |
| if new_wheels: | |
| # Check if any new wheels are universal2 | |
| universal2_wheels = [w for w in new_wheels if "universal2" in w.name] | |
| if universal2_wheels: | |
| print(f"β Success: {package_name} {version} (created {len(universal2_wheels)} universal2 wheels)") | |
| return True | |
| else: | |
| print(f"π¦ Success: {package_name} {version} (created {len(new_wheels)} wheels, but no universal2)") | |
| return True | |
| else: | |
| # uPip succeeded but created no wheels - likely pure Python | |
| if "Any wheel found" in result.stdout or "none-any" in result.stdout: | |
| print(f"π Success: {package_name} {version} (pure Python, no wheel needed)") | |
| else: | |
| print(f"π€ Success: {package_name} {version} (no wheels created, reason unclear)") | |
| return True | |
| else: | |
| # Check for the specific IndexError that indicates uPip's closest distribution bug | |
| if "IndexError: list index out of range" in result.stderr: | |
| print(f"β οΈ Skipped: {package_name} {version} (uPip closest distribution bug)") | |
| print(f" This is likely a pure Python package that doesn't need universal2 wheels") | |
| return True # Treat as success since these are usually pure Python packages | |
| else: | |
| print(f"β οΈ Failed: {package_name} {version}") | |
| print(f" stdout: {result.stdout}") | |
| print(f" stderr: {result.stderr}") | |
| return False | |
| except subprocess.TimeoutExpired: | |
| print(f"β° Timeout: {package_name} {version}") | |
| return False | |
| except Exception as e: | |
| print(f"β Error: {package_name} {version}: {e}") | |
| return False | |
| def main(): | |
| """Main function to create universal2 wheels for all dependencies.""" | |
| # Ensure we're in the right directory | |
| if not Path("poetry.lock").exists(): | |
| print("ERROR: Must run from directory containing poetry.lock") | |
| sys.exit(1) | |
| # Install poetry export plugin if needed | |
| if not install_poetry_export_plugin(): | |
| print("ERROR: Failed to install poetry-plugin-export") | |
| sys.exit(1) | |
| # Create destination directory | |
| wheelhouse = Path(".uwheels") | |
| wheelhouse.mkdir(exist_ok=True) | |
| # Fix uPip binary | |
| fix_upip_binary() | |
| # Export requirements using poetry export | |
| print("π Exporting platform-specific dependencies...") | |
| requirements_content = export_requirements() | |
| if not requirements_content: | |
| print("ERROR: Failed to export requirements from poetry") | |
| sys.exit(1) | |
| # Parse requirements.txt format | |
| packages = parse_requirements(requirements_content) | |
| print(f"π¦ Found {len(packages)} platform-appropriate packages") | |
| # Show what we're going to build | |
| print("π Packages to build universal2 wheels for:") | |
| for name, version in packages[:10]: # Show first 10 | |
| print(f" - {name} {version}") | |
| if len(packages) > 10: | |
| print(f" ... and {len(packages) - 10} more") | |
| # Create universal2 wheels for all packages | |
| success_count = 0 | |
| failed_packages = [] | |
| pure_python_count = 0 | |
| universal2_created_count = 0 | |
| for package_name, version in packages: | |
| # Count wheels before processing this package | |
| wheels_before = set(wheelhouse.glob("*universal2.whl")) | |
| success = create_universal2_wheel(package_name, version, str(wheelhouse)) | |
| if success: | |
| success_count += 1 | |
| # Count wheels after processing this package | |
| wheels_after = set(wheelhouse.glob("*universal2.whl")) | |
| new_universal2_wheels = wheels_after - wheels_before | |
| if new_universal2_wheels: | |
| universal2_created_count += 1 | |
| else: | |
| pure_python_count += 1 | |
| else: | |
| failed_packages.append((package_name, version)) | |
| print(f"\nπ Results:") | |
| print(f" β Total successful: {success_count}/{len(packages)}") | |
| print(f" π₯ Created universal2 wheels: {universal2_created_count}") | |
| print(f" π Pure Python (no wheel needed): {pure_python_count}") | |
| print(f" β Failed: {len(failed_packages)}") | |
| if failed_packages: | |
| print(f"\nβ Failed packages:") | |
| for name, version in failed_packages[:20]: # Show first 20 failures | |
| print(f" - {name} {version}") | |
| if len(failed_packages) > 20: | |
| print(f" ... and {len(failed_packages) - 20} more failures") | |
| # Clean up non-universal2 wheels | |
| print(f"\nπ§Ή Cleaning up non-universal2 wheels...") | |
| non_universal2_wheels = list(wheelhouse.glob("*.whl")) | |
| non_universal2_wheels = [w for w in non_universal2_wheels if "universal2" not in w.name] | |
| for wheel in non_universal2_wheels: | |
| wheel.unlink() | |
| print(f" Removed: {wheel.name}") | |
| # Show final wheelhouse contents | |
| universal2_wheels = list(wheelhouse.glob("*universal2.whl")) | |
| print(f"\nπ¦ Final wheelhouse contains {len(universal2_wheels)} universal2 wheels:") | |
| for wheel in sorted(universal2_wheels)[:15]: # Show first 15 | |
| print(f" {wheel.name}") | |
| if len(universal2_wheels) > 15: | |
| print(f" ... and {len(universal2_wheels) - 15} more") | |
| if len(universal2_wheels) == 0: | |
| print("β οΈ WARNING: No universal2 wheels were created!") | |
| # Don't exit with error - some packages might not have wheels or might be pure Python | |
| print("π‘ This might be expected if all dependencies are pure Python packages") | |
| print(f"\nβ Created universal2 wheelhouse with {len(universal2_wheels)} packages") | |
| print(f"π‘ Note: Pure Python packages don't need universal2 wheels and will be downloaded normally") | |
| if __name__ == "__main__": | |
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Downloads universal2 and pure Python wheels for all dependencies to .uwheels | |
| # Create/clean output directories | |
| mkdir -p .uwheels | |
| rm -rf .uwheels/* | |
| mkdir .uwheels/forge | |
| # Convert poetry.lock to requirements-local.txt | |
| poetry export --format requirements.txt --all-groups --without-hashes > requirements.txt | |
| cat requirements.txt | python setup/localize_requirements.py > requirements-local.txt | |
| BEST_UNIVERSAL2_PLATFORM=$(python -c "import packaging.tags; print([t.platform for t in packaging.tags.sys_tags() if 'universal2' in t.platform][0])") | |
| BEST_X86_64_PLATFORM=$(echo "$BEST_UNIVERSAL2_PLATFORM" | sed 's/universal2/x86_64/') | |
| BEST_ARM64_PLATFORM=$(echo "$BEST_UNIVERSAL2_PLATFORM" | sed 's/universal2/arm64/') | |
| # For each requirement, download the best universal2 or pure Python wheel | |
| for REQUIREMENT in $(cat requirements-local.txt); do | |
| # Download best universal2 or pure Python wheel, if available | |
| echo "Downloading: ${REQUIREMENT}" | |
| pip download --only-binary=:all: --dest .uwheels --no-deps "$REQUIREMENT" --platform "$BEST_UNIVERSAL2_PLATFORM" > /dev/null 2>&1 | |
| if [ $? -ne 0 ]; then | |
| REQUIREMENT_DONE=0 | |
| mkdir .uwheels/forge/$REQUIREMENT | |
| # Download best x86_64 wheel, or fallback to source package if not available | |
| pip download --only-binary=:all: --dest .uwheels/forge/$REQUIREMENT --no-deps "$REQUIREMENT" --platform "$BEST_X86_64_PLATFORM" > /dev/null 2>&1 | |
| if [ $? -ne 0 ]; then | |
| pip download --no-binary=:all: --dest .uwheels --no-deps "$REQUIREMENT" > /dev/null 2>&1 | |
| if [ $? -ne 0 ]; then | |
| echo "*** No wheel available for $REQUIREMENT on $BEST_X86_64_PLATFORM. No source package either." | |
| exit 1 | |
| fi | |
| REQUIREMENT_DONE=1 | |
| fi | |
| if [ $REQUIREMENT_DONE -eq 0 ]; then | |
| # Download best arm64 wheel | |
| pip download --only-binary=:all: --dest .uwheels/forge/$REQUIREMENT --no-deps "$REQUIREMENT" --platform "$BEST_ARM64_PLATFORM" > /dev/null 2>&1 | |
| if [ $? -ne 0 ]; then | |
| echo "*** No wheel available for $REQUIREMENT on $BEST_ARM64_PLATFORM." | |
| exit 1 | |
| fi | |
| # Merge the x86_64 and arm64 wheels into a universal2 wheel | |
| delocate-merge -w .uwheels .uwheels/forge/$REQUIREMENT/*.whl | |
| if [ $? -ne 0 ]; then | |
| echo "*** Could not merge wheels for $REQUIREMENT." | |
| exit 1 | |
| fi | |
| fi | |
| rm -rf .uwheels/forge/$REQUIREMENT | |
| fi | |
| done | |
| # Clean up | |
| rmdir .uwheels/forge | |
| rm requirements.txt requirements-local.txt |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment