sjmf/README.md

## README.md

      
    Raw
  

              README.md
            
          
    Monorepo Extractor

extract.sh

Let's say I have a private, monolithic git repo where I store my docker compose server configurations.
The folders are in a structure like this:
/ root
 |- project1/
 |--- compose.yml
 |--- optional.env
 |--- conf/
 |--- etc/
 |- project2/
 |- project3/
...
 |-- projectn/

This script performs the following actions:

Split the monorepo into individual repositories
Scan those repositories for secrets
Erase those secrets from the history

Tools

git-filter-repo

For extracting project folders with their history
BFG Repo-Cleaner
For removing secrets from history
Secret scanners: script supports:
ggshield (GitGuardian's CLI tool)
gitleaks


## extract.sh
#!/usr/bin/env bash

set -e  # Exit on error
set -u  # Exit on undefined variable

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
MAGENTA='\033[0;35m'
NC='\033[0m' # No Color

# Function to print colored output
print_error() {
    echo -e "${RED}ERROR: $1${NC}" >&2
}

print_success() {
    echo -e "${GREEN}✓ $1${NC}"
}

print_info() {
    echo -e "${YELLOW}ℹ $1${NC}"
}

print_phase() {
    echo -e "${BLUE}=== $1 ===${NC}"
}

print_warning() {
    echo -e "${MAGENTA}⚠ WARNING: $1${NC}"
}

# Function to check if a command exists
command_exists() {
    command -v "$1" >/dev/null 2>&1
}

# Check if required tools are installed
check_tools() {
    print_info "Checking required tools..."

    local missing_tools=()

    if ! command_exists git; then
        missing_tools+=("git")
    fi

    if ! command_exists git-filter-repo; then
        missing_tools+=("git-filter-repo")
    fi

    if [ ${#missing_tools[@]} -gt 0 ]; then
        print_error "Missing required tools: ${missing_tools[*]}"
        echo ""
        echo "Install them with:"
        for tool in "${missing_tools[@]}"; do
            case "$tool" in
                git-filter-repo)
                    echo "  pip install git-filter-repo"
                    echo "  or: brew install git-filter-repo (macOS)"
                    echo "  or: apt install git-filter-repo (Ubuntu/Debian)"
                    ;;
                git)
                    echo "  Install git from your package manager"
                    ;;
            esac
        done
        exit 1
    fi

    print_success "Required tools installed: git, git-filter-repo"
}

# Check if BFG is installed
check_bfg() {
    print_info "Checking for BFG Repo-Cleaner..."

    if command_exists bfg; then
        print_success "BFG is installed"
        return 0
    fi

    # Check for bfg.jar in common locations
    local bfg_locations=(
        "./bfg.jar"
        "$HOME/bfg.jar"
        "/usr/local/bin/bfg.jar"
    )

    for location in "${bfg_locations[@]}"; do
        if [ -f "$location" ]; then
            print_success "BFG found at: $location"
            echo "export BFG_JAR=\"$location\"" > /tmp/bfg_location.sh
            return 0
        fi
    done

    print_error "BFG Repo-Cleaner not found!"
    echo ""
    echo "Download it from: https://rtyley.github.io/bfg-repo-cleaner/"
    echo "  wget https://repo1.maven.org/maven2/com/madgag/bfg/1.14.0/bfg-1.14.0.jar -O bfg.jar"
    echo ""
    echo "Or install via package manager:"
    echo "  brew install bfg (macOS - requires Xcode)"
    echo "  apt install bfg (Ubuntu/Debian)"
    return 1
}

# Check if secret scanning tools are installed
check_secret_scanners() {
    print_info "Checking secret scanning tools..."

    local available_scanners=()
    local missing_scanners=()

    if command_exists ggshield; then
        # Check if ggshield is authenticated by checking API status
        if ggshield api-status >/dev/null 2>&1; then
            available_scanners+=("ggshield")
        else
            print_warning "ggshield is installed but not authenticated"
            echo "  Authenticate by running: ggshield auth login"
            echo "  Or set GITGUARDIAN_API_KEY environment variable"
            echo "  ggshield will be skipped during scanning"
            missing_scanners+=("ggshield (not authenticated)")
        fi
    else
        missing_scanners+=("ggshield")
    fi

    if command_exists gitleaks; then
        available_scanners+=("gitleaks")
    else
        missing_scanners+=("gitleaks")
    fi

    if [ ${#available_scanners[@]} -eq 0 ]; then
        print_error "No secret scanning tools found or configured!"
        echo ""
        echo "Install at least one:"
        echo "  pip install ggshield (then run: ggshield auth login)"
        echo "  brew install gitleaks (macOS)"
        echo "  or download from: https://github.com/gitleaks/gitleaks/releases"
        return 1
    fi

    print_success "Available scanners: ${available_scanners[*]}"

    if [ ${#missing_scanners[@]} -gt 0 ]; then
        print_info "Optional scanners not available: ${missing_scanners[*]}"
        echo "  Consider installing them for more thorough scanning:"
        for scanner in "${missing_scanners[@]}"; do
            case "$scanner" in
                ggshield*)
                    echo "    pip install ggshield"
                    echo "    ggshield auth login"
                    ;;
                gitleaks)
                    echo "    brew install gitleaks (macOS)"
                    echo "    or: https://github.com/gitleaks/gitleaks/releases"
                    ;;
            esac
        done
    fi

    return 0
}

# Function to get top-level directories in the repo
get_directories() {
    local repo_path="$1"

    # Get directories, excluding hidden ones and common non-project dirs
    find "$repo_path" -mindepth 1 -maxdepth 1 -type d \
        ! -name ".*" \
        ! -name "node_modules" \
        ! -name "__pycache__" \
        -exec basename {} \;
}

# Function to extract a single project
extract_project() {
    local source_repo="$1"
    local project_name="$2"
    local output_dir="$3"

    print_info "Extracting project: $project_name"

    # Clone the repository
    print_info "  Cloning repository to $output_dir..."
    if ! git clone "$source_repo" "$output_dir" 2>/dev/null; then
        print_error "Failed to clone repository for $project_name"
        return 1
    fi

    # CD into the cloned repo and run filter-repo
    print_info "  Running git filter-repo..."
    if ! (cd "$output_dir" && git filter-repo --path "$project_name/" --path-rename "$project_name/:" --force); then
        print_error "Failed to filter repository for $project_name"
        return 1
    fi

    print_success "Successfully extracted $project_name to $output_dir"
}

# Function to generate secrets.txt from scan results
generate_secrets_file() {
    local repo_dir="$1"
    local project_name="$2"
    local scan_results_dir="${repo_dir}/scan-results"
    local secrets_file="${repo_dir}/secrets.txt"

    print_info "Generating secrets.txt for $project_name..."

    if [ -f "$secrets_file" ]; then
        print_warning "  secrets.txt already exists - will append new secrets"
    fi

    local secrets_found=0
    local temp_secrets=$(mktemp)

    # Extract secrets from gitleaks JSON
    if [ -f "${scan_results_dir}/gitleaks-results.json" ]; then
        print_info "  Extracting secrets from gitleaks results..."

        # Use grep/sed to extract Secret field values (works without jq)
        grep -o '"Secret": "[^"]*"' "${scan_results_dir}/gitleaks-results.json" 2>/dev/null | \
            sed 's/"Secret": "//;s/"$//' >> "$temp_secrets" || true

        local gitleaks_count=$(grep -c '"Secret":' "${scan_results_dir}/gitleaks-results.json" 2>/dev/null || echo 0)
        if [ "$gitleaks_count" -gt 0 ]; then
            print_success "  Found $gitleaks_count secret(s) in gitleaks results"
            secrets_found=$gitleaks_count
        fi
    fi

    # Extract secrets from ggshield results
    if [ -f "${scan_results_dir}/ggshield-results.txt" ]; then
        print_info "  Extracting secrets from ggshield results..."

        # Extract the actual secret values (marked with |__apikey__|, |__password__|, etc.)
        # Look for lines that have the masking pattern with asterisks
        grep -E '^\s+[0-9]+\s+\|.*\*+.*\|' "${scan_results_dir}/ggshield-results.txt" 2>/dev/null | \
            sed -E "s/.*'([^']+)'.*/\1/; s/\"([^\"]+)\".*/\1/" | \
            grep -v '^\s*$' >> "$temp_secrets" || true

        # Also try to extract from lines that show the actual match
        grep -E "BASE64_AUTH=|PASSWORD=|API_KEY=|TOKEN=|SECRET=" \
            "${scan_results_dir}/ggshield-results.txt" 2>/dev/null | \
            grep -oE "=['\"]?[A-Za-z0-9+/=_-]{20,}['\"]?" | \
            sed "s/^=['\"]//; s/['\"]$//" >> "$temp_secrets" || true

        local temp_count=$(wc -l < "$temp_secrets" 2>/dev/null | tr -d ' \n' || echo 0)
        if [ "$temp_count" -gt "$secrets_found" ]; then
            local additional=$((temp_count - secrets_found))
            print_success "  Found $additional additional secret(s) in ggshield results"
            secrets_found=$temp_count
        fi
    fi

    # Remove duplicates and empty lines, append to secrets.txt
    if [ -s "$temp_secrets" ]; then
        sort -u "$temp_secrets" | grep -v '^\s*$' >> "$secrets_file"
        sort -u -o "$secrets_file" "$secrets_file"

        local final_count=$(wc -l < "$secrets_file" 2>/dev/null | tr -d ' \n' || echo 0)
        print_success "Generated secrets.txt with $final_count unique secret(s)"
        echo ""
        print_info "Review and edit: $secrets_file"
        echo "  Each line should contain one secret string to remove"
        echo "  You can add additional patterns or secrets manually"
        echo ""
    else
        print_warning "No secrets could be extracted automatically"
        echo "  You may need to create secrets.txt manually"
        echo "  Review: $scan_results_dir"
    fi

    rm -f "$temp_secrets"
}

# Function to scan a repository for secrets
scan_repository() {
    local repo_dir="$1"
    local project_name="$2"
    local scan_results_dir="${repo_dir}/scan-results"

    print_info "Scanning $project_name for secrets..."

    # Create scan results directory
    mkdir -p "$scan_results_dir"

    local found_secrets=false

    # Run ggshield if available
    if command_exists ggshield; then
        # Check if authenticated before running
        if ggshield api-status >/dev/null 2>&1; then
            print_info "  Running ggshield..."
            local ggshield_output="${scan_results_dir}/ggshield-results.txt"

            # Use absolute path for output file
            local abs_ggshield_output=$(cd "$(dirname "$ggshield_output")" && pwd)/$(basename "$ggshield_output")

            if (cd "$repo_dir" && ggshield secret scan repo . > "$abs_ggshield_output" 2>&1); then
                print_success "  ggshield: No secrets found"
            else
                print_error "  ggshield: Potential secrets detected!"
                found_secrets=true
                echo "    Results saved to: $ggshield_output"
            fi
        else
            print_info "  Skipping ggshield (not authenticated)"
        fi
    fi

    # Run gitleaks if available
    if command_exists gitleaks; then
        print_info "  Running gitleaks..."
        local gitleaks_output="${scan_results_dir}/gitleaks-results.json"
        local gitleaks_report="${scan_results_dir}/gitleaks-report.txt"

        # Use absolute paths for output files
        local abs_gitleaks_output=$(cd "$(dirname "$gitleaks_output")" && pwd)/$(basename "$gitleaks_output")
        local abs_gitleaks_report=$(cd "$(dirname "$gitleaks_report")" && pwd)/$(basename "$gitleaks_report")

        if (cd "$repo_dir" && gitleaks detect --source . --report-path "$abs_gitleaks_output" --verbose > "$abs_gitleaks_report" 2>&1); then
            print_success "  gitleaks: No secrets found"
        else
            print_error "  gitleaks: Potential secrets detected!"
            found_secrets=true
            echo "    Results saved to: $gitleaks_output"
            echo "    Report saved to: $gitleaks_report"
        fi
    fi

    if $found_secrets; then
        print_error "Secrets found in $project_name - review scan results before proceeding!"
        echo ""
        echo "  Review the results in: $scan_results_dir"
        return 1
    else
        print_success "No secrets detected in $project_name"
        return 0
    fi
}

# Function to run BFG on a repository
run_bfg() {
    local repo_dir="$1"
    local secrets_file="$2"

    if command_exists bfg; then
        bfg --replace-text "$secrets_file" "$repo_dir"
    elif [ -f "${BFG_JAR:-}" ]; then
        java -jar "$BFG_JAR" --replace-text "$secrets_file" "$repo_dir"
    elif [ -f "./bfg.jar" ]; then
        java -jar ./bfg.jar --replace-text "$secrets_file" "$repo_dir"
    else
        print_error "BFG not found!"
        return 1
    fi
}

# Function to clean a single repository
clean_repository() {
    local repo_dir="$1"
    local project_name="$2"

    print_info "Cleaning $project_name..."

    local secrets_file="${repo_dir}/secrets.txt"

    # Check if secrets.txt exists
    if [ ! -f "$secrets_file" ]; then
        print_info "  No secrets.txt found - skipping BFG"
        print_info "  If you need to remove secrets, create $secrets_file with strings to remove (one per line)"
    else
        print_info "  Found secrets.txt with $(wc -l < "$secrets_file" | tr -d ' \n') entries"
        print_info "  Running BFG Repo-Cleaner..."

        if ! run_bfg "$repo_dir" "$secrets_file"; then
            print_error "Failed to run BFG on $project_name"
            return 1
        fi

        print_success "  BFG completed"
    fi

    # Clean up with git reflog and gc
    print_info "  Expiring reflog..."
    if ! (cd "$repo_dir" && git reflog expire --expire=now --all); then
        print_error "Failed to expire reflog for $project_name"
        return 1
    fi

    print_info "  Running garbage collection (this may take a while)..."
    if ! (cd "$repo_dir" && git gc --prune=now --aggressive); then
        print_error "Failed to run gc for $project_name"
        return 1
    fi

    # Get repo size info
    local repo_size=$(du -sh "$repo_dir/.git" | cut -f1)
    print_success "  Repository cleaned! .git size: $repo_size"

    print_success "Successfully cleaned $project_name"
}

# Function to verify secrets are gone
verify_clean() {
    local repo_dir="$1"
    local project_name="$2"

    print_info "Verifying $project_name is clean..."

    local verification_dir="${repo_dir}/scan-results-post-clean"
    mkdir -p "$verification_dir"

    local issues_found=false

    # Run ggshield if available
    if command_exists ggshield; then
        # Check if authenticated before running
        if ggshield api-status >/dev/null 2>&1; then
            print_info "  Re-running ggshield..."
            local ggshield_output="${verification_dir}/ggshield-results.txt"

            # Use absolute path for output file
            local abs_ggshield_output=$(cd "$(dirname "$ggshield_output")" && pwd)/$(basename "$ggshield_output")

            if (cd "$repo_dir" && ggshield secret scan repo . > "$abs_ggshield_output" 2>&1); then
                print_success "  ggshield: Clean"
            else
                print_error "  ggshield: Still detecting potential secrets!"
                issues_found=true
            fi
        else
            print_info "  Skipping ggshield verification (not authenticated)"
        fi
    fi

    # Run gitleaks if available
    if command_exists gitleaks; then
        print_info "  Re-running gitleaks..."
        local gitleaks_output="${verification_dir}/gitleaks-results.json"
        local gitleaks_report="${verification_dir}/gitleaks-report.txt"

        # Use absolute paths for output files
        local abs_gitleaks_output=$(cd "$(dirname "$gitleaks_output")" && pwd)/$(basename "$gitleaks_output")
        local abs_gitleaks_report=$(cd "$(dirname "$gitleaks_report")" && pwd)/$(basename "$gitleaks_report")

        if (cd "$repo_dir" && gitleaks detect --source . --report-path "$abs_gitleaks_output" --verbose > "$abs_gitleaks_report" 2>&1); then
            print_success "  gitleaks: Clean"
        else
            print_error "  gitleaks: Still detecting potential secrets!"
            issues_found=true
        fi
    fi

    if $issues_found; then
        print_error "Verification failed for $project_name"
        echo "  Check: $verification_dir"
        return 1
    else
        print_success "Verification passed for $project_name"
        return 0
    fi
}

# Function to remove leftover temporary files
remove_scan_results() {
    find . -type d -name '*.bfg-report' -exec rm -rf {} +;
    find . -type d -name 'scan-results' -exec rm -rf {} +;
    find . -type d -name 'scan-results-post-clean' -exec rm -rf {} +;
    find . -type f -name 'secrets.txt' -exec rm {} +;
}

# Phase 1: Extract projects
phase1_extract() {
    local repo_path="$1"
    local repo_basename=$(basename "$repo_path")

    print_phase "PHASE 1: Extract Projects"
    echo ""

    # Get directories
    print_info "Scanning for project directories..."
    local directories=()
    while IFS= read -r dir; do
        directories+=("$dir")
    done < <(get_directories "$repo_path")

    if [ ${#directories[@]} -eq 0 ]; then
        print_error "No directories found in repository"
        exit 1
    fi

    print_success "Found ${#directories[@]} project(s):"
    for dir in "${directories[@]}"; do
        echo "  - $dir"
    done
    echo ""

    # Confirm with user
    read -p "Proceed with extraction? (y/N) " -r
    echo ""
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        print_info "Aborted by user"
        exit 0
    fi
    echo ""

    # Extract each project
    local success_count=0
    local fail_count=0

    for project in "${directories[@]}"; do
        local output_dir="${repo_basename}-${project}"

        if [ -d "$output_dir" ]; then
            print_error "Output directory already exists: $output_dir (skipping)"
            ((fail_count++))
            continue
        fi

        if extract_project "$repo_path" "$project" "$output_dir"; then
            ((success_count++))
            EXTRACTED_DIRS+=("$output_dir")
        else
            ((fail_count++))
        fi
        echo ""
    done

    # Summary
    echo "================================"
    print_success "Phase 1 complete!"
    echo "  Successful: $success_count"
    if [ $fail_count -gt 0 ]; then
        echo "  Failed: $fail_count"
    fi
    echo ""

    # Return success status
    return 0
}

# Phase 2: Scan for secrets
phase2_scan() {
    print_phase "PHASE 2: Scan for Secrets"
    echo ""

    print_info "This phase will scan each extracted repository for secrets."
    print_info "Scanning tools: ggshield, gitleaks (if available)"
    echo ""

    read -p "Proceed with secret scanning? (y/N/s to skip) " -r
    echo ""

    if [[ $REPLY =~ ^[Ss]$ ]]; then
        print_info "Skipping Phase 2"
        return 0
    fi

    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        print_info "Aborted by user"
        exit 0
    fi
    echo ""

    # Check secret scanners are available
    if ! check_secret_scanners; then
        print_error "Cannot proceed without secret scanning tools"
        exit 1
    fi
    echo ""

    # Scan each extracted repository
    local clean_count=0
    local secrets_found_count=0
    local repos_with_secrets=()

    # Read from global EXTRACTED_DIRS array
    for repo_dir in "${EXTRACTED_DIRS[@]}"; do
        if [ ! -d "$repo_dir" ]; then
            print_error "Directory not found: $repo_dir (skipping)"
            continue
        fi

        local project_name=$(basename "$repo_dir")

        if scan_repository "$repo_dir" "$project_name"; then
            ((clean_count++))
        else
            ((secrets_found_count++))
            repos_with_secrets+=("$repo_dir")
        fi
        echo ""
    done

    # Summary
    echo "================================"
    print_success "Phase 2 complete!"
    echo "  Clean repositories: $clean_count"

    if [ $secrets_found_count -gt 0 ]; then
        print_error "Repositories with potential secrets: $secrets_found_count"
        echo ""
        print_info "Repositories requiring attention:"
        for repo in "${repos_with_secrets[@]}"; do
            echo "  - $repo"
            echo "    Review: $repo/scan-results/"
        done
        echo ""

        # Offer to auto-generate secrets.txt for all repos with secrets
        local gen_reply=""
        read -p "Would you like to auto-generate secrets.txt files from scan results? (y/N) " gen_reply
        echo ""
        if [[ $gen_reply =~ ^[Yy]$ ]]; then
            echo ""
            for repo in "${repos_with_secrets[@]}"; do
                generate_secrets_file "$repo" "$(basename "$repo")"
            done
        else
            print_info "Skipping auto-generation. Create secrets.txt files manually with strings to remove"
        fi
        echo ""

        print_info "Next steps:"
        echo "  1. Review scan results in each repository's scan-results/ directory"
        echo "  2. Review/edit the secrets.txt files (auto-generated or manual)"
        echo "  3. Run Phase 3 to clean the repositories with BFG"
    else
        print_success "No secrets detected in any repository!"
        echo ""
        print_info "You can proceed directly to publishing these repositories."
    fi
}

# Phase 3: Clean repositories
phase3_clean() {
    print_phase "PHASE 3: Clean Repository History"
    echo ""

    print_warning "This phase will permanently modify git history!"
    print_info "Actions performed:"
    echo "  1. Run BFG Repo-Cleaner (if secrets.txt exists in each repo)"
    echo "  2. Expire reflog (git reflog expire --expire=now --all)"
    echo "  3. Aggressive garbage collection (git gc --prune=now --aggressive)"
    echo "  4. Re-scan to verify secrets are removed"
    echo ""
    print_info "Before proceeding, ensure you have:"
    echo "  - Reviewed all scan results from Phase 2"
    echo "  - Created secrets.txt files in repos that need cleaning"
    echo "  - Backed up any repositories you're uncertain about"
    echo ""

    read -p "Proceed with cleaning? (y/N/s to skip) " -r
    echo ""

    if [[ $REPLY =~ ^[Ss]$ ]]; then
        print_info "Skipping Phase 3"
        return 0
    fi

    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        print_info "Aborted by user"
        exit 0
    fi
    echo ""

    # Check if BFG is available
    if ! check_bfg; then
        print_error "Cannot proceed without BFG Repo-Cleaner"
        exit 1
    fi
    echo ""

    # Source BFG location if set
    if [ -f /tmp/bfg_location.sh ]; then
        source /tmp/bfg_location.sh
    fi

    # Clean each repository
    local success_count=0
    local fail_count=0
    local verified_count=0
    local verification_failed=()

    # Read from global EXTRACTED_DIRS array
    for repo_dir in "${EXTRACTED_DIRS[@]}"; do
        if [ ! -d "$repo_dir" ]; then
            print_error "Directory not found: $repo_dir (skipping)"
            ((fail_count++))
            continue
        fi

        local project_name=$(basename "$repo_dir")

        if clean_repository "$repo_dir" "$project_name"; then
            ((success_count++))
            echo ""

            # Verify cleaning
            if verify_clean "$repo_dir" "$project_name"; then
                ((verified_count++))
            else
                verification_failed+=("$repo_dir")
            fi
        else
            ((fail_count++))
        fi
        echo ""
    done

    # Summary
    echo "================================"
    print_success "Phase 3 complete!"
    echo "  Successfully cleaned: $success_count"
    echo "  Verification passed: $verified_count"

    if [ $fail_count -gt 0 ]; then
        print_error "Failed to clean: $fail_count"
    fi

    if [ ${#verification_failed[@]} -gt 0 ]; then
        print_warning "Verification failed for ${#verification_failed[@]} repositories:"
        for repo in "${verification_failed[@]}"; do
            echo "  - $repo"
            echo "    Check: $repo/scan-results-post-clean/"
        done
        echo ""
        print_info "These repositories may still contain secrets!"
        print_info "Review the verification results and re-run cleaning if needed."
    else
        print_success "All repositories verified clean!"
        echo ""
        print_info "Next steps:"
        echo "  1. Manually review a few commits in each repository"
        echo "  2. Create remote repositories on GitHub/GitLab"
        echo "  3. Push the cleaned repositories"
        echo "  4. Convert your monorepo to use submodules (Phase 4)"
    fi
}

# Main function
main() {
    # Check arguments
    if [ $# -ne 1 ]; then
        echo "Usage: $0 <path-to-monorepo-or-git-url>"
        echo ""
        echo "This script will:"
        echo "  Phase 1: Extract each directory into its own repository"
        echo "  Phase 2: Scan each repository for secrets"
        echo "  Phase 3: Clean repositories with BFG and verify"
        echo ""
        echo "Examples:"
        echo "  $0 /path/to/my-monorepo"
        echo "  $0 git@github.com:user/monorepo.git"
        echo "  $0 https://github.com/user/monorepo.git"
        exit 1
    fi

    local repo_input="$1"
    local repo_path=""
    local cloned_repo=false

    # Check all tools upfront before any cloning
    print_phase "Tool Check"
    echo ""

    check_tools
    echo ""

    check_secret_scanners || print_warning "Secret scanners missing - Phase 2 will be limited"
    echo ""

    check_bfg || print_warning "BFG not found - Phase 3 will require it"
    echo ""

    # Check if input is a URL (git@... or https://...)
    if [[ "$repo_input" =~ ^git@.*\.git$ ]] || [[ "$repo_input" =~ ^https?://.*\.git$ ]] || [[ "$repo_input" =~ ^https?://.*$ ]]; then
        print_info "Detected git URL: $repo_input"
        echo ""

        # Extract repo name from URL for directory name
        local repo_name=$(basename "$repo_input" .git)
        local clone_dir="./cloned-${repo_name}"

        if [ -d "$clone_dir" ]; then
            print_error "Clone directory already exists: $clone_dir"
            print_info "Please remove it or use a different location"
            exit 1
        fi

        print_info "Will clone to: $clone_dir"
        read -p "Proceed with cloning? (y/N) " -r
        echo ""

        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
            print_info "Aborted by user"
            exit 0
        fi

        print_info "Cloning repository..."
        if ! git clone "$repo_input" "$clone_dir"; then
            print_error "Failed to clone repository"
            exit 1
        fi

        print_success "Repository cloned successfully"
        repo_path="$clone_dir"
        cloned_repo=true
        echo ""
    else
        # Assume it's a local path
        repo_path="$repo_input"
    fi

    # Validate repository path
    if [ ! -d "$repo_path" ]; then
        print_error "Repository path does not exist: $repo_path"
        exit 1
    fi

    if [ ! -d "$repo_path/.git" ]; then
        print_error "Path is not a git repository: $repo_path"
        exit 1
    fi

    # Get absolute path
    repo_path=$(cd "$repo_path" && pwd)

    print_info "Repository: $repo_path"
    echo ""

    # Initialize global array for extracted directories
    EXTRACTED_DIRS=()

    # Phase 1: Extract
    phase1_extract "$repo_path"

    if [ ${#EXTRACTED_DIRS[@]} -eq 0 ]; then
        print_error "No repositories were extracted"
        exit 1
    fi

    # Phase 2: Scan
    phase2_scan

    # Phase 3: Clean
    phase3_clean

    # Final message
    echo ""
    print_success "Script complete! Review the output above for next steps."
}

# Run main function
main "$@"
No results found