Script to mirror a public gitea repo to a private instance, including releases

gitea_mirror.sh

#!/bin/bash

set -euo pipefail

usage() {
    echo "Usage: $0 [--dry-run] [--from-release=TAG] <source_repo_url> <destination_repo_url> <destination_token> <temp_folder>"
    echo ""
    echo "Options:"
    echo "  --dry-run               Show what would be done without making changes"
    echo "  --from-release=TAG      Sync releases starting from this tag (inclusive). Without this, only sync latest release."
    echo ""
    echo "Parameters:"
    echo "  source_repo_url         Source Gitea repository URL (e.g., https://gitea.com/user/repo)"
    echo "  destination_repo_url    Destination Gitea repository URL (e.g., https://private-gitea.com/user/repo)"
    echo "  destination_token       Access token for destination repository"
    echo "  temp_folder            Temporary folder for downloading release data"
    echo ""
    echo "Examples:"
    echo "  $0 https://gitea.com/user/repo https://private-gitea.com/user/repo token123 /tmp/mirror"
    echo "  $0 --from-release=v1.0.0 https://gitea.com/user/repo https://private-gitea.com/user/repo token123 /tmp/mirror"
    echo "  $0 --dry-run --from-release=v0.9.0 https://gitea.com/user/repo https://private-gitea.com/user/repo token123 /tmp/mirror"
    exit 1
}

DRY_RUN=false
FROM_RELEASE=""

# Parse options
while [[ $# -gt 0 ]]; do
    case $1 in
        --dry-run)
            DRY_RUN=true
            shift
            ;;
        --from-release=*)
            FROM_RELEASE="${1#*=}"
            shift
            ;;
        --from-release)
            FROM_RELEASE="$2"
            shift 2
            ;;
        -*)
            echo "Unknown option $1"
            usage
            ;;
        *)
            break
            ;;
    esac
done

if [ $# -ne 4 ]; then
    usage
fi

SOURCE_REPO_URL="$1"
DEST_REPO_URL="$2"
DEST_TOKEN="$3"
TEMP_FOLDER="$4"

extract_repo_info() {
    local url="$1"
    local base_url=$(echo "$url" | sed -E 's|^(https?://[^/]+).*|\1|')
    local repo_path=$(echo "$url" | sed -E 's|^https?://[^/]+/(.+)/?$|\1|')
    local api_url="${base_url}/api/v1/repos/${repo_path}"
    echo "$api_url"
}

SOURCE_API_URL=$(extract_repo_info "$SOURCE_REPO_URL")
DEST_API_URL=$(extract_repo_info "$DEST_REPO_URL")

echo "Source API URL: $SOURCE_API_URL"
echo "Destination API URL: $DEST_API_URL"
echo "Temp folder: $TEMP_FOLDER"

mkdir -p "$TEMP_FOLDER"

get_latest_release() {
    local api_url="$1"
    local token="${2:-}"

    local response
    if [ -n "$token" ]; then
        response=$(curl -s -H "Authorization: token $token" "${api_url}/releases")
    else
        response=$(curl -s "${api_url}/releases")
    fi

    local latest_release=$(echo "$response" | jq '.[0]')
    echo "$latest_release"
}

check_release_exists() {
    local api_url="$1"
    local token="$2"
    local tag_name="$3"

    local response=$(curl -s -H "Authorization: token $token" "${api_url}/releases/tags/${tag_name}" 2>/dev/null)
    local http_code=$(curl -s -w "%{http_code}" -o /dev/null -H "Authorization: token $token" "${api_url}/releases/tags/${tag_name}" 2>/dev/null)

    if [ "$http_code" = "404" ] || [ "$http_code" = "000" ]; then
        return 1
    else
        return 0
    fi
}

create_release() {
    local api_url="$1"
    local token="$2"
    local release_data="$3"

    local response=$(curl -s -X POST \
        -H "Authorization: token $token" \
        -H "Content-Type: application/json" \
        -d "$release_data" \
        "${api_url}/releases")

    echo "$response"
}

download_asset() {
    local download_url="$1"
    local output_path="$2"

    curl -L -o "$output_path" "$download_url"
}

upload_asset() {
    local api_url="$1"
    local token="$2"
    local release_id="$3"
    local file_path="$4"
    local filename="$5"

    local response=$(curl -s -X POST \
        -H "Authorization: token $token" \
        -F "attachment=@${file_path};filename=${filename}" \
        "${api_url}/releases/${release_id}/assets")

    echo "$response"
}

get_remote_branches() {
    local api_url="$1"
    local token="${2:-}"

    local auth_header=""
    if [ -n "$token" ]; then
        auth_header="-H \"Authorization: token $token\""
    fi

    local response
    if [ -n "$token" ]; then
        response=$(curl -s -H "Authorization: token $token" "${api_url}/branches")
    else
        response=$(curl -s "${api_url}/branches")
    fi

    echo "$response" | jq -r '.[].name' 2>/dev/null || echo ""
}

get_remote_tags() {
    local api_url="$1"
    local token="${2:-}"

    local response
    if [ -n "$token" ]; then
        response=$(curl -s -H "Authorization: token $token" "${api_url}/tags")
    else
        response=$(curl -s "${api_url}/tags")
    fi

    echo "$response" | jq -r '.[].name' 2>/dev/null || echo ""
}

get_all_releases() {
    local api_url="$1"
    local token="${2:-}"

    local response
    if [ -n "$token" ]; then
        response=$(curl -s -H "Authorization: token $token" "${api_url}/releases")
    else
        response=$(curl -s "${api_url}/releases")
    fi

    echo "$response"
}

clone_and_mirror_repo() {
    local source_url="$1"
    local dest_url="$2"
    local dest_token="$3"
    local temp_folder="$4"
    local dry_run="$5"

    local clone_dir="${temp_folder}/source_clone"
    local source_api_url="$(extract_repo_info "$source_url")"
    local dest_api_url="$(extract_repo_info "$dest_url")"

    if [ "$dry_run" = "true" ]; then
        echo "[DRY RUN] Would clone source repository: $source_url"

        echo "[DRY RUN] Comparing branches and tags..."
        local source_branches=$(get_remote_branches "$source_api_url")
        local dest_branches=$(get_remote_branches "$dest_api_url" "$dest_token")
        local source_tags=$(get_remote_tags "$source_api_url")
        local dest_tags=$(get_remote_tags "$dest_api_url" "$dest_token")

        echo "[DRY RUN] Source branches: $(echo "$source_branches" | wc -l)"
        echo "[DRY RUN] Destination branches: $(echo "$dest_branches" | wc -l)"
        echo "[DRY RUN] Source tags: $(echo "$source_tags" | wc -l)"
        echo "[DRY RUN] Destination tags: $(echo "$dest_tags" | wc -l)"

        local missing_branches=$(comm -23 <(echo "$source_branches" | sort) <(echo "$dest_branches" | sort))
        local missing_tags=$(comm -23 <(echo "$source_tags" | sort) <(echo "$dest_tags" | sort))

        if [ -n "$missing_branches" ]; then
            echo "[DRY RUN] Would push missing branches: $(echo "$missing_branches" | tr '\n' ' ')"
        else
            echo "[DRY RUN] All branches are up to date"
        fi

        if [ -n "$missing_tags" ]; then
            echo "[DRY RUN] Would push missing tags: $(echo "$missing_tags" | tr '\n' ' ')"
        else
            echo "[DRY RUN] All tags are up to date"
        fi

        echo "[DRY RUN] Fetching releases comparison..."
        if [ -n "${FROM_RELEASE:-}" ]; then
            echo "[DRY RUN] Filtering releases from: $FROM_RELEASE"
        else
            echo "[DRY RUN] Syncing only the latest release"
        fi

        local all_source_releases=$(get_all_releases "$source_api_url")
        local filtered_source_releases=$(filter_releases_from_tag "$all_source_releases" "${FROM_RELEASE:-}")
        local dest_releases=$(get_all_releases "$dest_api_url" "$dest_token")
        local source_release_tags=$(echo "$filtered_source_releases" | jq -r '.[].tag_name' 2>/dev/null || echo "")
        local dest_release_tags=$(echo "$dest_releases" | jq -r '.[].tag_name' 2>/dev/null || echo "")

        local missing_releases=$(comm -23 <(echo "$source_release_tags" | sort) <(echo "$dest_release_tags" | sort))

        if [ -n "$missing_releases" ]; then
            echo "[DRY RUN] Would create missing releases: $(echo "$missing_releases" | tr '\n' ' ')"
            echo "[DRY RUN] Total releases to sync: $(echo "$missing_releases" | wc -l)"
        else
            echo "[DRY RUN] All releases are up to date"
        fi

        echo "[DRY RUN] No changes made"
        return 0
    fi

    echo "Cloning source repository..."
    git clone "$source_url" "$clone_dir"

    cd "$clone_dir"

    local dest_base_url=$(echo "$dest_url" | sed -E 's|^(https?://[^/]+).*|\1|')
    local dest_repo_path=$(echo "$dest_url" | sed -E 's|^https?://[^/]+/(.+)/?$|\1|')
    local dest_git_url="${dest_base_url}/${dest_repo_path}.git"

    local dest_username=$(curl -s -H "Authorization: token $dest_token" "${dest_base_url}/api/v1/user" | jq -r '.login')
    local auth_dest_url=$(echo "$dest_git_url" | sed "s|://|://${dest_username}:${dest_token}@|")

    echo "Analyzing what needs to be synced..."

    # Check if destination remote already exists, if so remove it
    if git remote | grep -q "^destination$"; then
        git remote remove destination
    fi

    git remote add destination "$auth_dest_url"

    # Get remote branches and tags for comparison
    local source_branches=$(git branch -r | grep -v '/HEAD' | sed 's|origin/||' | sort)
    local dest_branches=$(get_remote_branches "$dest_api_url" "$dest_token" | sort)
    local source_tags=$(git tag | sort)
    local dest_tags=$(get_remote_tags "$dest_api_url" "$dest_token" | sort)

    # Find missing branches and tags
    local missing_branches=$(comm -23 <(echo "$source_branches") <(echo "$dest_branches"))
    local missing_tags=$(comm -23 <(echo "$source_tags") <(echo "$dest_tags"))

    # Push only missing branches
    if [ -n "$missing_branches" ]; then
        echo "Pushing missing branches..."
        echo "$missing_branches" | while read branch; do
            if [ -n "$branch" ]; then
                echo "Pushing branch: $branch"
                git push destination "$branch" || echo "Warning: Could not push branch $branch"
            fi
        done
    else
        echo "All branches are up to date"
    fi

    # Push only missing tags
    if [ -n "$missing_tags" ]; then
        echo "Pushing missing tags..."
        echo "$missing_tags" | while read tag; do
            if [ -n "$tag" ]; then
                echo "Pushing tag: $tag"
                git push destination "$tag" || echo "Warning: Could not push tag $tag"
            fi
        done
    else
        echo "All tags are up to date"
    fi

    cd - > /dev/null
}

filter_releases_from_tag() {
    local releases="$1"
    local from_tag="$2"

    if [ -z "$from_tag" ]; then
        # No filter, return only latest release (first in array)
        echo "$releases" | jq '.[0:1]'
        return 0
    fi

    # Find the position of the from_tag in the releases array
    local from_index=$(echo "$releases" | jq --arg tag "$from_tag" 'map(.tag_name) | index($tag)')

    if [ "$from_index" = "null" ]; then
        echo "Warning: Release tag '$from_tag' not found in source repository" >&2
        echo "[]"
        return 0
    fi

    # Return releases from beginning up to and including the specified tag
    # (since array is sorted newest to oldest, we want [0:index+1])
    local end_index=$((from_index + 1))
    echo "$releases" | jq --argjson end "$end_index" '.[0:$end]'
}

sync_missing_releases() {
    local source_api_url="$1"
    local dest_api_url="$2"
    local dest_token="$3"
    local temp_folder="$4"
    local dry_run="$5"
    local from_release="$6"

    echo "Synchronizing releases..."

    if [ -n "$from_release" ]; then
        echo "Filtering releases from: $from_release"
    else
        echo "Syncing only the latest release"
    fi

    local all_source_releases=$(get_all_releases "$source_api_url")
    local filtered_source_releases=$(filter_releases_from_tag "$all_source_releases" "$from_release")
    local dest_releases=$(get_all_releases "$dest_api_url" "$dest_token")

    local source_release_tags=$(echo "$filtered_source_releases" | jq -r '.[].tag_name' 2>/dev/null || echo "")
    local dest_release_tags=$(echo "$dest_releases" | jq -r '.[].tag_name' 2>/dev/null || echo "")

    local missing_releases=$(comm -23 <(echo "$source_release_tags" | sort) <(echo "$dest_release_tags" | sort))

    if [ -z "$missing_releases" ]; then
        echo "All releases are up to date"
        return 0
    fi

    if [ "$dry_run" = "true" ]; then
        echo "[DRY RUN] Would create missing releases: $(echo "$missing_releases" | tr '\n' ' ')"
        echo "[DRY RUN] Total releases to sync: $(echo "$missing_releases" | wc -l)"
        return 0
    fi

    echo "Found $(echo "$missing_releases" | wc -l) missing releases to sync"

    # Process each missing release in the order they appear in filtered releases
    echo "$filtered_source_releases" | jq -c '.[]' | while read -r release_data; do
        local tag_name=$(echo "$release_data" | jq -r '.tag_name')

        # Skip if this release already exists in destination
        if echo "$dest_release_tags" | grep -q "^${tag_name}$"; then
            echo "Release $tag_name already exists, skipping"
            continue
        fi

        echo "Syncing release: $tag_name"

        local name=$(echo "$release_data" | jq -r '.name')
        local body=$(echo "$release_data" | jq -r '.body')
        local draft=$(echo "$release_data" | jq -r '.draft')
        local prerelease=$(echo "$release_data" | jq -r '.prerelease')

        echo "Creating release: $name ($tag_name)"
        local create_release_data=$(jq -n \
            --arg tag_name "$tag_name" \
            --arg name "$name" \
            --arg body "$body" \
            --argjson draft "$draft" \
            --argjson prerelease "$prerelease" \
            '{
                tag_name: $tag_name,
                name: $name,
                body: $body,
                draft: $draft,
                prerelease: $prerelease
            }')

        local new_release=$(create_release "$dest_api_url" "$dest_token" "$create_release_data")
        local new_release_id=$(echo "$new_release" | jq -r '.id')

        if [ "$new_release_id" = "null" ]; then
            echo "Error creating release $tag_name:"
            echo "$new_release" | jq '.'
            continue
        fi

        echo "Release created with ID: $new_release_id"

        # Sync assets
        local assets=$(echo "$release_data" | jq -c '.assets[]?' 2>/dev/null)
        if [ -n "$assets" ]; then
            echo "Downloading and uploading assets for $tag_name..."
            echo "$release_data" | jq -c '.assets[]?' | while read -r asset; do
                local asset_name=$(echo "$asset" | jq -r '.name')
                local download_url=$(echo "$asset" | jq -r '.browser_download_url')

                echo "Processing asset: $asset_name"

                local asset_path="${temp_folder}/${asset_name}"
                download_asset "$download_url" "$asset_path"

                echo "Uploading asset: $asset_name"
                local upload_result=$(upload_asset "$dest_api_url" "$dest_token" "$new_release_id" "$asset_path" "$asset_name")

                if echo "$upload_result" | jq -e '.id' >/dev/null 2>&1; then
                    echo "Successfully uploaded: $asset_name"
                else
                    echo "Error uploading asset $asset_name:"
                    echo "$upload_result" | jq '.'
                fi

                rm -f "$asset_path"
            done
        else
            echo "No assets found for release $tag_name"
        fi
    done
}

main() {
    echo "Mirroring source code repository..."
    clone_and_mirror_repo "$SOURCE_REPO_URL" "$DEST_REPO_URL" "$DEST_TOKEN" "$TEMP_FOLDER" "$DRY_RUN"

    if [ "$DRY_RUN" = "true" ]; then
        exit 0
    fi

    echo "Mirroring releases..."
    sync_missing_releases "$SOURCE_API_URL" "$DEST_API_URL" "$DEST_TOKEN" "$TEMP_FOLDER" "$DRY_RUN" "$FROM_RELEASE"

    echo "Repository mirroring completed successfully!"
}

cleanup() {
    if [ -d "$TEMP_FOLDER" ]; then
        echo "Cleaning up temporary files..."
        rm -rf "$TEMP_FOLDER"
    fi
}

trap cleanup EXIT

if ! command -v jq >/dev/null 2>&1; then
    echo "Error: jq is required but not installed"
    exit 1
fi

if ! command -v curl >/dev/null 2>&1; then
    echo "Error: curl is required but not installed"
    exit 1
fi

if ! command -v git >/dev/null 2>&1; then
    echo "Error: git is required but not installed"
    exit 1
fi

main

To use this, create a repository on your private instance of Gitea, generate a token that has access to upload assets, create releases, push changes and run:

./gitea-mirror.sh \
    https://gitea.com/gitea/act_runner \
    https://10.0.9.5/gabriel/act_runner \
    YOUR_TOKEN_GOES_HERE \
    /tmp/temporary_git_mirror_path

The above will mirror the code and the latest release. To mirror more releases starting from a particular version, you can add the --from-release flag:

./gitea-mirror.sh \
    --from-release=v0.2.10 \
    https://gitea.com/gitea/act_runner \
    https://10.0.9.5/gabriel/act_runner \
    YOUR_TOKEN_GOES_HERE \
    /tmp/temporary_git_mirror_path

This will mirror v0.2.10 and newer.