benchmark_map_apis.sh

benchmark_map_apis.sh

#!/bin/bash

# Map API Benchmark Script
# Usage: MAPTILER_KEY=your_key ./benchmark_map_apis.sh

set -euo pipefail

# Check if MAPTILER_KEY is set
if [[ -z "${MAPTILER_KEY:-}" ]]; then
    echo "Error: MAPTILER_KEY environment variable is not set"
    echo "Usage: MAPTILER_KEY=your_key $0"
    exit 1
fi

# API URLs
MAPTILER_URL="https://api.maptiler.com/tiles/v3"
OPENFREEMAP_URL="https://tiles.openfreemap.org/planet/20250910_001001_pt"

# Statistics arrays
maptiler_ttfb=()
maptiler_total=()
openfreemap_ttfb=()
openfreemap_total=()

# Function to generate random coordinates for a given zoom level
generate_coords() {
    local zoom=$1
    local max_coord=$((2**zoom - 1))
    local x=$((RANDOM % (max_coord + 1)))
    local y=$((RANDOM % (max_coord + 1)))
    echo "$x $y"
}

# Function to benchmark a single API call
benchmark_api() {
    local url=$1
    local api_name=$2
    
    # Use curl with timing information
    local curl_output
    curl_output=$(curl -w "%{time_namelookup},%{time_connect},%{time_appconnect},%{time_pretransfer},%{time_starttransfer},%{time_total},%{http_code},%{size_download}" \
                      -s -o /dev/null \
                      --max-time 30 \
                      --connect-timeout 10 \
                      --user-agent "Map-API-Benchmark/1.0" \
                      "$url" 2>/dev/null || echo "0,0,0,0,0,0,000,0")
    
    # Check if we got valid output
    if [[ -z "$curl_output" ]]; then
        printf "%s: FAILED - No response from server\n" "$api_name" >&2
        echo "0 0"
        return
    fi
    
    IFS=',' read -r time_namelookup time_connect time_appconnect time_pretransfer time_starttransfer time_total http_code size_download <<< "$curl_output"
    
    # Check if we got valid timing data
    if [[ -z "$time_starttransfer" || -z "$time_total" ]]; then
        printf "%s: FAILED - Invalid timing data\n" "$api_name" >&2
        echo "0 0"
        return
    fi
    
    # Convert to milliseconds (handle potential empty values)
    local ttfb_ms=0
    local total_ms=0
    if [[ "$time_starttransfer" != "0" && -n "$time_starttransfer" ]]; then
        ttfb_ms=$(awk "BEGIN {printf \"%.0f\", $time_starttransfer * 1000}")
    fi
    if [[ "$time_total" != "0" && -n "$time_total" ]]; then
        total_ms=$(awk "BEGIN {printf \"%.0f\", $time_total * 1000}")
    fi
    
    # Print results to stderr so it doesn't interfere with return value
    printf "%s: TTFB=%sms, Total=%sms, HTTP=%s, Size=%s bytes\n" \
           "$api_name" "$ttfb_ms" "$total_ms" "$http_code" "$size_download" >&2
    
    # Return values for statistics (only if successful)
    if [[ "$http_code" =~ ^2[0-9][0-9]$ && "$ttfb_ms" -gt 0 ]]; then
        echo "$ttfb_ms $total_ms"
    else
        echo "0 0"
    fi
}

# Function to calculate statistics
calculate_stats() {
    local arr_name=$1
    local name=$2
    
    # Get array size using eval (safer approach)
    local arr_size
    eval "arr_size=\${#${arr_name}[@]}"
    
    if [[ $arr_size -eq 0 ]]; then
        echo "$name: No successful requests"
        return
    fi
    
    # Build array values manually to avoid eval with array expansion
    local values=()
    local i
    for ((i=0; i<arr_size; i++)); do
        local val
        eval "val=\${${arr_name}[$i]}"
        values+=("$val")
    done
    
    local sum=0
    local min=${values[0]}
    local max=${values[0]}
    
    for val in "${values[@]}"; do
        sum=$((sum + val))
        if [[ $val -lt $min ]]; then min=$val; fi
        if [[ $val -gt $max ]]; then max=$val; fi
    done
    
    local avg=$((sum / ${#values[@]}))
    
    # Calculate median
    local sorted=($(printf '%s\n' "${values[@]}" | sort -n))
    local len=${#sorted[@]}
    local median
    if [[ $((len % 2)) -eq 0 ]]; then
        median=$(((sorted[len/2-1] + sorted[len/2]) / 2))
    else
        median=${sorted[len/2]}
    fi
    
    printf "%s: Min=%sms, Max=%sms, Avg=%sms, Median=%sms, Count=%s\n" \
           "$name" "$min" "$max" "$avg" "$median" "${#values[@]}"
}

echo "=== Map API Benchmark ==="
echo "Testing with 20 random coordinate pairs"
echo "APIs:"
echo "1. MapTiler: $MAPTILER_URL"
echo "2. OpenFreeMap: $OPENFREEMAP_URL"
echo ""

# Generate 20 random test cases
for i in {1..20}; do
    # Generate random zoom level (reasonable range for tile APIs)
    zoom=$((RANDOM % 15 + 1))  # zoom levels 1-15
    
    # Generate random coordinates for this zoom level
    coords=($(generate_coords $zoom))
    x=${coords[0]}
    y=${coords[1]}
    
    echo "=== Test $i/20: z=$zoom, x=$x, y=$y ==="
    
    # Test MapTiler API
    maptiler_url="$MAPTILER_URL/$zoom/$x/$y.pbf?key=$MAPTILER_KEY"
    result_output=$(benchmark_api "$maptiler_url" "MapTiler")
    result=($result_output)
    if [[ ${#result[@]} -eq 2 && ${result[0]} -ne 0 ]]; then
        maptiler_ttfb+=(${result[0]})
        maptiler_total+=(${result[1]})
    fi
    
    # Test OpenFreeMap API
    openfreemap_url="$OPENFREEMAP_URL/$zoom/$x/$y.pbf"
    result_output=$(benchmark_api "$openfreemap_url" "OpenFreeMap")
    result=($result_output)
    if [[ ${#result[@]} -eq 2 && ${result[0]} -ne 0 ]]; then
        openfreemap_ttfb+=(${result[0]})
        openfreemap_total+=(${result[1]})
    fi
    
    echo ""
    
    # Small delay between tests to be respectful to the APIs
    sleep 0.5
done

echo "=== FINAL STATISTICS ==="
echo ""
echo "MapTiler API Results:"
calculate_stats maptiler_ttfb "  TTFB"
calculate_stats maptiler_total "  Total Time"
echo ""
echo "OpenFreeMap API Results:"
calculate_stats openfreemap_ttfb "  TTFB"
calculate_stats openfreemap_total "  Total Time"
echo ""

# Success rate
maptiler_success_rate=$((${#maptiler_ttfb[@]} * 100 / 20))
openfreemap_success_rate=$((${#openfreemap_ttfb[@]} * 100 / 20))

echo "Success Rates:"
echo "  MapTiler: ${#maptiler_ttfb[@]}/20 (${maptiler_success_rate}%)"
echo "  OpenFreeMap: ${#openfreemap_ttfb[@]}/20 (${openfreemap_success_rate}%)"
echo ""
echo "=== Benchmark Complete ==="