generate_progress_report.py

import urllib.request
from datetime import datetime

import wandb
import yaml
from yaml.loader import SafeLoader

atari_ids = [
    # "AdventureNoFrameskip-v4",
    # "AirRaidNoFrameskip-v4",
    # "AlienNoFrameskip-v4",
    # "AmidarNoFrameskip-v4",
    # "AssaultNoFrameskip-v4",
    # "AsterixNoFrameskip-v4",
    # "AsteroidsNoFrameskip-v4",
    # "AtlantisNoFrameskip-v4",
    # "BankHeistNoFrameskip-v4",
    # "BattleZoneNoFrameskip-v4",
    "BeamRiderNoFrameskip-v4",
    # "BerzerkNoFrameskip-v4",
    # "BowlingNoFrameskip-v4",
    # "BoxingNoFrameskip-v4",
    "BreakoutNoFrameskip-v4",
    # "CarnivalNoFrameskip-v4",
    # "CentipedeNoFrameskip-v4",
    # "ChopperCommandNoFrameskip-v4",
    # "CrazyClimberNoFrameskip-v4",
    # "DefenderNoFrameskip-v4",
    # "DemonAttackNoFrameskip-v4",
    # "DoubleDunkNoFrameskip-v4",
    # "ElevatorActionNoFrameskip-v4",
    "EnduroNoFrameskip-v4",
    # "FishingDerbyNoFrameskip-v4",
    # "FreewayNoFrameskip-v4",
    # "FrostbiteNoFrameskip-v4",
    # "GopherNoFrameskip-v4",
    # "GravitarNoFrameskip-v4",
    # "HeroNoFrameskip-v4",
    # "IceHockeyNoFrameskip-v4",
    # "JamesbondNoFrameskip-v4",
    # "JourneyEscapeNoFrameskip-v4",
    # "KangarooNoFrameskip-v4",
    # "KrullNoFrameskip-v4",
    # "KungFuMasterNoFrameskip-v4",
    # "MontezumaRevengeNoFrameskip-v4",
    # "MsPacmanNoFrameskip-v4",
    # "NameThisGameNoFrameskip-v4",
    # "PhoenixNoFrameskip-v4",
    # "PitfallNoFrameskip-v4",
    "PongNoFrameskip-v4",
    # "PooyanNoFrameskip-v4",
    # "PrivateEyeNoFrameskip-v4",
    "QbertNoFrameskip-v4",
    # "RiverraidNoFrameskip-v4",
    # "RoadRunnerNoFrameskip-v4",
    # "RobotankNoFrameskip-v4",
    "SeaquestNoFrameskip-v4",
    # "SkiingNoFrameskip-v4",
    # "SolarisNoFrameskip-v4",
    "SpaceInvadersNoFrameskip-v4",
    # "StarGunnerNoFrameskip-v4",
    # "TennisNoFrameskip-v4",
    # "TimePilotNoFrameskip-v4",
    # "TutankhamNoFrameskip-v4",
    # "UpNDownNoFrameskip-v4",
    # "VentureNoFrameskip-v4",
    # "VideoPinballNoFrameskip-v4",
    # "WizardOfWorNoFrameskip-v4",
    # "YarsRevengeNoFrameskip-v4",
    # "ZaxxonNoFrameskip-v4",
]


api = wandb.Api()
runs = api.runs("openrlbenchmark/sb3")

runs = [{"status": run._state, "env": run.config["env"], "algo": run.config["algo"]} for run in runs]

algo_ids = sorted(["a2c", "ddpg", "ppo_lstm", "qrdqn", "td3", "trpo", "ars", "dqn", "ppo", "sac", "tqc"])

n_runs = {}

for algo_id in algo_ids:
    url = f"https://raw.githubusercontent.com/DLR-RM/rl-baselines3-zoo/master/hyperparams/{algo_id}.yml"
    with urllib.request.urlopen(url) as f:
        data = yaml.load(f, Loader=SafeLoader)

    env_ids = sorted(list(data.keys()))
    if "atari" in env_ids:
        env_ids.remove("atari")
        env_ids.extend(atari_ids)

    for env_id in ["donkey-generated-track-v0", "A1Walking-v0", "A1Jumping-v0", "NeckEnvRelative-v2", "RocketLander-v0", "PandaStack-v1"]:
        if env_id in env_ids:
            env_ids.remove(env_id)
    
    if algo_id in ["tqc", "sac"]:
        env_ids.remove("CarRacing-v0") # Param in rl-zoo require external code. Skip them for now.

    n_runs[algo_id] = {env_id: 0 for env_id in env_ids}

    for env_id in env_ids:
        for i in range(len(runs)):
            if runs[i]["env"] == env_id and runs[i]["algo"] == algo_id and runs[i]["status"] in {"running", "finished"}:
                n_runs[algo_id][env_id] += 1

tot_runs = 0
tot_desired_runs = 0

text = ""
for algo_id in n_runs.keys():
    text += f"""### {algo_id.upper()}

Use `--algo {algo_id}`.

"""
    text += "| Environment | Runs |\n"
    text += "|:----------- |:---- |\n"
    for env_id in n_runs[algo_id].keys():
        n_run = min(n_runs[algo_id][env_id], 10)
        tot_runs += n_run
        tot_desired_runs += 10
        black_squares = n_run * "⬛"
        white_squares = (10 - n_run) * "⬜"
        text += f"| {env_id} | {black_squares}{white_squares} {str(n_runs[algo_id][env_id]).rjust(2, ' ')}/10 |\n"
    text += "\n"

percentage = round(tot_runs / tot_desired_runs * 100, 1)
nb_black_squares = int(percentage / 5)

text = f"""This report was last updated on {datetime.now().strftime("%c")}. To generate it, use [this python script](https://gist.github.com/qgallouedec/34571aa5ecdef2cc23ce7c609bcea4bc).

Total benchmark progress:
{nb_black_squares * "⬛"}{(20 - nb_black_squares) * "⬜"} {tot_runs}/{tot_desired_runs} ({percentage}%)

Link to openrlbenchmark: https://wandb.ai/openrlbenchmark/sb3

### To contribute

Install [RL Baselines3 Zoo](https://github.com/DLR-RM/rl-baselines3-zoo)

```shell
pip install rl_zoo3 --upgrade
pip install wandb tensorboard
```

and run the following command

```shell
python -m rl_zoo3.train --algo <ALGO> --env <ENV> --eval-episodes 20 --n-eval-envs 5 --track --wandb-entity openrlbenchmark --wandb-project-name sb3
```

{text}
"""
with open("progress.md", "w") as file:
    file.write(text)

Thanks!