chessgamelogs.py

import requests
import re
import os
from concurrent.futures import ProcessPoolExecutor
import atexit

URL = "https://www.chessgames.com/perl/nph-chesspgn?text=1&gid="


def send_request(url=None, number=None):
    assert url is not None or number is not None, "Url or number cannot be none!"
    url = url + str(number)
    r = requests.get(url)
    if r.status_code == 200:
        return r.text


def save_file(folder_name, file, number):
    path = os.path.join('output', folder_name)
    if folder_name not in os.listdir('output'):
        os.mkdir(path)
    with open(os.path.join(path, f'{number}.pgn'), 'w') as f:
        f.write(file)


def main(number):
    global last_number
    last_number = number
    if 'output' not in os.listdir('.'):
        os.mkdir('output')
    try:
        r = send_request(URL, number)
    except requests.exceptions.ConnectionError:
        print(f"Couldn't reach the number:{number}")
        return
    if r is None:
        print(f'No game with this ID: {number}')
    eco = re.search("\[ECO \"([A-Z][0-9]{2})\"\]", r)
    folder_name = eco.group(1) if eco else 'UNKNOWN'
    save_file(folder_name, r, number)
    print(f'Saved: {number}')


@atexit.register
def on_close():
    with open('last_number.txt', 'w') as f:
        f.write(str(last_number))


if __name__ == "__main__":
    if 'last_number.txt' in os.listdir('.'):
        with open('last_number.txt', 'r') as f:
            start_range = f.readline
    else:
        start_range = 1_200_000
    for i in range(start_range, 1_300_000, 1_000):
        with ProcessPoolExecutor(max_workers=32) as executor:
            last_number = i
            executor.map(main, range(i, i + 1_000))