worker.py

#!/usr/bin/env python3
import os, time, subprocess, requests, socket

SERVER = os.environ["JOB_SERVER_URL"]
INSTANCE_ID = os.environ.get("INSTANCE_ID", socket.gethostname())
POLL = int(os.environ.get("JOB_POLL_SECONDS", "20"))

print(f"[worker] instance={INSTANCE_ID}")
print(f"[worker] server={SERVER}")

while True:
    r = requests.get(
        f"{SERVER}/next",
        params={"instance_id": INSTANCE_ID},
        timeout=15,
    )
    job = r.json()

    if job.get("status") == "empty":
        time.sleep(POLL)
        continue

    for i in job.get("inputs", []):
        subprocess.check_call([
            "curl", "-fL", i["url"], "-o", i["local"]
        ])

    rc = subprocess.call(job["cmd"], shell=True)

    for o in job.get("outputs", []):
        subprocess.check_call([
            "curl", "-f", "-X", "PUT",
            "--upload-file", o["local"], o["url"]
        ])

    time.sleep(POLL)