Script to count the git hosts used on ContentDB; tested w/ python 3.11

cdbgithosters.py

# Script to count the git hosts used on ContentDB; tested w/ python 3.11
import requests, json, re, time, operator

hosts = {}
pkgnum = 0

def val(key):
	if key in hosts.keys():
		return hosts[key]
	else:
		return 0
	
def regex_url(url): # https://regex101.com/r/cYN1MT/1
	split = re.split("https?://(www\.)?([a-zA-Z0-9]+)(\.[a-zA-Z0-9.-]+)", url)
	return split[2] + split[3]

request = requests.get("https://content.minetest.net/api/packages/")
packages = json.loads(request.content)

for package in packages:
	pkgurl = f"https://content.minetest.net/api/packages/{package['author']}/{package['name']}"
	pkgnum = pkgnum + 1
	print(f"Check package #{pkgnum}: {pkgurl}")

	req = requests.get(pkgurl)
	pkg = None
	try:
		pkg = json.loads(req.content)
	except:
		print(f"Error in decoding response for {pkgurl} (maybe too many requests)")
		#print(pkg.content) # debug, enable if needed
		time.sleep(5)
		continue

	if pkg["repo"] != None:
		host = regex_url(pkg["repo"])
		hosts[host] = val(host) + 1
	else:
		hosts["No repo"] = val("No repo") + 1
	
	time.sleep(1.2) # ratelimit

hosters = dict(sorted(hosts.items(), key = operator.itemgetter(1), reverse = True))

for host in hosters:
	print(f"{host} Packages: {hosters[host]}")