Script for fetching a list of the regions geofabrik generates

get-geofabrik-regions.ts

/**
 * All of these regions should have a `.poly` and a `-latest.osm.pbf` file
 * associated to them.
 */
import * as cheerio from "npm:cheerio"

const baseURL = "https://download.geofabrik.de/"
const regions = new Set()

// antarctica has no children, so not in baseRegions, but add to regions
const baseRegions = [
  "australia-oceania",
  "asia",
  "africa",
  "central-america",
  "europe",
  "north-america",
  "south-america",
]

baseRegions.concat(["antarctica"])
  .forEach(region => regions.add(region))

const responses = baseRegions.map(fetchRegion)

await Promise.all(responses)
const regionsArray = [...regions];
Deno.writeTextFile('regions.txt', regionsArray.sort().join("\n") + "\n")

async function fetchRegion(region: string) {
  console.log('fetching: ', region)
  const url = baseURL + "/" + region
  const resp = await fetch(url)
  const html = await resp.text()
  const { directories, polyFiles } = await parseGeofabrikDirectory(html)
  polyFiles
    .forEach(file => regions.add(region + "/" + file.replace(".poly", "")))

  const childrenResp = directories.map(
    (name) => {
      const childName = name.replace(/^\//, "").replace(/\/$/, "")
      if (!regions.has(childName)) {
        return fetchRegion(region + "/" + childName)
      }
    }
  )
  await Promise.all(childrenResp)
}

async function parseGeofabrikDirectory(html: string) {
  const $ = cheerio.load(html)
  const rows = $("tr")

  const directories: string[] = []
  const polyFiles: string[] = []

  rows.each((_, row) => {
    const href = $(row).find("td a").first().attr("href")

    if (!href) return

    if (href.endsWith("/") && (href !== "/") && !href.endsWith("-updates/")) {
      directories.push(href)
    }
    else if (href.endsWith(".poly")) {
      polyFiles.push(href.replace("/", ""))
    }
  })

  return {
    directories,
    polyFiles
  }
}

regions.txt

africa
africa/algeria
africa/angola
africa/benin
africa/botswana
africa/burkina-faso
africa/burundi
africa/cameroon
africa/canary-islands
africa/cape-verde
africa/central-african-republic
africa/chad
africa/comores
africa/congo-brazzaville
africa/congo-democratic-republic
africa/djibouti
africa/egypt
africa/equatorial-guinea
africa/eritrea
africa/ethiopia
africa/gabon
africa/ghana
africa/guinea
africa/guinea-bissau
africa/ivory-coast
africa/kenya
africa/lesotho
africa/liberia
africa/libya
africa/madagascar
africa/malawi
africa/mali
africa/mauritania
africa/mauritius
africa/morocco
africa/mozambique
africa/namibia
africa/niger
africa/nigeria
africa/rwanda
africa/saint-helena-ascension-and-tristan-da-cunha
africa/sao-tome-and-principe
africa/senegal-and-gambia
africa/seychelles
africa/sierra-leone
africa/somalia
africa/south-africa
africa/south-africa-and-lesotho
africa/south-sudan
africa/sudan
africa/swaziland
africa/tanzania
africa/togo
africa/tunisia
africa/uganda
africa/zambia
africa/zimbabwe
antarctica
asia
asia/afghanistan
asia/armenia
asia/azerbaijan
asia/bangladesh
asia/bhutan
asia/cambodia
asia/china
asia/china/anhui
asia/china/beijing
asia/china/chongqing
asia/china/fujian
asia/china/gansu
asia/china/guangdong
asia/china/guangxi
asia/china/guizhou
asia/china/hainan
asia/china/hebei
asia/china/heilongjiang
asia/china/henan
asia/china/hong-kong
asia/china/hubei
asia/china/hunan
asia/china/inner-mongolia
asia/china/jiangsu
asia/china/jiangxi
asia/china/jilin
asia/china/liaoning
asia/china/macau
asia/china/ningxia
asia/china/qinghai
asia/china/shaanxi
asia/china/shandong
asia/china/shanghai
asia/china/shanxi
asia/china/sichuan
asia/china/tianjin
asia/china/tibet
asia/china/xinjiang
asia/china/yunnan
asia/china/zhejiang
asia/east-timor
asia/gcc-states
asia/india
asia/india/central-zone
asia/india/eastern-zone
asia/india/north-eastern-zone
asia/india/northern-zone
asia/india/southern-zone
asia/india/western-zone
asia/indonesia
asia/indonesia/java
asia/indonesia/kalimantan
asia/indonesia/maluku
asia/indonesia/nusa-tenggara
asia/indonesia/papua
asia/indonesia/sulawesi
asia/indonesia/sumatra
asia/iran
asia/iraq
asia/israel-and-palestine
asia/japan
asia/japan/chubu
asia/japan/chugoku
asia/japan/hokkaido
asia/japan/kansai
asia/japan/kanto
asia/japan/kyushu
asia/japan/shikoku
asia/japan/tohoku
asia/jordan
asia/kazakhstan
asia/kyrgyzstan
asia/laos
asia/lebanon
asia/malaysia-singapore-brunei
asia/maldives
asia/mongolia
asia/myanmar
asia/nepal
asia/north-korea
asia/pakistan
asia/philippines
asia/russia-asian-part
asia/south-korea
asia/sri-lanka
asia/syria
asia/taiwan
asia/tajikistan
asia/thailand
asia/turkmenistan
asia/uzbekistan
asia/vietnam
asia/yemen
australia-oceania
australia-oceania/american-oceania
australia-oceania/australia
australia-oceania/cook-islands
australia-oceania/fiji
australia-oceania/ile-de-clipperton
australia-oceania/kiribati
australia-oceania/marshall-islands
australia-oceania/micronesia
australia-oceania/nauru
australia-oceania/new-caledonia
australia-oceania/new-zealand
australia-oceania/niue
australia-oceania/palau
australia-oceania/papua-new-guinea
australia-oceania/pitcairn-islands
australia-oceania/polynesie-francaise
australia-oceania/samoa
australia-oceania/solomon-islands
australia-oceania/tokelau
australia-oceania/tonga
australia-oceania/tuvalu
australia-oceania/vanuatu
australia-oceania/wallis-et-futuna
central-america
central-america/bahamas
central-america/belize
central-america/costa-rica
central-america/cuba
central-america/el-salvador
central-america/guatemala
central-america/haiti-and-domrep
central-america/honduras
central-america/jamaica
central-america/nicaragua
central-america/panama
europe
europe/albania
europe/alps
europe/andorra
europe/austria
europe/azores
europe/belarus
europe/belgium
europe/bosnia-herzegovina
europe/britain-and-ireland
europe/bulgaria
europe/croatia
europe/cyprus
europe/czech-republic
europe/dach
europe/denmark
europe/estonia
europe/faroe-islands
europe/finland
europe/france
europe/france/alsace
europe/france/aquitaine
europe/france/auvergne
europe/france/basse-normandie
europe/france/bourgogne
europe/france/bretagne
europe/france/centre
europe/france/champagne-ardenne
europe/france/corse
europe/france/franche-comte
europe/france/guadeloupe
europe/france/guyane
europe/france/haute-normandie
europe/france/ile-de-france
europe/france/languedoc-roussillon
europe/france/limousin
europe/france/lorraine
europe/france/martinique
europe/france/mayotte
europe/france/midi-pyrenees
europe/france/nord-pas-de-calais
europe/france/pays-de-la-loire
europe/france/picardie
europe/france/poitou-charentes
europe/france/provence-alpes-cote-d-azur
europe/france/reunion
europe/france/rhone-alpes
europe/georgia
europe/germany
europe/germany/baden-wuerttemberg
europe/germany/baden-wuerttemberg/freiburg-regbez
europe/germany/baden-wuerttemberg/karlsruhe-regbez
europe/germany/baden-wuerttemberg/stuttgart-regbez
europe/germany/baden-wuerttemberg/tuebingen-regbez
europe/germany/bayern
europe/germany/bayern/mittelfranken
europe/germany/bayern/niederbayern
europe/germany/bayern/oberbayern
europe/germany/bayern/oberfranken
europe/germany/bayern/oberpfalz
europe/germany/bayern/schwaben
europe/germany/bayern/unterfranken
europe/germany/berlin
europe/germany/brandenburg
europe/germany/bremen
europe/germany/hamburg
europe/germany/hessen
europe/germany/mecklenburg-vorpommern
europe/germany/niedersachsen
europe/germany/nordrhein-westfalen
europe/germany/nordrhein-westfalen/arnsberg-regbez
europe/germany/nordrhein-westfalen/detmold-regbez
europe/germany/nordrhein-westfalen/duesseldorf-regbez
europe/germany/nordrhein-westfalen/koeln-regbez
europe/germany/nordrhein-westfalen/muenster-regbez
europe/germany/rheinland-pfalz
europe/germany/saarland
europe/germany/sachsen
europe/germany/sachsen-anhalt
europe/germany/schleswig-holstein
europe/germany/thueringen
europe/great-britain
europe/great-britain/united-kingdom
europe/greece
europe/guernsey-jersey
europe/hungary
europe/iceland
europe/ireland-and-northern-ireland
europe/isle-of-man
europe/italy
europe/italy/centro
europe/italy/isole
europe/italy/nord-est
europe/italy/nord-ovest
europe/italy/sud
europe/kosovo
europe/latvia
europe/liechtenstein
europe/lithuania
europe/luxembourg
europe/macedonia
europe/malta
europe/moldova
europe/monaco
europe/montenegro
europe/netherlands
europe/netherlands/drenthe
europe/netherlands/flevoland
europe/netherlands/friesland
europe/netherlands/gelderland
europe/netherlands/groningen
europe/netherlands/limburg
europe/netherlands/noord-brabant
europe/netherlands/noord-holland
europe/netherlands/overijssel
europe/netherlands/utrecht
europe/netherlands/zeeland
europe/netherlands/zuid-holland
europe/norway
europe/poland
europe/poland/dolnoslaskie
europe/poland/kujawsko-pomorskie
europe/poland/lodzkie
europe/poland/lubelskie
europe/poland/lubuskie
europe/poland/malopolskie
europe/poland/mazowieckie
europe/poland/opolskie
europe/poland/podkarpackie
europe/poland/podlaskie
europe/poland/pomorskie
europe/poland/slaskie
europe/poland/swietokrzyskie
europe/poland/warminsko-mazurskie
europe/poland/wielkopolskie
europe/poland/zachodniopomorskie
europe/portugal
europe/romania
europe/serbia
europe/slovakia
europe/slovenia
europe/spain
europe/spain/andalucia
europe/spain/aragon
europe/spain/asturias
europe/spain/cantabria
europe/spain/castilla-la-mancha
europe/spain/castilla-y-leon
europe/spain/cataluna
europe/spain/ceuta
europe/spain/extremadura
europe/spain/galicia
europe/spain/islas-baleares
europe/spain/la-rioja
europe/spain/madrid
europe/spain/melilla
europe/spain/murcia
europe/spain/navarra
europe/spain/pais-vasco
europe/spain/valencia
europe/sweden
europe/switzerland
europe/turkey
europe/ukraine
europe/united-kingdom
europe/united-kingdom/england
europe/united-kingdom/england/bedfordshire
europe/united-kingdom/england/berkshire
europe/united-kingdom/england/bristol
europe/united-kingdom/england/buckinghamshire
europe/united-kingdom/england/cambridgeshire
europe/united-kingdom/england/cheshire
europe/united-kingdom/england/cornwall
europe/united-kingdom/england/cumbria
europe/united-kingdom/england/derbyshire
europe/united-kingdom/england/devon
europe/united-kingdom/england/dorset
europe/united-kingdom/england/durham
europe/united-kingdom/england/east-sussex
europe/united-kingdom/england/east-yorkshire-with-hull
europe/united-kingdom/england/essex
europe/united-kingdom/england/gloucestershire
europe/united-kingdom/england/greater-london
europe/united-kingdom/england/greater-manchester
europe/united-kingdom/england/hampshire
europe/united-kingdom/england/herefordshire
europe/united-kingdom/england/hertfordshire
europe/united-kingdom/england/isle-of-wight
europe/united-kingdom/england/kent
europe/united-kingdom/england/lancashire
europe/united-kingdom/england/leicestershire
europe/united-kingdom/england/lincolnshire
europe/united-kingdom/england/london/enfield
europe/united-kingdom/england/merseyside
europe/united-kingdom/england/norfolk
europe/united-kingdom/england/north-yorkshire
europe/united-kingdom/england/northamptonshire
europe/united-kingdom/england/northumberland
europe/united-kingdom/england/nottinghamshire
europe/united-kingdom/england/oxfordshire
europe/united-kingdom/england/rutland
europe/united-kingdom/england/shropshire
europe/united-kingdom/england/somerset
europe/united-kingdom/england/south-yorkshire
europe/united-kingdom/england/staffordshire
europe/united-kingdom/england/suffolk
europe/united-kingdom/england/surrey
europe/united-kingdom/england/tyne-and-wear
europe/united-kingdom/england/warwickshire
europe/united-kingdom/england/west-midlands
europe/united-kingdom/england/west-sussex
europe/united-kingdom/england/west-yorkshire
europe/united-kingdom/england/wiltshire
europe/united-kingdom/england/worcestershire
europe/united-kingdom/scotland
europe/united-kingdom/wales
north-america
north-america/canada
north-america/canada/alberta
north-america/canada/british-columbia
north-america/canada/manitoba
north-america/canada/new-brunswick
north-america/canada/newfoundland-and-labrador
north-america/canada/northwest-territories
north-america/canada/nova-scotia
north-america/canada/nunavut
north-america/canada/ontario
north-america/canada/prince-edward-island
north-america/canada/quebec
north-america/canada/saskatchewan
north-america/canada/yukon
north-america/greenland
north-america/mexico
north-america/us
north-america/us-midwest
north-america/us-northeast
north-america/us-pacific
north-america/us-south
north-america/us-west
north-america/us/alabama
north-america/us/alaska
north-america/us/arizona
north-america/us/arkansas
north-america/us/california
north-america/us/california/norcal
north-america/us/california/socal
north-america/us/colorado
north-america/us/connecticut
north-america/us/delaware
north-america/us/district-of-columbia
north-america/us/florida
north-america/us/georgia
north-america/us/hawaii
north-america/us/idaho
north-america/us/illinois
north-america/us/indiana
north-america/us/iowa
north-america/us/kansas
north-america/us/kentucky
north-america/us/louisiana
north-america/us/maine
north-america/us/maryland
north-america/us/massachusetts
north-america/us/michigan
north-america/us/minnesota
north-america/us/mississippi
north-america/us/missouri
north-america/us/montana
north-america/us/nebraska
north-america/us/nevada
north-america/us/new-hampshire
north-america/us/new-jersey
north-america/us/new-mexico
north-america/us/new-york
north-america/us/north-carolina
north-america/us/north-dakota
north-america/us/ohio
north-america/us/oklahoma
north-america/us/oregon
north-america/us/pennsylvania
north-america/us/puerto-rico
north-america/us/rhode-island
north-america/us/south-carolina
north-america/us/south-dakota
north-america/us/tennessee
north-america/us/texas
north-america/us/us-virgin-islands
north-america/us/utah
north-america/us/vermont
north-america/us/virginia
north-america/us/washington
north-america/us/west-virginia
north-america/us/wisconsin
north-america/us/wyoming
south-america
south-america/argentina
south-america/bolivia
south-america/brazil
south-america/brazil/centro-oeste
south-america/brazil/nordeste
south-america/brazil/norte
south-america/brazil/sudeste
south-america/brazil/sul
south-america/chile
south-america/colombia
south-america/ecuador
south-america/guyana
south-america/paraguay
south-america/peru
south-america/suriname
south-america/uruguay
south-america/venezuela