Script for downloading the google ngram dataset

download-google-ngram.sh

#!/bin/bash -e

LANGUAGES=(ger eng)
NGRAM_SIZES=(1 2 3)

for LANGUAGE in "${LANGUAGES[@]}"; do
    echo "Download the files using wget into $LANGUAGE/"
    for SIZE in "${NGRAM_SIZES[@]}"; do
        (
            echo "$LANGUAGE/$SIZE-gram"
            mkdir -p "$LANGUAGE/$SIZE-gram"
            cd "$LANGUAGE/$SIZE-gram"
            URLS=$(
                curl -s "https://storage.googleapis.com/books/ngrams/books/20200217/$LANGUAGE/$LANGUAGE-$SIZE-ngrams_exports.html" |
                grep -oP 'href="\K[^"]+\.gz' |
                sort -h
            )
            for URL in $URLS; do
                wget --continue -q --show-progress "$URL"
            done
        )
    done
done