Cronjob ideally in daily to block known web bot, only when web server / reverse proxy is running on docker

block-web-bot

#!/bin/bash

IPSET=/usr/sbin/ipset
IPSET_NAME="blocked_ips"
IP_FILE="/tmp/blocked_ips.txt"
DOCKER_CHAIN="DOCKER-USER"

declare -A IPTABLES_BINARY=(
    ["/usr/sbin/iptables"]="iptables v4"
    ["/usr/sbin/ip6tables"]="iptables v6"
)

curl https://raw.githubusercontent.com/AnTheMaker/GoodBots/refs/heads/main/all.ips --output $IP_FILE

# Create ipset (flush if exists)
$IPSET destroy $IPSET_NAME 2>/dev/null
$IPSET destroy $IPSET_NAME-v6 2>/dev/null
$IPSET flush $IPSET_NAME 2>/dev/null
$IPSET flush $IPSET_NAME-v6 2>/dev/null
$IPSET create $IPSET_NAME hash:net
$IPSET create $IPSET_NAME-v6 hash:net family inet6

for iptables in "${!IPTABLES_BINARY[@]}"; do
  comment="${IPTABLES_BINARY[$iptables]}"
  if $iptables -L $DOCKER_CHAIN &>/dev/null; then
    echo "Will $comment rules at top of DOCKER chain"
    echo "Populate ipset with data set for $comment"
    grep -vE '^\s*#|^\s*$' "$IP_FILE" | grep --invert-match ":" | while read -r ip; do
      $IPSET add $IPSET_NAME "$ip"
    done
    while true; do
      line_num=$($iptables -L $DOCKER_CHAIN --line-numbers | grep DROP | awk '{print $1}' | head -n 1)
      if [[ -z "$line_num" ]]; then
        break
      fi
      echo "Deleting DROP rule number $line_num in DOCKER chain."
      $iptables -D $DOCKER_CHAIN "$line_num"
    done

    $iptables -I $DOCKER_CHAIN 1 -m set --match-set $IPSET_NAME src -p tcp --dport 80 -j DROP
    $iptables -I $DOCKER_CHAIN 2 -m set --match-set $IPSET_NAME src -p tcp --dport 443 -j DROP
  fi
done

echo "Blocked IPs from $IP_FILE using ipset $IPSET_NAME"

block-web-bot.txt

# Script will download a bot ips from github, insert in a ipset and reference it to iptables
# ipset is needed -> `apt install ipset`
# Should work with ipv4 and ipv6, not able to test with ipv6, docker is not ready for it yet
# at boot DOCKER iptables chain will be erased, you can add @reboot in crontable or wait 1 day if it's in cron.daily

After few minutes of use

sudo iptables -vnL DOCKER --line-numbers
Chain DOCKER (3 references)
num   pkts bytes target     prot opt in     out     source               destination
1       25  1500 DROP       6    --  *      *       0.0.0.0/0            0.0.0.0/0            match-set blocked_ips src tcp dpt:80
2      106  6360 DROP       6    --  *      *       0.0.0.0/0            0.0.0.0/0            match-set blocked_ips src tcp dpt:443
...
6       71  4168 ACCEPT     6    --  !br-aba74de1b897 br-aba74de1b897  0.0.0.0/0            172.19.0.11          tcp dpt:443
7       13   684 ACCEPT     6    --  !br-aba74de1b897 br-aba74de1b897  0.0.0.0/0            172.19.0.11          tcp dpt:80