server-setup.md

Cloud Server Setup Guide

Initial Server Setup

1. System Preparation

# Update system
rm -f /var/lib/man-db/auto-update
apt update && apt upgrade -y

# Install essential packages
export DEBIAN_FRONTEND=noninteractive
apt install -y git vim htop

2. Setup Dotfiles for Root

Using my preferred bash and vim config.

git clone https://github.com/lhermann/dotfiles.git ~/.dotfiles
ln -fs ~/.dotfiles/.bash_aliases ~/.bash_aliases
ln -fs ~/.dotfiles/.bash_env ~/.bash_env
ln -fs ~/.dotfiles/.bashrc ~/.bashrc
ln -fs ~/.dotfiles/.bash_profile ~/.bash_profile
ln -fs ~/.dotfiles/.vimrc ~/.vimrc
ln -fs ~/.dotfiles/.tmux.conf ~/.tmux.conf
ln -fs ~/.dotfiles/.gitconfig ~/.gitconfig
source ~/.bashrc

3. Install Docker

# Add Docker's official GPG key
apt-get install -y ca-certificates curl gnupg
install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
chmod a+r /etc/apt/keyrings/docker.gpg

# Add the repository to Apt sources
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null

# Install Docker
apt update
apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

# Enable services
systemctl enable docker.service
systemctl enable containerd.service
systemctl start docker

4. Add Swap Space (for memory-constrained servers)

fallocate -l 4G /swapfile
chmod 600 /swapfile
mkswap /swapfile
swapon /swapfile
echo '/swapfile none swap sw 0 0' | tee -a /etc/fstab
sysctl vm.swappiness=10
echo 'vm.swappiness=10' | tee -a /etc/sysctl.conf

5. Install and Configure Nginx

# Install nginx
apt install -y nginx
systemctl enable nginx.service

Example nginx reverse-proxy for load-balanced node instances:

# Load Balancer
upstream backend_load_balanced {
  hash $http_cf_connecting_ip consistent;  # Sticky sessions based on client IP
  server 127.0.0.1:3001 max_fails=3 fail_timeout=30s;
  server 127.0.0.1:3002 max_fails=3 fail_timeout=30s;
  server 127.0.0.1:3003 max_fails=3 fail_timeout=30s;
  server 127.0.0.1:3004 max_fails=3 fail_timeout=30s;
}

map $http_upgrade $connection_upgrade {
  default upgrade;
  '' close;
}

# HTTP Server (Port 80)
server {
  listen 80;
  listen [::]:80;
  server_name <<ENTER_DOMAIN_HERE>>;

  charset utf-8;
  root /dev/null;

  # Force the latest IE version
  add_header "X-UA-Compatible" "IE=Edge";

  # Prevent clients from accessing hidden files (starting with a dot)
  location ~* /\.(?!well-known\/) {
    deny all;
  }

  # Prevent clients from accessing to backup/config/source files
  location ~* (?:\.(?:bak|conf|dist|fla|in[ci]|log|psd|sh|sql|sw[op])|~)$ {
    deny all;
  }

  # Let's Encrypt challenge location (works for both HTTP and HTTPS)
  location /.well-known/acme-challenge/ {
    root /var/www/letsencrypt;
    try_files $uri $uri/ =404;
  }

  location / {
    proxy_pass http://backend_load_balanced;
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Real-Request $request;
    proxy_set_header X-Real-URI $request_uri;
    proxy_set_header X-Forwarded-For $remote_addr;
    proxy_set_header X-Forwarded-Proto http;
    proxy_set_header X-Forwarded-Host $host;
    proxy_set_header X-Forwarded-Server $host;

    # WebSocket specific settings
    proxy_cache_bypass $http_upgrade;
    proxy_read_timeout 86400;
    proxy_send_timeout 86400;
    proxy_connect_timeout 86400;
  }
}

# HTTPS Server (Port 443)
server {
  listen 443 ssl;
  listen [::]:443 ssl;
  http2 on;
  server_name <<ENTER_DOMAIN_HERE>>;

  # SSL Configuration
  ssl_certificate /etc/letsencrypt/live/<<ENTER_DOMAIN_HERE>>/fullchain.pem;
  ssl_certificate_key /etc/letsencrypt/live/<<ENTER_DOMAIN_HERE>>/privkey.pem;
  include /etc/letsencrypt/options-ssl-nginx.conf;
  ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;

  charset utf-8;
  root /dev/null;

  # Force the latest IE version
  add_header "X-UA-Compatible" "IE=Edge";

  # Prevent clients from accessing hidden files (starting with a dot)
  location ~* /\.(?!well-known\/) {
    deny all;
  }

  # Prevent clients from accessing to backup/config/source files
  location ~* (?:\.(?:bak|conf|dist|fla|in[ci]|log|psd|sh|sql|sw[op])|~)$ {
    deny all;
  }

  # Let's Encrypt challenge location (works for both HTTP and HTTPS)
  location /.well-known/acme-challenge/ {
    root /var/www/letsencrypt;
    try_files $uri $uri/ =404;
  }

  # Health check endpoint
  location /nginx-health {
    access_log off;
    return 200 "healthy\n";
    add_header Content-Type text/plain;
  }

  location / {
    proxy_pass http://backend_load_balanced;
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Real-Request $request;
    proxy_set_header X-Real-URI $request_uri;
    proxy_set_header X-Forwarded-For $remote_addr;
    proxy_set_header X-Forwarded-Proto https;
    proxy_set_header X-Forwarded-Host $host;
    proxy_set_header X-Forwarded-Server $host;

    # WebSocket specific settings
    proxy_cache_bypass $http_upgrade;
    proxy_read_timeout 86400;
    proxy_send_timeout 86400;
    proxy_connect_timeout 86400;
  }
}

6. Configure DNS Nameservers

Use Cloudflare and Google nameserver because I ran into problems with default Hetzner nameservers in the past.

mkdir -p /etc/resolvconf/resolv.conf.d/
echo 'nameserver 1.1.1.1' >> /etc/resolvconf/resolv.conf.d/head
echo 'nameserver 8.8.8.8' >> /etc/resolvconf/resolv.conf.d/head

# Apply changes
if command -v resolvconf &> /dev/null; then
    resolvconf -u
else
    echo 'nameserver 1.1.1.1' > /etc/resolv.conf
    echo 'nameserver 8.8.8.8' >> /etc/resolv.conf
fi

# Verify DNS resolution
nslookup redis-11762.c1.us-east1-2.gce.cloud.redislabs.com

Application Setup

1. Create Node User and Deploy Key

To run the docker containers on a dedicated user, and not root, for security.

# Create node user with docker access
useradd -m -s /bin/bash -d /home/node -G docker node
mkdir -p /var/server
chown -R node:node /var/server

# Switch to node user
su - node

# Setup dotfiles for node user
git clone https://github.com/lhermann/dotfiles.git ~/.dotfiles
ln -fs ~/.dotfiles/.bash_aliases ~/.bash_aliases
ln -fs ~/.dotfiles/.bash_env ~/.bash_env
ln -fs ~/.dotfiles/.bashrc ~/.bashrc
ln -fs ~/.dotfiles/.bash_profile ~/.bash_profile
ln -fs ~/.dotfiles/.vimrc ~/.vimrc
ln -fs ~/.dotfiles/.tmux.conf ~/.tmux.conf
ln -fs ~/.dotfiles/.gitconfig ~/.gitconfig
echo "cd /var/server/" >> ~/.bashrc

# Copy SSH keys from root to node user for easy access
cp /root/.ssh/authorized_keys /home/node/.ssh/
chown node:node /home/node/.ssh/authorized_keys

# Generate SSH deploy key
mkdir -p ~/.ssh && cd ~/.ssh
ssh-keygen -t ed25519 -f deploy_key_ed25519 -N ""
cat <<EOF > ~/.ssh/config
Host github-deploy-key
  Hostname        github.com
  IdentityFile    ~/.ssh/deploy_key_ed25519
EOF

# Display public key (add to GitHub repository deploy keys)
cat ~/.ssh/deploy_key_ed25519.pub

# Add GitHub to known hosts
ssh-keyscan -t ed25519 github.com >> ~/.ssh/known_hosts

# Return to root
exit

2. Clone Repositories

# As node user
su - node

# Clone staging repository
mkdir -p /var/server/stage
cd /var/server/stage
git clone -b staging git@github-deploy-key:stagetimerio/stagetimer.git .

# Clone production repository
mkdir -p /var/server/prod
cd /var/server/prod
git clone -b master git@github-deploy-key:stagetimerio/stagetimer.git .

# Create runtime directories
mkdir -p /var/server/stage/.run /var/server/stage/.secrets
mkdir -p /var/server/prod/.run /var/server/prod/.secrets
chmod 700 /var/server/stage/.secrets /var/server/prod/.secrets

# Return to root
exit

SSL Certificate Setup

1. Install Certbot

apt install -y python3 python3-dev python3-venv libaugeas-dev gcc
python3 -m venv /opt/certbot/
/opt/certbot/bin/pip install --upgrade pip
/opt/certbot/bin/pip install certbot certbot-nginx certbot-dns-cloudflare
ln -sf /opt/certbot/bin/certbot /usr/bin/certbot

2. Setup Automatic Renewal

echo "0 0,12 * * * root /opt/certbot/bin/python -c 'import random; import time; time.sleep(random.random() * 3600)' && sudo certbot renew -q" | tee -a /etc/crontab > /dev/null

3. Create Cloudflare API Credentials

I use cloudflare as DNS and this will allow certbot to provision wildcard certificates.

touch /etc/letsencrypt/cloudflare.ini
chmod 600 /etc/letsencrypt/cloudflare.ini
vim /etc/letsencrypt/cloudflare.ini

# Add the following (replace with your API token):
# dns_cloudflare_api_token = your_api_token_here

4. Download SSL Configuration Files

Since we manage nginx configs manually, download the required SSL files:

# Download Let's Encrypt SSL options file
curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf -o /etc/letsencrypt/options-ssl-nginx.conf

# Download Mozilla's recommended DH parameters (2048-bit)
curl -s https://ssl-config.mozilla.org/ffdhe2048.txt -o /etc/letsencrypt/ssl-dhparams.pem

5. Provision Certificates

certbot certonly \
  --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \
  -d <<ENTER_DOMAIN_HERE>> \
  --agree-tos \
  --no-eff-email \
  --email me@example.com

6. Verify and Restart Nginx

nginx -t
systemctl restart nginx

Application Management

View Logs

# View container logs
docker logs container-name-or-id

# Follow logs
docker logs -f container-name-or-id

Debug Inside Container

# Run interactive shell in container
docker run --rm --volume /path/to/dir/on/host:/path/to/dir/in/container --workdir /path/to/dir/in/container -it node:20 /bin/bash

Clean Up Docker

# Remove unused containers and images
docker system prune

# Remove all unused images
docker image prune -a

Update Certbot

/opt/certbot/bin/pip install --upgrade certbot certbot-nginx certbot-dns-cloudflare

Monitoring

System Information

# System version
uname -a
lsb_release -a

# Check memory and swap
free -h

# Check disk usage
df -h

# View installed packages
apt list --installed

# Check Docker containers
docker ps
docker stats

Troubleshooting

Login as Node User

# Option 1: SSH directly as node user (authorized_keys copied during setup)
ssh node@<server-ip>

# Option 2: SSH as root then switch
ssh root@<server-ip>
su - node

Restart Services

# Restart nginx
systemctl restart nginx

# Restart Docker containers
docker compose restart

# Restart specific container
docker restart stage-app1-1

Check Service Status

# Check nginx
systemctl status nginx

# Check Docker
systemctl status docker

# Check container health
docker ps
docker inspect stage-app1-1 --format='{{.State.Health.Status}}'

Common Issues

• SSL certificate errors: Check /etc/letsencrypt/cloudflare.ini for correct API token
• Container won't start: Check logs with docker logs <container-name>
• Memory issues: Check swap is enabled with free -h
• DNS resolution: Verify nameservers in /etc/resolv.conf