CloudWatch Agent Deployment Scripts for Production

1-install-cloudwatch.sh

#!/bin/bash
# Run this ON the instance via Session Manager
# Just connect to the instance and paste this whole thing

set -e

echo "=== Installing CloudWatch Agent ==="

# Detect region from instance metadata
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
REGION=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)

echo "Region: $REGION"
echo "Instance: $INSTANCE_ID"

# Determine environment from region
if [ "$REGION" == "us-east-1" ]; then
    ENV="prod"
elif [ "$REGION" == "us-east-2" ]; then
    ENV="staging"
else
    ENV="unknown"
fi

echo "Environment: $ENV"

# Install agent
yum install -y amazon-cloudwatch-agent

# Write config
cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF
{
  "agent": {
    "run_as_user": "root",
    "region": "$REGION"
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/httpd/signingorder-ssl-access.log",
            "log_group_name": "/signingorder/$ENV/apache/access",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          },
          {
            "file_path": "/var/log/httpd/signingorder-ssl-error.log",
            "log_group_name": "/signingorder/$ENV/apache/error",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          },
          {
            "file_path": "/var/www/vhosts/signingorder.com/system/core/logs/log-*.log",
            "log_group_name": "/signingorder/$ENV/codeigniter",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          }
        ]
      }
    }
  }
}
EOF

# Start agent
systemctl enable amazon-cloudwatch-agent
systemctl restart amazon-cloudwatch-agent

# Verify
echo ""
echo "=== Verifying ==="
systemctl status amazon-cloudwatch-agent --no-pager | head -10

echo ""
echo "=== DONE ==="
echo ""
echo "Logs will appear at:"
echo "  https://$REGION.console.aws.amazon.com/cloudwatch/home?region=$REGION#logsV2:log-groups"
echo ""
echo "Now run the image script in CloudShell:"
echo "  ./2-image-and-update.sh $INSTANCE_ID"

2-image-and-update.sh

#!/bin/bash
# Run this in CloudShell after installing CloudWatch on the instance
# Usage: ./2-image-and-update.sh <instance-id>

INSTANCE_ID=$1
REGION="us-east-1"

if [ -z "$INSTANCE_ID" ]; then
    echo "Usage: $0 <instance-id>"
    echo ""
    echo "List instances:"
    aws ec2 describe-instances \
        --filters "Name=instance-state-name,Values=running" \
        --query "Reservations[].Instances[].[InstanceId, Tags[?Key=='Name'].Value | [0], InstanceType]" \
        --output table \
        --region $REGION
    exit 1
fi

echo "=== Creating AMI from $INSTANCE_ID ==="

# Get instance name
INSTANCE_NAME=$(aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" "Name=key,Values=Name" --query "Tags[0].Value" --output text --region $REGION)
echo "Instance: $INSTANCE_NAME"

# Generate AMI name
TIMESTAMP=$(date +%Y%m%d-%H%M)
AMI_NAME="so-prod-cloudwatch-${TIMESTAMP}"

echo "Creating AMI: $AMI_NAME (no reboot)"

AMI_ID=$(aws ec2 create-image \
    --instance-id "$INSTANCE_ID" \
    --name "$AMI_NAME" \
    --description "Production with CloudWatch - from $INSTANCE_NAME" \
    --no-reboot \
    --region $REGION \
    --query "ImageId" \
    --output text)

echo "AMI ID: $AMI_ID"
echo ""
echo "Waiting for AMI... (2-5 minutes)"

aws ec2 wait image-available --image-ids "$AMI_ID" --region $REGION

echo "AMI ready!"
echo ""

# Find ASG
ASG_NAME=$(aws autoscaling describe-auto-scaling-instances \
    --instance-ids "$INSTANCE_ID" \
    --region $REGION \
    --query "AutoScalingInstances[0].AutoScalingGroupName" \
    --output text 2>/dev/null)

if [ "$ASG_NAME" == "None" ] || [ -z "$ASG_NAME" ]; then
    echo "Instance not in ASG."
    echo "AMI created: $AMI_ID"
    echo ""
    echo "Manually update your launch template with this AMI."
    exit 0
fi

echo "ASG: $ASG_NAME"

# Get launch template
LT_ID=$(aws autoscaling describe-auto-scaling-groups \
    --auto-scaling-group-names "$ASG_NAME" \
    --region $REGION \
    --query "AutoScalingGroups[0].LaunchTemplate.LaunchTemplateId" \
    --output text)

if [ "$LT_ID" == "None" ] || [ -z "$LT_ID" ]; then
    echo "No launch template on ASG."
    echo "AMI created: $AMI_ID"
    exit 0
fi

LT_NAME=$(aws ec2 describe-launch-templates --launch-template-ids "$LT_ID" --region $REGION --query "LaunchTemplates[0].LaunchTemplateName" --output text)
CURRENT_VER=$(aws ec2 describe-launch-templates --launch-template-ids "$LT_ID" --region $REGION --query "LaunchTemplates[0].LatestVersionNumber" --output text)

echo "Launch Template: $LT_NAME (current v$CURRENT_VER)"
echo ""
echo "Creating new version with AMI $AMI_ID..."

NEW_VER=$(aws ec2 create-launch-template-version \
    --launch-template-id "$LT_ID" \
    --source-version "$CURRENT_VER" \
    --launch-template-data "{\"ImageId\":\"$AMI_ID\"}" \
    --version-description "CloudWatch agent - $TIMESTAMP" \
    --region $REGION \
    --query "LaunchTemplateVersion.VersionNumber" \
    --output text)

echo "New version: $NEW_VER"
echo "Setting as default..."

aws ec2 modify-launch-template \
    --launch-template-id "$LT_ID" \
    --default-version "$NEW_VER" \
    --region $REGION > /dev/null

echo ""
echo "========================================="
echo "DONE!"
echo "========================================="
echo ""
echo "AMI:              $AMI_ID"
echo "Launch Template:  $LT_NAME v$NEW_VER (default)"
echo "ASG:              $ASG_NAME"
echo ""
echo "New instances will use this AMI."
echo ""
echo "To update existing instances:"
echo "  aws autoscaling start-instance-refresh --auto-scaling-group-name $ASG_NAME --region $REGION"

install-cloudwatch-all.sh

#!/bin/bash
# Install CloudWatch Agent on ALL running instances in one shot
# Run this in CloudShell
#
# Usage:
#   ./install-cloudwatch-all.sh              # Dry run - show what would happen
#   ./install-cloudwatch-all.sh --execute    # Actually do it

REGION="us-east-1"
EXECUTE=false

if [ "$1" == "--execute" ]; then
    EXECUTE=true
fi

echo "=== CloudWatch Agent Mass Deployment ==="
echo "Region: $REGION"
echo ""

# Get all running instances
echo "=== Running Instances ==="
INSTANCES=$(aws ec2 describe-instances \
    --filters "Name=instance-state-name,Values=running" \
    --query "Reservations[].Instances[].InstanceId" \
    --output text \
    --region $REGION)

# Show them
aws ec2 describe-instances \
    --filters "Name=instance-state-name,Values=running" \
    --query "Reservations[].Instances[].[InstanceId, Tags[?Key=='Name'].Value | [0], InstanceType, PrivateIpAddress]" \
    --output table \
    --region $REGION

INSTANCE_COUNT=$(echo $INSTANCES | wc -w)
echo ""
echo "Total: $INSTANCE_COUNT instances"
echo ""

if [ "$EXECUTE" != "true" ]; then
    echo "DRY RUN - No changes made"
    echo ""
    echo "To install CloudWatch on ALL these instances, run:"
    echo "  $0 --execute"
    echo ""
    exit 0
fi

echo "=== EXECUTING - Installing CloudWatch on all $INSTANCE_COUNT instances ==="
echo ""

# The install script - auto-detects region and environment
INSTALL_SCRIPT='#!/bin/bash
set -e

# Get metadata
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
REGION=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)

# Environment from region
if [ "$REGION" == "us-east-1" ]; then
    ENV="prod"
elif [ "$REGION" == "us-east-2" ]; then
    ENV="staging"
else
    ENV="unknown"
fi

echo "Installing CloudWatch Agent on $INSTANCE_ID ($ENV)"

# Install
yum install -y amazon-cloudwatch-agent 2>/dev/null || dnf install -y amazon-cloudwatch-agent 2>/dev/null || apt-get install -y amazon-cloudwatch-agent 2>/dev/null

# Config
cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF
{
  "agent": {
    "run_as_user": "root",
    "region": "$REGION"
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/httpd/signingorder-ssl-access.log",
            "log_group_name": "/signingorder/$ENV/apache/access",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          },
          {
            "file_path": "/var/log/httpd/signingorder-ssl-error.log",
            "log_group_name": "/signingorder/$ENV/apache/error",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          },
          {
            "file_path": "/var/www/vhosts/signingorder.com/system/core/logs/log-*.log",
            "log_group_name": "/signingorder/$ENV/codeigniter",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          }
        ]
      }
    }
  }
}
EOF

# Start
systemctl enable amazon-cloudwatch-agent
systemctl restart amazon-cloudwatch-agent

echo "CloudWatch Agent installed on $INSTANCE_ID"
'

# Send to all instances at once
echo "Sending SSM command to all instances..."

COMMAND_ID=$(aws ssm send-command \
    --document-name "AWS-RunShellScript" \
    --targets "Key=instanceids,Values=$(echo $INSTANCES | tr ' ' ',')" \
    --parameters "commands=[\"$INSTALL_SCRIPT\"]" \
    --timeout-seconds 300 \
    --region $REGION \
    --query "Command.CommandId" \
    --output text)

echo "Command ID: $COMMAND_ID"
echo ""
echo "Waiting for completion..."

sleep 10

# Check status
echo ""
echo "=== Results ==="
aws ssm list-command-invocations \
    --command-id "$COMMAND_ID" \
    --region $REGION \
    --query "CommandInvocations[].[InstanceId, Status, StatusDetails]" \
    --output table

echo ""
echo "To check detailed output for a specific instance:"
echo "  aws ssm get-command-invocation --command-id $COMMAND_ID --instance-id <instance-id> --region $REGION"
echo ""
echo "View logs at:"
echo "  https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups"