CloudWatch deployment scripts

image-machine.sh

#!/bin/bash
# Image a production machine and update its ASG launch template
#
# Usage:
#   ./image-machine.sh                      # List all running machines
#   ./image-machine.sh i-XXXXX              # Image (new version NOT default)
#   ./image-machine.sh i-XXXXX --default    # Image and SET as default

REGION="us-east-1"
SET_DEFAULT=false

if [ "$2" == "--default" ]; then
    SET_DEFAULT=true
fi

list_machines() {
    echo "=== Running Production Machines ==="
    echo ""
    printf "%-22s %-45s %-25s\n" "INSTANCE ID" "NAME" "ASG"
    printf "%-22s %-45s %-25s\n" "-----------" "----" "---"

    aws ec2 describe-instances \
        --filters "Name=instance-state-name,Values=running" \
        --query "Reservations[].Instances[].[InstanceId, Tags[?Key=='Name'].Value | [0], Tags[?Key=='aws:autoscaling:groupName'].Value | [0]]" \
        --output text \
        --region $REGION | while read -r id name asg; do
            printf "%-22s %-45s %-25s\n" "$id" "${name:-No Name}" "${asg:-None}"
        done

    echo ""
    echo "Usage: $0 <instance-id>"
}

image_machine() {
    INSTANCE_ID="$1"

    echo "=== Getting Instance Info ==="

    # Get instance name
    INSTANCE_NAME=$(aws ec2 describe-tags \
        --filters "Name=resource-id,Values=$INSTANCE_ID" "Name=key,Values=Name" \
        --query "Tags[0].Value" \
        --output text \
        --region $REGION)

    echo "Instance: $INSTANCE_NAME ($INSTANCE_ID)"

    # Get ASG
    ASG_NAME=$(aws autoscaling describe-auto-scaling-instances \
        --instance-ids "$INSTANCE_ID" \
        --query "AutoScalingInstances[0].AutoScalingGroupName" \
        --output text \
        --region $REGION 2>/dev/null)

    if [ -z "$ASG_NAME" ] || [ "$ASG_NAME" == "None" ]; then
        echo "WARNING: Instance not in an ASG"
        echo "Will create AMI but cannot update launch template"
        ASG_NAME=""
    else
        echo "ASG: $ASG_NAME"
    fi

    # Determine AMI name based on instance name
    TIMESTAMP=$(date +"%Y%m%d %H%M")

    if echo "$INSTANCE_NAME" | grep -qi "cron"; then
        AMI_NAME="SO PROD CRON $TIMESTAMP"
    elif echo "$INSTANCE_NAME" | grep -qi "beta"; then
        AMI_NAME="SO BETA Web Node $TIMESTAMP"
    elif echo "$INSTANCE_NAME" | grep -qi "canary"; then
        AMI_NAME="SO Prod Canary $TIMESTAMP"
    else
        AMI_NAME="SO PROD ELB Web Node $TIMESTAMP"
    fi

    echo "AMI Name: $AMI_NAME"
    echo ""

    # Create AMI
    echo "=== Creating AMI (no reboot) ==="

    AMI_ID=$(aws ec2 create-image \
        --instance-id "$INSTANCE_ID" \
        --name "$AMI_NAME" \
        --description "Created from $INSTANCE_NAME" \
        --no-reboot \
        --region $REGION \
        --query "ImageId" \
        --output text)

    echo "AMI ID: $AMI_ID"
    echo ""
    echo "Waiting for AMI to be available..."

    aws ec2 wait image-available --image-ids "$AMI_ID" --region $REGION

    echo "AMI ready!"
    echo ""

    # If no ASG, we're done
    if [ -z "$ASG_NAME" ]; then
        echo "=== DONE ==="
        echo "AMI: $AMI_ID"
        echo "No ASG - manually update launch template if needed"
        exit 0
    fi

    # Get launch template from ASG
    echo "=== Updating Launch Template ==="

    LT_INFO=$(aws autoscaling describe-auto-scaling-groups \
        --auto-scaling-group-names "$ASG_NAME" \
        --query "AutoScalingGroups[0].LaunchTemplate" \
        --output json \
        --region $REGION)

    LT_ID=$(echo "$LT_INFO" | jq -r '.LaunchTemplateId // empty')

    if [ -z "$LT_ID" ]; then
        echo "ASG doesn't use a launch template"
        echo ""
        echo "=== DONE ==="
        echo "AMI: $AMI_ID"
        exit 0
    fi

    LT_NAME=$(aws ec2 describe-launch-templates \
        --launch-template-ids "$LT_ID" \
        --query "LaunchTemplates[0].LaunchTemplateName" \
        --output text \
        --region $REGION)

    CURRENT_VERSION=$(aws ec2 describe-launch-templates \
        --launch-template-ids "$LT_ID" \
        --query "LaunchTemplates[0].LatestVersionNumber" \
        --output text \
        --region $REGION)

    DEFAULT_VERSION=$(aws ec2 describe-launch-templates \
        --launch-template-ids "$LT_ID" \
        --query "LaunchTemplates[0].DefaultVersionNumber" \
        --output text \
        --region $REGION)

    echo "Launch Template: $LT_NAME ($LT_ID)"
    echo "Current Latest: v$CURRENT_VERSION"
    echo "Current Default: v$DEFAULT_VERSION"
    echo ""

    # Create new version (NOT setting as default)
    echo "Creating new version with AMI $AMI_ID..."

    NEW_VERSION=$(aws ec2 create-launch-template-version \
        --launch-template-id "$LT_ID" \
        --source-version "$CURRENT_VERSION" \
        --launch-template-data "{\"ImageId\":\"$AMI_ID\"}" \
        --version-description "CloudWatch agent - $TIMESTAMP" \
        --region $REGION \
        --query "LaunchTemplateVersion.VersionNumber" \
        --output text)

    echo "New Version: v$NEW_VERSION"
    echo ""

    # Set as default if requested
    if [ "$SET_DEFAULT" == "true" ]; then
        echo "Setting v$NEW_VERSION as default..."
        aws ec2 modify-launch-template \
            --launch-template-id "$LT_ID" \
            --default-version "$NEW_VERSION" \
            --region $REGION > /dev/null

        echo ""
        echo "========================================="
        echo "DONE - SET AS DEFAULT"
        echo "========================================="
        echo ""
        echo "AMI:              $AMI_ID"
        echo "Launch Template:  $LT_NAME"
        echo "New Version:      v$NEW_VERSION (NOW DEFAULT)"
        echo "Previous Default: v$DEFAULT_VERSION"
        echo "ASG:              $ASG_NAME"
        echo ""
        echo "New instances will use this AMI."
        echo ""
        echo "To rollback:"
        echo "  aws ec2 modify-launch-template --launch-template-id $LT_ID --default-version $DEFAULT_VERSION --region $REGION"
    else
        echo "========================================="
        echo "DONE - NOT SET AS DEFAULT"
        echo "========================================="
        echo ""
        echo "AMI:              $AMI_ID"
        echo "Launch Template:  $LT_NAME"
        echo "New Version:      v$NEW_VERSION (NOT default)"
        echo "Default Version:  v$DEFAULT_VERSION (unchanged)"
        echo "ASG:              $ASG_NAME"
        echo ""
        echo "To make default:"
        echo "  aws ec2 modify-launch-template --launch-template-id $LT_ID --default-version $NEW_VERSION --region $REGION"
        echo ""
        echo "Or re-run with: $0 $INSTANCE_ID --default"
    fi
}

# Main
if [ -z "$1" ]; then
    list_machines
else
    image_machine "$1"
fi

PRODUCTION-INSTALL-NOW.sh

#!/bin/bash
# Run this in PRODUCTION CloudShell (us-east-1)
# Installs CloudWatch on one web server and the CRON server
#
# PRODUCTION = UBUNTU (uses apt-get, /var/log/apache2/)
# STAGING = Amazon Linux (uses yum, /var/log/httpd/)

REGION="us-east-1"

# The install script - base64 encoded to avoid escaping hell
# NOTE: Uses Ubuntu paths (/var/log/apache2/) for production
INSTALL_SCRIPT=$(cat << 'SCRIPTEOF' | base64 -w 0
#!/bin/bash
set -e
echo "=== Installing CloudWatch Agent ==="

# Install CloudWatch Agent (Ubuntu)
apt-get update
apt-get install -y amazon-cloudwatch-agent

# Create config with UBUNTU paths
# Production uses /var/log/apache2/ not /var/log/httpd/
cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF'
{
  "agent": {
    "run_as_user": "root",
    "region": "us-east-1",
    "metrics_collection_interval": 60
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/apache2/stagingsigningorder.com-access.log",
            "log_group_name": "/signingorder/prod/apache/access",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          },
          {
            "file_path": "/var/log/apache2/stagingsigningorder.com-error.log",
            "log_group_name": "/signingorder/prod/apache/error",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          },
          {
            "file_path": "/var/www/vhosts/signingorder.com/system/core/logs/log-*.log",
            "log_group_name": "/signingorder/prod/codeigniter",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 14
          }
        ]
      }
    }
  },
  "metrics": {
    "namespace": "SigningOrder/prod",
    "metrics_collected": {
      "disk": {
        "measurement": ["used_percent", "free"],
        "metrics_collection_interval": 60,
        "resources": ["/"]
      },
      "mem": {
        "measurement": ["mem_used_percent", "mem_available"],
        "metrics_collection_interval": 60
      },
      "cpu": {
        "measurement": ["cpu_usage_active"],
        "metrics_collection_interval": 60,
        "totalcpu": true
      }
    }
  }
}
EOF

systemctl enable amazon-cloudwatch-agent
systemctl restart amazon-cloudwatch-agent
sleep 3
echo "=== Status ==="
systemctl status amazon-cloudwatch-agent --no-pager | head -10
SCRIPTEOF
)

echo "=== Installing CloudWatch Agent ==="
echo "NOTE: Production is UBUNTU - using /var/log/apache2/ paths"
echo ""

# Web Server
WEB_INSTANCE="i-0c26cdaf3185186c4"
echo "Installing on Web Server: $WEB_INSTANCE"

WEB_CMD=$(aws ssm send-command \
    --instance-ids "$WEB_INSTANCE" \
    --document-name "AWS-RunShellScript" \
    --parameters "commands=[\"echo $INSTALL_SCRIPT | base64 -d | bash\"]" \
    --timeout-seconds 300 \
    --region $REGION \
    --query "Command.CommandId" \
    --output text)

echo "Web Command ID: $WEB_CMD"

# CRON Server
CRON_INSTANCE="i-01056a9e0c52dac85"
echo "Installing on CRON Server: $CRON_INSTANCE"

CRON_CMD=$(aws ssm send-command \
    --instance-ids "$CRON_INSTANCE" \
    --document-name "AWS-RunShellScript" \
    --parameters "commands=[\"echo $INSTALL_SCRIPT | base64 -d | bash\"]" \
    --timeout-seconds 300 \
    --region $REGION \
    --query "Command.CommandId" \
    --output text)

echo "CRON Command ID: $CRON_CMD"

echo ""
echo "Waiting 30 seconds..."
sleep 30

echo ""
echo "=== Web Server Result ==="
aws ssm get-command-invocation --command-id "$WEB_CMD" --instance-id "$WEB_INSTANCE" --region $REGION --query "[Status, StandardOutputContent]" --output text 2>/dev/null || echo "Still running..."

echo ""
echo "=== CRON Server Result ==="
aws ssm get-command-invocation --command-id "$CRON_CMD" --instance-id "$CRON_INSTANCE" --region $REGION --query "[Status, StandardOutputContent]" --output text 2>/dev/null || echo "Still running..."

echo ""
echo "=== View Logs ==="
echo "https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups"
echo ""
echo "Log groups to look for:"
echo "  /signingorder/prod/apache/access"
echo "  /signingorder/prod/apache/error"
echo "  /signingorder/prod/codeigniter"