WIP splunk patch and reboot fabric script

inventory.yml

---
# list of node groups
service:
  splunk:
    config:
      verify_tls: False
      splunk_cm_api_url: 'https://splunk-cluster-master01.domain.tld:8089'
      splunk_status_check_path: 'services/cluster/master/status?output_mode=json'
      splunk_set_mm_path: 'services/cluster/master/control/default/maintenance?output_mode=json'
    roles:
      lm:
        - "splunk-license-master01.domain.tld"
      ds:
        - "splunk-deployment-server01.domain.tld"
      cm:
        - "splunk-cluster-master01.domain.tld"
      shc:
        - "splunk-search-head01.domain.tld"
        - "splunk-search-head02.domain.tld"
        - "splunk-search-head03.domain.tld"
      idx:
        - "splunk-indexer01.domain.tld"
        - "splunk-indexer02.domain.tld"
        - "splunk-indexer03.domain.tld"

patch_and_reboot.py

import requests
import urllib3
import yaml
from requests.auth import HTTPBasicAuth
from yaml.loader import SafeLoader
from time import sleep
from fabric import Connection, Config
from paramiko.ssh_exception import NoValidConnectionsError

# Read inventory and config data from file
inv_file_path = 'inventory.yml'
with open(inv_file_path) as inv_file:
    inv_data = yaml.load(inv_file, Loader=SafeLoader)

roles = inv_data['service']['splunk']['roles']
config = inv_data['service']['splunk']['config']

# Disable bad cert warnings when config['verify_tls'] is false
verify_tls = config['verify_tls']
if not verify_tls:
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# Account credentials
username = '' 
password = ''
rest_user = ''
rest_user_pass = ''


def connect(host, user, passwd=''):
    """
    Setup SSH connection to a host

    :param host: Hostname of an instance to connect to
    :param user: Username to use when connecting
    :param passwd: Password to use for username
    :return: A connection object
    """
    override = Config(overrides={'sudo': {'password': passwd}})
    ret = Connection(host=host, user=user, config=override, connect_kwargs={"password": passwd})
    return ret


def join_api_url(api_base_url, api_path):
    """
    Simple join of a base url and an api path

    :param api_base_url: String representing the base url to use when connecting
    :param api_path: String representing the api path
    :return: A joined url with path
    """
    return api_base_url + '/' + api_path


def check_status(con):
    """
    Check status by verifying Splunkd is running using sudo

    :param con: Connection obj
    :return: True if is running else false
    """
    try:
        status = con.sudo('su - splunk -c "/opt/splunk/bin/splunk status"', hide=True)
        if 'is running' in status.stdout:
            return True
        else:
            return False
    except (ConnectionError, NoValidConnectionsError):
        return False


def patch_os(con):
    """
    Run the command to patch the OS using sudo.

    :param con: Connection obj
    :return: Result from command
    """
    try:
        ret = con.sudo('yum update -y', hide=True)
        return ret
    except (ConnectionError, NoValidConnectionsError):
        ret = 'Connection error aborting'
        return ret


def cmd_wait(con, run_cmd):
    """
    Command will be retried until success unless range value is reached

    :param con: Connection obj
    :param run_cmd: Command to run
    :return: Result from command
    """
    # May take up to 5 minutes
    sleep(5)
    ret = 'Command took too long to finish'
    for _ in range(25):
        try:
            ret = con.run(run_cmd, hide=True)
            break
        except (ConnectionError, NoValidConnectionsError):
            sleep(10)

    return ret


def check_maintenance_mode(api_url, user, user_pass, verify=False):
    """
    Check if maintenance mode is enabled, or disabled based on the value set for maintenance_mode

    :param api_url: String representing the base url to use when connecting
    :param user: Rest User to use when connecting
    :param user_pass: Password to use for Rest User
    :param verify: Verify TLS
    :return: Enabled, disabled, or unknown depending on requests value
    """
    try:
        ret = requests.get(api_url, auth=HTTPBasicAuth(user, user_pass), verify=verify)
        if ret.json()['entry'][0]['content']['maintenance_mode']:
            return 'enabled'
        else:
            return 'disabled'
    except (ConnectionError, NoValidConnectionsError):
        return 'unknown'


def set_maintenance_mode(api_set_mm, user, user_pass, data, verify=False):
    """
    Set maintenance mode status based on mode passed in as data

    :param api_set_mm: String representing the base url to use when connecting
    :param user: Rest User to use when connecting
    :param user_pass: Password to use for Rest User
    :param data: Mode value is True or False
    :param verify: Verify TLS
    :return: Result from API
    """
    try:
        ret = requests.post(api_set_mm, auth=HTTPBasicAuth(user, user_pass), data=data, verify=verify)
        return ret
    except (ConnectionError, NoValidConnectionsError):
        ret = 'Connection error aborting'
        return ret


# Join API url and path for maintenance status check
check_status_url = join_api_url(config['splunk_cm_api_url'], config['splunk_status_check_path'])

# False is the expected value but we return disabled
if check_maintenance_mode(check_status_url, rest_user, rest_user_pass, verify_tls) == 'disabled':
    # Perform checks to verify ready to start
    for role in roles.keys():
        for hostname in roles[role]:
            # Create a new session and check Splunkd status
            session = connect(hostname, username, password)
            if check_status(session):
                print('[OK] Splunkd is running: ' + hostname)
            else:
                print('[FAIL] Status Check: ' + hostname)
                # Close session and exit
                session.close()
                exit(1)

            # Close session and continue
            session.close()
else:
    # Maintenance mode is already enabled.
    # We should find out why and try again
    print('[FAIL] Maintenance mode already enabled')
    exit(1)

print('[OK] All status checks passed... Proceeding')

# Join API url and path for setting maintenance mode value
splunk_api_set_mm = join_api_url(config['splunk_cm_api_url'], config['splunk_set_mm_path'])

# Status checks have passed, do the updates
for role in roles.keys():
    # Set maintenance mode status to true
    # Only needs to be done for the indexers
    if role == 'idx':
        rest_data = {'mode': 'true'}
        res = set_maintenance_mode(splunk_api_set_mm, rest_user, rest_user_pass, rest_data, verify_tls)
        if check_maintenance_mode(check_status_url, rest_user, rest_user_pass, verify_tls) != 'enabled':
            print('[FAIL] Maintenance mode not enabled:')
            exit(1)

    # Iterate through list of hosts assigned to this role
    for hostname in roles[role]:
        # Create session
        session = connect(hostname, username, password)

        # Update and restart host if patches applied
        is_updated = patch_os(session)
        if 'Complete!' in is_updated.stdout and 'Nothing to do' not in is_updated.stdout:
            if role == 'idx':
                # Take splunk offline
                pass

            is_rebooted = session.sudo('nohup sudo -b bash -c "sleep 5 && reboot"', hide=True)
            if is_rebooted.return_code == 0:
                is_running = cmd_wait(session, 'systemctl status Splunkd.service')
                if is_running.return_code == 0:
                    print('[OK] Splunkd is running: ' + hostname)
                    print('[OK] Reboot successful: ' + hostname)
                else:
                    print('[FAIL] Status Check: ' + hostname)
        else:
            print('[OK] No updates: ' + hostname)

        # Close session
        session.close()

    # Set maintenance mode status to false
    # Only needs to be done for the indexers
    if role == 'idx':
        rest_data = {'mode': 'false'}
        res = set_maintenance_mode(splunk_api_set_mm, rest_user, rest_user_pass, rest_data, verify_tls)
        if check_maintenance_mode(check_status_url, rest_user, rest_user_pass, verify_tls) != 'disabled':
            print('[FAIL] Maintenance mode not disabled:')
            exit(1)

print('[OK] All hosts finished. Complete!')

requirements.txt

fabric==2.6.0
invoke==1.6.0
PyYAML==6.0
requests==2.26.0
urllib3~=1.26.7
paramiko~=2.8.0