Shelley monitoring

setup_shelley_monitoring.sh

#!/bin/bash
#---------------------------------------------------------------------
# File:    setup_shelley_monitoring.sh
# Created: 2019/10/17
# Creator: ilap
#=====================================================================
# DESCRIPTION:
#
# This script downloads and configures the required files 
# for monitoring a Shelley node by using grafana/prometheus.
#

clean_up () {
    echo "Cleaning up...." >&2
    rm -rf "${TMP_DIR}"
    RES=$1
    exit ${RES:=127}
}

message() {
    echo -e "$*" >&2
    exit 127
}

get_idx () {
    case $OSTYPE in
        "darwin"*)
            IDX=0
        ;;
        "linux-gnu"*)
            if [[ $HOSTTYPE == *"x86_64"* ]]; then
                IDX=1
            elif [[ $HOSTTYPE == *"arm"* ]]; then
                IDX=2
            else
                message "The $HOSTTYPE  is not supported"
            fi
        ;;
        *)
            message "The \"$OSTYPE\" OS is not supported"
        ;;
    esac
    echo $IDX
}

dl() {
    DL_URL="${1}"
    OUTPUT="${TMP_DIR}/`basename \"${DL_URL}\"`"
    shift
    
    TO_DIR="${@}"
    
    case ${DL} in
        *"wget"*)
        wget --no-check-certificate --output-document="${OUTPUT}" "${DL_URL}";;
        *)
        ( cd ${TMP_DIR} && curl -JOL "${DL_URL}" --silent );;
    esac
}

#### MAIN
if [ -z "$1" ] ; then
    message "usage: `basename $0` <project path> # e.g. ... ./Shelley"
fi

PROJ_DIR="`pwd`/${1}"
PROJ_NAME="`basename ${1}`"

export TMP_DIR=`mktemp -d "/tmp/${PROJ_NAME}.XXXXXXXX"`

CURL=`which curl`
WGET=`which wget`
JCLI=`which jcli`
DL=${CURL:=$WGET}

if  [ -z "$DL" -o -z "`which pip3`" -o -z "$JCLI" ]; then
    message 'You need to have 'wget' or 'curl', 'jcli' and 'pip3' to be installed\nand accessable by PATH environment to continue...\nExiting.'
fi

# Obtain parameters.
IP=127.0.0.1
PORT=3101
export IP PORT

while :
do
    
    read -p "What is the ip of the node (default:${IP})? " ip
    read -p "What is the port of the REST api of the node running on ${IP:="${ip}"}'s (default: ${PORT})? " port
    echo "Is this correct? http://${ip:-"${IP}"}:${port:-"${PORT}"}"
    read -p "Do you want to continue? [Y/n/q] " answer
    
    case ${answer:="Y"} in
        [yY]*)
            IP=${ip:-"${IP}"}
            PORT=${port:-"${PORT}"}
            break;;
        [nN]* )
            continue;;
        [qQ]* )
            exit;;
        * )
            echo "Please enter [yY](es), [nN](o) or [qQ](quit).";;
    esac
done

ARCHS=("darwin-amd64" "linux-amd64"  "linux-armv6")
IDX=`get_idx`

PROM_VER=2.13.0
PROM_URL="https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}/prometheus-${PROM_VER}.${ARCHS[IDX]}.tar.gz"

GRAF_VER=6.4.3
GRAF_URL="https://dl.grafana.com/oss/release/grafana-${GRAF_VER}.${ARCHS[IDX]}.tar.gz"

NIX_URL="https://raw.githubusercontent.com/input-output-hk/jormungandr-nix/master/nixos/jormungandr-monitor"

MON_PY="monitor.py"
GRAF_DB="grafana.json"

trap clean_up  SIGHUP SIGINT SIGQUIT SIGTRAP SIGABRT SIGTERM

echo -e "Downloading prometheus..." >&2
dl $PROM_URL

echo -e "Downloading grafana..." >&2
dl $GRAF_URL

echo -e "Downloading jormungandr monitoring scripts for prometheus" >&2
dl ${NIX_URL}/$MON_PY
dl ${NIX_URL}/$GRAF_DB

cd ${TMP_DIR} && mkdir -p "${PROJ_DIR}"/{exporters,prometheus,grafana}

PROM_DIR="${PROJ_DIR}/prometheus"
GRAF_DIR="${PROJ_DIR}/grafana"
tar zxC "${PROM_DIR}" -f *prome*gz --strip-components 1
tar zxC "${GRAF_DIR}" -f *graf*gz --strip-components 1

echo -e "Configuring components" >&2
cp -pr ${MON_PY} "${PROJ_DIR}"/exporters && chmod +x "${PROJ_DIR}"/exporters/*
cp -pr ${GRAF_DB} "${PROJ_DIR}"/
sed -i -e 's#@jcli@#'"$JCLI"'#g' "${PROJ_DIR}"/exporters/${MON_PY}
sed -i -e 's@\(/usr/bin/env python\)@\13@' "${PROJ_DIR}"/exporters/${MON_PY}
pip3 install ipython python-dateutil prometheus_client >/dev/null

cd ${PROJ_DIR}

EXPORTER_IP=localhost
EXPORTER_PORT=9100
sed -i -e 's@\(^scrape_configs:.*\)@\1\
  - job_name: '\''jormungandr'\''\
    static_configs:\
    - targets: ['\'${EXPORTER_IP}:${EXPORTER_PORT}\'']@g'  "${PROM_DIR}"/prometheus.yml

cat > start_all.sh <<EOF
#!/bin/bash

	#1. exporter
	PORT=${EXPORTER_PORT}
	JORMUNGANDR_API="http://${IP}:${PORT}/api"
	# Addresses to monitor e.g.
	# ADDRESSES="ta1..... ta1..... ta1...."
	ADDRESSES=""
	export PORT JORMUNGANDR_API ADDRESSES
	"${PROJ_DIR}"/exporters/monitor.py &
	sleep 3

	#2. Prometheus
	"${PROM_DIR}"/prometheus --config.file="${PROM_DIR}"/prometheus.yml &
	sleep 3

	#3. Grafana
	#vi conf/defaults.ini
	cd "${GRAF_DIR}"
	./bin/grafana-server web
EOF

chmod a+rx start_all.sh

echo -e "Installation completed

You need to do the following to configure grafana:
0. Start the required services by \"./${PROJ_NAME}/start_all.sh\"
  - Keep in mind this startup script is very simple and has limited capabilities
  - check the monitor script (http://${EXPORTER_IP}:${EXPORTER_PORT})
1. Login to grafana as admin/admin (http://localhost:3000)
2. Add "prometheus" (all lowercase) datasource (http://localhost:9090)
3. Create a new dashboard by importing 'grafana.json' (left plus sign).

Enjoy...
" >&2

clean_up 0

Well ... I think I've things working for now. Thanks for the script. It helped a lot.

@Isc3 can you share the changes you made on OSX? thx

I never got 'ss' working. But for now I can get some of these metrics into prometheus and grafana.

I had to pip3 install prometheus_client and py-dateutil python modules.

I basically changed the bash env to python3, commented out everything having to do with "ss". And changed the @lsof@ and @wc@ instances to full paths to each. This just allowed me to run 'python3 monitor.py'. I already have Prometheus/Grafana running. I didn't use the start_all.sh script.
I think that's all of it.
Hopefully this helps.

#!/usr/bin/env python3 <-------- changed from python to python3

JORMUNGANDR_API = os.getenv('JORMUNGANDR_RESTAPI_URL',
os.getenv('JORMUNGANDR_API', 'http://127.0.0.1:<MY_NODE_LISTEN_PORT>/api')) <------- node rest listen port

NODE_METRICS = [
"blockRecvCnt",
"connections",
"lastBlockDate",
"lastBlockEpoch",
"lastBlockFees",
"lastBlockHash",
"lastBlockHeight",
"lastBlockSlot",
"lastBlockSum",
"lastBlockTime",
"lastBlockTx",
"recvq", <------------------ commented out
"txRecvCnt",
"uptime",

Decorate function with metric.
@JORMUNGANDR_METRICS_REQUEST_TIME.time()
def process_jormungandr_metrics():
# Process jcli returned metrics
metrics = jcli_rest(['node', 'stats', 'get'])
lsof = subprocess.Popen(
('/usr/sbin/lsof', '-nPi', ':3000', '-sTCP:ESTABLISHED'), <---------------- Changed @lsof@ to direct path
stdout=subprocess.PIPE)
wc = subprocess.check_output(('/usr/bin/wc', '-l'), stdin=lsof.stdout). <-------------- Changed @wc@ to direct path
lsof.wait()
metrics['connections'] = int(wc, 10)

ss = subprocess.run(('@ss@', '-plntH', '( sport = :3000 )'), stdout=subprocess.PIPE).  <--------------comment out
recvq = ss.stdout.split()[1]                <--------------comment out
metrics['recvq'] = int(recvq, 10)       <--------------comment out

def jcli_rest(args):
flags = ['--host', JORMUNGANDR_API, '--output-format', 'json']
params = ['/usr/local/bin/jcli', 'rest', 'v0'] + args + flags <----------- Put in direct path to my jcli binary
result = subprocess.run(params, stdout=subprocess.PIPE)
return json.loads(result.stdout)