#! /bin/bash
#
#  Check of QT test status reboot if not available
#  Author: Chris Gaukroger JDSU Plymouth UK 20091127
#
#msg=`printf "This is the extra check for the watchdog.\n"`
#logger -is -t "watchdog_check" $msg
#
# called by watchdog check_qtapp.sh at 30 intervals
# This is the place for extra checks and can be used for diagnosing problems
#
# IMPORTANT
# Return the RETVAL updated if necessary
#
# In this file:
#   if RETVAL = "BACKGROUND" this is the background test for curl with long timeout (see below)
#   if RETVAL not 0 exit immediately as already failed
#   if RETVAL = 0 then use this script
#      wait for START_UPTIME before any check as webserver, etc not running
#      only check every MOD_UPTIME in seconds as may take time for request and avoids
#         multiple requests
#   To review running set verbose=1
#
# Notes: when test.py is restarted curl test status can take up to ~32 s
#        This script runs normally from watchdog every 30 s.
#        Check of test status is every 5 minutes but needs to finish in 30 s
#        so we do not have two versions running.
#        If curl timeout after 20 s probably doing restart test.py so use
#        background test of long timeout. Must reboot if this fails
#

# RETVAL is from calling script so can alter it
RETVAL=$1

# To enable more output use verbose=1
verbose=0

# Start testing after all startup OK: say 10 Minutes 10*60
declare -i START_UPTIME=600
# Only check every 5 minutes so do not have multiple requests if slow
declare -i MOD_UPTIME=300
# so only do 1 test
declare -i MOD_MAX=30
# Standard CURL timeout (can be changed if LONG_TIMEOUT
declare -i CURL_TIMEOUT=20
# Fixed directory path so not reliant on environment variables (this file)
SCRIPTS_DIR="/opt/qt/scripts"
declare check_qtxxx_file="${SCRIPTS_DIR}/check_qtxxx.sh"
declare check_qt_state_file="${SCRIPTS_DIR}/check_qt_state.py"
# background test
background=0

# log to stderr, /usr/etc/watchdog.log and /var/log/messages
# Use background so cannot stop script
mylogger()
{
   logger -is -p daemon.notice -t "watchdog_check_qtxxx" "$*" &
}

# Verbose level logging
mylogger_verbose()
{
   msg=$1
   if [ ${verbose} -ne 0 ]; then
      mylogger $msg
   fi
}

#mylogger_verbose "Entered with RETVAL=${RETVAL}"

# if BACKGROUND test then do not exit early
if [ "${RETVAL}" == "BACKGROUND" ]; then
   CURL_TIMEOUT=300
   mylogger_verbose "BACKGROUND test curl use timeout=${CURL_TIMEOUT}"
   RETVAL=0      # good return for now
   background=1
else
   if [ ${RETVAL} -ne 0 ]; then
      # if already in failure mode exit without logging in case this stops exit
      exit ${RETVAL}
   fi
fi


# get uptime from /proc/uptime
uptime=$(</proc/uptime)
uptime=${uptime%%.*}

mod_time=$(( ${uptime}%${MOD_UPTIME} ))
minutes=$(( uptime/60%60 ))
seconds=$(( uptime%60 ))

if [ ${verbose} -ne 0 ]; then
   hours=$(( uptime/60/60%24 ))
   days=$(( uptime/60/60/24 ))
fi

mylogger_verbose "${days} days, ${hours} hours, ${minutes} minutes, ${seconds} seconds: mod_time=${mod_time}"

#
# strstr string string_to_find
# return 0 if found else 1
#
strstr()
{
   #mylogger_verbose "strstr '$1' '$2'"
   [ "$1" = "$2" ] && return 0
   slice=${1#*$2*}
   #mylogger_verbose "strstr slice='$slice'"
   [ "$slice" = "$1" ] && return 1

   return 0
}

# Wait for a while from startup for all agents to be running
if [ ${uptime} -gt ${START_UPTIME} ]; then
   if [ ${background} -eq 0 ]; then
      if [ ${minutes} -eq 0 ] && [ ${seconds} -lt 25 ]; then
         # do this every hour
         # check qt state
         if [ -e ${check_qt_state_file} ]; then
            ${check_qt_state_file} &
         fi
      fi
   elif [ ${mod_time} -lt ${MOD_MAX} ] || [ ${background} -eq 1 ]; then
      # Do test on the time increment
      mylogger_verbose "uptime ${uptime} seconds: Greater than ${START_UPTIME}: Do check"

      # use IP address so no lookup in host table
      response=`curl -x "" --max-time ${CURL_TIMEOUT} -sS http://127.0.0.1/status/test.xml`
      test_status=$?
      mylogger_verbose "test status response ${response}"
      if [ ${test_status} -eq 0 ]; then
         # check response contains "/status" so valid status
         key="/status"
         if strstr "$response" "$key"; then
            # key in string
            mylogger_verbose "test status request OK"
         else  # key not in string
            mylogger_verbose "test status timeout=${CURL_TIMEOUT} invalid response ${response}"
            # if in background and curl fails reboot
            if [ ${background} -eq 1 ]; then
               mylogger "test status timeout=${CURL_TIMEOUT} invalid response ${response}"
               mylogger "test status failure: not valid status: Force reboot failure and reboot"
               # Check cpu status as extra help for fault
               response=`curl -x "" --max-time ${CURL_TIMEOUT} -sS http://127.0.0.1/status/cpu.xml`
               cpu_status=$?
               mylogger "cpu status response ${response}"
               mylogger "cpu status=${cpu_status} test status=${test_status}"
               mylogger "Background test status response failure QT->reboot"
               reboot
               sleep 600  # wait for reboot to happen
               mylogger "Reboot failure"  # This should never be seen
            else
               mylogger_verbose "test status failure: invalid status: do background test status"
               ${check_qtxxx_file} "BACKGROUND" &
            fi
         fi
      else # test status failure
         if [ ${background} -eq 1 ]; then
            # if in background and curl fails reboot
            # Error in getting test status
            mylogger "test status failure: Force reboot failure and reboot"
            # Check cpu status as extra help for fault
            response=`curl -x "" --max-time ${CURL_TIMEOUT} -sS http://127.0.0.1/status/cpu.xml`
            cpu_status=$?
            mylogger "cpu status timeout=${CURL_TIMEOUT} response ${response}"
            mylogger "cpu status=${cpu_status} test status=${test_status}"
            mylogger "Background test status failure QT->reboot"
            reboot
            sleep 600  # wait for reboot to happen
            mylogger "Reboot failure"  # This should never be seen
         else
            mylogger_verbose "test status failure: do background test status"
            # restart script in background with long curl timeout
            ${check_qtxxx_file} "BACKGROUND" &
         fi
      fi
   else
      mylogger_verbose "uptime ${uptime} seconds: Waiting for next check when uptime%${MOD_UPTIME}<${MOD_MAX} ${mod_time}"
   fi
else
   mylogger_verbose "uptime ${uptime} seconds: Waiting for ${START_UPTIME} before starting check"
fi

mylogger_verbose "Extra Script executed return RETVAL=${RETVAL}"

exit ${RETVAL}