#! /bin/ksh

###############################################################################
# errmsg
###############################################################################
#
# Echo arguments.
#
###############################################################################

errmsg()
{
    if [ "$1" = "-n" ] ; then
        shift
        gettext "$*"            # no newline
    else
        gettext "$*\\n"
    fi
}

###############################################################################
# get_pid
###############################################################################
#
# Return the pid(s) of the processes with the given name.  A null string is
# returned if the process isn't running.
#
###############################################################################

get_pid()
{
   case $1 in
      em_ob_corba_rgw | em_ob_corba_mgw | em_ob_corba_epr | em_ob_corba_eds |\
      em_vb_corba_rgw | em_vb_corba_mgw | em_vb_corba_epr | em_vb_corba_eds |\
      em_io_corba_rgw | em_io_corba_mgw | em_io_corba_epr | em_io_corba_eds )
         short_daemon=$1
         ps -eo pid,comm  | grep "$short_daemon" | sed -e 's/^  *//' -e 's/ .*//'
      ;;
	em_mpa_sunmc | em_jdmkfwd)
		if [ $1 = "em_mpa_sunmc" ] ; then
	 		pid_file=/var/tmp/em_mpa_sunmc.pid
		elif [ $1 = "em_jdmkfwd" ] ; then
			pid_file=/tmp/jdmk_fwd.pid
		fi
		if [ -r $pid_file ] ; then
			process_pid=`cat $pid_file`
			ps -p $process_pid >/dev/null 2>&1
			if [  $? -ne 0 ] ; then
				process_pid=
			fi
		else
			process_pid=
		fi
		echo $process_pid
		;;

	*)
		ps -e -o pid,args | /usr/bin/grep -v "grep"| /usr/bin/grep -w $1 |  sed -e 's/^  *//' -e 's/ .*//'
      ;;
   esac
}

###############################################################################
#
#  Locks
#
###############################################################################
CreateLock() {

   cp /dev/null /tmp/$1
   chmod 666    /tmp/$1
}

CheckLock() {

typeset -i ticks

ticks=0
while [[ -f /tmp/$1 ]] ; do
   if [[ $ticks -gt $TIMEOUT ]]; then
      return 1
   fi
   sleep 5
   ticks=$(($ticks+1))
done
return 0
}

###############################################################################
# restart
###############################################################################
restart() {

arg=$1
case $arg in
	oninit )
	   return 1
	   ;;
	em_auxdb )
	   return 1
	   ;;
	em_autoexd )
	   em_autoexd -config $EM_RUNTIME/conf/em_autoexd.cfg > $debug_device 2>&1
	   ;;
	em_ncam )
	   em_ncam &
	   ;;
	em_mis )
	   return 1
	   ;;
	em_login )
           CreateLock login-not-ready

	   em_login &

           if CheckLock login-not-ready ; then
              return 0
           else 
              return 1
           fi
	   ;;
	em_ns_server )
	   # Here we start the Names Server system.
           CreateLock ns-not-ready

           em_ns_server &

           # Wait for the lock file to be removed.  This tells us NS is
           # done initializing.
           if CheckLock ns-not-ready ; then
              return 0
           else 
              return 1
           fi
	   ;;
	em_eds )
           if ps -fe | fgrep source | fgrep -s em_eds ; then
              # eds -src is alive, so start a snk
              CreateLock eds-not-ready
              em_eds &
              if CheckLock login-not-ready ; then
                 return 0
              else 
                 return 1
              fi
           else
              # eds -src is dead; restart platform
              return 1
           fi
	   ;;
	em_log2hist )
	   em_log2hist &
	   ;;
	em_toposrv )
	   sh <<- 'EOF'
		trap "exit 0" 1
		em_toposrv -k &
		wait
		EOF
	   ;;
	em_log )
           CreateLock log-not-ready
	   em_log -d &
	   # log removes the lock 
           if CheckLock log-not-ready ; then
              return 0
           else 
              return 1
           fi
	   ;;
	em_snmp-trap )
	   em_trapd start &
	   ;;
	em_mpa_snmp | em_mpa_rpc )
#           Kill_Proc $arg
	   /etc/rc2.d/S98ipmpa start > $debug_device 2>&1
	   ;;
	em_mpa_sunmc )
		if pkginfo -q SUNWemsmc ; then
			em_mpa_sunmc &
			sunmc_pid_file=/var/tmp/em_mpa_sunmc.pid
			until [ -r $sunmc_pid_file ]
			do
				sleep 1
			done 
		fi
		;;
	em_mpa_jdmk | em_jdmkfwd )
           Kill_Proc $arg
	   /etc/rc2.d/S98jdmkmpa start > $debug_device 2>&1
	   ;;
	em_snmfwd )
	   em_snmfwd &
	   ;;
	em_cmip )
#           Kill_Proc $arg
	   /etc/rc2.d/S98cmipmpa start > $debug_device 2>&1
	   ;;
	em_nnmpa )
	   em_nnmpa &
	   ;;
	em_purged )
	   em_purged
	   ;;
	jme_jre )
#           Kill_Proc $arg
	   $EM_HOME/bin/jme_services -start
	   ;;
      em_ob_corba_rgw | em_ob_corba_mgw | em_ob_corba_epr | em_ob_corba_eds |\
      em_vb_corba_rgw | em_vb_corba_mgw | em_vb_corba_epr | em_vb_corba_eds |\
      em_io_corba_rgw | em_io_corba_mgw | em_io_corba_epr | em_io_corba_eds )
           Kill_Proc $arg
	   $EM_HOME/bin/em_cgw_services -start_no_kill
	   ;;
	em_datad )
	   $EM_HOME/bin/em_datad &
	   ;;
	em_autod )
	   em_autod &
	   ;;
	em_srm )
	   $EM_HOME/bin/em_srm &
	   ;;
esac
return 0
}

############################################################################
# Function Kill_Proc {arg}
############################################################################
# arg: processname
# Kill process {arg}
############################################################################

Kill_Proc() {

sig=HUP
proc=$1

	case $proc in
        em_ob_corba_rgw | em_ob_corba_mgw | em_ob_corba_epr | em_ob_corba_eds |\
        em_vb_corba_rgw | em_vb_corba_mgw | em_vb_corba_epr | em_vb_corba_eds |\
        em_io_corba_rgw | em_io_corba_mgw | em_io_corba_epr | em_io_corba_eds )
		if pkginfo -q SUNWemcgr ; then
            	   $EM_HOME/bin/em_cgw_services -stop -force
    		fi
	   ;;
	   jme_jre ) 
		if pkginfo -q SUNWemjme ; then
		   $EM_HOME/bin/jme_services -stop
		fi
	   ;;
	   em_mpa_snmp | em_mpa_rpc )
		if pkginfo -q SUNWemipa ; then
		   /etc/rc2.d/S98ipmpa stop > $debug_device 2>&1
		fi
	   ;;
	   em_mpa_jdmk | em_jdmkfwd )
		if pkginfo -q SUNWemjmk ; then
		   /etc/rc2.d/S98jdmkmpa stop > $debug_device 2>&1
		fi
	   ;;
	   em_snmp-trap )
		if [ -x $EM_HOME/bin/em_trapd ]; then
		   $EM_HOME/bin/em_trapd stop
		fi
	   ;;
	   oninit )
		kill_database $sig
	   ;;
	   obj_op1 | obj_op2 )
	   # do nothing
	   ;;
	   em_ncam | em_datad | em_autod | em_srm | \
           em_snmfwd | em_nnmpa | \
           em_purged | em_login | em_mis | em_log2hist | \
	   em_log | em_toposrv | em_autoexd | em_cmip | \
	   em_eds | em_ns_server | \
	   em_auxdb ) 
                pid=`get_pid $proc`
		kill -$sig $pid > /dev/null 2>&1
	   ;;
	esac
  
   return 0
}

###############################################################################
#
# Read_escalation_file
#
###############################################################################
Read_escalation_file() {

if [[ -r $ESCALATION_FILE ]] ; then
   i=1
   while [[ $i -le 4 ]] ; do
      act=`nawk -v lvl=$i '  
           $1 == "^#.*" { next }
           $0 == "" { next }
           $1 == lvl { 
             if ( NF > 1 ) {
                print substr($0, index($0, $2))
             }
           }' $ESCALATION_FILE`
      if [[ -n "$act" ]]; then
         ESCALATION_ACTION[$i]="$act"
      fi
      i=$(($i+1))
   done
fi

}

###############################################################################
# escalation_level
###############################################################################
escalation_level() {
   if [[ -f /tmp/emha_escalate4 ]] ; then
     echo 4
     return
   fi
   if [[ -f /tmp/emha_escalate3 ]] ; then
     echo 3
     return
   fi
   if [[ -f /tmp/emha_escalate2 ]] ; then
     echo 2
     return
   fi
   if [[ -f /tmp/emha_escalate1 ]] ; then
     echo 1
     return
   fi
   echo 0
   return

}

###############################################################################
#
#           MAIN
#
###############################################################################

# Fail if the required EM package is not installed.
if pkginfo -q SUNWemalb ; then
    em_dir=`pkgparam SUNWemalb BASEDIR`
    PATH=${PATH}:$em_dir/SUNWconn/em/bin
    em_dir=$em_dir/SUNWconn/em/bin
else
    exit 1
fi


# default actions
ESCALATION_ACTION[1]="em_services -start"
ESCALATION_ACTION[2]="em_services -stop -force"
ESCALATION_ACTION[3]="em_services -stop -force"
ESCALATION_ACTION[4]="em_services -stop -force"

# Set up EM environment.
. $em_dir/emenv.sh

if [[ -z $MON_DISPLAY ]]; then
   if tty -s ; then
      MON_DISPLAY=/dev/tty
   else
      MON_DISPLAY=/dev/console
   fi
fi

debug_device=/dev/null

ESCALATION_FILE=$em_dir/escalation_info

# Time out for the locks.  Unit=5secs
typeset -i TIMEOUT=60

typeset -i i
typeset -i col

sleep 10

# Read escalation action file
Read_escalation_file

esc_level=`escalation_level`
if [[ $esc_level -gt 1 ]]; then
	services_pid=`get_pid em_services`
	if [ ! -z $services_pid ] ; then
		kill -9 $services_pid
	fi
fi
if [[ $esc_level -gt 0 ]]; then
  errmsg "fixit: At escalation level $esc_level ..." > $MON_DISPLAY
  errmsg "fixit: executing ${ESCALATION_ACTION[$esc_level]}" > $MON_DISPLAY
  ${ESCALATION_ACTION[$esc_level]} &
  exit 0
fi


# Get list of daemons started by em_services
list=$(</tmp/emha_daemon_list)
col=1
i=1
for x in $list ; do
   if [[ col -eq 1 ]] ; then
      daemon_list[$i]=$x
      col=0
   else
      number[$i]=$x
      col=1
      i=$(($i+1))
   fi
done
  
Daemons=$(($i-1))

# Find which daemons are dead
dead_list=
i=1
while [[ $i -le $Daemons ]]; do
   d=${daemon_list[$i]}
   n=${number[$i]}
	pid=`get_pid $d`
   if [[ -z $pid ]]; then
      if [[ $n -gt 0 ]] ; then
         dead_list="$dead_list $d"
      fi
   else
      set $pid
      if [[ $# -lt $n ]] ; then
         dead_list="$dead_list $d"
      fi
   fi
i=$(($i+1))
done

#echo "fixit: $dead_list" > $MON_DISPLAY

for p in $dead_list; do
   restart $p
   stat=$?
   case $stat in
      0) 
         errmsg "fixit: At escalation level 0 ..." > $MON_DISPLAY
         errmsg $p restarted > $MON_DISPLAY
      ;;
      1)
         errmsg "fixit: At escalation level 0 ..." > $MON_DISPLAY
         errmsg "fixit: executing ${ESCALATION_ACTION[1]}" > $MON_DISPLAY
         ${ESCALATION_ACTION[1]} &
         exit 1
      ;;
   esac
done

exit 0
