#!/sbin/sh

#pragma ident "@(#)reconf_ener	1.109   95/08/17 SMI"

#
# 	Copyright (C) 1994 Sun Microsystems, Inc.
#

#
# reconf_ener - Energizer cluster reconfiguration program.
#

#
# XXX - TODO
#
# More robust handling of temporary files
#
# Cleanup file names and location of temporary files
#

# setenv TRACE_PDB to enable debugging traces on either stdout
# or the log file (depending on who/what invokes reconf_ener)

export PATH
PATH=/usr/sbin:/usr/bin

pre="SUNWcluster.reconf"

log_trace() {
	if [ -n "$TRACE_PDB" ]; then
		echo "# + $cmd: $*" >&3;
	fi
}

log_trace_end() {
	if [ -n "$TRACE_PDB" ]; then
		echo "# - $cmd: $*" >&3;
	fi
}

init() {
	log_trace init

	mybin=/opt/SUNWcluster/bin
	myetc=/etc/opt/SUNWcluster
	myvar=/var/opt/SUNWcluster

#
# Ok if we are running on a version of the os that contained
# a bundled version of the SSA packages then we will use 
# /usr/sbin/ssaadm, otherwise we will use our standard
# /opt/SUNWssa/bin/ssacli
#
	if [ -x /usr/sbin/ssaadm ]; then
	 SSACLI=/usr/sbin/ssaadm
	else
	 SSACLI=/opt/SUNWssa/bin/ssacli
	fi
	PATH=${PATH}:${mybin}

        logfile=${myvar}/pdbadmin.log		# needs to be in sync with pdbadmin

        cdbfile=${myetc}/conf/${clustname}.cdb
	cdbfilter=${myetc}/conf/cdb.filter
        tmpdir=`${mybin}/cdbmatch env.tmpdir ${cdbfile}`

	if [ -z "$tmpdir" ]; then tmpdir=${myvar}; fi

	ccm_selected_net_file=`${mybin}/cdbmatch ccm.script.net.file ${cdbfile}`

	 DOINGSTOPFLAG=${tmpdir}/didstopprogs			# hooks for execution of user cluster applications
	  RESERVEDCTLS=${tmpdir}/ssa_is_reserved		# control disk reservation
	FORCESTARTFLAG=${tmpdir}/do_not_use_ssa_reservations	# ok to continue without majority quorum?
	 ISRUNNINGFLAG=${tmpdir}/cluster_is_running		# is cluster is already online
	  RSUMDRUNNING=${tmpdir}/rsumd_is_running		# is rsumd running

	#
	# On a two-node cluster, the quorum disk protocol interacts with CMM to
	# identify and remove stale nodes from the cluster. This is necessary
	# because CMM doesn't use majority voting to determine the most recent
	# reconfiguration sequence number. We use a modified algorithm that
	# requires a node to keep in stable storage the last sequence number seen
	# at the beginning of cmmstep1 in stable storage. When a node joins starts
	# up, CMM resets its reconfiguration sequence number to this value.
	#

	  CMMSEQNUM=`${mybin}/cdbmatch cmm.sequencefile ${cdbfile}`

	if [ -z "$CMMSEQNUM"   ]; then   CMMSEQNUM=/var/opt/SUNWcluster/cmm/reconf-seqnum.${clustname}; fi

	if [ ! -d $tmpdir ]; then mkdir -p $tmpdir; fi

	log_trace_end init
}

# This function verifies that the required env variables are set by clustd.
# Otherwise, it will issue appropriate clustm requests to set them.
validate_env_vars()
{
	if [ "X${CURRNODES}" = "X" ]; then
		currnodes=`${mybin}/clustm getcurrmembers $clustname`
	else
		currnodes=${CURRNODES}
	fi

	if [ "X${LOCALNODEID}" = "X" ]; then
		localnodeid=`${mybin}/clustm getlocalnodeid $clustname`
	else
		localnodeid=${LOCALNODEID}
	fi

	if [ "X${SEQNUM}" = "X" ]; then
		seqnum=`${mybin}/clustm getseqnum $clustname`
	else
		seqnum=${SEQNUM}
	fi

	if [ "X${RESTART_CCMD}" = "X" ]; then
		RESTART_CCMD="Y"
	fi
}

build_ssa_path_file()
{
        Tmp=/var/tmp/ssapath.$$
        Tmp2=/var/tmp/ssapath2.$$
	touch $Tmp2
	PLUTOFILE=`eval enmatch ssa.file`
 	DEVICEDIR=`eval enmatch ssa.devdir`
        findssa $Tmp >/dev/null 
	log_trace build_ssa_path_file
	ctllist=`eval enmatch ctlreserve.node.ctllist`
	(
	  cd $DEVICEDIR	
		for ctl in . ${ctllist}; do
	         if [ "." = $ctl ]; then continue; fi
		 sn=`echo $ctl | sed -e 's/^\(....\)0*/SUNW,pln@[ab]0*\1,/'`
		 realpath=`/bin/grep -i $sn $Tmp |tee -a $Tmp2`
		 /bin/rm -rf $ctl
		 /bin/ln -s $realpath $ctl
		done
	)
	mv $Tmp2 $PLUTOFILE
	/bin/rm -rf $Tmp $Tmp2
	log_trace_end build_ssa_path_file
}

get_real_ssa_path()
{
 Tmp=/var/tmp/Tmp.$$
 serial_num=$1
 log_trace get_real_ssa_path
 DEVICEDIR=`eval enmatch ssa.devdir`
 PLUTOFILE=`eval enmatch ssa.file`
 cd $DEVICEDIR
 realpath=`/bin/ls -l $1 | sed -e 's/^.*->//'`
 echo $realpath
# sn=`echo $serial_num | awk '{print substr($1,5)}'`
## if /bin/grep -i $sn $PLUTOFILE >$Tmp 2>/dev/null
# then
#   cat $Tmp
# else
##   return 0
# fi
 /bin/rm -rf $Tmp
 log_trace_end get_real_ssa_path
}

############################################################################

_usage() {
	echo "Usage: ${prog} [-a] [-f] startnode cluster_name"
	echo "       ${prog} [-a]      stopnode  cluster_name"
	echo "       ${prog} reldisks cluster_name"
	echo ""
	echo "       The [startnode|stoptnode] sub-commands are invoked"
	echo "       manually by the system administrator or automatically"
	echo "       from /etc/rc3.d scripts via the 'pdbadmin' script."
	echo ""
	echo "       The reldisks sub-command is invoked manually by the"
	echo "       system administrator, usually by the 'pdbadmin' script."
	exit 2

	# The following are not public entrypoints

	echo ""
	echo "       ${prog} [cmmstart|cmmstop|cmmabort]  cluster_name"
	echo "       ${prog} [cmmstep0-9|cmmreturn]       cluster_name"
	echo ""
	echo "       The [cmmstart|cmmstepN|cmmstop|cmmabort|cmmreturn]"
	echo "       commands are invoked by the cluster membership"
	echo "       monitor during cluster state transitions."
	exit 2
}

# lookup a value in the pdb configuration file
enmatch() {
	${mybin}/cdbmatch $* ${cdbfile} || \
		(log_error "$pre.4001" "cdbmatch $* ${cdbfile} failed" 1>&2; return 1)
}

# wrapper for some unix commands that reports errors to syslog
unix_cmd() {
	(
	set +e
	errfile=${tmpdir}/err.$$
	result=`$* 2>${errfile}`
	status=$?
	if [ ${status} != 0 ]; then
		log_error "$pre.4003" "'$*' failed: `cat ${errfile}`"
		rm -f ${errfile}
		set -e
		exit 1
	fi
	rm -f ${errfile}
	set -e
	exit 0
	)
}

# networks utilities
set_network_vars() {
	log_trace set_network_vars
	myuname=`uname -n`
	if [ `enmatch cluster.node.0.hostname` = ${myuname} ]; then
		myid=0
	elif [ `enmatch cluster.node.1.hostname` = ${myuname} ]; then
		myid=1
	else
		log_info "$pre.4010" "${clustname} node ${myuname} is not defined as part of this cluster in the ${clustname}.cdb file"
		exit 1
	fi

	str="enmatch cluster.node.${myid}.hahost.0"
	set -- `eval $str`
	hahost_ipaddrs0=$1

	netmask=`enmatch cluster.subnet_mask`

	# -1 means no net is in use; -2 means we don't know ...
	if [ -r ${ccm_selected_net_file} ]; then
		selected_net=`cat ${ccm_selected_net_file}`
		if [ "X${selected_net}" = "X" ]; then
			/bin/rm -f ${ccm_selected_net_file}
			selected_net="-2";
		elif [ ${selected_net} -ne -1 -a \
			${selected_net} -ne 0 -a ${selected_net} -ne 1 ]; then
			/usr/bin/rm -f ${ccm_selected_net_file}
			selected_net="-2"
		fi
	else
		selected_net="-2"
	fi
	log_trace_end set_network_vars
}


# start networks
start_networks() {
	log_trace start_networks

	set_network_vars

	str="enmatch cluster.node.${myid}.if.0"
	set -- `eval $str`
	phost_interface0=$1
	str="enmatch cluster.node.${myid}.phost.0"
	set -- `eval $str`
	phost_ipaddrs0=$1

	str="enmatch cluster.node.${myid}.if.1"
	set -- `eval $str`
	phost_interface1=$1
	str="enmatch cluster.node.${myid}.phost.1"
	set -- `eval $str`
	phost_ipaddrs1=$1

	# if no history was available, try net zero
	if [ ${selected_net} = -1 -o ${selected_net} = -2 ]; then
		selected_net=0;
	fi

	str="enmatch cluster.node.${myid}.haiface.0"
	set -- `eval $str`
	hahost_interface0=$1
	str="enmatch cluster.node.${myid}.haiface.1"
	set -- `eval $str`
	hahost_interface1=$1

	str="enmatch cluster.node.${myid}.haiface.${selected_net}"
	set -- `eval $str`
	hahost_interface=$1

	unix_cmd /sbin/ifconfig $phost_interface0 plumb
	unix_cmd /sbin/ifconfig $phost_interface1 plumb

	unix_cmd /sbin/ifconfig $hahost_interface0 0.0.0.0 down
	unix_cmd /sbin/ifconfig $hahost_interface1 0.0.0.0 down

	unix_cmd /sbin/ifconfig $phost_interface0 0.0.0.0 down
	unix_cmd /sbin/ifconfig $phost_interface1 0.0.0.0 down

	unix_cmd /sbin/ifconfig $phost_interface0 $phost_ipaddrs0  \
		netmask $netmask broadcast + -trailers private up
	unix_cmd /sbin/ifconfig $phost_interface1 $phost_ipaddrs1  \
		netmask $netmask broadcast + -trailers private up

	#
	# Map the ha network on the first physical link. ccm will remap
	# the ha network if the link is found faulty.
	#
	unix_cmd /sbin/ifconfig $hahost_interface $hahost_ipaddrs0  \
		netmask $netmask broadcast + -trailers private up
	sleep 1

	echo ${selected_net} > ${ccm_selected_net_file}
	${mybin}/fsync ${ccm_selected_net_file}

	ifvar=phost_interface${selected_net}
	ifname=`eval echo \\$$ifvar`
	# log_info "$pre.1020" "${clustname} net ${selected_net} ($ifname) selected (default)"

	log_trace_end start_networks
}


# select netowrk "n"
select_network() {
	log_trace select_network $1
	netnumber=$1

	str="enmatch cluster.node.${myid}.haiface.${netnumber}"
	set -- `eval $str`
	hahost_interface=$1

	# Leave a trail in case we are killed after ifconfig but before 
	# getting the chance to write the network id on the disk.
	echo -2 > ${ccm_selected_net_file}
	${mybin}/fsync ${ccm_selected_net_file}

	unix_cmd /sbin/ifconfig ${hahost_interface} $hahost_ipaddrs0 \
		netmask $netmask broadcast + private up

	echo ${netnumber} > ${ccm_selected_net_file}
	${mybin}/fsync ${ccm_selected_net_file}

	str="enmatch cluster.node.${myid}.if.${netnumber}"
	set -- `eval $str`
	ifname=$1
	log_info "$pre.1030" "${clustname} net ${netnumber} ($ifname) selected"
	log_trace_end select_network $1
}


# deselect netowrk "n"
deselect_network() {
	netnumber=$1
	newnetnumber=$2

	log_trace select_network $netnumber

	str="enmatch cluster.node.${myid}.haiface.${netnumber}"
	set -- `eval $str`
	hahost_interface=$1

	unix_cmd /sbin/ifconfig ${hahost_interface} 0.0.0.0 down

	echo ${newnetnumber} > ${ccm_selected_net_file}
	${mybin}/fsync ${ccm_selected_net_file}

	str="enmatch cluster.node.${myid}.if.${netnumber}"
	set -- `eval $str`
	ifname=$1

	log_info "$pre.5010" "${clustname} net ${netnumber} ($ifname) de-selected"
	log_trace_end select_network $netnumber
}

ccmevent_cmd () {
	log_trace ccmevent_cmd

	set_network_vars

	# Note: the variables below are passes to reconf_ener as environment
	# variables by ccmd.
	#
	# acthosts : list a nodes currently in the cluster.
	#
	# select :  The id of the net (if any) currently in use by the 
	#		other host.
	#
	# top    : The list of nets thru which I can reach the maximum
	#		number of hosts.  These nets are eligible for
	#		being selected.
	# failed : Any net which is not in the "top" list.
	# topnets: The number of nets in the "top" list.

	acthosts=${ACTHOSTS}
	select=${SELECT}
	top=${TOP}
	topnets=${TOPNETS}
	failed=${FAILED}

	# valid network state can only be determined if there is
	# more than one active node
	if [ ${acthosts} = 1 ]; then
		return
	fi

	# if a net is already in use by the other host and is eligible, that 
	# is the one to pick.
	if [ ${select} != -1 ]; then
		for i in $failed; do
			if [ ${i} = ${select} ]; then
				select=-1
			fi
		done
	fi

	# if the current net can be used, don't change nets (keep using it).
	if [ ${select} = -1 ]; then
		for i in $top; do
			if [ ${i} = ${selected_net} ]; then
				select=${selected_net}
			fi
		done
	fi

	if [ ${select} = -1 ]; then	# nope, selected net has failed,
					# need to pick a new selected net
					# from any of the available networks
		if [ ${topnets} != 0 ]; then
			# pick the first in the list of good networks
			set -- $top
			select=$1
		else
			log_info "$pre.4020" "${clustname} no active interconnect networks"
			# A non-zero exit code will cause CCM to force a
			# reconfiguration.  If we are dealing with a 
			# persistent problem, after cmm.reconfig.max attempts
			# clustd will abort one of the nodes in the cluster
			# and try again.
			exit 8
		fi
	fi


	# Up/Down state reporting is done by the ccmd

	# There is a window in select_network where if the script is killed
	# after the ifconfig but before writting the net id to disk where
	# we really don't know which net is in use.  If that happens, we 
	# make sure that all nets except the selected one are put out of use.
	# The same holds if the file usually used to store the active net
	# id can not be located.
	# -1 means no net is in use; -2 means we don't know ...

	# bring down the old network if it is different
	if [ ! ${select} = ${selected_net} ]; then
		if [ ${selected_net} = -2 ]; then
			for i in $failed $top; do
				if [ ${select} != ${i} ]; then
					deselect_network ${i} ${select}
				fi
			done
		# only if the net was actually up
		elif [ ${selected_net} != -1 ]; then
			deselect_network ${selected_net} -1
			selected_net=-1
		fi
	fi

	# bring up the new net
	if  [ ${select} != -1 ]; then
		select_network ${select}
	fi

	log_trace_end ccmevent_cmd
	exit 0
}

# if $1 != 0 kill ccmd and wait for it to release all its resources
# else check to make sure that ccmd is alive.  If not, set RESTART_CCMD="Y".
kill_ccmd() {
        pfile=`enmatch ccm.pidfile`

	RESTART_CCMD="Y"

        if [ -f ${pfile} ]; then
                pid=`cat ${pfile}`
		set $1 ${pid}
		if [ "$2" != "" -a "$2" != "0" ]; then
			if [ "$1" != "0" ]; then
				while /usr/bin/kill -USR2 -$2 1>/dev/null 2>&1
					do
					sleep 1
					done
			elif [ ! /usr/bin/kill -0 $2 1>/dev/null 2>&1 ]; then
				RESTART_CCMD="N"
			fi
		fi
                /usr/bin/rm -f ${pfile}
        fi
        # log_info "$pre.2040" "${clustname} terminated ccmd"
}

# XXX - should we specify the reservation timeouts in the cdb file?
# reserve a Pluto controller (don't fail if controller is not accessible)
reserve_1_ctl() {
	ctl=$1
	${mybin}/timed_run 3 ${SSACLI} reserve ${ctl} 1>/dev/null 2>&1 || return 0
}

# release a Pluto controller (don't fail if controller is not accessible)
release_1_ctl() {
	ctl=$1
	${mybin}/timed_run 3 ${SSACLI} release ${ctl} 1>/dev/null 2>&1 || return 0
}

# reserve a quorum Pluto controller
reserve_quorum_ctl() {
	log_trace reserve_quorum_ctl
	quorum_ctl=`eval enmatch ctlreserve.node.quorumctl`
	if [ -n "$quorum_ctl" ]; then
		log_info "$pre.1010" "${clustname} reserving $quorum_ctl as quorum controller"
		if [ ${quorum_ctl} ]; then
			real_quorum_ctl=`get_real_ssa_path ${quorum_ctl}`
			${mybin}/timed_run 3 \
				${SSACLI} reserve ${real_quorum_ctl} 1>/dev/null 2>&1 || ( \
		log_info "$pre.4040" "${clustname} failed to reserve $real_quorum_ctl as quorum controller"
		echo "Warning: could not reserve the SSA quorum controller (${real_quorum_ctl})" >&3
					exit 1)
		fi
	fi
	log_trace_end reserve_quorum_ctl
}

# release a quorum Pluto controller
release_quorum_ctl() {
	log_trace release_quorum_ctl
	quorum_ctl=`eval enmatch ctlreserve.node.quorumctl`
	if [ -n "${quorum_ctl}" ]; then
		real_quorum_ctl=`get_real_ssa_path $quorum_ctl`
		release_1_ctl ${real_quorum_ctl}
	fi
	log_trace_end release_quorum_ctl
}

# reserve all Pluto controllers
reserve_all_ctls() {
	log_trace reserve_all_ctls
	ctllist=`eval enmatch ctlreserve.node.ctllist`
	(
		for ctl in . ${ctllist}; do
			if [ "." = $ctl ]; then continue; fi
			rctl=`get_real_ssa_path $ctl`
			reserve_1_ctl ${rctl} &
		done
		wait
	)
	log_trace_end reserve_all_ctls
}

# release all Pluto controllers
release_all_ctls() {
	log_trace release_all_ctls
	ctllist=`eval enmatch ctlreserve.node.ctllist`
	(
		for ctl in . ${ctllist}; do
			if [ "." = $ctl ]; then continue; fi
			rctl=`get_real_ssa_path $ctl`
			release_1_ctl ${rctl} &
		done
		wait
	)
	log_trace_end release_all_ctls
}

check_cdbfile() {
	for i in $*
	do
		if [ $i = `uname -n` ]; then
			return 0
		fi

		set +e
		${mybin}/checkrfile -p ${cdbfilter} -h $i -f ${cdbfile}
		status=$?
		set -e

		if [ ${status} = 2 ]
		then ${mybin}/clustm reconfigure ${clustname} >/dev/null 2>&1
		elif [ ${status} = 1 ]
		then exit 1
		fi
	done
}

stop_rsumd() {
	if [ -r ${RSUMDRUNNING} ]; then
		set +e
		${mybin}/rsumd_stop
		set -e
		/bin/rm -f ${RSUMDRUNNING}
	fi
}

start_rsumd() {
	if [ ! -r ${RSUMDRUNNING} ]
	then
		${mybin}/rsumd -F ${cdbfilter}
		/usr/bin/touch ${RSUMDRUNNING}
	fi
}

cvm() {
	if [ -f /usr/sbin/vxclust ]
	then
		if [ $1 = "stop" ]
		then
			priocntl -e -c RT /usr/sbin/vxclust $*
		else
			/usr/sbin/vxclust $*
		fi
	fi
}

# join the local node with an Energizer cluster
startnode_cmd() {
	log_trace startnode
	#
	# XXX - here we should do sanity checks to assure that the node
	# 	is well configured for joining the cluster.

	# check that cluster is not already running
	if [ -f ${ISRUNNINGFLAG} ]; then
		# check if clustd is really running
		${mybin}/timed_run -q 3 ${mybin}/clustm getstate ${clustname} >/dev/null 2>&1 && \
		(
			echo "    This node is already running as part of the ${clustname} cluster" >&3
			exit 1
		)
	fi

	nodename=`eval /bin/uname -n`
	numnodes=`enmatch cmm.nodes`
	i=0;
	while [ $i -lt $numnodes ]; do
		if [ `enmatch cluster.node.$i.hostname` = ${nodename} ]; then
			break;
		fi
		i=`expr $i + 1`
	done
	log_info "$pre.1150" "Starting PDB; node $i ($nodename) joining the ${clustname} cluster."
	echo "Starting SPARCcluster PDB software - joining the ${clustname} cluster." >&3

	touch ${ISRUNNINGFLAG}

	# cleanup stale files
        pfile=`enmatch ccm.pidfile`
	if [ -f ${pfile} ]; then
		/bin/rm -f ${pfile}
	fi
	if [ -f ${ccm_selected_net_file} ]; then
		/bin/rm -f ${ccm_selected_net_file}
	fi

	start_networks

	# avoid confusion over stale sequence numbers
	/usr/bin/rm -f ${CMMSEQNUM}

	# start the dlmd if necessary
        pfile=`enmatch dlm.pidfile`
        if [ ! -s "$pfile" -o `ps -e | grep dlmd | wc -l` -eq 0 ]; then
		# log_info "$pre.2050" "${clustname} starting dlmd"
		${mybin}/dlmd -f ${cdbfile} > /dev/console 2>&1
	fi

	# Rebuild the ccm.ssa file here.
	build_ssa_path_file

	${mybin}/clustd -f ${cdbfile}	# stdout/err are redirected by caller to the logfile

	/bin/rm -f ${DOINGSTOPFLAG}

	if [ "${forcestart}" = 1 ]; then
		touch ${FORCESTARTFLAG}
		log_info "$pre.2030" "${clustname} cluster started with -f (force) option"
	else
		/bin/rm -rf ${FORCESTARTFLAG}
	fi


	# unless the '-a' flag was specified, wait for the node to do
	# the first reconfiguration.
	if [ "${async}" != 1 ]; then
		while [ -f  ${ISRUNNINGFLAG} ]; do
			# check if clustd is still running
			state=`${mybin}/timed_run -q 10 ${mybin}/clustm getstate ${clustname} 2>/dev/null` ||\
				 exit 1
			if [ "$state" = "end" ]; then
				break
			else
				sleep 1
			fi
		done
	fi
	if [ ! -f  ${ISRUNNINGFLAG} ]; then
		exit 1
	# else
	#	 echo "This node is now running as part of the ${clustname} cluster." >&3
	fi

	log_trace_end startnode
}

# called from cluster membership monitor "start" transition
cmmstart_cmd() {
        (((${mybin}/dlmctl start $clustname `enmatch dlm.start_timeout`) || echo $? >> /${myvar}/dlmctl.start) &
        ((cvm start $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.start)&
        wait)
        if [ -s /${myvar}/dlmctl.start ] || [ -s /${myvar}/cvm.start ]
        then
                /bin/rm -f /${myvar}/dlmctl.start /${myvar}/cvm.start
                exit 1
        fi

	stop_rsumd
	start_rsumd
}

# pdb cluster graceful shutdown
stopnode_cmd() {
	log_trace stopnode

	if [ ! -f ${ISRUNNINGFLAG} ]; then
		echo "The SPARCcluster PDB software is not currently running on this node." >&3
		# log_info "$pre.4060" "${clustname} cluster is not currently running on this node"
		exit 1	# for bugID 1166404
	fi

	echo "Stopping the SPARCcluster PDB software - leaving the ${clustname} cluster" >&3


        cvm stop $clustname ${cdbfile}
	touch ${DOINGSTOPFLAG}

	kill_ccmd 1

	validate_env_vars
	${mybin}/clustm stop ${clustname} this

	# unless the '-a' flag was specified, wait for the node to shutdown
	if [ "${async}" != 1 ]; then
		while [ -f ${ISRUNNINGFLAG} ]; do
			sleep 1
		done
	fi
	echo "This node is no longer running as part of the ${clustname} cluster." >&3
	if [ "$currnodes" = "$localnodeid" ]; then
		echo "The ${clustname} cluster has no active hosts." >&3
	fi
	# log_info "$pre.4070" "${clustname} cluster is stopped on this node"
	log_trace_end stopnode
}

# called from cluster membership monitor "abort" and "stop" transitions
cmmabort_cmd() {
	log_trace cmmabort_cmd

	stop_rsumd

	log_info "SUNWcluster.cvm.5010" "disabling cluster volume manager shared access mode"

	pfile=`enmatch dlm.pidfile`
        (
	if [ ! -f ${DOINGSTOPFLAG} ]; then
		((cvm abort $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.abort)&
	fi

        if [ -s "$pfile" ]
        then
		((${mybin}/dlmctl abort $clustname `enmatch dlm.abort_timeout` \
                   ${pfile}) || echo $? >> /${myvar}/dlmctl.abort)&
        fi
        wait)

        if [ -s /${myvar}/dlmctl.abort ] || [ -s /${myvar}/cvm.abort ]
        then
                /bin/rm -f /${myvar}/dlmctl.abort /${myvar}/cvm.abort
                exit 1
        fi

	# The dlmd pid file is not removed if the dlmctl abort command
	# above was not successful.
	if [ -s ${pfile} ]; then
		/bin/rm -f ${pfile}
	fi

	if [ -f ${RESERVEDCTLS} ]; then
		release_all_ctls
		release_quorum_ctl
		/bin/rm -rf ${RESERVEDCTLS}
	fi

	/bin/rm -rf ${ISRUNNINGFLAG}

	kill_ccmd 1

	if [  -f ${DOINGSTOPFLAG} ]; then
		# note that we print cmmstop instead of cmmabort not
		# to scare the user.
		log_info "$pre.4080" "${clustname} cluster stopped on this node (`uname -n`)"
	else
		log_info "$pre.4090" "${clustname} cluster aborted on this node (`uname -n`)"
	fi

	log_trace_end cmmabort_cmd
}

# called from cluster membership monitor "return" transition
cmmreturn_cmd() {
        (((${mybin}/dlmctl return $clustname `enmatch dlm.return_timeout`) \
                || echo $? >> /${myvar}/dlmctl.return)&
        ((cvm return $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.return)&
        wait)
        if [ -s /${myvar}/dlmctl.return ] || [ -s /${myvar}/cvm.return ]
        then
                /bin/rm -f /${myvar}/dlmctl.return /${myvar}/cvm.return
                exit 1
        fi

	start_rsumd
}

# called from cluster membership monitor "step1" transition
cmmstep1_cmd() {
	validate_env_vars

        set -- $currnodes
        names=""
        while [ ! -z "$*" ]; do
                thisname=`enmatch cluster.node.$1.hostname`
                  names="$names $thisname"
                shift;
        done

	check_cdbfile ${names}

	# log_info "$pre.1110" "${clustname} reconfiguration sequence ${seqnum} started with nodes ${currnodes}"
	# log_info "$pre.1120" "${clustname} reconfiguration ${seqnum} started on ${names}"

	if [ "${RESTART_CCMD}X" = "YX" ]; then
		kill_ccmd 1
	else
		kill_ccmd 0
	fi

	if [ "${currnodes}" = "${localnodeid}" ]; then

		if [ -f ${FORCESTARTFLAG} ]; then
			log_info "$pre.1140" "${clustname} cluster bypassing reservation of quorum controller"
		else
			reserve_quorum_ctl
		fi
		touch ${RESERVEDCTLS}
		reserve_all_ctls
	else
		# the cluster includes both nodes

		# log_info "$pre.1130" "${clustname} starting ccmd"
		if [ "${RESTART_CCMD}X" = "YX" ]; then
			${mybin}/ccmd -w -m ${localnodeid} -c \
				${cdbfile} ${currnodes} &
		fi

		if [ -f ${RESERVEDCTLS} ]; then
			release_all_ctls
			release_quorum_ctl
			/bin/rm -rf ${RESERVEDCTLS}
		fi
	fi
	/bin/rm -rf ${FORCESTARTFLAG}
        (((${mybin}/dlmctl step1_1 $clustname `enmatch dlm.step1_1_timeout`) \
                || echo $? >> /${myvar}/dlmctl.step1) &
        ((cvm step1 $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.step1) &
        wait)
        if [ -s /${myvar}/dlmctl.step1 ] || [ -s /${myvar}/cvm.step1 ]
        then
                /bin/rm -f /${myvar}/dlmctl.step1 /${myvar}/cvm.step1
                exit 1
        fi
}

# called from cluster membership monitor "step2" transition
# A TCP bug makes it necessary to break out the dlmd's step1 into two steps
cmmstep2_cmd() {
	${mybin}/dlmctl step1_2 $clustname `enmatch dlm.step1_2_timeout`
}

# called from cluster membership monitor "step3" transition
cmmstep3_cmd() {
	validate_env_vars
	stop_rsumd

        (((${mybin}/dlmctl step2 $clustname `enmatch dlm.step2_timeout`) \
                || echo $? >> /${myvar}/dlmctl.step2) &
        ((cvm step2 $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.step2) &
        wait)
        if [ -s /${myvar}/dlmctl.step2 ] || [ -s /${myvar}/cvm.step2 ]
        then
                /bin/rm -f /${myvar}/dlmctl.step2 /${myvar}/cvm.step2
                exit 1
        fi
}

# called from cluster membership monitor "step4" transition
cmmstep4_cmd() {
        (((${mybin}/dlmctl step3 $clustname `enmatch dlm.step3_timeout`) \
                || echo $? >> /${myvar}/dlmctl.step3) &
        ((cvm step3 $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.step3)&
        wait)
        if [ -s /${myvar}/dlmctl.step3 ] || [ -s /${myvar}/cvm.step3 ]
        then
                /bin/rm -f /${myvar}/dlmctl.step3 /${myvar}/cvm.step3
                exit 1
        fi
}

# called from cluster membership monitor "step5" transition
cmmstep5_cmd() {
       (((${mybin}/dlmctl step4 $clustname `enmatch dlm.step4_timeout`) \
                || echo $? >> /${myvar}/dlmctl.step4) &
        ((cvm step4 $clustname ${cdbfile}) || echo $? >> /${myvar}/cvm.step4) &
        wait)
        if [ -s /${myvar}/dlmctl.step4 ] || [ -s /${myvar}/cvm.step4 ]
        then
                /bin/rm -f /${myvar}/dlmctl.step4 /${myvar}/cvm.step4
                exit 1
        fi
	log_info "SUNWcluster.cvm.6010" "cluster volume manager shared access mode enabled"
}

# called from cluster membership monitor "step6" transition
cmmstep6_cmd() {
	${mybin}/dlmctl step5 $clustname `enmatch dlm.step5_timeout`
}

# called from cluster membership monitor "step7" transition
cmmstep7_cmd() {
	return 0		# step7 is reserved for future use
}

# called from cluster membership monitor "step8" transition
cmmstep8_cmd() {
	return 0		# step8 is reserved for future use
}

# called from cluster membership monitor "step9" transition
cmmstep9_cmd() {
	set +e
	validate_env_vars

	numnodes=`enmatch cmm.nodes`
	_thisnode=`expr $numnodes - 1`	# 0 .. n-1
	while [ ${_thisnode} -ge 0 ]; do
	 	set -- $currnodes
	 	_foundit="NO"
	 	while [ ! -z "$*" ]; do
	 		if [ $1 -eq ${_thisnode} ]; then
				_foundit="YES"
			fi
	 		shift;
	 	done
	 	name=`enmatch cluster.node.${_thisnode}.hostname`
	 	if [ "NO" = "${_foundit}" ]; then
	 		log_info "$pre.1930" "${clustname} node ${_thisnode} (${name}) is not a cluster member"
		else
			log_info "$pre.1920" "${clustname} node ${_thisnode} (${name}) is a cluster member"
	 	fi
	 	_thisnode=`expr ${_thisnode} - 1`
	done

	log_info "$pre.1940" "${clustname} cluster reconf #${seqnum} finished"
	set -e
}
#
# end of shell functions
#

#####################################################################################

# get program options
set -- `getopt af $*`
if [ $? != 0 ]; then
	_usage
fi
for i in  $*; do
	case $i in
	-a) async=1; shift ;;
	-f) forcestart=1; shift ;;
	--) shift; break;;
	esac
done

prog=$0
cmd=$1
clustname=$2

init $*

set -e				# exit shell script on error

#
# Dispatch the call.
#
# We are carefully passing file descriptor 2 from "startnode" to clustd
# and then from clustd to the "cmm???" calls.
#
# file descriptor #3 is the original stdout if you need to send
# messages to the interactive user.
#
case ${cmd} in
	# interactive commands
	startnode)		startnode_cmd    3>&1 1>>${logfile} 2>&1 ;;
	stopnode)		stopnode_cmd     3>&1 1>>${logfile} 2>&1 ;;

	# async commands - not interactive
	cmmstop | cmmabort)	cmmabort_cmd     3>&1 1>>${logfile} 2>&1 ;;
	cmmstart | cmmreturn | cmmstep[1-9]) \
				eval ${cmd}_cmd  3>&1 1>>${logfile} 2>&1 ;;
	ccmevent)		ccmevent_cmd     3>&1 1>>${logfile} 2>&1 ;;
	reldisks)		release_all_ctls 3>&1 1>>${logfile} 2>&1 ;;

	*)			_usage				    ;;
esac
exit 0
