#!/usr/bin/perl -I/opt/SUNWstade21/lib
#<copyright>
# ----------------------------------------------------------
# Sun Proprietary/Confidential Code
# Copyright 2001, Sun Microsystems, Inc. All rights reserved.
# ----------------------------------------------------------
#</copyright>

#
#  $Id: rasagent,v 1.80 2002/09/26 16:53:12 ccadieux Exp $
#
use strict 'vars', 'subs';
use Getopt::Std;
#use CC::Email;
#use CC::Agent;
use Util;
use Util::Http;
use Timelapse;
use PDM;
use Events;
use PDM::ConfigFile;
use Modules;
use Scheduler;
use Data::Dumper;
use Agent;
use Debug;
use System;
use PDM;
use TO;
use UNIVERSAL qw( isa ) ;
#
#  -T       : test mode, will not email, will not update cache
#  -e <#>   : Which Events directory to store events in
#  -v       : version
#  -f       : force: run agent anyway
#  -F <file>: run against a dfferent config file.
#  -P       : print output, no email
#  -O       : send everything, no cache
#  -H       : all health test are BAD
#  -d<level>: debug
#  -D       : directory for System DATA and rasagent.conf file
#  -?       : Help
#  -s       : do not save the data found
#  -c       : running from the cron.
#

# GLOBALS
use vars qw(
   $util @AGENTS $MASTER_LOC $RASPORT  %opts 
   $VERSION $SIGTERM  $HOME $DATA $CACHE $PROMPT  $BIN
   $MAILER $MESSAGE_FILE  %DONE $CONF
   $EMAIL0 $HBA  $verbose  $CRON_LOC
   $LINE_CNT $TIMETEST  $SLAVE $LOCAL_DATA
   $PIDfile $renv $devices  $hosts $frus $RESET $STORAGE
   $startDate $curr_year  $RAS_SERVER_EMAIL 
   $FORCE $device $last_config  $notifs
   @t300_log  @THR_log @A5000_log  @thresholds @data_pings
   %SFOFFL %WARNSSD
   %SFCRCWARN  %SFOFFTOWARN %SFDMAWARN %SFRESET %RETRYELS %RETRYSF
   %TOELS  %SFTOELS %DDOFFL %DDCRCWARN
   $HOST_ID $HOST_NAME
   $GOLDEN $STOR_PROCS
   %HMAP  %AMAP $SE_CFG
);


##################################
#     SIGNALS
##################################

#   sub die_signal {
#     my($sig) = @_;
#     print "Caught a signal SIG$sig -- aborting \n";
#     exit(1);
#   }
#   
#   sub catch_signal {
#     my($sig) = @_;
#     Debug->print3("Caught a signal SIG$sig") if ($sig ne "CHLD");
#   }
#   
#   $SIG{CHLD} = \&catch_signal; # 'IGNORE';
#   $SIG{TERM} = \&catch_signal;
#   $SIG{INT}  = \&die_signal;
#   
#   foreach my $sig ('SEGV', 'QUIT','INT','ILL','TRAP','IOT','ABRT',
#                     'EMT','FPE','KILL','BUS') {
#       $SIG{$sig} = \&die_signal;
#   }

##################################     SIGNALS


$SE_CFG = "/var/adm/log/SEcfglog";

if ($ENV{OPTS}) {
  @ARGV = split(/ +/, $ENV{OPTS});
}

if (!getopts("HAfeD:hcX?Pd:TvO", \%opts)) {
    die("Abort: $Getopt::Std::ERROR \n");
}

$opts{d} = 2 if ($opts{T} && !$opts{d});
$verbose = 1 if ($opts{T} && $opts{X});
$FORCE   = 1 if ($opts{f});
my $AUDIT   = 1 if ($opts{A});
System->set_bad_health(1) if ($opts{H});

System->set_audit($AUDIT);

my $force_refresh;

# RESET LOCALE

$ENV{LANG} = 'C';
$ENV{LC_ALL} = 'C';

System->set_testMode($opts{T} || 0);
my($e) = $opts{e} || "2";
System->set_eventDir($e);

my($op);
$op .= "Test on," if ($opts{T});
$op .= "Debug on," if ($opts{d});
$op .= "Print only," if ($opts{P});
if ($op) {
  chop($op);
  print STDERR "Options: $op\n";
}

&version if ($opts{v});
&help if ($opts{"?"} || $opts{"h"});

my($local_p) = $INC[0];
my($ix) = rindex($local_p, "/");

$HOME   = $opts{D} || substr($local_p,0,$ix);

System->set_home($HOME);
unlink "$HOME/DATA/snapshot.log";

if ($opts{c}) {  # from crons
  $CRON_LOC = "$HOME/System/cron_started";
  if (!-f $CRON_LOC) {
     open(O, ">" . $CRON_LOC); 
     
     print O Util->get_today;
     close(O);
     exit;
  }
}

Debug->level($opts{d});
my $LB = Labels->read('rasagent');

if (! -f "$HOME/DATA/start" && Debug->level()  < 2) { # start the ras-agent
   Debug->print2($LB->{no_start});
   exit;
}

$VERSION   = "1.0";
$EMAIL0    = "rasagent\@nscc.central.sun.com";
$TIMETEST  = 0;
$startDate = "";   # Where to start reading in the message file
$SIGTERM   = 15;
$DATA      = "$HOME/DATA";
$BIN       = "$HOME/bin";
$PIDfile   = "$DATA/pid";
$CACHE     = $DATA;
$CONF      = $opts{F} || "$DATA/rasagent.conf";

System->set_config($CONF);
System->set_schemaVersion("1.0");
System->set_runId(time);

$HOST_ID     = System->hostid;
$HOST_NAME   = System->hostname;

System->set_ipno(Util->name2ip($HOST_NAME));


$PROMPT       =  '/\w*:\/:\<\d+\>/';

my($today)    = Util->today;


if (Debug->level() > 2) {
   print $LB->expand('new_snapshot', "/DATA/snapshot.log");
}

($renv, $devices, $hosts, $notifs) = PDM::ConfigFile->read();

System->set_renv($renv);
System->set_configDevices($devices);

$renv->{"timeout.luxadm"} = 200 if (!$renv->{"timeout.luxadm"});
$renv->{"timeout.rm6"}    = 200 if (!$renv->{"timeout.rm6"});
$renv->{"timeout.discman"} = 600 if (!$renv->{"timeout.discman"});
$renv->{"solution_prefix"} = Util->ipPrefix(System->ifconfig("dmfe1"));


$MAILER              = $renv->{mailer} || "/usr/bin/mail";
$renv->{hostid}      = $HOST_ID;

#$renv->{logfile}    = "/var/adm/messages" if (!$renv->{logfile});

$RASPORT             = System->getConfigPort($HOME);
System->set_rasport($RASPORT);

$MASTER_LOC          = Util->findMaster();

my($s1) = ($MASTER_LOC) ? "SLAVE, MASTER=>$MASTER_LOC" : "MASTER";

$RAS_SERVER_EMAIL =  $renv->{email};

$util = Util->new({  
            data  => $DATA, 
            home  => $HOME, 
            renv  => $renv 
                 });
  
if (!$opts{T} && ($opts{d} < 2)) {
    if ($renv->{frequency} && !$util->is_Xmins("frequency.cache", $renv->{frequency}/2)) {
       if (!$opts{c}) {  # not from crons
           my($elapsed) = sprintf("%.1f", $Util::Xmin_value);
           print STDERR $LB->expand('freq_err', $renv->{frequency}, $elapsed) . "\n";
       }
       exit;
    }
}

print STDERR $LB->expand('running', $CONF, $today, $s1) . "\n";

if (&cron_conflicts($#$devices+1) ) {
   die($LB->{cron_conflict});
}

my($st_found);
if (!$FORCE) {
  if (($st_found= Agent->stor_conflicts()) ) {
     Debug->print2($LB->expand('diag_conflict',  $st_found));
  }
}
System->set_stFound($st_found);

$renv->{role} = "M" if (!-f System->get_home() . "/DATA/MASTER" && !$renv->{role} );

if ($renv->{role} eq "AM") {
   my $active = Util->file2string("ALTACTIVE");
   my $M = $MASTER_LOC || $active;
   my $alive;
   if (!$M) {
     Debug->err(TEXT => $LB->{no_master});
     exit(1);
   } else {
     $alive = Util->ping($M);
   }
   my $new_master;

   if (!$alive &&  !$active) { # take over
      Debug->print2($LB->expand('cannot_ping', $MASTER_LOC));

      $new_master = Util->name2ip($renv->{hostname}) || $renv->{hostname};
      Events->saveEvent("AM+",$renv->{hostname}, $MASTER_LOC);

      Util->string2file($MASTER_LOC, "ALTACTIVE");
      Util->clearMaster();
      $MASTER_LOC = undef;

   } elsif ($alive && $active) {  # release
      $new_master = $active;
      my $info = Util::Http->getCommand($new_master,"INFO1", 10);
      if ($info =~ /version/) {
        Events->saveEvent("AM-", $renv->{hostname} , $active);

        Util->setMaster($new_master);
        unlink System->get_home() . "/DATA/ALTACTIVE";
        $MASTER_LOC = $new_master;
        Debug->print2("Can ping $M, giving-up master...");
      }

   }
   if ($new_master) {
      foreach my $h (@$hosts) {
        next if ($h->{role} eq "AM" || $h->{role} eq "M");
        my $ip = $h->{ipno} || Util->name2ip($h->{hostname});
        my($err, $ans) = Util::Http->saveFile($ip, "MASTER", $new_master, 20);
      }
      PDM::ConfigFile->write($renv, $devices, $hosts, $notifs);
   }

}

if (-f $SE_CFG) {
  my ($enter, $exit,  $l, $enter_command, $enter_list);
  my (%H, $l);
  if (open(O, "/bin/ls /opt/SUNWsecfg/etc/.[stv]*lock 2>/dev/null|")) {
    while ($l = <O>) {
      $enter_list .= substr($l, 6, -6) . ",";
    }
    close(O);
  }
  if ($enter_list) {
     my $now = Util->get_today(); 
     Debug->print2("SE_CFG: Locks found on $enter_list");
     my $a1 = Util->shortHostname($renv->{hostname}) . "-" . $enter_list;
     System->set_se_conflict($a1); # skip this component;
  }
}
  
Debug->print(PID => $LB->{create_pid});

if (open(O,">$PIDfile")) {
  print O $$;
  close(O);
} else {
  Debug->print(PID =>  $LB->expand('cannot_write_pid', $PIDfile));
  exit;
}

#
# RUN FRUSTAT/MESSAGES EVERY TIME, FRU LIST EVERY DAY

$curr_year  = (localtime)[5] + 1900;


my($pdm) = PDM->new({  dir => "$HOME/DATA",
                      renv => $renv ,
                   devices => $devices,
                     hosts => $hosts,
                    notifs => $notifs,
                    });


Scheduler->cleanUp(2); # remove 2 days old tests

my $agent_list = &list_agents($pdm);

Debug->dump('PDM', $pdm, 1);

my $LH = &load_map();

&load_health_monitors($pdm, $LH);  # only INITIAL or FINAL

$pdm->initialHealth;  # check if slaves work and get their events.

my($mod, @OUT, %OUTPUT,  $ras_flag, %LOADED);


foreach my $modname (@$agent_list) {
   if ($st_found) {
      if ( index("MESSAGE,STOOLS4", $modname) < 0 ) {
         Debug->print2("\n" . $LB->expand('exe_agent', $modname));
         Debug->print2($LB->{diag_skip});
         next;
      }
   }
   if (!Agent->category_selected($modname) && 
        index("TOPO,SE", $modname) < 0) {
      Debug->print2("\n" . $LB->{skip} . " $modname.");
      next;
   }
   if (!eval "require \"Agent/${modname}.pm\"") {
        Debug->err(LOADING => "$modname: $@ \n");
        next;
   }
   
   Debug->print1("\n" . $LB->expand('exe_agent', $modname) );

   next if (!&load_health($modname) );
   my $f = "Agent::$modname";
   $mod = $f->new();

   $mod->init({  data  => $DATA, 
                util  => $util,
                today => $today, 
                pdm   => $pdm, 
               todayY => Util->today("YMDH"),
               hostid => $HOST_ID,
             hostname => $HOST_NAME,
                opt_T => $opts{T},
                opt_O => $opts{O},
                renv  => $renv, 
                hosts => $hosts,
               master => $MASTER_LOC,
                home  => $HOME
             });

   if ($mod->can('RUN')) {
      $mod->RUN($ras_flag);

   } else {
      Debug->err('RUN_MISSING', ref($mod) );   
   }
}

$pdm->finalHealth;  # final health: heartbeat..

#############################################
#      PROVIDERS
#############################################

$DB::single = 1;
Debug->print2("\n" . $LB->{exe_pro});

my($eventCount) = $pdm->getEventCount;

Debug->print2("$eventCount new Events");

if ($MASTER_LOC) {  # a slave
  $pdm->storeMessages;
  #$pdm->push($MASTER_LOC);

} else {
  my $providers = Modules->load("Provider");

  foreach my $p (@$providers) {
    my $pro = "Provider::$p";
    my $f = $pro->new($pdm);
    
    if ($f->can('RUN')) {
       Debug->print1("\nProvider $p ");
       $f->RUN($pdm);
    } else {
       Debug->err('RUN_MISSING', ref($pro) );   
    }
  }
}

if ( !$opts{s}) {
    Agent->save_all_caches($util);  # save all caches.
    $pdm->serialize();
}
#
# force-push if the master moved
#

Report->pushReportsList($MASTER_LOC, $force_refresh);

if ($MASTER_LOC) {  # a slave
  State->push($MASTER_LOC, $force_refresh);
  # TO->push(               $MASTER_LOC, $force_refresh);
  # Catalog->push(          $MASTER_LOC, $force_refresh);
}
Timelapse->save_all;

Util->set_Xmins("frequency.cache");
Debug->print(PID => $LB->{remove_pid});
unlink $PIDfile;

exit;  


##########################################################
#                  SUBROUTINES
##########################################################

sub load_map {
  my $D  = System->get_home() . "/lib/Health";
  my $grep = (-x "/usr/bin/grep")? "/usr/bin/grep" : "/bin/grep";
  my @LH = `$grep reportRequest $D/*.pm`;

  foreach my $l (@LH) {
     my($fn, $rest) = split(/\:/, $l, 2);
     my $ix = rindex($fn, "/");
     $fn = substr($fn,$ix+1) if ($ix > 0);
     if ($rest =~ /Report::CAT_(\w+)/) {
       $HMAP{$1} .= substr($fn,0,-3) . "|";
     }
  }

  my $D  = System->get_home() . "/lib/Agent";
  my @LA = `$grep Report::CAT_ $D/*.pm`;

  foreach my $l (@LA) {
     my($fn, $rest) = split(/\:/, $l, 2);
     my $ix = rindex($fn, "/");
     $fn = substr($fn,$ix+1) if ($ix > 0);
     if ($rest =~ /Report::CAT_(\w+)/) {
       $AMAP{substr($fn,0,-3)} .= $1 . "|";
     }
  }
  return \@LH;
}

sub load_health {
   my($modname) = @_;
   my $repcat1 = $AMAP{$modname};
   chop($repcat1);
   my @repcatlist = split(/\|/, $repcat1);
   foreach my $c (@repcatlist) {
      my $hm1 = $HMAP{$c};
      chop($hm1);
      my @hmlist = split(/\|/, $hm1);
      foreach my $h (@hmlist) {
         if (! $LOADED{$h} ) {
            $LOADED{$h} = 1;
            Debug->print2("Loading Health '$h'.");
            if (eval "require \"Health/$h.pm\"") {
                my $f = "Health::$h";
                $f->new($pdm);  
            } else {
                print STDERR "Error load_health_monitor: $@ \n";
                return undef;
            }
         }
      }
   }
   return 1;
}

#################################################################

sub slave_log_read {
  my($file) = @_;
  my($l, $out);

  if (open(O2,$file)) {
     while ($l = <O2>) {
        $out .= $l;
     }
     close(O2); 
     unlink($file);
  }
  $out;
}



sub load_health_monitors {
  my($pdm, $list) = @_;
  my($health_monitor, @mods,$d, $mod);
  my($hm, $d);
  my($list2) = ",";
  foreach my $l (@$list) {
      my($fn, $rest) = split(/\:/, $l, 2);
      if ($l =~ /INITIAL/ || $l =~ /FINAL/) {
          $list2 .= substr($fn,0,-3) . ",";
      }
  }

  my $mods = Modules->read("Health", "Slave");

  push(@$mods, 'Slave') if (!$MASTER_LOC);  # slave_health only runs on the master

  my($pre) = System->get_home() . "/lib/Health";

  foreach $health_monitor (@$mods) {
      next if (index($list2, $health_monitor) < 0);
      Debug->print2("Loading Health '$health_monitor'.");
      if (eval "require \"Health/${health_monitor}.pm\"") {
         my $f = "Health::$health_monitor";
         $hm = $f->new($pdm);   # create hm, it will register with the pdm.
      } else {
         print STDERR "Error load_health_monitor: $@ \n";
      }
  }

}


@AGENTS = ();

sub list_agents {
  my($pdm) = @_;
  my($mod, @mods,$d, $mod0, @mods2, $san);

  my $mods = Modules->read("Agent");

  my($san, @mods2);
 #  pus topo and san at the end of the agent list
  foreach my $m (sort @$mods) {
     if ($m eq 'TOPO' || $m eq "HOST" || $m eq 'SAN' || $m eq "SE") {
        $san = 1;
     } else {
        push(@mods2, $m);
     }
  }
  if ($san) {
     push(@mods2, 'TOPO');
     push(@mods2, 'HOST');
     push(@mods2, 'SAN');
     push(@mods2, 'SE');
  }
  return \@mods2;
}


sub NUload_agents {
  my($pdm) = @_;
  my($mod, @mods,$d, $mod0);

  my $mods = Modules->read("Agent");

  my($san, @mods2);
 #  pus topo and san at the end of the agent list
  foreach my $m (@$mods) {
     if ($m eq 'TOPO' || $m eq "HOST" || $m eq 'SAN' || $m eq 'SE') {
        $san = 1;
     } else {
        push(@mods2, $m);
     }
  }
  if ($san) {
     push(@mods2, 'TOPO');
     push(@mods2, 'HOST');
     push(@mods2, 'SAN');
     push(@mods2, 'SE');
  }
  
  my($pre) = System->get_home() . "/lib/Agent";

  foreach $mod (@mods2) {
     Debug->print2("Loading Agent  '$mod' .");
     if (eval "require \"Agent/${mod}.pm\"") {
        push(@AGENTS, $mod->new);
     } else {
        print STDERR "Error in load_agents: $@ \n";
     }
  }
}


sub format_dev_info {
  my($device) = @_;
  my($info);
  if ($device->{serial}) {
     $info = "S" . $device->{serial};
  } else {
     $info = "N" . $device->{type} . "-" . $device->{_name} . "-" . 
                   $device->{name} . "-" . $device->{ip};
  }
  $info;
}

#
# NOT USED
#
#sub process_message_file {
#    my ($device, $starts, $file)  = @_;
#    my($current, $inode, $l, @now, $startline, $line, @a, $today, $cnt, $x, $dev_info);
#    my($id) = $device->{_name};
#
#    $today = &today;
#    $dev_info = &format_dev_info($device);
#
#    if ($starts->{$id}) {
#      $startline = $starts->{$id};
#    } else {
#      my($y) = substr($today,0,3) . sprintf("%2.2d", substr($today,3,2) - 1) .
#               substr($today,5);
#      $startline = $y;
#    }
#    open(F1, $file); $cnt=0;
#    my(@save);
#    while ($line = <F1>) {
#        chop($line);
#        last if ($line eq "exit"); # for tests only
#
#        @a = split(/ /, $line,6);
#        $current = sprintf("%2.2d-%2.2d %s", Util->MTH($a[0]), $a[1], $a[2]);
#
#        next if ($current gt $today); # if too big, probably the wrong year.
#        if ($current gt $startline) {
#          if (Util->include($RAS_OPTIONS, "l")) {
#             $cnt++;
#             read_errors_warnings($line, $dev_info);
#          }
#        } elsif ($LASTRUN && $current ge $LASTRUN) { # find 24 hours for thresholds
#          push(@save, $line);
#        }
#    }
#    close(F1);
#    if ($LASTRUN && $STORAGE ne "T3" && Util->include($RAS_OPTIONS,"t") ) {  
#       %SFOFFL = %WARNSSD = %SFCRCWARN = %SFOFFTOWARN = %SFDMAWARN = %SFRESET = ();
#       %RETRYELS = %RETRYSF = %TOELS = %SFTOELS = %DDOFFL = %DDCRCWARN = ();
#       $x = 0;
#       while ($x <= $#save) {
#          read_thresholds(\$x, \@save);
#       }
##       &process_thresholds($TH, $dev_info);
#    }
#    return $current;
#}



sub read_errors_warnings {
  my($line, $dev_info) = @_;
  my($name,$type, $no1, $no2,$rest);
  my($new_line, $here, $hostname, @a, $d1);
  my($m_type, @b, $l);

  @a = split(/ /, $line,6);
  $d1 = sprintf("%2.2d-%2.2d %s", Util->MTH($a[0]), $a[1], $a[2]);

  $new_line = "$curr_year-$d1 $a[3] $a[4] $a[5]";
  $here = 1;
  $hostname = $a[3];
#
# LOOKING FOR PATTERNS
#
  if ($line =~ /\[A5000:.*:ERR:/) {
        push(@A5000_log, 
            &line_add( type => "Error", 
                  date  => "$curr_year-$d1",
                  stype => "A",
                  device => $dev_info,
                  unit => $a[3],
                  comp => '',
                  ctype => '',
                  text => $line, source => '',
                ));

  } elsif ($line =~ /\]: W: (.*)/) {
        ($name,$type, $no1, $no2, $rest) = &get_info($1);
        push(@t300_log, 
            &line_add( type => "Warning", 
                  date  => "$curr_year-$d1",
                  stype => "T",
                  device => $dev_info,
                  unit => $a[3],
                  comp => $name,
                  ctype => $type,
                  text => $a[5], source => $a[4],
                ));
  
  } elsif ($line =~ /\]: E: (.*)/) {
        $m_type = "Error";
        ($name,$type, $no1, $no2,$rest) = &get_info($1);
        push(@t300_log, 
            &line_add( type => "Error", 
                  date => "$curr_year-$d1",
                  stype => "T",
                  unit => $a[3],
                  device => $dev_info,
                  comp => $name,
                  ctype => $type,
                  text => $a[5], source => $a[4],
                ));

  } elsif ($line =~ / N: ([\w:]+) (.+)/ ) {
     if ($2 eq "Enabled") {
        $m_type = "Notice";
        ($name,$type, $no1, $no2,$rest) = &get_info($1);

        push(@t300_log, 
            &line_add( type => "Notice", 
                  date => "$curr_year-$d1",
                  stype => "T",
                  unit => $a[3],
                  device => $dev_info,
                  comp => $name,
                  ctype => $type,
                  text => $a[5], source => $a[4],
                ));
     }
  }
}

sub process_message_file0 {
    my ($eofile, $fh)  = @_;
    my(@now, $today, $d1, $line, $linedate, @a);

    @now = localtime; $now[4]++; 
    $today = sprintf("%2.2d-%2.2d %2.2d:%2.2d:%2.2d", $now[4],$now[3], 
                      $now[2], $now[1], $now[0]);
    $today     = "12-03 17:13:00" if ($TIMETEST);

    while ($line = <$fh>) {
        chop($line);
	$_ = $line;
        last if ($line eq "exit");

        @a = split(/ /, $line,6);
        $d1 = sprintf("%2.2d-%2.2d %s", Util->MTH($a[0]), $a[1], $a[2]);
        next if ($d1 gt $today);

        $line =~ /^(\w+\s+\d+\s+\d+:\d+:\d+)/;
        $linedate = str2time("$1 $curr_year");
        next if ($linedate < $startDate);

        process_one_line($line);
    }
}



sub line_add {
  my(%a) = @_;
  my($o, $tx);

  $tx = $a{text};
  $tx =~ s/"/'/g;

  $LINE_CNT++;
  $o =<<EOF;
<line no="$LINE_CNT"
 device="$a{device}"
 mess_type="$a{type}"
 comp="$a{comp}" 
 date="$a{date}" 
 stype="$a{stype}"
 unit="$a{unit}" 
 source="$a{source}"
 comp_type="$a{ctype}"
 text="$tx"
/>
EOF
    
  return $o;
}



sub match_hostname {

    my $line = shift;
    my $hostname;
    ($hostname) = $line =~ /\w+\s+\d+\s+\d+:\d+:\d+\s(\S+)/;
    if ( $hostname ) {
        return $hostname;
    }
    ($hostname) = $line =~ /\w+\s+\d+\s+\d+:\d+:\d+\s(\[\d+.\d+.\d+.\d+.\d+.\d+\])/;
    if ( $hostname ) {
        return $hostname;
    }
    return "INVALID HOSTNAME";
}

sub hb_event {
    my ( $renv) = @_;
    my ($storage_type, $mess);

    my $host_name = `/usr/bin/hostname`; chomp($host_name);
    my $host_info = `/usr/bin/uname -a`; chomp($host_info);
    my $host_id   = `/bin/hostid`; chomp($host_id);

    $mess =<<EOF;
<event event_type=\"heartbeat\" hostid="$host_id"  host="$host_name" host_info="$host_info" customer="$renv->{customer}"
/>
EOF
    return $mess;
}


sub mail_message {
  my($to, $subject,  $out) = @_;
  my($header, $secure_email, $key, $sig);
  $key = "12345678901234567890123456";

  if (!-x $MAILER) {
    Debug->err('CANNOT_EXECUTE', $MAILER);
    return 0;
  }
  $header =<<EOF;
From: Storage Automatted Diagnostic Environment
Subject: $subject

EOF
   $sig = $renv->{signature} || "0";
   $out = "Customer: $renv->{cust_no}
Contract: $renv->{contract}
$out";

   if (!($secure_email = CC::Email::encrypt_email($out , $key, $sig ))) {
      Debug->err('ENCRYPT_BUG', $CC::Email::error);
      return 0;
   } else {
      if ($to) {
         open(O, "|$MAILER $to");
         print O $header;
         print O $secure_email;
         close(O);
      }
   }
   return(1);
}



# YYYY-MM-DD HH:MM:SS
sub get_today {

  my(@date) = localtime(time); 
  $date[4]++; $date[5] += 1900;

  sprintf("%2.2d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d",
    $date[5], $date[4],$date[3], $date[2], $date[1], $date[0]);
}

sub version {
  my($v) = `/usr/bin/cat /opt/SUNWstade/System/config"`;
  print "
  Storage Automated Diagnostic Environement $v \n";
  exit;

}

sub help {
  print "
  USAGE: rasagent 

   -T          : test mode (will not save cache or email)
   -d <level>  : Debug (2=most trace, 3=more details, write snapshot.log in /DATA)
   -D <dir>    : home Directory , used to test slaves on the same machine.
   -? -h       : Help
   -f          : Force, run all agents.
   -P          : Print only, no email
   -s          : do not save the data found
    
";
  exit;
}




# 0 = OK
# 1 = exit

sub cron_conflicts {
  my($device_cnt) = @_;
  my($cnt, $l, $created, $PID, $pspid, @a);

  return 0 if (!-f $PIDfile);  # no pidfile

  open(O,$PIDfile); $PID = <O>; close(O);
  $cnt = 0;
  my $sig = $SIGTERM;
  while (1) {
     last if (!$PID);
     $cnt++;
     if ($cnt > 3) {
        Debug->err('FAILING_TO_KILL',$PID, $pspid);
        return 1;
     }
     $sig = 9 if ($cnt == 3);
     my ($pspid, $children) = Util->findProcesses($PID);

     if ($pspid eq $PID) {
       $created =  (stat($PIDfile))[9];
       if (time - $created > (600*$device_cnt)) {
         Debug->err(CRON_CONFLICT => "killing $pspid, @$children");
         kill $sig, $pspid;
         kill $sig, @$children if ($#$children >= 0);
         sleep(1);
       } else {
         Debug->print('RASAGENT_RUNNING', $pspid);
         return 1;
       }
     } elsif (!$pspid) {
       unlink $PIDfile;
       Debug->print('PID' => "Remove File");
       return 0;
     } elsif ($pspid ne $PID) {
       Debug->err('WRONG_PROGRAM', $pspid);
       kill $sig, $pspid if ($pspid =~ /\d+/);
     }
  }
  return 0; # OK
}




sub datetime {
  my(@date) = localtime(time); $date[4]++; $date[5] += 1900;
  sprintf("%2.2d-%2.2d-%d %2.2d:%2.2d:%2.2d",
           $date[4],$date[3],$date[5],$date[2],$date[1],$date[0]);
}



###########################################################
#
#    $SBUS_SOCAL_HBAS TRUE detected, FALSE not found
#    $PCI_QL2100_HBAS TRUE detected, FALSE not found
#    $PCI_QL2200_HBAS TRUE detected, FALSE not found

sub get_fcal_hbas {
    my($sbus_socal_platform, $pci_platform, $pci_platform) ;
    my($pci_ql2100_platform, $pci_ql2200_platform, $ifp_driver);
    my(%INFO);

    # type of Host Bus Adapters(s)
    $INFO{SBUS_SOCAL_HBAS} = 0;
    $INFO{PCI_QL2100_HBAS} = 0;
    $INFO{PCI_QL2200_HBAS} = 0;
    $pci_platform = 0;

    chomp( $sbus_socal_platform = `/usr/sbin/prtconf -D | grep "SUNW,socal"` );
    if ( $sbus_socal_platform ) {
        $INFO{SBUS_SOCAL_HBAS} = 1;
    }

    chomp( $pci_platform = `/usr/sbin/prtconf -pv | grep "vendor-id:  00001077"` );
    if ( $pci_platform ) {
        chomp( $pci_ql2100_platform = `/usr/sbin/prtconf -pv | grep "device-id:  00002100"` );
        if ( $pci_ql2100_platform ) {
            $INFO{PCI_QL2100_HBAS} = 1;
        }
        chomp( $pci_ql2200_platform = `/usr/sbin/prtconf -pv | grep "device-id:  00002200"` );
        if ( $pci_ql2200_platform ) {
            $INFO{PCI_QL2200_HBAS} = 1;
        }
        # make sure there is an ifp driver for PCI HBA.
        # test for QLOGIC board with SUNW,ifp name property
        chomp( $ifp_driver = `/usr/sbin/prtconf -D | grep "SUNW,ifp"` );
        if ( ! $ifp_driver ) {
            # test for QLOGIC board with scsi name property
            chomp( $ifp_driver = `/usr/sbin/prtconf -D | grep "scsi" | grep "ifp"` );
        }
        if ( ! $ifp_driver ) {
            if ( $INFO{PCI_QL2100_HBAS} ) {
                $INFO{PCI_QL2100_HBAS} = 0;
            }
            if ( $INFO{PCI_QL2200_HBAS} ) {
                $INFO{PCI_QL2200_HBAS} = 0;

            }
        }
    }
    return \%INFO;
}


#    $HBA = "S"  - for SBUS
#    $HBA = "P"  - for Qlogic 2100 PCI
#    $HBA = "Q"  - for Qlogic 2200 PCI

sub select_fcal_hbas {
  my($info) = @_;

    if ( !$info->{SBUS_SOCAL_HBAS}  && !$info->{PCI_QL2100_HBAS} && 
         !$info->{PCI_QL2200_HBAS} ) {
        Debug->warning('NO_HBA');
    }
    if  ( ( $info->{SBUS_SOCAL_HBAS} && $info->{PCI_QL2100_HBAS}) ||
          ( $info->{SBUS_SOCAL_HBAS} && $info->{PCI_QL2200_HBAS}) ||
          ( $info->{PCI_QL2100_HBAS} && $info->{PCI_QL2200_HBAS} ) ) {

        Debug->warning('MULTIPLE_HBAS');
    }
}



sub match_hostname {

    my $line = shift;
    my $hostname;
    ($hostname) = $line =~ /\w+\s+\d+\s+\d+:\d+:\d+\s(\S+)/;
    if ( $hostname ) {
        return $hostname;
    }
    ($hostname) = $line =~ /\w+\s+\d+\s+\d+:\d+:\d+\s(\[\d+.\d+.\d+.\d+.\d+.\d+\])/;
    if ( $hostname ) {
        return $hostname;
    }
    return "INVALID HOSTNAME";
}





