#!/usr/bin/perl -I/opt/SUNWstade/lib 
use System;
use Util;
use strict;
use Getopt::Std;
use Data::Dumper;
use Scheduler;
use System;
use PDM::ConfigFile;
use Report;

use vars qw (%opts $DEBUG %PROMPTS $YESNO $GUI $Qfile $single_isl $Config );

#
# all possible prompts
#
# insert - used for loopback only
# remove - remove current FRU/loopback 
# replace - replace the current FRU with a NEW FRU
# restore - restore the original FRU
# ra - Recommend Action

%PROMPTS = 
  (
    restore_orignal_components => "\nRestore Orignal Components:\n",
    multiple_suspects => "\nlinktest could not isolate to a single failing FRU\nTry a single recommended action listed below:\n",
    insert_port_loopback => "Insert FC Loopback Cable into %s: %s, port: %s",
    remove_port_loopback => "Remove FC Loopback Cable from %s: %s, port: %s",
    detected_single_isl => "Detected single ISL (Single Point of Failure) on %s: %s, port: %s",
    ra_detected_single_isl => "Recommened action: Add ISL to zone containing %s: %s, port: %s",
    #detected_single_path_to_sw_ports => "Only one communication path to switch ports. This will prevent linktest from isolating FRUs in the remote %s: %s, port: %s",
    #ra_add_enet_connection => "Recommened action: Add enet connect to %s: %s and redisover and retest",
    #is_enet => "Is there an ethernet communication path to %s: %s, port: %s",
    suspect_switch => "Suspect FC %s: %s",
    ra_suspect_switch => "Recommened action: Replace FC %s: %s",
    suspect_port => "Suspect FC %s: %s, port: %s",
    ra_suspect_port => "Recommended action: reconfigure %s: %s using a port other than port: %s",
    suspect_replaced_gbic => "Suspect REPLACED FC GBIC in %s: %s, port: %s",
    ra_suspect_replaced_gbic => "Recommended action: Retry linktest with known good FC GBIC in %s: %s, port: %s",
    functional_port => "FC %s: %s, and port: %s is Functional",
    remove_port_cable => "Remove FC Cable from %s: %s, port: %s",
    replace_port_cable => "Insert a NEW FC Cable into %s: %s, port: %s",
    restore_port_cable => "Restore ORIGINAL FC Cable into %s: %s, port: %s",
    suspect_port_cable => "Suspect ORIGINAL FC Cable from %s: %s, port: %s",
    functional_port_cable => "FC Cable in %s: %s, port: %s is Functional",
    remove_port_gbic => "Remove FC GBIC from %s: %s, port: %s",
    replace_port_gbic => "Insert a NEW FC GBIC into %s: %s, port: %s",
    restore_port_gbic => "Restore ORIGINAL FC GBIC into %s: %s, port: %s",
    suspect_port_gbic => "Suspect ORIGINAL FC GBIC in %s: %s, port: %s",
    functional_port_gbic => "FC GBIC in %s: %s, port: %s is Functional",
    insert_device_loopback => "Insert FC Loopback Cable into %s: %s, port: %s",
    remove_device_loopback => "Remove FC Loopback Cable from %s: %s, port: %s",
    functional_device => "%s: %s, port: %s is Functional",
    remove_device_cable => "Remove FC Cable from %s: %s, port: %s",
    replace_device_cable => "Insert a NEW FC Cable into %s: %s, port: %s",
    restore_device_cable => "Restore ORIGINAL FC Cable into %s: %s, port: %s",
    suspect_device_cable => "Suspect ORIGINAL FC Cable from %s: %s, port: %s",
    functional_device_cable => "FC Cable in %s: %s, port: %s is Functional",
    remove_device_gbic => "Remove FC GBIC from %s: %s, port: %s",
    replace_device_gbic => "Insert a NEW FC GBIC into %s: %s, port: %s",
    restore_device_gbic => "Restore ORIGINAL FC GBIC into %s: %s, port: %s",
    suspect_device_gbic => "Suspect ORIGINAL FC GBIC from %s: %s, port: %s",
    functional_device_gbic => "FC GBIC in %s: %s, port: %s is Functional",
    remove_device_mia => "Remove MIA from %s: %s, port: %s",
    replace_device_mia => "Insert a NEW MIA into %s: %s, port: %s",
    restore_device_mia => "Restore ORIGINAL MIA into %s: %s, port: %s",
    suspect_device_mia => "Suspect ORIGINAL MIA from %s: %s, port: %s",
    functional_device_mia => "MIA in %s: %s, port: %s is Functional",
    remove_device_coupler => "Remove FC Coupler from %s: %s, port: %s",
    replace_device_coupler => "Insert a NEW FC Coupler into %s: %s, port: %s",
    restore_device_coupler => "Restore ORIGINAL FC Coupler into %s: %s, port: %s",
    suspect_device_coupler => "Suspect ORIGINAL FC Coupler from %s: %s, port: %s",
    functional_device_coupler => "FC Coupler in %s: %s, port: %s is Functional",
    remove_device_adapter => "Remove FC Adapter cable from %s: %s, port: %s",
    replace_device_adapter => "Insert a NEW FC Adapter cable into %s: %s, port: %s",
    restore_device_adapter => "Restore ORIGINAL FC Adapter cable into %s: %s, port: %s",
    suspect_device_adapter => "Suspect ORIGINAL FC Adapter cable from %s: %s, port: %s",
    functional_device_adapter => "FC Adapter cable in %s: %s, port: %s is Functional",
    remove_hba_cable    => "Remove FC Cable from %s: %s",
    replace_hba_cable   => "Insert a NEW FC Cable from %s: %s",
    restore_hba_cable   => "Restore ORIGINAL FC Cable into %s: %s",
    suspect_hba_cable   => "Suspect ORIGINAL FC Cable from %s: %s",
    functional_hba_cable => "FC Cable in %s: %s is Functional",
    remove_hba_gbic     => "Remove FC GBIC from %s: %s",
    replace_hba_gbic    => "Insert a NEW FC GBIC into %s: %s",
    restore_hba_gbic    => "Restore ORIGINAL FC GBIC into %s: %s",
    suspect_hba_gbic    => "Suspect ORIGINAL FC GBIC from %s: %s",
    functional_hba_gbic => "FC GBIC in %s: %s is Functional",
    insert_hba_loopback => "Insert FC Loopback Cable into %s: %s",
    remove_hba_loopback => "Remove FC Loopback Cable from %s: %s",
    suspect_hba         => "Suspect %s: %s",
    functional_hba      => "ORIGINAL %s: %s is Functional",
    ra_a5ktest          => "Recommened action: execute functional test a5ktest on %s: %s",
    ra_hbatest          => "Recommened action: execute functional HBA test on %s: %s",
    ra_daktest          => "Recommened action: execute functional test daktest on %s: %s",
    ra_fcdiskktest      => "Recommened action: execute functional test fcdisktest on %s: %s",
    ra_a3500fctest      => "Recommened action: execute functional test a3500fctest on %s: %s",
    ra_a3500rm6         => "Recommened action: execute rm6 healthcheck on %s: %s",
    ra_fctapetest       => "Recommened action: execute functional test fctapetest on %s: %s",
    ra_t3_ofdg          => "Recommened action: execute functional test t3ofdg on %s: %s",
    ra_t3test           => "Recommened action: execute functional test t3test on %s: %s",
    no_password         => "Password required in Maintain Device: Telnet Password device %s: %s",
    ra_password         => "Recommened action: enter password in Maintain Device: Telnet Password for device %s: %s",
  );

System->set_home("/opt/SUNWstade");
System->set_rasport("7654");
$Config = PDM::ConfigFile->read();
my $renv = $Config->renv();
System->set_renv($renv);
print "running on $renv->{hostname} \n";

$YESNO = "y=Yes|n=No";

$DEBUG=0;

my $SUCCESS=1;
my $FOUND_SUSPECT_FRU=2;
my $PROBLEM_UNDETECTED=99;

# exit status. These align w/the StorADE test structure.
my $ERROR = -1;
my $COMPLETED = 0;
my $ABORTED = 143;

if (!getopts("I:p:T:svW:a:b:h", \%opts)) {
  die("Abort: $Getopt::Std::ERROR \n");
}

if ($opts{h}) {
  usage();
  exit( $ERROR );
}
$Qfile = $opts{W};
$GUI   = 1 if ($Qfile);

my $option_verbose        = 1 if ($opts{v});
my $single_isl            = $opts{I};
my $option_pattern        = $opts{p};
my $option_pattern_type   = $opts{T};

if (!$opts{a} || !$opts{b}) {
  print "Error: need 2 nodes to run.\n\n";
  usage();
  exit( $ERROR );
}

# default timeout for tests: 30 minutes.
# long enough for almost any test.

my $test_timeout = 60*30;

# a switchtest of "all" takes approximately
# 275 minutes to run. pad it out to 360 minutes

if ($option_pattern_type eq "all") {
  $test_timeout = 360*60;
}

my $node_as = $opts{a};
my $node_bs = $opts{b};

my ($node_a, $node_b) = &parse($node_as, $node_bs);

# node_a:  {
#          'bitMode' => 32|64,
#          'monHost' => 'ccadieux.central.sun.com',
#          'portWWN' => 'wwn',
#          'ip' => '1.1.1.1',
#          'type' => 't3',
#          'port' => '0'
#        };
#node_b:  {
#          'monHost' => 'rasd2',
#          'ip' => '1.1.1.1',
#          'type' => 'switch',
#          'WWN' => 111,
#          'fcaddr' => 108600,
#          'port' => 2
#        };

if ( $DEBUG == 1) {
  debug_header( $node_a, $node_b, "main" );
  my $dev = $Config->deviceByKey( $node_a->{key} );
  print "node_a class info\n";
  print Dumper($dev);
  my $dev = $Config->deviceByKey( $node_b->{key} );
  print "node_b class info\n";
  print Dumper($dev);
}


if ($GUI) {
  select(STDOUT); $|=1; 
}

###########################################################
# Isolation code makes assumptions about node a  and node b.
#
# Swap the nodes to match assumptions when appropriate.
#
# Test both end nodes of a link segment when possible.
#
# If a end-node (I/O device) is detected and we are NOT direct connect
# do not test it. The path used for I/O in those cases is NOT determinstic.
#
# If devices can NOT be tested due to insufficient test capablities, report
# it and exit.
##########################################################

  debug_header( $node_a, $node_b, "main" );

  # 0 = can't execute linktest
  # 1 = test 1st node
  # 2 = test both nodes
  my $test_both_nodes;  

  my $node_swap;
  my $single_enet_path;
  my $answer;
  my $port_type;

  # isolation code makes assumptions about node a  and node b
  # swap the nodes to match assumptions when appropriate

  if ( $node_a->{type} eq 'switch' && $node_b->{type} eq 'switch' ) {
    $test_both_nodes = 2;

    # remote port must be last in sequence
    if ( $node_a->{fcaddr} ) {
      $node_swap = $node_a;
      $node_a = $node_b;
      $node_b = $node_swap;
    }

    if ( $single_isl == 1 ) {
      prompt( 'detected_single_isl', $node_a );
      prompt( 'ra_detected_single_isl', $node_a );
    }

  } elsif ( $node_a->{type} eq 'brocade' && $node_b->{type} eq 'brocade' ) {
    # brocade capabilities are TBD
    $test_both_nodes = 2;

  # For now, if either port is a Vicom, use p2p isolation
  } elsif ( $node_a->{type} eq 'switch' && $node_b->{type} eq 've' ) {
    # need vicom drop to support running switch port test and/or vicom
    # port test
    $test_both_nodes = 0;

  } elsif ( $node_a->{type} eq 've' && $node_b->{type} eq 'switch' ) {
    # diagnose from switch to ve
    # need vicom drop to support running switch port test and/or vicom
    # port test
    $node_swap = $node_a;
    $node_a = $node_b;
    $node_b = $node_swap;
    $test_both_nodes = 0;

  } elsif ( $node_a->{type} eq 'hba' && $node_b->{type} eq 'switch' ) {
    # what about SOC+
    $test_both_nodes = 2;

  } elsif ( $node_a->{type} eq 'switch' && $node_b->{type} eq 'hba' ) {
    # diagnose from HBA to switch
    # what about SOC+
    $node_swap = $node_a;
    $node_a = $node_b;
    $node_b = $node_swap;
    $test_both_nodes = 2;

  } elsif ( $node_a->{type} eq 'hba' && $node_b->{type} eq 'brocade' ) {
    # brocade capabilities are TBD
    $test_both_nodes = 1;

  } elsif ( $node_a->{type} eq 'brocade' && $node_b->{type} eq 'hba' ) {
    # diagnose from HBA to switch
    # brocade capabilities are TBD
    $node_swap = $node_a;
    $node_a = $node_b;
    $node_b = $node_swap;
    $test_both_nodes = 1;

  } elsif ( $node_a->{type} eq 'switch' ) {
    # diagnose switch port to device, do NOT test the I/O device

    # if switch to T3, check the switch for F port. If F do not test.
    # The T3 in F (fabric) mode does NOT support echo
    if ( $node_b->{type} eq 't3' ) {
      (my $readerr, my $rep1) = Report->readReport("switch:$node_a->{key}", "*");
      if ($rep1) {
        my $v = $rep1->value();
        $port_type = $v->{"port.$node_a->{port}.type"};
      } else {
        program_error();
      }
    }

    if ( $port_type eq "F_Port" ) {
      $test_both_nodes = 0;
    } else {
      $test_both_nodes = 1;
    }

  } elsif ( $node_b->{type} eq 'switch' ) {
    # diagnose switch port to device, do NOT test the I/O device
    $node_swap = $node_a;
    $node_a = $node_b;
    $node_b = $node_swap;
    $test_both_nodes = 1;

  } elsif ( $node_a->{type} eq 'brocade' ) {
    # diagnose switch port to device, do NOT test the I/O device

    # if switch to T3, check the switch for F port. If F do not test.
    # The T3 in F (fabric) mode does NOT support echo
    if ( $node_b->{type} eq 't3' ) {
      (my $readerr, my $rep1) = Report->readReport("brocade:$node_a->{key}", "*");
      if ($rep1) {
        my $v = $rep1->value();
        $port_type = $v->{"port.$node_a->{port}.type"};
      } else {
        program_error();
      }
    }

    if ( $port_type eq "f-port" ) {
      $test_both_nodes = 0;
    } else {
      $test_both_nodes = 1;
    }

  } elsif ( $node_b->{type} eq 'brocade' ) {
    # diagnose switch port to device
    # brocade capabilities are TBD
    $node_swap = $node_a;
    $node_a = $node_b;
    $node_b = $node_swap;
    $test_both_nodes = 1;

  } elsif ( $node_a->{type} eq 'hba' ) {
    # direct connect, test using HBA and not the I/O device
    $test_both_nodes = 1;
  } elsif ( $node_b->{type} eq 'hba' ) {
    # direct connect, test using HBA and not the I/O device
    # make the HBA the testable unit
    $node_swap = $node_a;
    $node_a = $node_b;
    $node_b = $node_swap;
    $test_both_nodes = 1;
  } else {
    device_unsupported_msg();
    exit( $ERROR );
  }

debug_header( $node_a, $node_b, "main" );

if ( $test_both_nodes == 0 ) {
  if ( ( $port_type eq "F_Port" ) || ( $port_type eq "f-port" ) ) {
    t3_f_port_detected( $node_a->{type}, $node_b->{type} );
  } else {
    untestable_link( $node_a->{type}, $node_b->{type} );
  }
}

# change the t3 type to accurately reflect the device and frus
# T3b has a model number starting with 501-5710-0
# T3 has a model number starting with 375-0084-0

if ( $node_b->{type} eq 't3' ) {
  if ($node_b->{ctrl_model} =~ /501/ ) {
    $node_b->{type} = "t3b";
  }
}

my $status;
my $out;

start_linktest_msg( $node_a, $node_b );

if ( $test_both_nodes > 0 ) {
  if ( $node_a->{driver} eq 'ifp' ) {
    $out = start_diagnostic( $node_b, "E" );
  } else {
    $out = start_diagnostic( $node_a, "E" );
  }

  if ( $out->{rc} != 0 ) {
    $status = link_isolation( $node_a, $node_b, "a" );
    if ( $status == $FOUND_SUSPECT_FRU ) {
      stop_linktest_msg( $node_a, $node_b );
      exit( $COMPLETED );
    }
    failed_linktest_msg( $node_a, $node_b );
    exit( $ERROR ); # Always exit if isolation failed
  }
}
 
if ( $test_both_nodes == 2 ) {
  $out = start_diagnostic( $node_b );

  if ( $out->{rc} != 0 ) {
    $status = link_isolation( $node_a, $node_b, "b" );
    if ( $status == $FOUND_SUSPECT_FRU ) {
      stop_linktest_msg( $node_a, $node_b );
      exit( $COMPLETED );
    }
    failed_linktest_msg( $node_a, $node_b );
    exit( $ERROR ); # Always exit if isolation failed
  }
} else {
  if ( $node_a->{driver} ne 'ifp' ) {
    skip_io_device( $node_b->{type} );
  }
}

stop_linktest_msg( $node_a, $node_b );
exit( $COMPLETED );


#######################################
#     LIBRARIES
#######################################


#
# link_isolation is entered if and only if one of the 2 node's functional
# test fail in main (linktest).
#
# dispatches correct high level isolation algorithm based on 2 node types
# only one side of the link is isolated, if NTF set the failing node
# to the far side (node b) and return
#

sub link_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "link_isolation" );

  my $status;

  # isolation code makes assumptions about node a  and node b
  # the nodes should have been swaped by main() when appropriate
  # dispatch high level algorithm for node to node isolation

  if ( $node_a->{type} eq 'switch' && $node_b->{type} eq 'switch' ) {
    $status = port2port_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_a->{type} eq 'brocade' && $node_b->{type} eq 'brocade' ) {
    $status = port2port_link_isolation( $node_a, $node_b, $failed_node );

  # For now, if either port is a Vicom, use p2p isolation
  } elsif ( $node_a->{type} eq 'switch' && $node_b->{type} eq 've' ) {
    $status = port2port_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_a->{type} eq 've' && $node_b->{type} eq 'switch' ) {
    # should never get if node swap worked
    program_error();

  } elsif ( $node_a->{type} eq 'hba' && $node_b->{type} eq 'switch' ) {
    $status = hba2port_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_a->{type} eq 'switch' && $node_b->{type} eq 'hba' ) {
    # should never get if node swap worked
    program_error();

  } elsif ( $node_a->{type} eq 'hba' && $node_b->{type} eq 'brocade' ) {
    $status = hba2port_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_a->{type} eq 'brocade' && $node_b->{type} eq 'hba' ) {
    # should never get if node swap worked
    program_error();

  } elsif ( $node_a->{type} eq 'switch' ) {
    $status = port2device_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_b->{type} eq 'switch' ) {
    # should never get if node swap worked
    program_error();

  } elsif ( $node_a->{type} eq 'brocade' ) {
    $status = port2device_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_b->{type} eq 'brocade' ) {
    # should never get if node swap worked
    program_error();

  } elsif ( $node_a->{type} eq 'hba' ) {
    $status = hba2device_link_isolation( $node_a, $node_b, $failed_node );

  } elsif ( $node_b->{type} eq 'hba' ) {
    # should never get if node swap worked
    program_error();

  } else {
    device_unsupported_msg();
    exit( $ERROR );
  }

}

#
# isolate failing FRU in any 2 ISL (Inter Switch Link) connection
# isolate both sides of the ISL at this level of dispatch
#

sub port2port_link_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "port2port_isolation" );

  my $answer;
  my $status;

  my $dev_a = $Config->deviceByKey( $node_a->{key} );
  my $dev_b = $Config->deviceByKey( $node_b->{key} );
  if ( $dev_a->{ip} eq $dev_b->{ip} ) {
    if ( $single_isl == 1 ) {
      $status = single_isl_port_node_isolation( $node_a, $node_b, $failed_node );
      return $status;
    }
  }
  if ( $failed_node eq "a" ) {
    $status = port_node_isolation( $node_a, $node_b, $failed_node );
    if ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    } elsif ( $status == $SUCCESS ) {
      # port tested OK; try device
      # could be a cable or interface on other device
      $failed_node = "b";
    } elsif ( $status == $PROBLEM_UNDETECTED ) {
      # could be a cable or interface on device
      $failed_node = "b";
    }
  }

  if ( $failed_node eq "b" ) {
    $status = port_node_isolation( $node_b, $node_a, $failed_node );
    if ( $status == $PROBLEM_UNDETECTED ) {
      # Device tested OK; unable to isolate problem 
      return $status;
    } elsif ( $status == $SUCCESS ) {
      # Device tested OK; should no get here 
      return $status;
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }

}

#
# isolate failing FRU in any HBA (Host Bus Adapter) port 
# to switch connection
# node_a MUST be the HBA port
#

sub hba2port_link_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "hba2port_link_isolation" );

  my $answer;
  my $status;

  if ( $failed_node eq "a" ) {
    $status = hba_node_isolation( $node_a, $node_b, $failed_node );
    if ( $status == $SUCCESS ) {
      # HBA tested OK; try device
      # could be a cable or interface on port
      $failed_node = "b";
    } elsif ( $status == $PROBLEM_UNDETECTED ) {
      # could be a cable or interface on device
      $failed_node = "b";
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }

  if ( $failed_node eq "b" ) {
    $status = port_node_isolation( $node_b, $node_a, $failed_node );
    if ( $status == $PROBLEM_UNDETECTED ) {
      # Device tested OK; unable to isolate problem 
      return $status;
    } elsif ( $status == $SUCCESS ) {
      # Device tested OK; should no get here 
      return $status;
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }

}

#
# isolate failing FRU in any FC switch/vicom port 
# to device connection
# node_a MUST be the switch port
#

sub port2device_link_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "port2device_link_isolation" );

  my $answer;
  my $status;

  if ( $failed_node eq "a" ) {
    $status = port_node_isolation( $node_a, $node_b, $failed_node );
    if ( $status == $SUCCESS ) {
      # port tested OK; try device
      # could be a cable or interface on device
      $failed_node = "b";
    } elsif ( $status == $PROBLEM_UNDETECTED ) {
      # could be a cable or interface on device
      $failed_node = "b";
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }

  # problem was NOT dectected on the port side, could be cable
  # or FC interconnects (or i/o device
  # use the port test to isolate FC interconnects

  if ( $failed_node eq "b" ) {
    $status = device_interconnect_isolation( $node_a, $node_b, $failed_node  );
    if ( $status == $SUCCESS ) {
      # port tested OK; try device
      $failed_node = "b";
    } elsif ( $status == $PROBLEM_UNDETECTED ) {
      $failed_node = "b";
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }


}

#
# isolate failing FRU in any HBA port 
# to device connection
# node_a must be the HBA port
#

sub hba2device_link_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "hba2device_link_isolation" );

  my $status;
  my $answer;

  if ( $failed_node eq "a" ) {
    $status = hba_node_isolation( $node_a, $node_b, $failed_node );
    if ( $status == $SUCCESS ) {
      # HBA tested OK; try device
      # could be a cable or interface on device
      $failed_node = "b";
    } elsif ( $status == $PROBLEM_UNDETECTED ) {
      # could be a cable or interface on device
      $failed_node = "b";
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }

  if ( $failed_node eq "b" ) {
    $status = device_interconnect_isolation( $node_a, $node_b, $failed_node );
    if ( $status == $PROBLEM_UNDETECTED ) {
      # Device tested OK; unable to isolate problem 
      return $status;
    } elsif ( $status == $SUCCESS ) {
      # Device tested OK; should no get here 
      return $status;
    } elsif ( $status == $FOUND_SUSPECT_FRU ) {
      return $status;
    }
  }
}

#
# isolate failing FRU in any HBA port 
# node_a must be the HBA port
#

sub hba_node_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "hba_node_isolation" );

  my $answer;
  my $status;
  my $out;

  if ( $node_a->{driver} eq "ifp" ) {
    prompt( 'remove_hba_cable', $node_a );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      $out = start_diagnostic( $node_a, "E" );
    } else {
      prompt( 'restore_hba_cable', $node_a );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed, the HBA node is OK
      # test the next node, could be a cable problem
      prompt( 'restore_hba_cable', $node_a );
      prompt( 'functional_hba', $node_a );
      $answer = ask_continue();
      if ( $answer ne "n" ) {
        return $SUCCESS;
      } else {
        premature_exit();
      }
    } else {
      prompt( 'restore_hba_cable', $node_a );
      prompt( 'suspect_hba', $node_a );
      return $FOUND_SUSPECT_FRU;
    }
  }

  prompt( 'remove_hba_cable', $node_a );
  prompt( 'insert_hba_loopback', $node_a );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $node_a, "E" );
  } else {
    prompt( 'remove_hba_loopback', $node_a );
    prompt( 'restore_hba_cable', $node_a );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a loopback, the HBA node is OK
    # test the next node, could be a cable problem
    prompt( 'remove_hba_loopback', $node_a );
    prompt( 'restore_hba_cable', $node_a );
    prompt( 'functional_hba', $node_a );
    return $SUCCESS;
  }

  if ( $node_a->{driver} eq "socal" ) {

    prompt( 'remove_hba_loopback', $node_a );
    prompt( 'replace_hba_gbic', $node_a );
    prompt( 'insert_hba_loopback', $node_a );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      $out = start_diagnostic( $node_a, "E" );
    } else {
      prompt( 'remove_hba_loopback', $node_a );
      prompt( 'restore_hba_gbic', $node_a );
      prompt( 'restore_hba_cable', $node_a );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed with a new GBIC. The GBIC is the suspect.
      prompt( 'remove_hba_loopback', $node_a );
      prompt( 'restore_hba_cable', $node_a );
      prompt( 'suspect_hba_gbic', $node_a );
      retest_msg();
      return $FOUND_SUSPECT_FRU;
    } else {
      premature_exit();
    }

  }

  prompt( 'remove_hba_loopback', $node_a );
  prompt( 'restore_hba_gbic', $node_a );
  prompt( 'restore_hba_cable', $node_a );
  prompt( 'suspect_hba', $node_a );
  return $FOUND_SUSPECT_FRU;

}

#
# isolate a5k FC interconnect FRUs 
# using port test if "PORT"
# using I/O of "IO"
#

sub a5k_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "a5k_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    $diagnostic_node = $node_b;
  }

  # try the GBIC first

  prompt( 'remove_device_cable', $node_b );
  prompt( 'replace_device_gbic', $node_b );
  prompt( 'restore_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_gbic', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new GBIC. The GBIC is the suspect.
    prompt( 'suspect_device_gbic', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  prompt( 'remove_device_cable', $node_b );
  prompt( 'restore_device_gbic', $node_b );
  prompt( 'replace_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the GBIC or the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_a5ktest', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

# isolate fcdisk FC interconnect FRUs 
# using port test if "PORT"
# using I/O of "IO"

sub fcdisk_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "fcdisk_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    $diagnostic_node = $node_b;
  }

  # try the GBIC first

  prompt( 'remove_device_cable', $node_b );
  prompt( 'replace_device_gbic', $node_b );
  prompt( 'restore_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_gbic', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new GBIC. The GBIC is the suspect.
    prompt( 'suspect_device_gbic', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  prompt( 'remove_device_cable', $node_b );
  prompt( 'restore_device_gbic', $node_b );
  prompt( 'replace_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the GBIC or the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_fcdisktest', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

sub daktari_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "daktari_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    $diagnostic_node = $node_b;
  }

  # try the GBIC first

  prompt( 'remove_device_cable', $node_b );
  prompt( 'replace_device_gbic', $node_b );
  prompt( 'restore_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_gbic', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new GBIC. The GBIC is the suspect.
    prompt( 'suspect_device_gbic', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  prompt( 'remove_device_cable', $node_b );
  prompt( 'restore_device_gbic', $node_b );
  prompt( 'replace_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the GBIC or the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_dakdisktest', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

sub t3_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "t3_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    $diagnostic_node = $node_b;
  }

  # try the MIA first

  prompt( 'remove_device_cable', $node_b );
  prompt( 'replace_device_mia', $node_b );
  prompt( 'restore_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    # utilize the port diagnostic
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_mia', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new MIA. The MIA is the suspect.
    prompt( 'suspect_device_mia', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  prompt( 'remove_device_cable', $node_b );
  prompt( 'restore_device_mia', $node_b );
  prompt( 'replace_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    # utilize the port diagnostic
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the MIA or the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_t3_ofdg', $node_b );
  prompt( 'ra_t3test', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

sub t3b_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "t3b_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;
  my $t3b_direct = "y";

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    $diagnostic_node = $node_b;
  }

  # ask the user if it's a direct cable or FC Coupler w/Adaptor
  $answer = ask_t3b_direct();
  if ( $answer eq "n" ) {

    # try coupler first

    prompt( 'remove_device_cable', $node_b );
    prompt( 'replace_device_coupler', $node_b );
    prompt( 'restore_device_cable', $node_b );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      $out = start_diagnostic( $diagnostic_node, "E" );
    } else {
      prompt( 'remove_device_cable', $node_b );
      prompt( 'restore_device_coupler', $node_b );
      prompt( 'restore_device_cable', $node_b );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed with a new coupler. The coupler is the suspect.
      prompt( 'suspect_device_coupler', $node_b );
      retest_msg();
      return $FOUND_SUSPECT_FRU;
    }
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_coupler', $node_b );
    prompt( 'replace_device_cable', $node_b );

    # try adapter

    prompt( 'remove_device_cable', $node_b );
    prompt( 'replace_device_adapter', $node_b );
    prompt( 'restore_device_cable', $node_b );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      $out = start_diagnostic( $diagnostic_node, "E" );
    } else {
      prompt( 'remove_device_cable', $node_b );
      prompt( 'restore_device_adapter', $node_b );
      prompt( 'restore_device_cable', $node_b );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed with a new adapter. The adapter is the suspect.
      prompt( 'suspect_device_adapter', $node_b );
      retest_msg();
      return $FOUND_SUSPECT_FRU;
    }
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_adapter', $node_b );
    prompt( 'replace_device_cable', $node_b );

  } else {

    prompt( 'remove_device_cable', $node_b );
    prompt( 'replace_device_cable', $node_b );
  }

  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the MIA or the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_t3_ofdg', $node_b );
  prompt( 'ra_t3test', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

sub a3500fc_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "a3500fc_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    $diagnostic_node = $node_b;
  }

  prompt( 'remove_device_cable', $node_b );
  prompt( 'replace_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_a3500fctest', $node_b );
  prompt( 'ra_rm6', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

sub fctape_node_isolation
{
  my ( $node_a, $node_b, $failed_node, $mode ) = @_;

  debug_header( $node_a, $node_b, "fctape_node_isolation" );

  my $diagnostic_node;
  my $answer;
  my $status;
  my $out;

  if ( $mode eq "PORT" ) {
    $diagnostic_node = $node_a;
  } else {
    if ( $node_a->{driver} eq 'ifp' ) {
      skip_io_device( $node_a->{type} );
    }
    $diagnostic_node = $node_b;
  }

  prompt( 'remove_device_cable', $node_b );
  prompt( 'replace_device_cable', $node_b );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $diagnostic_node, "E" );
  } else {
    prompt( 'remove_device_cable', $node_b );
    prompt( 'restore_device_cable', $node_b );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new cable. The cable is the suspect.
    prompt( 'suspect_device_cable', $node_b );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # it wasn't the CABLE. 
  prompt( 'multiple_suspects', $node_b );
  prompt( 'ra_fctapetest', $node_b );
  prompt( 'ra_hbatest', $node_a );

  return $PROBLEM_UNDETECTED;

}

sub port_node_isolation
{
  my ( $node, $other_node, $failed_node ) = @_;

  debug_header( $node, $other_node, "port_node_isolation" );

  my $answer;
  my $status;
  my $out;

  prompt( 'remove_port_cable', $node );
  prompt( 'insert_port_loopback', $node );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $node );
  } else {
    prompt( 'remove_port_loopback', $node );
    prompt( 'restore_port_cable', $node );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a loopback, the switch port node is OK
    # test the next node, could be a cable problem
    prompt( 'remove_port_loopback', $node );
    prompt( 'functional_port', $node );
    if ( $failed_node ne "b" ) {
      $failed_node == "b";
      return $SUCCESS;
    }

  } else {

    prompt( 'remove_port_loopback', $node );
    prompt( 'replace_port_gbic', $node );
    prompt( 'insert_port_loopback', $node );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      $out = start_diagnostic( $node );
    } else {
      prompt( 'remove_port_loopback', $node );
      prompt( 'restore_port_gbic', $node );
      prompt( 'restore_port_cable', $node );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed with a new GBIC. The GBIC is the suspect.
      prompt( 'remove_port_loopback', $node );
      prompt( 'restore_port_cable', $node );
      prompt( 'suspect_port_gbic', $node );
      retest_msg();
      return $FOUND_SUSPECT_FRU;
    } else {
      # the test failed with a new GBIC. The port and/or switch is suspect.

      prompt( 'restore_orignal_components', $node );

      prompt( 'remove_port_loopback', $node );
      prompt( 'restore_port_cable', $node );
      prompt( 'restore_port_gbic', $node );

      prompt( 'multiple_suspects', $node );

      prompt( 'suspect_switch', $node );
      prompt( 'ra_suspect_switch', $node );

      prompt( 'suspect_port', $node );
      prompt( 'ra_suspect_port', $node );

      prompt( 'suspect_replaced_gbic', $node );
      prompt( 'ra_suspect_replaced_gbic', $node );

      mulitple_frus_msg();
      return $PROBLEM_UNDETECTED;
    }

  }

  if ( $failed_node eq "b" ) {
    # node a passed all loopback testing and node b passed loopback
    # try CABLE
    prompt( 'replace_port_cable', $node );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      if ( $node->{type} eq 'brocade' && $other_node->{type} eq 'hba' ) {
        $out = start_diagnostic( $other_node, "E" );
      } else {
        $out = start_diagnostic( $node );
      }
    } else {
      prompt( 'remove_port_cable', $node );
      prompt( 'restore_port_cable', $node );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed with a new CABLE. The CABLE is the suspect.
      prompt( 'suspect_port_cable', $node );
      retest_msg();
      return $FOUND_SUSPECT_FRU;
    } else {
      premature_exit();
    }
  }
 
  tbd_msg();
  return $PROBLEM_UNDETECTED;

}

sub single_isl_port_node_isolation
{
  my ( $node, $other_node, $failed_node ) = @_;

  debug_header( $node, $other_node, "single_isl_port_node_isolation" );

  my $answer;
  my $status;
  my $out;

  prompt( 'remove_port_cable', $node );
  prompt( 'insert_port_loopback', $node );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $node );
  } else {
    prompt( 'remove_port_loopback', $node );
    prompt( 'restore_port_cable', $node );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a loopback, the switch port node is OK
    # test the next node, could be a cable problem
    prompt( 'functional_port', $node );
  } else {
    prompt( 'remove_port_loopback', $node );
    prompt( 'replace_port_gbic', $node );
    prompt( 'insert_port_loopback', $node );
    $answer = ask_continue();
    if ( $answer ne "n" ) {
      $out = start_diagnostic( $node );
    } else {
      prompt( 'remove_port_loopback', $node );
      prompt( 'restore_port_gbic', $node );
      prompt( 'restore_port_cable', $node );
      premature_exit();
    }
    if ( $out->{rc} == 0 ) {
      # the test passed with a new GBIC. The GBIC is the suspect.
      prompt( 'remove_port_loopback', $node );
      prompt( 'restore_port_cable', $node );
      prompt( 'suspect_port_gbic', $node );
      retest_msg();
      return $FOUND_SUSPECT_FRU;
    } 
  }

  # try remote GBIC
  prompt( 'remove_port_loopback', $node );
  prompt( 'restore_port_cable', $node );
  prompt( 'replace_port_gbic', $other_node );
  prompt( 'restore_port_cable', $other_node );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $node );
  } else {
    prompt( 'restore_port_gbic', $other_node );
    prompt( 'restore_port_cable', $other_node );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new remote GBIC. The GBIC is the suspect.
    prompt( 'suspect_port_gbic', $other_node );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  }

  # node a passed all loopback testing and node b passed loopback
  # try CABLE
  prompt( 'replace_port_cable', $other_node );
  $answer = ask_continue();
  if ( $answer ne "n" ) {
    $out = start_diagnostic( $node );
  } else {
    prompt( 'remove_port_cable', $other_node );
    prompt( 'restore_port_cable', $other_node );
    premature_exit();
  }
  if ( $out->{rc} == 0 ) {
    # the test passed with a new CABLE. The CABLE is the suspect.
    prompt( 'suspect_port_cable', $other_node );
    retest_msg();
    return $FOUND_SUSPECT_FRU;
  } else {

      # the test failed after trying all hot swap FRUs . 
      # The port and/or switch is suspect.
      prompt( 'restore_orignal_components', $node );

      prompt( 'remove_port_cable', $node );
      prompt( 'restore_port_cable', $node );

      prompt( 'multiple_suspects', $node );

      prompt( 'suspect_switch', $node );
      prompt( 'ra_suspect_switch', $node );

      prompt( 'suspect_port', $node );
      prompt( 'ra_suspect_port', $node );

      prompt( 'suspect_replaced_gbic', $node );
      prompt( 'ra_suspect_replaced_gbic', $node );

      mulitple_frus_msg();
      return $PROBLEM_UNDETECTED;
  }
 
  tbd_msg();
  return $PROBLEM_UNDETECTED;

}
# isolate FRUs in a port to device configuration utilizing the
# available port diagnostic

sub device_interconnect_isolation
{
  my ( $node_a, $node_b, $failed_node ) = @_;

  debug_header( $node_a, $node_b, "device_interconnect_isolation" );

  my $answer;
  my $status;
  my $out;
  my $mode = "PORT";

  if ( $node_a->{driver} eq 'ifp' ) {
    $mode = "IO";
  }

  if ( $node_b->{type} eq "a5k" ) {
    $status = a5k_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } elsif ( $node_b->{type} eq "fcdisk" ) {
    $status = fcdisk_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } elsif ( $node_b->{type} eq "dakdisk" ) {
    $status = daktari_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } elsif ( $node_b->{type} eq "t3" ) {
      $status = t3_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } elsif ( $node_b->{type} eq "t3b" ) {
      $status = t3b_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } elsif ( $node_b->{type} eq "a3500fc" ) {
    $status = a3500fc_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } elsif ( $node_b->{type} eq "tape" ) {
    $status = fctape_node_isolation( $node_a, $node_b, $failed_node, $mode );
  } else {
    device_unsupport_msg();
  }

  return $status;

}

sub parse {
  my($node_as, $node_bs) = @_;
  my @args;
  my(%node_a, %node_b);
  
  @args = split(/\|/, $node_as);
  foreach my $v (@args) {
    my($name, $value) = split(/=/, $v);
    $node_a{$name} = $value;
  }
  
  @args = split(/\|/, $node_bs);
  foreach my $v (@args) {
    my($name, $value) = split(/=/, $v);
    $node_b{$name} = $value;
  }
  return (\%node_a, \%node_b);
}

sub start_diagnostic
{
  my($node, $mode) = @_;
  my $test;
  my $options;
  my $sparcv9;
  my $option_user_pattern;
  my $estimated_time = 0;

  # for now use all default options when possible.
  # will need a interface to get "selected" options from the UI

  my $verbose;
  if ( ( $option_verbose == 1 ) || $DEBUG == 1 ) {
     $verbose = "-v"; 
  }

  my $wb_patterns = $option_pattern_type;
  if ( $option_pattern_type == "user" ) {
    $option_user_pattern = "|userpattern=$option_pattern";
  }

#node a:  {
          #'monHost' => '172.20.104.3',
          #'key' => '100000c0dd0085c3',
          #'name' => 'crash3-011',
          #'ip' => '172.20.104.12',
          #'type' => 'switch',
          #'WWN' => '100000c0dd0085c3',
          #'fcaddr' => '0x104000',
          #'port' => 3
        #};
#node b:  {
          #'monHost' => '172.20.104.3',
          #'key' => '100000c0dd008869',
          #'name' => 'crash3-012',
          #'ip' => '172.20.104.12',
          #'type' => 'switch',
          #'WWN' => '100000c0dd008869',
          #'fcaddr' => '',
          #'port' => 4
        #};
#node_a class info
              #'_name' => 'device5',
                 #'name' => 'crash3-011',
                 #'type' => 'switch',
                 #'hostIpno' => '172.20.104.3',
                 #'ipno' => '172.20.104.11',
                 #'class' => 'switch.switch',
                 #'ip' => '172.20.104.11',
                 #'key' => '100000c0dd0085c3',
                 #'wwn' => '100000c0dd0085c3',
                 #'active' => 'Y',
                 #'wwn2' => '100000c0dd0085c3',
                 #'host' => 'crash3.central.sun.com'
               #}, 'ConfigDev' );
#node_b class info
              #'_name' => 'device6',
                 #'name' => 'crash3-012',
                 #'type' => 'switch',
                 #'hostIpno' => '172.20.104.3',
                 #'ipno' => '172.20.104.12',
                 #'class' => 'switch.switch',
                 #'ip' => '172.20.104.12',
                 #'key' => '100000c0dd008869',
                 #'wwn' => '100000c0dd008869',
                 #'active' => 'Y',
                 #'wwn2' => '100000c0dd008869',
                 #'host' => 'crash3.central.sun.com'
               #}, 'ConfigDev' );

  # Qlogic Switch
  if ( $node->{type} eq 'switch' ) {
    # Code potentially for 2.1; NOT the brocade T-patch
    # if the switch has an IP, use it instead of IP:fcaddr
    # my $switch_ip;
    #my $dev = $Config->deviceByKey( $node->{key} );
    # utilize the database IP instead of the discman IP, if found
    #if ( defined( $dev->{ip} ) && ( $dev->{ip} ne $node->{ip} ) ) {
    #  $switch_ip = "$dev->{ip}:";
    #} else {
    #  $switch_ip = "$node->{ip}:$node->{fcaddr}";
    #}
    $test = "switchtest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{port}:$node->{ip}:$node->{fcaddr}|selectpattern=$wb_patterns" . $option_user_pattern . '"';
    #$options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{port}:$switch_ip|selectpattern=$wb_patterns" . $option_user_pattern . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 1;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = "4-7";
    } else {
      $estimated_time = "40-70";
    }

  # Brocade Silkworm Switch
  } elsif ( $node->{type} eq 'brocade' ) {
    my $dev = $Config->deviceByKey( $node->{key} );
    my $user_password = Util->decode($dev->{telnet});
    if ( !defined( $user_password ) ) {
      prompt( 'no_password', $node );
      prompt( 'ra_password', $node );
      ra_exit();
    }
    $test = "brocadetest";
    # options to be implemented when brocade supports users patterns
    #$options = "$verbose -f -o " . '"' . "dev=$node->{port}:$node->{ip}|passwd=$user_password|selectpattern=$wb_patterns" . $option_user_pattern . '"';
    $options = "$verbose -f -o " . '"' . "dev=$node->{port}:$node->{ip}|passwd=$user_password" . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 1;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = 7;
    } else {
      $estimated_time = 70;
    }
  # Vicom
  } elsif ( $node->{type} eq 've' ) {
    $test = "veporttest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|selectpattern=$wb_patterns" . $option_user_pattern . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 1;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = 7;
    } else {
      $estimated_time = 70;
    }
  # hbas (need type for Qlogic 22XX, 2100, soc+
  } elsif ( $node->{type} eq 'hba' ) {
    if ( $node->{driver} eq 'fp' ) {
      $test = "qlctest";
      if ( $mode eq "E" ) {
        $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|run_connect=Yes|selftest=Disable|mbox=Disable|checksum=Disable|ilb_10=Disable|ilb=Disable|elb=Enable|iterations=100|xcnt=65536|selectpattern=$wb_patterns" . $option_user_pattern . '"';
      } elsif ( $mode eq "I" ) {
        $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|run_connect=Yes|selftest=Disable|mbox=Disable|checksum=Disable|ilb_10=Enable|ilb=Enable|iterations=10|xcnt=65536|selectpattern=$wb_patterns" . $option_user_pattern . '"';
      } elsif ( $mode eq "C" ) {
        $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|run_connect=Yes|selftest=Enable|mbox=Enable|checksum=Enable|ilb_10=Enable|ilb=Enable|elb=Disable|iterations=10000|xcnt=10|selectpattern=$wb_patterns" . $option_user_pattern . '"';
      } else {
        tbd_msg();
        premature_exit();
      }
      if ($option_pattern_type eq "user") {
        $estimated_time = 1;
      } elsif ($option_pattern_type eq "critical") {
        $estimated_time = 1;
      } else {
        $estimated_time = 2;
      }
    } elsif ( $node->{driver} eq 'ifp' ) {
      $sparcv9 = ($node->{bitMode} == 64) ? "sparcv9/" : "";
      $test = "ifptest";
      $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|mbox=e|fwrevcheck=e|checksum=e|modrevcheck=e" . '"';
      $estimated_time = 1;
    } elsif ( $node->{driver} eq 'socal' ) {
      $test = "socaltest";
      if ( $mode eq "E" ) {
        $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|ilb=disable|elb=disable|lbf=enable|iterations=100|xcnt=65536|selectpattern=$wb_patterns" . $option_user_pattern . '"';
      } elsif ( $mode eq "I" ) {
        $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|ilb=disable|elb=enable|lbf=disable|selectpattern=$wb_patterns" . $option_user_pattern . '"';
      } elsif ( $mode eq "C" ) {
        $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{path}|ilb=enable|elb=disable|lbf=disable|selectpattern=$wb_patterns" . $option_user_pattern . '"';
      } else {
        tbd_msg();
        premature_exit();
      }
      if ($option_pattern_type eq "user") {
        $estimated_time = 1;
      } elsif ($option_pattern_type eq "critical") {
        $estimated_time = 1;
      } else {
        $estimated_time = 4;
      }
    }
  # no loopback for these devices, must run I/O, port test or HBA lbf 
  # should determine parent for best choice, use I/O for now
  } elsif ( $node->{type} eq 'a5k' ) {
    $test = "a5ktest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{logical}|partition=2|selftest=Enable|wrdevbuf=Enable|selectpattern=$wb_patterns|rawsub=Enable|method=SyncIO+AsyncIO|rawcover=1|rawiosize=0x32768" . $option_user_pattern . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 2;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = 3;
    } else {
      $estimated_time = 4;
    }
  } elsif ( $node->{type} eq 't3' ) {
    $test = "t3test";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{logical}|partition=2|rawsub=e|method=SyncIO+AsyncIO|rawcover=1|rawiosize=0x32768" . $option_user_pattern . '"';
    $estimated_time = 5;
  } elsif ( $node->{type} eq 'a3500fc' ) {
    $test = "a3500fctest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{logical}|partition=2|wrdevbuf=Disable|rawsub=Enable|method=SyncIO+AsyncIO|rawcover=1|rawiosize=0x32768" . $option_user_pattern . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 2;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = 3;
    } else {
      $estimated_time = 4;
    }
  } elsif ( $node->{type} eq 'tape' ) {
    $test = "fctapetest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{logical}|" . '"';
    $estimated_time = 5;
  } elsif ( $node->{type} eq 'fcdisk' ) {
    $test = "fcdisktest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{logical}|partition=2|selftest=Enable|wrdevbuf=Enable|selectpattern=$wb_patterns|rawsub=Enable|method=SyncIO+AsyncIO|rawcover=1|rawiosize=0x32768" . $option_user_pattern . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 2;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = 3;
    } else {
      $estimated_time = 4;
    }
  } elsif ( $node->{type} eq 'dakdisk' ) {
    $test = "daktest";
    $options = "$verbose -q -i1 -f -o " . '"' . "dev=$node->{logical}|partition=2|selftest=Enable|wrdevbuf=Enable|selectpattern=$wb_patterns|rawsub=Enable|method=SyncIO+AsyncIO|rawcover=1|rawiosize=0x32768" . $option_user_pattern . '"';
    if ($option_pattern_type eq "user") {
      $estimated_time = 2;
    } elsif ($option_pattern_type eq "critical") {
      $estimated_time = 3;
    } else {
      $estimated_time = 4;
    }
  } else {
    tbd_msg();
    premature_exit();
  }

  my $diagnostic = "/Diags/bin/$sparcv9$test $options";

  start_test_msg( $node, $test, $options, $verbose, $estimated_time );
  my $out = Scheduler->syncRun($node->{monHost}, "$diagnostic", $test_timeout, { progressHandler => \&printit });

  if ( defined($out) ) {
    stop_test_msg( $node, $test, $out, $verbose );
  } else {
    print( "Program Error: Status not returned from remote test.\n" );
    program_error();
  }
  return $out;
}

sub printit {
  my($sofar) = @_;
  print $sofar;
}

#  get_response("the question", "y=yes|n=no");

sub get_response {
  my ($question, $opt) = @_;
  my $response;

  if ($GUI) {
    print $question . " ?\n";
    $response = Scheduler->QandA($Qfile, $opt, 60*60); # wait 60 mins max
    $response = "n" if (!defined($response));
    return( $response );
    
  } else {
    $opt =~ s/\|/, /g;
    print "$question [$opt]";
    $response = <STDIN>;
    chop( $response );
    return( $response );
  }
}


sub retest_msg
{
  print( "\n" );
  print( "Retest to verify FRU replacement.\n" );
}

sub ra_exit
{
  print( "\n" );
  print( "Perform a recommeded action and restart linktest.\n");
  exit( $ERROR );
}

sub mulitple_frus_msg
{
  print( "\n" );
  print( "linktest could not isolate to a single hot swap FRU.\n" );
  print( "Perform a recommeded action and restart linktest.\n");
}

sub device_unsupported_msg
{
  print( "Device type unknown or error detecting device.\n" );
  print( "Isolation code TBD: report to developer!\n" );
  print( "Failing FRU not isolated!\n" );
}

sub tbd_msg
{
  print( "Isolation code TBD: report to developer!\n" );
  print( "Failing FRU not isolated!\n" );
}

sub program_error
{
  print( "Program Error: Isolation terminated pre-maturely!\n" );
  print( "Failing FRU not isolated!\n" );
  exit( $ERROR );
}

sub premature_exit
{
  print( "Isolation terminated pre-maturely!\n" );
  print( "Probable cause: User Request\n" );
  print( "Probable cause: Timed out waiting for user input\n" );
  print( "Failing FRU not isolated!\n" );
  exit( $ABORTED );
}

sub skip_io_device
{
  my($device) = @_;

  print( "\nSkipping $device.\n" );
  print( "Required FC loopback test capabilities are not supported by this device.\n" );
}

sub t3_f_port_detected
{
  my($dev_a, $dev_b) = @_;

  print( "linktest does not support testing of $dev_a F port connected to $dev_b.\n" );
  print( "Required FC loopback test capabilities are not supported by the T3 in Fabric mode.\n" );
  print(  "Reference the appropriate Trouble Shooting Guide for Manual FRU Isolation and addititional information.\n" );
  exit( $ERROR );
}

sub untestable_link
{
  my($dev_a, $dev_b) = @_;

  print( "Test terminated!\n" );
  print( "linktest does not support testing of $dev_a connected to $dev_b.\n" );
  print( "Required FC loopback test capabilities are not supported by one or both devices.\n" );
  print(  "Reference the appropriate Trouble Shooting Guide for Manual FRU Isolation and addititional information.\n" );
  exit( $ERROR );
}

sub ask_continue
{
  my $answer = get_response( "Continue Isolation", $YESNO);
  return $answer;
}

sub ask_t3b_direct
{
  my $answer = get_response( "T3B: Direct cable", $YESNO);
  return $answer;
}


#################################################
#   PROMPT MODULE
#
#   usage:  prompt('functional_hba', $node_a);
#
#################################################

sub prompt {
  my($tag, $dev) = @_;
 
  my $device;
  my $port;
  my $fcaddr;

  my $class_info = $Config->deviceByKey( $dev->{key} );

# '_name' => 'device3',
#'name' => '172.20.67.167',
#'type' => 'brocade',
#'ipno' => '172.20.67.167',
#'class' => 'switch.brocade',
#'ip' => '172.20.67.167',
#'key' => '1000006069201efc',
#'wwn' => '1000006069201efc',
#'active' => 'Y',
#'telnet' => '%70%61%73%73%77%6f%72%64',
#'host' => ''

# '_name' => 'device2',
# 'name' => 'Leno',
# 'type' => 'a5k',
# 'class' => 'storage.a5k',
# 'key' => '5080020000032510',
# 'hba' => 'S',
# 'wwn' => '5080020000032510',
# 'active' => 'Y',
# 'host' => ''

#'_name' => 'device1',
#'name' => '172.20.67.168',
#'type' => 't3',
#'ctrl_model' => '375-0084-02-JJ11',
#'ipno' => '172.20.67.168',
#'class' => 'storage.t3',
#'ip' => '172.20.67.168',
#'key' => 'slr-mi.370-3990-01-e-f0.016101',
#'wwn' => '50020f2300003ee5',
#'active' => 'Y',
#'telnet' => 'diags',
#'wwn2' => '50020f2300003d2c',
#'host' => ''
 
  if ( $dev->{type} eq "switch" ) {
    $device = $dev->{WWN};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "brocade" ) {
    $device = $dev->{WWN};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "hba" ) {
    $device = $dev->{path};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "a5k" ) {
    $device = $dev->{logical};
    if ( $dev->{port} == 0 ) {
      $port = "A0";
    } elsif ( $dev->{port} == 1 ) {
      $port = "B0";
    } elsif ( $dev->{port} == 2 ) {
      $port = "A1";
    } elsif ( $dev->{port} == 3 ) {
      $port = "B1";
    } else {
      $port   = "undefined";
    }
  } elsif ( $dev->{type} eq "t3" ) {
    $device = $dev->{logical};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "t3b" ) {
    $device = $dev->{logical};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "a3500fc" ) {
    $device = $dev->{logical};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "fcdisk" ) {
    $device = $dev->{logical};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "dakdisk" ) {
    $device = $dev->{logical};
    $port   = $dev->{port};
  } elsif ( $dev->{type} eq "tape" ) {
    $device = $dev->{logical};
    $port   = $dev->{port};
  } else {
    $device = $dev->{type};
    $port   = $dev->{port};
  }
  if ( !defined( $port ) ) {
    $port = 0;
  }
  if ( $class_info->{name} ) {
    $device .= " (" . $class_info->{name} . ")";
  }
  my $p = $PROMPTS{$tag};

  if ($p) {
     $p = $p . "\n";
     print( sprintf( $p, $dev->{type}, $device, $port ) );
  } else {
     print( "Internal error: Cannot find prompt $tag\n" );
     error( "Cannot find prompt $tag" );
  }
}

sub error {
  my($err) = @_;

  open(OO, ">>" . System->get_home() . "/log/linktest.log");
  print OO $err . "\n";
  close(OO);
}
  
sub start_linktest_msg
{
  my( $node_a, $node_b ) = @_;

  my $device = $node_a->{type};
  if ( $node_a->{type} eq 'hba' ) {
    $device = $node_a->{driver};
  }

  print( "linktest started on FC interconnect: $device to $node_b->{type}\n" );

}

sub stop_linktest_msg
{
  my( $node_a, $node_b ) = @_;

  print( "linktest completed on FC interconnect: $node_a->{type} to $node_b->{type}\n" );
}

sub failed_linktest_msg
{
  my( $node_a, $node_b ) = @_;

  print( "linktest failed on FC interconnect: $node_a->{type} to $node_b->{type}\n" );
}


sub start_test_msg
{
  my( $node, $test, $options, $verbose, $estimated_time) = @_;

  my $name;

  if ( $node->{type} eq "switch" ) {
    $name = "$node->{type} $node->{WWN} port $node->{port}";
  } else {
    $name = "$node->{type} port $node->{port}";
  }

  if ( $node->{type} eq 'brocade' ) {
    print( "linktest -Pattern- option unsupported and ignored for this device.\n" );
  }

  print( "$test started on $name\n" );

  # now that verbose is working, this is not req'd 
  #if ( $estimated_time != 0 ) {
    #print( "Estimated test time $estimated_time minute(s)\n" );
  #}
  #if ( $verbose ) {
  #  print "$test options: $options\n";
  #}
}

sub stop_test_msg
{
  my( $node, $test, $out, $verbose ) = @_;

  #if ( $verbose ) {
  #  print( "$out->{data}\n" );
  #}

  print( "$test " );
  if ( $out->{rc} == 0 ) {
    print( "completed successfully\n" );
  } else {
    print( "failed\n" );
  }

  if ( $verbose ) {
    print( "error code: $out->{rc}\n" );
  }

}

sub usage {
  tbd_msg(); 
  print "Usage: linktest -a node_a -b node_b\n";
  print " linktest -a \"type=t3|logical=/dev/...|ip=1.1.1.1|port=0|portWWN=wwn|monHost=rasd2\" \n";
  print "          -a \"type=switch|ip=1.1.1.1|port=2|fcaddr=108600|WWN=111|monHost=rasd2\"\n";
  print "          -a \"type=vicom|ip=1.1.1.1|login=v1|port=0|WWN=111|portWWN=111|monHost=rasd2\"\n";
  print "          -a \"driver=socal|type=hba|registerName=qlc0|path=/device...|ip=datahost.central\" \n";
  print "\n";
}

sub debug_header
{
  my( $node_a, $node_b, $function_name ) = @_;

  if ( $DEBUG == 1) {
    print "$function_name ()\n";
    print "node a: type is $node_a->{type}, ip is $node_a->{path} \n";
    print "node a: ";
    print substr(Dumper($node_a),7);
    print "node b: ";   
    print substr(Dumper($node_b),7);
  }
}

