674c7e029b
git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@6293 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd
563 lines
14 KiB
Perl
Executable File
563 lines
14 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
|
|
BEGIN
|
|
{
|
|
$::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat';
|
|
}
|
|
use lib "$::XCATROOT/lib/perl";
|
|
use Sys::Syslog;
|
|
use xCAT::Table;
|
|
use xCAT::Utils;
|
|
use xCAT::NetworkUtils;
|
|
use xCAT_plugin::ipmi;
|
|
use xCAT_monitoring::monitorctrl;
|
|
use Socket;
|
|
use Data::Dumper;
|
|
#use strict;
|
|
|
|
|
|
#-------------------------------------------------------------------------------
|
|
=head1 xcat_traphandler
|
|
=head2 Description
|
|
Script for SNMP trap handling.
|
|
|
|
=cut
|
|
#-------------------------------------------------------------------------------
|
|
# admin needs to create a mail aliase called alerts
|
|
# put "alerts: emailadd,emailaddr.." to /etc/aliases file
|
|
# then run newaliases command
|
|
my $MAILTO="alerts";
|
|
|
|
my $message;
|
|
my $briefmsg;
|
|
my $node1;
|
|
my $info;
|
|
my $severity_type;
|
|
my $severity;
|
|
my $appname;
|
|
my $id;
|
|
my $message1;
|
|
my $errsrc;
|
|
|
|
#get settings
|
|
my $EMAIL=0;
|
|
my $LOG=0;
|
|
my $IGNORE=0;
|
|
my $DB=0;
|
|
my %hashI=();
|
|
my %hashE=();
|
|
my %hashL=();
|
|
my %hashR=();
|
|
my %hashD=();
|
|
my %hashRUN=();
|
|
my %settings=xCAT_monitoring::monitorctrl->getPluginSettings("snmpmon");
|
|
my $i=$settings{'ignore'};
|
|
if ($i) { %hashI =parseSettings($i); }
|
|
my $e=$settings{'email'};
|
|
if ($e) { %hashE=parseSettings($e); }
|
|
my $l=$settings{'log'};
|
|
if ($l) { %hashL=parseSettings($l); }
|
|
my $d=$settings{'db'};
|
|
if ($d) { %hashD=parseSettings($d); }
|
|
|
|
foreach my $k (keys %settings) {
|
|
if ($k =~ /runcmd(\d*)/) {
|
|
my $r=$settings{$k};
|
|
my %hashTemp=parseSettings($r);
|
|
$hashR{$k}=\%hashTemp;
|
|
$hashRUN{$k}=0;
|
|
}
|
|
}
|
|
|
|
# read host name
|
|
my $host=<STDIN>;
|
|
chomp($host);
|
|
|
|
# read the host ip
|
|
my $ip=<STDIN>;
|
|
chomp($ip);
|
|
|
|
# read uptime
|
|
my $uptimeline=<STDIN>;
|
|
chomp($uptimeline);
|
|
my @a=split(/ /, $uptimeline);
|
|
my $oid=shift @a;
|
|
my $value=join(' ', @a);
|
|
$message .= " $oid=$value\n";
|
|
|
|
# read trapid
|
|
my $trapidline=<STDIN>;
|
|
chomp($trapidline);
|
|
@a=split(/ /, $trapidline);
|
|
$oid=shift @a;
|
|
$value=join(' ', @a);
|
|
$message .= " $oid=$value\n";
|
|
checkWithOid($oid, $value);
|
|
#print "I=$IGNORE,E=$EMAIL, L=$LOG, R=$RUNCMD D=$DB\n";
|
|
if ($IGNORE) { exit 0;}
|
|
|
|
#for ipmi traps, the values is: SNMPv2-SMI::enterprises.3183.1.1.0.x or
|
|
# RFC1155-SMI::enterprises.3183.1.1.0.x, where x is the ipmi trap id.
|
|
my $trapid;
|
|
if ($value =~ /enterprises\.3183\.1\.1\.0\.(.*)/) { $trapid=$1; }
|
|
|
|
#print "trapid=$trapid\n";
|
|
|
|
|
|
|
|
my $holder;
|
|
my $begin=1;
|
|
my $pair;
|
|
my $temp;
|
|
while ($temp=<STDIN>) {
|
|
chomp($temp);
|
|
my $temp1=$temp; #save the one with quotes
|
|
$temp1 =~ s/\"//g;
|
|
my $c1=length($temp);
|
|
my $c2=length($temp1);
|
|
|
|
|
|
if (($c1-$c2)%2 == 0) {
|
|
if ($begin==1) { # single line
|
|
$pair=$temp;
|
|
}
|
|
else { # middle multi-line
|
|
$pair .= " $temp";
|
|
next;
|
|
}
|
|
} else { # multi line
|
|
if ($begin==1) {
|
|
$pair=$temp; #start of multi-line
|
|
$begin=0;
|
|
next;
|
|
} else {
|
|
$pair .= " $temp"; #end of multi-line
|
|
$begin=1;
|
|
}
|
|
}
|
|
|
|
@a=split(/ /, $pair);
|
|
$oid=shift @a;
|
|
$value=join(' ', @a);
|
|
$value =~ s/^"//;
|
|
$value =~ s/"$//;
|
|
|
|
#for BladeCenter MM traps and RSA II traps, creat a brief message
|
|
if ($oid =~ /BLADESPPALT-MIB::spTrapAppId|RSASPPALT-MIB::ibmSpTrapAppId/) {
|
|
$briefmsg .= " App ID: $value\n";
|
|
$appname=$value;
|
|
}
|
|
elsif ($oid =~ /BLADESPPALT-MIB::spTrapAppType|RSASPPALT-MIB::ibmSpTrapAppType/) {
|
|
$briefmsg .= " App Alert Type: $value\n";
|
|
$id=$value;
|
|
}
|
|
elsif ($oid =~ /BLADESPPALT-MIB::spTrapMsgText|RSASPPALT-MIB::ibmSpTrapMsgText/) {
|
|
$briefmsg .= " Message: $value\n";
|
|
$message1=$value;
|
|
}
|
|
elsif ($oid =~ /BLADESPPALT-MIB::spTrapBladeName/) {
|
|
my $temp="$value";
|
|
$temp =~ /^\"(.*)\"/;
|
|
if ($1) {
|
|
$briefmsg .= " Blade Name: $value\n";
|
|
$node1=$1;
|
|
}
|
|
}
|
|
elsif (($oid =~ /BLADESPPALT-MIB::spTrapSourceId/)) {
|
|
$briefmsg .= " Error Source=$value\n";
|
|
$errsrc=$value;
|
|
}
|
|
elsif ($oid =~ /BLADESPPALT-MIB::spTrapPriority|RSASPPALT-MIB::ibmSpTrapPriority/) {
|
|
# Critical Alert(0), Major(1), Non-Critical Alert(2), System Alert(4),
|
|
# Recovery Alert(8), Informational Only Alert(255)
|
|
if ($value==0) {
|
|
$severity="Critical Alert";
|
|
$severity_type="Critical";
|
|
} elsif ($value==1) {
|
|
$severity="Major Alert";
|
|
$severity_type="Critical";
|
|
} elsif ($value==2) {
|
|
$severity="Non-Critical Alert";
|
|
$severity_type="Warning";
|
|
} elsif ($value==4) {
|
|
$severity="System Alert";
|
|
$severity_type="Warning";
|
|
} elsif ($value==8) {
|
|
$severity="Recovery Alert";
|
|
$severity_type="Informational";
|
|
} elsif ($value==255) {
|
|
$severity="Informational Alert";
|
|
$severity_type="Informational";
|
|
}
|
|
}
|
|
elsif ($oid =~ /enterprises\.3183\.1\.1\.1/) { #IPMI PRTs (traps)
|
|
$node1=$host;
|
|
#$node1 =~ s/(-(eth|man)\d+)?(\..*)?$//;
|
|
|
|
my $ip1=$ip;
|
|
$ip1 =~ /(\d+\.\d+\.\d+\.\d+)/;
|
|
$ip1=$1;
|
|
|
|
# get the host name if it is unknown
|
|
if ($node1 =~/<UNKNOWN>/) {
|
|
my $name = xCAT::NetworkUtils->gethostname($ip1);
|
|
if ($name) {
|
|
$node1=$name;
|
|
$host=$name;
|
|
my @shorthost = split(/\./, $node1);
|
|
$node1=$shorthost[0];
|
|
}
|
|
}
|
|
|
|
#print "node1=$node1\n";
|
|
#node1 is the bmc name, we need the node that bmc connects to to call xCAT
|
|
my $realnode;
|
|
my $ipmitab = xCAT::Table->new('ipmi');
|
|
if (defined($ipmitab)) {
|
|
my @tmp1=$ipmitab->getAllNodeAttribs(['node','bmc']);
|
|
if (@tmp1 && (@tmp1 > 0)) {
|
|
foreach(@tmp1) {
|
|
if ($_->{bmc} eq $node1) { $realnode=$_->{node}; last;}
|
|
}
|
|
}
|
|
$ipmitab->close();
|
|
}
|
|
if ($realnode) {$node1=$realnode;}
|
|
#print "node1=$node1\n";
|
|
|
|
|
|
# make the vlaue into a hex array for decoding
|
|
$value =~ s/\"//g;
|
|
my @varray=split(/\s+/, $value);
|
|
#print "varray=@varray\n";
|
|
foreach (@varray) { $_=hex($_); }
|
|
my $error = xCAT_plugin::ipmi->decodealert($trapid, $node1, @varray);
|
|
$briefmsg .= $error;
|
|
$message .= " decodedipmialert=$error\n";
|
|
|
|
#severity value from decodealert are:
|
|
#LOG,INFORMATION,OK,CRITICAL,NON-RECOVERABLE,MONITOR and UNKNOWN-SEVERITY
|
|
$severity_type="Warning";
|
|
if ($error) {
|
|
my @tempArray=split(/\:/, $error);
|
|
$severity=$tempArray[0];
|
|
if ($severity eq "LOG") {#in fact this is called "unspecifiled"
|
|
$severity_type="Warning";
|
|
}
|
|
elsif($severity eq "INFORMATION") {
|
|
$severity_type="Informational";
|
|
}
|
|
elsif($severity eq "OK") {
|
|
$severity_type="Informational";
|
|
}
|
|
elsif($severity eq "CRITICAL") {
|
|
$severity_type="Critical";
|
|
}
|
|
elsif($severity eq "NON-RECOVERABLE") {
|
|
$severity_type="Critical";
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
$message .= " $oid=$value\n";
|
|
|
|
#check agains the settings
|
|
$value =~ s/\"//g;
|
|
checkWithOid($oid, $value);
|
|
#print "I=$IGNORE,E=$EMAIL, L=$LOG, R=$RUNCMD, D=$DB\n";
|
|
if ($IGNORE) { exit 0;}
|
|
}
|
|
|
|
if (!$severity_type) { $severity_type="Warning"; }
|
|
if ((!$IGNORE) && exists($hashI{$severity_type})) {
|
|
#print "ignore, exit\n";
|
|
exit 0;
|
|
}
|
|
if ((!$EMAIL) && exists($hashE{$severity_type})) { $EMAIL=1; }
|
|
if ((!$LOG) && exists($hashL{$severity_type})) { $LOG=1; }
|
|
if ((!$DB) && exists($hashD{$severity_type})) { $DB=1; }
|
|
foreach my $k (keys %hashRUN) {
|
|
if (($hashRUN{$k} == 0) && exists($hashR{$k}->{$severity_type})) { $hashRUN{$k} = 1; }
|
|
}
|
|
#print "I=$IGNORE,E=$EMAIL, L=$LOG, R=$RUNCMD, D=$DB\n";
|
|
|
|
#email 'alerts' aliase
|
|
if ($EMAIL || ((keys(%hashE)==0) && ($severity_type =~/Critical|Warning/))) {
|
|
#($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
|
|
#$datetime=sprintf "%2d-%02d-%04d %02d:%02d:%02d", $mon+1,$mday,$year+1900,$hour,$min,$sec;
|
|
#my $head="SNMP $severity received from $host($ip) on $datetime\n$briefmsg\n";
|
|
my $head="SNMP $severity received from $host($ip)\n$briefmsg\n";
|
|
if ($node1) { $info= getMoreInfo($node1);}
|
|
my $middle="Trap details:\n$message\n";
|
|
|
|
#email the full message to the alerts aliase
|
|
my $cmd="echo \'$info$head\n$middle\' \| mail -s \"$severity_type: Cluster SNMP Alert\!\" $MAILTO";
|
|
$ENV{'XCATCFG'}="";
|
|
`$cmd`;
|
|
}
|
|
|
|
#log to syslog if needed. default is log all
|
|
$ENV{'XCATCFG'}="";
|
|
if ($LOG || (keys(%hashL)==0)) {
|
|
my $head="SNMP $severity received from $host($ip).";
|
|
my $body;
|
|
if ($briefmsg) { $body=$briefmsg;}
|
|
else { $body=$message; }
|
|
|
|
openlog("xCATMon Event","","local4");
|
|
if ($severity_type eq "Informational") {
|
|
syslog("local4|info", "$head\n$body\n");
|
|
} else {
|
|
syslog("local4|err", "$head\n$body\n");
|
|
}
|
|
closelog();
|
|
}
|
|
|
|
#save to eventlog table in xCAT database
|
|
if ($DB) {
|
|
my $head="SNMP $severity received from $host($ip)\n$briefmsg\n";
|
|
if (($node1)&& (!$info)) { $info= getMoreInfo($node1);}
|
|
my $middle="Trap details:\n$message\n";
|
|
my $event={
|
|
eventtype => 'event',
|
|
monitor => 'snmpmon',
|
|
monnode => $host,
|
|
node => $node1? $node1:"",
|
|
application => $appname,
|
|
component => $errsrc ,
|
|
id =>$id,
|
|
severity => $severity_type,
|
|
message => $message1,
|
|
rawdata => "$info$head\n$middle",
|
|
};
|
|
my @a=();
|
|
push(@a, $event);
|
|
xCAT::Utils->logEventsToDatabase(\@a);
|
|
}
|
|
|
|
#run user defined commands if needed.
|
|
foreach my $k (keys %hashRUN) {
|
|
if ($hashRUN{$k} == 1) {
|
|
$k =~ /runcmd(\d*)/;
|
|
my $scripts=$settings{"cmds$1"};
|
|
while ($scripts =~ s/^([^,]+)(,)*//) {
|
|
my $cmd="echo \'host=$host\nip=$ip\n$message\n\' \| $1";
|
|
`$cmd`;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
#--------------------------------------------------------------------------------
|
|
=head3 getMoreInfo
|
|
This function returns the node module/type, serial number, position etc.
|
|
Arguments:
|
|
node-- name of the node.
|
|
Returns:
|
|
A string with node info ready to display.
|
|
=cut
|
|
#--------------------------------------------------------------------------------
|
|
sub getMoreInfo {
|
|
my $node=shift;
|
|
my $pos;
|
|
my $vpd;
|
|
|
|
# get module name and serial number from the xCAT DB.
|
|
my $ref;
|
|
my $table=xCAT::Table->new("vpd", -create =>1);
|
|
if ($table) {
|
|
my $ref = $table->getNodeAttribs($node, ['serial', 'mtm']);
|
|
if ($ref) {
|
|
$vpd .= " Type/Mudule: ";
|
|
if ($ref->{mtm}) { $vpd .= $ref->{mtm};}
|
|
$vpd .= "\n";
|
|
$vpd .= " Serial Number: ";
|
|
if ($ref->{serial}) { $vpd .= $ref->{serial};}
|
|
$vpd .= "\n";
|
|
}
|
|
$table->close();
|
|
}
|
|
|
|
# get the info from rinv command if nothing in the vpd table
|
|
if (!$vpd) {
|
|
my $result=`XCATBYPASS=Y $::XCATROOT/bin/rinv $node all 2>&1 | egrep -i '(model|serial)' | grep -v Univ`;
|
|
if ($? == 0) {#success
|
|
chomp($result);
|
|
my @b=split(/\n/, $result);
|
|
foreach (@b) {
|
|
s/^(.*)\:(.*)\:(.*)$/$2: $3/;
|
|
$vpd .= " $_\n";
|
|
}
|
|
}
|
|
}
|
|
if (!$vpd) {
|
|
$vpd .= " Type/Mudule: \n";
|
|
$vpd .= " Serial Number: \n";
|
|
}
|
|
|
|
|
|
#get the position
|
|
my $table1=xCAT::Table->new("nodepos", -create =>1);
|
|
if ($table1) {
|
|
my $ref1 = $table1->getNodeAttribs($node, ['rack', 'u', 'chassis', 'slot', 'room', 'comments']);
|
|
if (($ref1) && ($ref1->{room})) { $pos .= " Room: " . $ref1->{room}. "\n"; }
|
|
if(($ref1) && ($ref1->{rack})) { $pos .= " Rack: ". $ref1->{rack}. "\n";}
|
|
if(($ref1) && ($ref1->{u})) { $pos .= " Unit: " . $ref1->{u} . "\n"; }
|
|
if(($ref1) && ($ref1->{chassis})) { $pos .= " Chassis: " . $ref1->{chassis} . "\n";}
|
|
if(($ref1) && ($ref1->{slot})) { $pos .= " Slot: " . $ref1->{slot} . "\n";}
|
|
if(($ref1) && ($ref1->{comments})) { $pos .= " Comments: " . $ref1->{comments} . "\n";}
|
|
|
|
$pos .= "\n";
|
|
|
|
$table1->close();
|
|
}
|
|
|
|
if (($pos) || ($vpd)) {
|
|
return " Node: $node\n$vpd$pos\n";
|
|
}
|
|
|
|
return "";
|
|
}
|
|
|
|
|
|
#--------------------------------------------------------------------------------
|
|
=head3 parseSettings
|
|
This function takes a setting string which looks like "key1=value1,key2=value2..."
|
|
and returns a hash (key1=>value1, key2=>value2...).
|
|
Arguments:
|
|
setting the setting string.
|
|
Returns:
|
|
A hash.
|
|
=cut
|
|
#--------------------------------------------------------------------------------
|
|
sub parseSettings {
|
|
my $settings=shift;
|
|
my %ret=();
|
|
while (($settings =~ s/^(Informational|Warning|Critical|All|None)()()(,)*//) ||
|
|
($settings =~ s/^([^\=]+)(=~)(\"[^\"]+\")(,)*//) ||
|
|
($settings =~ s/^([^\=]+)(=~)([^\"\,]+)(,)*//) ||
|
|
($settings =~ s/^([^\=]+)(=)(\"[^\"]+\")(,)*//) ||
|
|
($settings =~ s/^([^\=]+)(=)([^\"\,]+)(,)*//)) {
|
|
my $val='eq';
|
|
if ($2 eq "=~") { $val='=~';}
|
|
if (exists($ret{$1}{$val})) {
|
|
my $pa=$ret{$1}{$val};
|
|
push(@$pa, $3);
|
|
}
|
|
else {
|
|
$ret{$1}{$val}=[$3];
|
|
}
|
|
}
|
|
|
|
#print Dumper(%ret);
|
|
return %ret;
|
|
}
|
|
|
|
#--------------------------------------------------------------------------------
|
|
=head3 checkWithOid
|
|
This function checks the input strings with the setting to see what
|
|
actions need to be done for this event.
|
|
Arguments:
|
|
oid the oid string
|
|
value the value string
|
|
Returns:
|
|
none. The variables $EMAIL, $LOG, $IGNORE and $RUNCMD may be changed.
|
|
=cut
|
|
#--------------------------------------------------------------------------------
|
|
sub checkWithOid {
|
|
my $o=shift;
|
|
my $v=shift;
|
|
|
|
sub checking {
|
|
my $hashX=shift;
|
|
my $o=shift;
|
|
my $v=shift;
|
|
|
|
if (exists($hashX->{'All'})) { return 1; }
|
|
if (exists($hashX->{'None'})) { return 0; }
|
|
|
|
my @a_oid=split('::', $o);
|
|
my $new_o=$o;
|
|
if (@a_oid == 2) { $new_o=$a_oid[1]; }
|
|
#print "o=$o, new_o=$new_o v=$v\n";
|
|
|
|
#check for 'contains'
|
|
if (exists($hashX->{$o})) {
|
|
my $pa= $hashX->{$o}{'=~'};
|
|
foreach(@$pa) {
|
|
if ($v =~ /$_/) {return 1; }
|
|
}
|
|
}
|
|
if (exists($hashX->{$new_o})) {
|
|
my $pa= $hashX->{$new_o}{'=~'};
|
|
foreach(@$pa) {
|
|
if ($v =~ /$_/) {return 1; }
|
|
}
|
|
}
|
|
|
|
#check for 'equals'
|
|
if (exists($hashX->{$o})) {
|
|
my $pa= $hashX->{$o}{'eq'};
|
|
foreach(@$pa) {
|
|
if ($_ eq $v) { return 1; }
|
|
}
|
|
}
|
|
if (exists($hashX->{$new_o})) {
|
|
my $pa= $hashX->{$new_o}{'eq'};
|
|
foreach(@$pa) {
|
|
if ($_ eq $v) { return 1; }
|
|
}
|
|
}
|
|
|
|
|
|
return 0;
|
|
}
|
|
|
|
if ((!$IGNORE) && (keys(%hashI)>0)) {
|
|
$IGNORE=checking(\%hashI, $o, $v);
|
|
if ($IGNORE) { return;}
|
|
}
|
|
|
|
if ((!$EMAIL) && (keys(%hashE)>0)) {
|
|
$EMAIL=checking(\%hashE, $o, $v);
|
|
}
|
|
|
|
if ((!$LOG) && (keys(%hashL)>0)) {
|
|
$LOG=checking(\%hashL, $o, $v);
|
|
}
|
|
|
|
if ((!$DB) && (keys(%hashD)>0)) {
|
|
$DB=checking(\%hashD, $o, $v);
|
|
}
|
|
|
|
foreach my $k (keys %hashRUN) {
|
|
if ($hashRUN{$k} == 0) { $hashRUN{$k} = checking($hashR{$k}, $o, $v); }
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|