update for monitor enhancement to set up end to end performance monitor by rmcmon-plugin

git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@3592 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd
This commit is contained in:
ellen56 2009-06-17 06:45:38 +00:00
parent 274edcb7bf
commit 6ccc776d32
3 changed files with 533 additions and 43 deletions

View File

@ -13,6 +13,7 @@ use Socket;
use xCAT::Utils;
use xCAT::GlobalDef;
use xCAT_monitoring::monitorctrl;
use xCAT_monitoring::rmcmetrix;
use xCAT::MsgUtils;
#print "xCAT_monitoring::rmcmon loaded\n";
1;
@ -152,7 +153,16 @@ sub start {
}
}
} #if ($callback)
my @metrixconf = xCAT_monitoring::rmcmetrix::get_metrix_conf();
my @rmetrixcmd = ();
while(@metrixconf){
my ($rsrc, $rname, $attrlist, $minute);
$rsrc = shift @metrixconf;
$rname = shift @metrixconf;
$attrlist = shift @metrixconf;
$minute = shift @metrixconf;
push @rmetrixcmd, "$rsrc $rname $attrlist $minute";
}
if ($scope) {
#get a list of managed nodes
$result=`/usr/bin/lsrsrc-api -s IBM.MngNode::::Name 2>&1`;
@ -165,6 +175,13 @@ sub start {
}
chomp($result);
my @rmc_nodes=split(/\n/, $result);
my @svc_nodes = ();
my $node = undef;
foreach $node (@rmc_nodes){
if(xCAT::Utils->isSN($node)){
push @svc_nodes, $node;
}
}
#start the rmc daemons for its children
if (@rmc_nodes > 0) {
@ -174,8 +191,22 @@ sub start {
reportError( $result, $callback);
}
}
#add to cron task
if(@svc_nodes > 0){
my $nodestring=join(',', @svc_nodes);
foreach (@rmetrixcmd){
$result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring $::XCATROOT/sbin/rmcmon/rmcmetrixmon init $_ 2>&1`;
}
if(($result)){
reportError( $result, $callback);
}
}
}
foreach (@rmetrixcmd){
xCAT::Utils->runcmd("XCATBYPASS=Y $::XCATROOT/sbin/rmcmon/rmcmetrixmon init $_", 0);
}
if ($callback) {
my $rsp={};
$rsp->{data}->[0]="$localhostname: done.";
@ -254,7 +285,7 @@ sub stop {
my $localhostname=hostname();
system("$::XCATROOT/sbin/rmcmon/rmcmetrixmon clean");
my $result;
chomp(my $pid= `/bin/ps -ef | /bin/grep rmcd | /bin/grep -v grep | /bin/awk '{print \$2}'`);
if ($pid){
@ -332,6 +363,10 @@ sub stop {
if (@nodes_to_stop > 0) {
my $nodestring=join(',', @nodes_to_stop);
$result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring $::XCATROOT/sbin/rmcmon/rmcmetrixmon clean 2>&1`;
if($result){
reportError($result, $callback);
}
#$result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring stopsrc -s ctrmc 2>&1`;
$result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring "/bin/ps -ef | /bin/grep rmcd | /bin/grep -v grep | /bin/awk '{if (\\\$2>0) system(\\\"stopsrc -s ctrmc\\\")}' 2>&1"`;
@ -1415,5 +1450,170 @@ sub getPostscripts {
return $ret;
}
sub showmetrix($rrddir, $attrs, $start_time, $end_time)
{
my($rrddir, $attrs, $start_time, $end_time) = @_;
my $result = [];
my $output = undef;
my @files = ();
my $file = undef;
my @attrlist = split /,/,$attrs;
my $attr = undef;
my $line = undef;
my @namelist = ();
my @timelist = ();
my %hash = {};
my $name = undef;
my $timestamp = undef;
my $sum = undef;
my $num = undef;
foreach $attr (@attrlist) {
@namelist = ();
@timelist = ();
%hash = {};
$output = `ls -A $rrddir/$attr*`;
@files = split /\n/, $output;
foreach $file (@files) {
if($file =~ /$attr\.rrd$/){
$name = "$attr";
} elsif ($file =~ /${attr}_(\S+)\.rrd$/) {
$name = $1;
}
push @namelist, $name;
$output = xCAT_monitoring::rrdutil::RRD_fetch($file, $start_time, $end_time);
$line = pop(@$output);
if($line =~ /ERROR/){
push @$result, $line;
next;
} else {
push @$output, $line;
}
foreach $line (@$output){
if($line =~ /NaNQ/){
next;
} elsif ($line =~ /^(\d+): (\S+) (\S+)/){
$timestamp = $1;
$sum = $2;
$num = $3;
if(! grep {/$timestamp/} @timelist){
push @timelist, $timestamp;
}
$hash{$name}{$timestamp} = sprintf "%.4f:%d", $sum, $num;
} elsif ($line =~ /^(\d+): (\S+)/){
$timestamp = $1;
$sum = $2;
if(! grep {/$timestamp/} @timelist){
push @timelist, $timestamp;
}
$hash{$name}{$timestamp} = sprintf "%.4f", $sum;
}
}
}
$line = join "\t", (@namelist);
$line = " ".$line;
push @$result, $line;
@timelist = sort @timelist;
foreach $timestamp (@timelist){
$line = localtime($timestamp)." ";
foreach $name (@namelist){
if(exists $hash{$name}{$timestamp}){
$line =$line."$hash{$name}{$timestamp}\t";
} else {
$line = $line."-\t";
}
}
push @$result, $line;
}
}
return $result;
}
#--------------------------------------------------------------------------------
=head3 show
This function configures the cluster for the given nodes.
This function is called when moncfg command is issued or when xcatd starts
on the service node. It will configure the cluster to include the given nodes within
the monitoring doamin.
Arguments:
p_nodes -- a pointer to an arrays of nodes to be added for monitoring. none means all.
scope -- the action scope, it indicates the node type the action will take place.
0 means localhost only.
2 means localhost and nodes,
callback -- the callback pointer for error and status displaying. It can be null.
Returns:
(error code, error message)
=cut
#--------------------------------------------------------------------------------
sub show {
print "rmcmon:show called\n";
no strict 'refs';
my ($noderef, $sum, $time, $attrs, $pe, $callback) = @_;
my $rsp = {};
my $localhostname=hostname();
my $start_time = undef;
my $end_time = undef;
my $node = undef;
my $output = undef;
my $rrddir = undef;
$end_time = `date +%s`;
if($time =~ /(\d+)-(\d+)/){
$start_time = $end_time - $1 * 60;
$end_time = $end_time - $2 * 60;
} else {
$start_time = $end_time - $time * 60;
}
#the identification of this node
my $isSV=xCAT::Utils->isServiceNode();
if(!$isSV && ($sum&0x2)){
push @$noderef, $localhostname;
}
print "$localhostname: @$noderef\n";
$sum &= 0x1;
if($sum){
foreach $node (@$noderef){
if($node eq $localhostname){
if($isSV){
$rrddir = "/var/rrd/summary";
} else {
$rrddir = "/var/rrd/cluster";
my @metrixconf = xCAT_monitoring::rmcmetrix::get_metrix_conf();
my $rmetrixcmd = undef;
while(@metrixconf){
my ($rsrc, $rname, $attrlist, $minute);
$rsrc = shift @metrixconf;
$rname = shift @metrixconf;
$attrlist = shift @metrixconf;
$minute = shift @metrixconf;
$rmetrixcmd = "/opt/xcat/sbin/rmcmon/rmcmetrixmon sum $rsrc $attrlist $minute";
xCAT::Utils->runcmd($rmetrixcmd, 0);
}
}
} else {
$rrddir = "/var/rrd/$node";
}
$output = &showmetrix($rrddir, $attrs, $start_time, $end_time);
push @{$rsp->{data}}, "\n$node-sum:";
push @{$rsp->{data}}, @$output;
}
} else {
foreach $node (@$noderef){
$rrddir = "/var/rrd/$node";
$output = &showmetrix($rrddir, $attrs, $start_time, $end_time);
push @{$rsp->{data}}, "\n$node:";
push @{$rsp->{data}}, @$output;
}
}
$callback->($rsp);
return (0, "");
}
#my $nref = ["hv8plus02", "hv8plus03"];
#my $cb = {};
#&show($nref, '0', '100', 'RecByteRate,XmitByteRate', 0, $cb);

View File

@ -1368,41 +1368,63 @@ sub getNodeConfData {
return %ret;
}
#--------------------------------------------------------------------------------
=head3 show
This function show performance for the given nodes.
Arguments:
names -- a pointer to an array of monitoring plug-in names. If non is specified,
all the plug-ins registered in the monitoring table will be notified.
p_nodes -- a pointer to an arrays of nodes to be added for monitoring. none means all.
sum --
0 indicates to show performance of specified node.
1 indicates to show sum of performance of one or more management domain which managed by node specified in the noderange.
2 means to show performance of specified node and MN.
3 means to show sum of performance of whole cluster and one or more management domain which managed by node specified in the noderange.
time -- data in a range of time to show
attrs -- data of specified attrbuites to show
pe -- TODO;
callback -- the callback pointer for error and status displaying. It can be null.
Returns:
ret a hash with plug-in name as the keys and the an arry of
[return code, error message] as the values.
=cut
#--------------------------------------------------------------------------------
sub show {
my $nameref=shift;
if ($nameref =~ /xCAT_monitoring::monitorctrl/) {
$nameref=shift;
}
my ($noderef, $sum, $time, $attrs, $pe, $callback) = @_;
my %ret=();
my @product_names=@$nameref;
my %all=getAllRegs();
if (@product_names == 0) {
@product_names=keys(%all);
}
foreach(@product_names) {
if (exists($all{$_})) {
my $file_name="$::XCATROOT/lib/perl/xCAT_monitoring/$_.pm";
my $module_name="xCAT_monitoring::$_";
#load the module in memory
eval {require($file_name)};
if ($@) {
my @ret3=(1, "The file $file_name cannot be located or has compiling errors.\n");
$ret{$_}=\@ret3;
next;
}
undef $SIG{CHLD};
#initialize and start monitoring
no strict "refs";
if (defined(${$module_name."::"}{show})) {
my @ret1 = ${$module_name."::"}{show}->($noderef, $sum, $time, $attrs, $pe, $callback);
$ret{$_}=\@ret1;
}
} else {
$ret{$_}=[1, "Monitoring plug-in module $_ is not registered."];
}
}
return %ret;
}

View File

@ -46,6 +46,7 @@ sub handled_commands {
monrm => "monctrlcmds",
moncfg => "monctrlcmds",
mondecfg => "monctrlcmds",
monshow => "monctrlcmds",
}
}
@ -67,7 +68,7 @@ sub preprocess_request
my @requests=();
if (($command eq "monstart") || ($command eq "monstop") || ($command eq "moncfg") || ($command eq "mondecfg") ) {
if (($command eq "monstart") || ($command eq "monstop") || ($command eq "moncfg") || ($command eq "mondecfg") || ($command eq "monshow")) {
my @a_ret; #(0, $modulename, $nodestatutmon, $scope, \@nodes)
if ($command eq "monstart") {
@a_ret=preprocess_monstart($args, $callback);
@ -75,8 +76,10 @@ sub preprocess_request
@a_ret=preprocess_monstop($args, $callback);
} elsif ($command eq "moncfg") {
@a_ret=preprocess_moncfg($args, $callback);
} else {
} elsif ($command eq "mondecfg") {
@a_ret=preprocess_mondecfg($args, $callback);
} elsif ($command eq "monshow") {
@a_ret=preprocess_monshow($args, $callback);
}
if ($a_ret[0] != 0) {
@ -89,6 +92,16 @@ sub preprocess_request
my $file_name="$::XCATROOT/lib/perl/xCAT_monitoring/$pname.pm";
my $module_name="xCAT_monitoring::$pname";
undef $SIG{CHLD};
if(($command eq "monshow") && (@$allnodes==0)){
my $reqcopy = {%$req};
push @{$reqcopy->{module}}, $a_ret[1];
push @{$reqcopy->{priv}}, $a_ret[2];
push @{$reqcopy->{priv}}, $a_ret[3];
push @{$reqcopy->{priv}}, $a_ret[5];
push @{$reqcopy->{priv}}, $a_ret[6];
push @requests, $reqcopy;
return \@requests;
}
#initialize and start monitoring
no strict "refs";
my $mon_hierachy;
@ -115,6 +128,38 @@ sub preprocess_request
foreach(@hostinfo) {$iphash{$_}=1;}
if (!$isSV) { $iphash{'noservicenode'}=1;}
# if($command eq "monshow"){
# my $keynosvcnode = undef;
# foreach (@mon_servers){
# my @server_pair=split(':', $_);
# my $sv=$server_pair[0];
# my $mon_nodes=$mon_hierachy->{$_};
# if ((!$mon_nodes) || (@$mon_nodes ==0)) { next; }
# if(($sv eq 'noservicenode') && ($a_ret[2] & 0x1)){
# $keynosvcnode = $_;
# my $i = 0;
# foreach (@$mon_nodes){
# if(xCAT::Utils->isSN($_)){
# push @{$mon_hierachy->{$_}}, $_;
# push @mon_servers, $_;
# delete $mon_nodes->[$i];
# }
# $i++;
# }
# }
# }
# if(($keynosvcnode) && ($a_ret[2] & 0x2)){
# if(@{$mon_hierachy->{$keynosvcnode}} == 0){
# my $reqcopy = {%$req};
# push @{$reqcopy->{module}}, $a_ret[1];
# push @{$reqcopy->{priv}}, $a_ret[2];
# push @{$reqcopy->{priv}}, $a_ret[3];
# push @{$reqcopy->{priv}}, $a_ret[5];
# push @{$reqcopy->{priv}}, $a_ret[6];
# push @requests, $reqcopy;
# }
# }
# }
foreach (@mon_servers) {
#service node come in pairs, the first one is the monserver adapter that facing the mn,
# the second one is facing the cn. we use the first one here
@ -129,13 +174,20 @@ sub preprocess_request
if ($isSV) { next; } #if the command is issued on the monserver, only handle its children.
$reqcopy->{'_xcatdest'}=$sv;
my $rsp2={};
$rsp2->{data}->[0]="sending request to $sv...";
$rsp2->{data}->[0]="sending request to $sv..., ".join(',', @$mon_nodes);
$callback->($rsp2);
}
push @{$reqcopy->{module}}, $a_ret[1];
push @{$reqcopy->{nodestatmon}}, $a_ret[2];
push @{$reqcopy->{scope}}, $a_ret[3];
if($command eq "monshow"){
push @{$reqcopy->{priv}}, $a_ret[2];
push @{$reqcopy->{priv}}, $a_ret[3];
push @{$reqcopy->{priv}}, $a_ret[5];
push @{$reqcopy->{priv}}, $a_ret[6];
} else {
push @{$reqcopy->{nodestatmon}}, $a_ret[2];
push @{$reqcopy->{scope}}, $a_ret[3];
}
push @{$reqcopy->{nodeinfo}}, join(',', @$mon_nodes);
push @requests, $reqcopy;
}
@ -194,7 +246,10 @@ sub process_request {
}
elsif ($command eq "mondecfg") {
return mondecfg($request, $callback, $doreq);
} else {
}
elsif ($command eq "monshow") {
return monshow($request, $callback, $doreq);
} else {
my $rsp={};
$rsp->{data}->[0]= "unsupported command: $command.";
$callback->($rsp);
@ -1551,7 +1606,220 @@ sub mondecfg
return 0;
}
#--------------------------------------------------------------------------------
=head3 preprocess_monshow
This function handles the syntax checking for monshow command.
Arguments:
callback - the pointer to the callback function.
args - The format of the args is:
[-h|--help|-v|--version] or
name noderange [-s] [-t time] [-a attributes] [-o pe]
where
name is the monitoring plug-in name. For example: rmcmon. Only for rmcmon currently.
noderange a range of nodes to be showed for.
-s shows the summary data only, default is not setting
-t specify to display data in a range of time, default is 1h before until now
-a specfiy the attribute of RMC resource
Returns:
(0, $modulename, $sum, $time, \@nodes, $attrs, $pe) for success.
(1, "") for unsuccess. The error messages are returns through the callback pointer.
=cut
#--------------------------------------------------------------------------------
sub preprocess_monshow
{
my $args=shift;
my $callback=shift;
# subroutine to display the usage
sub monshow_usage
{
my $cb=shift;
my $error=shift;
my $rsp={};
$rsp->{data}->[0]= "Usage:";
$rsp->{data}->[1]= " monshow name noderange [-s] [-t time] [-a attributes] [-o pe]";
$rsp->{data}->[2]= " monshow [-h|--help|-v|--version]";
$rsp->{data}->[3]= " name is the name of the monitoring plug-in module to be invoked.";
$rsp->{data}->[4]= " Only for rmcmon currently";
$rsp->{data}->[5]= " noderange is a range of nodes to be showed for.";
$rsp->{data}->[6]= " -s option, shows the sum data only, default is not setting.";
$rsp->{data}->[7]= " -t option, specify to display data in a range of time, default is last 1h";
$rsp->{data}->[8]= " -a option, specfiy the attribute of RMC resource, default is all specified in monsetting.";
# $cb->($rsp);
if($error){
xCAT::MsgUtils->message("E", $rsp, $callback);
} else {
xCAT::MsgUtils->message("D", $rsp, $callback);
}
}
@ARGV=();
if ($args) { @ARGV=@{$args} ; }
# parse the options
if(!GetOptions(
'h|help' => \$::HELP,
'v|version' => \$::VERSION,
's' => \$::SUMMARY,
't=s' => \$::TIME,
'a=s' => \$::ATTRS,
'o=s' => \$::PE))
{
&monshow_usage($callback, 1);
return (1, "");
}
# display the usage if -h or --help is specified
if ($::HELP) {
&monshow_usage($callback, 0);
return (1, "");
}
# display the version statement if -v or --verison is specified
if ($::VERSION)
{
my $rsp={};
$rsp->{data}->[0]= xCAT::Utils->Version();
$callback->($rsp);
return (1, "");
}
if(@ARGV != 2) {
&monshow_usage($callback, 1);
return (1, "");
}
my $pname="";
my $sum=0;
my $time = 60;
my @nodes=();
my $attrs=undef;
my $pe = 0;
if($::SUMMARY) {$sum=1;}
if($::TIME) {$time=$::TIME;}
if($::ATTRS) {
$attrs=$::ATTRS;
} else {
my $conftable = xCAT::Table->new('monsetting');
my @metrixconf = $conftable->getAttribs({'name'=>'rmcmon'}, ('key','value'));
foreach (@metrixconf){
my $key = $_->{key};
my $value = $_->{value};
my $temp = undef;
if($key =~ /^rmetrics/){
if($value =~ /\]/){
($temp, $value) = split /\]/, $value;
}
($value, $temp) = split /:/, $value;
if($attrs){
$attrs = "$attrs,$value";
} else {
$attrs = $value;
}
}
}
}
if($::PE) {$pe=$::PE;}
$pname=$ARGV[0];
my $noderange=$ARGV[1];
@nodes = noderange($noderange);
my @temp = nodesmissed;
if(@temp && xCAT::Utils->isMN()){
my $localhost = hostname();
my $i = 0;
foreach (@temp) {
if($_ eq $localhost){
$sum |= 0x2;
splice @temp, $i, 1;
last;
}
$i++;
}
}
if (xCAT::Utils->isMN() && @temp) {
my $rsp={};
$rsp->{data}->[0]= "Invalid nodes in noderange:".join(',',@temp);
xCAT::MsgUtils->message("E", $rsp, $callback);
return (1, "");
}
my $file_name="$::XCATROOT/lib/perl/xCAT_monitoring/$pname.pm";
if (!-e $file_name) {
my $rsp={};
$rsp->{data}->[0]="File $file_name does not exist.";
xCAT::MsgUtils->message("E", $rsp, $callback);
return (1, "");
} else {
#load the module in memory
eval {require($file_name)};
if ($@) {
my $rsp={};
$rsp->{data}->[0]="The file $file_name has compiling errors:\n$@\n";
xCAT::MsgUtils->message("E", $rsp, $callback);
return (1, "");
}
}
my $table=xCAT::Table->new("monitoring", -create => 1,-autocommit => 1);
if ($table) {
my $found=0;
my $tmp1=$table->getAllEntries();
if (defined($tmp1) && (@$tmp1 > 0)) {
foreach(@$tmp1) {
if ($pname eq $_->{name}) {
$found=1;
last;
}
}
}
if (!$found) {
my $rsp={};
$rsp->{data}->[0]="$pname cannot be found in the monitoring table.";
xCAT::MsgUtils->message("E", $rsp, $callback);
$table->close();
return (1, "");
}
$table->close();
} else {
my $rsp={};
$rsp->{data}->[0]="Failed to open the monitoring table.";
xCAT::MsgUtils->message("E", $rsp, $callback);
return (1, "");
}
return (0, $pname, $sum, $time, \@nodes, $attrs, $pe);
}
#--------------------------------------------------------------------------------
=head3 monshow
This function configures the cluster performance for the given nodes.
Arguments:
request -- a hash table which contains the command name and the arguments.
callback -- the callback pointer for error and status displaying. It can be null.
Returns:
0 for success. The output is returned through the callback pointer.
1. for unsuccess. The error messages are returns through the callback pointer.
=cut
#--------------------------------------------------------------------------------
sub monshow
{
my $request=shift;
my $callback=shift;
my $pname=$request->{module}->[0];
my $nodeinfo=$request->{nodeinfo}->[0];
my $sum=$request->{priv}->[0];
my $time=$request->{priv}->[1];
my $attrs=$request->{priv}->[2];
my $pe=$request->{priv}->[3];
my @nodes=split(',', $nodeinfo);
xCAT_monitoring::monitorctrl->show([$pname], \@nodes, $sum, $time, $attrs, $pe, $callback);
return 0;
}