From 6ccc776d32ea62568526c0d81abd352ff92ec8fc Mon Sep 17 00:00:00 2001 From: ellen56 Date: Wed, 17 Jun 2009 06:45:38 +0000 Subject: [PATCH] update for monitor enhancement to set up end to end performance monitor by rmcmon-plugin git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@3592 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd --- xCAT-rmc/plugin/rmcmon.pm | 204 ++++++++++++- .../lib/xcat/monitoring/monitorctrl.pm | 92 +++--- xCAT-server/lib/xcat/plugins/monctrlcmds.pm | 280 +++++++++++++++++- 3 files changed, 533 insertions(+), 43 deletions(-) diff --git a/xCAT-rmc/plugin/rmcmon.pm b/xCAT-rmc/plugin/rmcmon.pm index 464e127e5..fa7beca93 100644 --- a/xCAT-rmc/plugin/rmcmon.pm +++ b/xCAT-rmc/plugin/rmcmon.pm @@ -13,6 +13,7 @@ use Socket; use xCAT::Utils; use xCAT::GlobalDef; use xCAT_monitoring::monitorctrl; +use xCAT_monitoring::rmcmetrix; use xCAT::MsgUtils; #print "xCAT_monitoring::rmcmon loaded\n"; 1; @@ -152,7 +153,16 @@ sub start { } } } #if ($callback) - + my @metrixconf = xCAT_monitoring::rmcmetrix::get_metrix_conf(); + my @rmetrixcmd = (); + while(@metrixconf){ + my ($rsrc, $rname, $attrlist, $minute); + $rsrc = shift @metrixconf; + $rname = shift @metrixconf; + $attrlist = shift @metrixconf; + $minute = shift @metrixconf; + push @rmetrixcmd, "$rsrc $rname $attrlist $minute"; + } if ($scope) { #get a list of managed nodes $result=`/usr/bin/lsrsrc-api -s IBM.MngNode::::Name 2>&1`; @@ -165,6 +175,13 @@ sub start { } chomp($result); my @rmc_nodes=split(/\n/, $result); + my @svc_nodes = (); + my $node = undef; + foreach $node (@rmc_nodes){ + if(xCAT::Utils->isSN($node)){ + push @svc_nodes, $node; + } + } #start the rmc daemons for its children if (@rmc_nodes > 0) { @@ -174,8 +191,22 @@ sub start { reportError( $result, $callback); } } + + #add to cron task + if(@svc_nodes > 0){ + my $nodestring=join(',', @svc_nodes); + foreach (@rmetrixcmd){ + $result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring $::XCATROOT/sbin/rmcmon/rmcmetrixmon init $_ 2>&1`; + } + if(($result)){ + reportError( $result, $callback); + } + } } + foreach (@rmetrixcmd){ + xCAT::Utils->runcmd("XCATBYPASS=Y $::XCATROOT/sbin/rmcmon/rmcmetrixmon init $_", 0); + } if ($callback) { my $rsp={}; $rsp->{data}->[0]="$localhostname: done."; @@ -254,7 +285,7 @@ sub stop { my $localhostname=hostname(); - + system("$::XCATROOT/sbin/rmcmon/rmcmetrixmon clean"); my $result; chomp(my $pid= `/bin/ps -ef | /bin/grep rmcd | /bin/grep -v grep | /bin/awk '{print \$2}'`); if ($pid){ @@ -332,6 +363,10 @@ sub stop { if (@nodes_to_stop > 0) { my $nodestring=join(',', @nodes_to_stop); + $result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring $::XCATROOT/sbin/rmcmon/rmcmetrixmon clean 2>&1`; + if($result){ + reportError($result, $callback); + } #$result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring stopsrc -s ctrmc 2>&1`; $result=`XCATBYPASS=Y $::XCATROOT/bin/xdsh $nodestring "/bin/ps -ef | /bin/grep rmcd | /bin/grep -v grep | /bin/awk '{if (\\\$2>0) system(\\\"stopsrc -s ctrmc\\\")}' 2>&1"`; @@ -1415,5 +1450,170 @@ sub getPostscripts { return $ret; } +sub showmetrix($rrddir, $attrs, $start_time, $end_time) +{ + my($rrddir, $attrs, $start_time, $end_time) = @_; + my $result = []; + my $output = undef; + my @files = (); + my $file = undef; + my @attrlist = split /,/,$attrs; + my $attr = undef; + my $line = undef; + my @namelist = (); + my @timelist = (); + my %hash = {}; + my $name = undef; + my $timestamp = undef; + my $sum = undef; + my $num = undef; + + foreach $attr (@attrlist) { + @namelist = (); + @timelist = (); + %hash = {}; + $output = `ls -A $rrddir/$attr*`; + @files = split /\n/, $output; + foreach $file (@files) { + if($file =~ /$attr\.rrd$/){ + $name = "$attr"; + } elsif ($file =~ /${attr}_(\S+)\.rrd$/) { + $name = $1; + } + push @namelist, $name; + $output = xCAT_monitoring::rrdutil::RRD_fetch($file, $start_time, $end_time); + $line = pop(@$output); + if($line =~ /ERROR/){ + push @$result, $line; + next; + } else { + push @$output, $line; + } + foreach $line (@$output){ + if($line =~ /NaNQ/){ + next; + } elsif ($line =~ /^(\d+): (\S+) (\S+)/){ + $timestamp = $1; + $sum = $2; + $num = $3; + if(! grep {/$timestamp/} @timelist){ + push @timelist, $timestamp; + } + $hash{$name}{$timestamp} = sprintf "%.4f:%d", $sum, $num; + } elsif ($line =~ /^(\d+): (\S+)/){ + $timestamp = $1; + $sum = $2; + if(! grep {/$timestamp/} @timelist){ + push @timelist, $timestamp; + } + $hash{$name}{$timestamp} = sprintf "%.4f", $sum; + } + } + } + $line = join "\t", (@namelist); + $line = " ".$line; + push @$result, $line; + @timelist = sort @timelist; + foreach $timestamp (@timelist){ + $line = localtime($timestamp)." "; + foreach $name (@namelist){ + if(exists $hash{$name}{$timestamp}){ + $line =$line."$hash{$name}{$timestamp}\t"; + } else { + $line = $line."-\t"; + } + } + push @$result, $line; + } + } + + return $result; +} +#-------------------------------------------------------------------------------- +=head3 show + This function configures the cluster for the given nodes. + This function is called when moncfg command is issued or when xcatd starts + on the service node. It will configure the cluster to include the given nodes within + the monitoring doamin. + Arguments: + p_nodes -- a pointer to an arrays of nodes to be added for monitoring. none means all. + scope -- the action scope, it indicates the node type the action will take place. + 0 means localhost only. + 2 means localhost and nodes, + callback -- the callback pointer for error and status displaying. It can be null. + Returns: + (error code, error message) +=cut +#-------------------------------------------------------------------------------- +sub show { + print "rmcmon:show called\n"; + no strict 'refs'; + my ($noderef, $sum, $time, $attrs, $pe, $callback) = @_; + my $rsp = {}; + my $localhostname=hostname(); + my $start_time = undef; + my $end_time = undef; + my $node = undef; + my $output = undef; + my $rrddir = undef; + + $end_time = `date +%s`; + if($time =~ /(\d+)-(\d+)/){ + $start_time = $end_time - $1 * 60; + $end_time = $end_time - $2 * 60; + } else { + $start_time = $end_time - $time * 60; + } + + #the identification of this node + my $isSV=xCAT::Utils->isServiceNode(); + + if(!$isSV && ($sum&0x2)){ + push @$noderef, $localhostname; + } + print "$localhostname: @$noderef\n"; + $sum &= 0x1; + + if($sum){ + foreach $node (@$noderef){ + if($node eq $localhostname){ + if($isSV){ + $rrddir = "/var/rrd/summary"; + } else { + $rrddir = "/var/rrd/cluster"; + my @metrixconf = xCAT_monitoring::rmcmetrix::get_metrix_conf(); + my $rmetrixcmd = undef; + while(@metrixconf){ + my ($rsrc, $rname, $attrlist, $minute); + $rsrc = shift @metrixconf; + $rname = shift @metrixconf; + $attrlist = shift @metrixconf; + $minute = shift @metrixconf; + $rmetrixcmd = "/opt/xcat/sbin/rmcmon/rmcmetrixmon sum $rsrc $attrlist $minute"; + xCAT::Utils->runcmd($rmetrixcmd, 0); + } + } + } else { + $rrddir = "/var/rrd/$node"; + } + $output = &showmetrix($rrddir, $attrs, $start_time, $end_time); + push @{$rsp->{data}}, "\n$node-sum:"; + push @{$rsp->{data}}, @$output; + } + } else { + foreach $node (@$noderef){ + $rrddir = "/var/rrd/$node"; + $output = &showmetrix($rrddir, $attrs, $start_time, $end_time); + push @{$rsp->{data}}, "\n$node:"; + push @{$rsp->{data}}, @$output; + } + } + $callback->($rsp); + return (0, ""); +} + +#my $nref = ["hv8plus02", "hv8plus03"]; +#my $cb = {}; +#&show($nref, '0', '100', 'RecByteRate,XmitByteRate', 0, $cb); diff --git a/xCAT-server/lib/xcat/monitoring/monitorctrl.pm b/xCAT-server/lib/xcat/monitoring/monitorctrl.pm index d5a23988e..c9d5e6e83 100644 --- a/xCAT-server/lib/xcat/monitoring/monitorctrl.pm +++ b/xCAT-server/lib/xcat/monitoring/monitorctrl.pm @@ -1368,41 +1368,63 @@ sub getNodeConfData { return %ret; } +#-------------------------------------------------------------------------------- +=head3 show + This function show performance for the given nodes. + Arguments: + names -- a pointer to an array of monitoring plug-in names. If non is specified, + all the plug-ins registered in the monitoring table will be notified. + p_nodes -- a pointer to an arrays of nodes to be added for monitoring. none means all. + sum -- + 0 indicates to show performance of specified node. + 1 indicates to show sum of performance of one or more management domain which managed by node specified in the noderange. + 2 means to show performance of specified node and MN. + 3 means to show sum of performance of whole cluster and one or more management domain which managed by node specified in the noderange. + time -- data in a range of time to show + attrs -- data of specified attrbuites to show + pe -- TODO; + callback -- the callback pointer for error and status displaying. It can be null. + Returns: + ret a hash with plug-in name as the keys and the an arry of + [return code, error message] as the values. +=cut +#-------------------------------------------------------------------------------- +sub show { + my $nameref=shift; + if ($nameref =~ /xCAT_monitoring::monitorctrl/) { + $nameref=shift; + } + my ($noderef, $sum, $time, $attrs, $pe, $callback) = @_; + my %ret=(); + my @product_names=@$nameref; + my %all=getAllRegs(); + if (@product_names == 0) { + @product_names=keys(%all); + } - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + foreach(@product_names) { + if (exists($all{$_})) { + my $file_name="$::XCATROOT/lib/perl/xCAT_monitoring/$_.pm"; + my $module_name="xCAT_monitoring::$_"; + #load the module in memory + eval {require($file_name)}; + if ($@) { + my @ret3=(1, "The file $file_name cannot be located or has compiling errors.\n"); + $ret{$_}=\@ret3; + next; + } + undef $SIG{CHLD}; + #initialize and start monitoring + no strict "refs"; + if (defined(${$module_name."::"}{show})) { + my @ret1 = ${$module_name."::"}{show}->($noderef, $sum, $time, $attrs, $pe, $callback); + $ret{$_}=\@ret1; + } + } else { + $ret{$_}=[1, "Monitoring plug-in module $_ is not registered."]; + } + } + return %ret; +} diff --git a/xCAT-server/lib/xcat/plugins/monctrlcmds.pm b/xCAT-server/lib/xcat/plugins/monctrlcmds.pm index 4d4640f17..50cdc50e7 100644 --- a/xCAT-server/lib/xcat/plugins/monctrlcmds.pm +++ b/xCAT-server/lib/xcat/plugins/monctrlcmds.pm @@ -46,6 +46,7 @@ sub handled_commands { monrm => "monctrlcmds", moncfg => "monctrlcmds", mondecfg => "monctrlcmds", + monshow => "monctrlcmds", } } @@ -67,7 +68,7 @@ sub preprocess_request my @requests=(); - if (($command eq "monstart") || ($command eq "monstop") || ($command eq "moncfg") || ($command eq "mondecfg") ) { + if (($command eq "monstart") || ($command eq "monstop") || ($command eq "moncfg") || ($command eq "mondecfg") || ($command eq "monshow")) { my @a_ret; #(0, $modulename, $nodestatutmon, $scope, \@nodes) if ($command eq "monstart") { @a_ret=preprocess_monstart($args, $callback); @@ -75,8 +76,10 @@ sub preprocess_request @a_ret=preprocess_monstop($args, $callback); } elsif ($command eq "moncfg") { @a_ret=preprocess_moncfg($args, $callback); - } else { + } elsif ($command eq "mondecfg") { @a_ret=preprocess_mondecfg($args, $callback); + } elsif ($command eq "monshow") { + @a_ret=preprocess_monshow($args, $callback); } if ($a_ret[0] != 0) { @@ -89,6 +92,16 @@ sub preprocess_request my $file_name="$::XCATROOT/lib/perl/xCAT_monitoring/$pname.pm"; my $module_name="xCAT_monitoring::$pname"; undef $SIG{CHLD}; + if(($command eq "monshow") && (@$allnodes==0)){ + my $reqcopy = {%$req}; + push @{$reqcopy->{module}}, $a_ret[1]; + push @{$reqcopy->{priv}}, $a_ret[2]; + push @{$reqcopy->{priv}}, $a_ret[3]; + push @{$reqcopy->{priv}}, $a_ret[5]; + push @{$reqcopy->{priv}}, $a_ret[6]; + push @requests, $reqcopy; + return \@requests; + } #initialize and start monitoring no strict "refs"; my $mon_hierachy; @@ -115,6 +128,38 @@ sub preprocess_request foreach(@hostinfo) {$iphash{$_}=1;} if (!$isSV) { $iphash{'noservicenode'}=1;} +# if($command eq "monshow"){ +# my $keynosvcnode = undef; + # foreach (@mon_servers){ +# my @server_pair=split(':', $_); +# my $sv=$server_pair[0]; +# my $mon_nodes=$mon_hierachy->{$_}; +# if ((!$mon_nodes) || (@$mon_nodes ==0)) { next; } +# if(($sv eq 'noservicenode') && ($a_ret[2] & 0x1)){ +# $keynosvcnode = $_; +# my $i = 0; +# foreach (@$mon_nodes){ +# if(xCAT::Utils->isSN($_)){ +# push @{$mon_hierachy->{$_}}, $_; +# push @mon_servers, $_; +# delete $mon_nodes->[$i]; +# } +# $i++; +# } +# } +# } +# if(($keynosvcnode) && ($a_ret[2] & 0x2)){ +# if(@{$mon_hierachy->{$keynosvcnode}} == 0){ +# my $reqcopy = {%$req}; +# push @{$reqcopy->{module}}, $a_ret[1]; +# push @{$reqcopy->{priv}}, $a_ret[2]; +# push @{$reqcopy->{priv}}, $a_ret[3]; +# push @{$reqcopy->{priv}}, $a_ret[5]; +# push @{$reqcopy->{priv}}, $a_ret[6]; +# push @requests, $reqcopy; +# } +# } +# } foreach (@mon_servers) { #service node come in pairs, the first one is the monserver adapter that facing the mn, # the second one is facing the cn. we use the first one here @@ -129,13 +174,20 @@ sub preprocess_request if ($isSV) { next; } #if the command is issued on the monserver, only handle its children. $reqcopy->{'_xcatdest'}=$sv; my $rsp2={}; - $rsp2->{data}->[0]="sending request to $sv..."; + $rsp2->{data}->[0]="sending request to $sv..., ".join(',', @$mon_nodes); $callback->($rsp2); } push @{$reqcopy->{module}}, $a_ret[1]; - push @{$reqcopy->{nodestatmon}}, $a_ret[2]; - push @{$reqcopy->{scope}}, $a_ret[3]; + if($command eq "monshow"){ + push @{$reqcopy->{priv}}, $a_ret[2]; + push @{$reqcopy->{priv}}, $a_ret[3]; + push @{$reqcopy->{priv}}, $a_ret[5]; + push @{$reqcopy->{priv}}, $a_ret[6]; + } else { + push @{$reqcopy->{nodestatmon}}, $a_ret[2]; + push @{$reqcopy->{scope}}, $a_ret[3]; + } push @{$reqcopy->{nodeinfo}}, join(',', @$mon_nodes); push @requests, $reqcopy; } @@ -194,7 +246,10 @@ sub process_request { } elsif ($command eq "mondecfg") { return mondecfg($request, $callback, $doreq); - } else { + } + elsif ($command eq "monshow") { + return monshow($request, $callback, $doreq); + } else { my $rsp={}; $rsp->{data}->[0]= "unsupported command: $command."; $callback->($rsp); @@ -1551,7 +1606,220 @@ sub mondecfg return 0; } +#-------------------------------------------------------------------------------- +=head3 preprocess_monshow + This function handles the syntax checking for monshow command. + Arguments: + callback - the pointer to the callback function. + args - The format of the args is: + [-h|--help|-v|--version] or + name noderange [-s] [-t time] [-a attributes] [-o pe] + where + name is the monitoring plug-in name. For example: rmcmon. Only for rmcmon currently. + noderange a range of nodes to be showed for. + -s shows the summary data only, default is not setting + -t specify to display data in a range of time, default is 1h before until now + -a specfiy the attribute of RMC resource + Returns: + (0, $modulename, $sum, $time, \@nodes, $attrs, $pe) for success. + (1, "") for unsuccess. The error messages are returns through the callback pointer. +=cut +#-------------------------------------------------------------------------------- +sub preprocess_monshow +{ + my $args=shift; + my $callback=shift; + # subroutine to display the usage + sub monshow_usage + { + my $cb=shift; + my $error=shift; + my $rsp={}; + $rsp->{data}->[0]= "Usage:"; + $rsp->{data}->[1]= " monshow name noderange [-s] [-t time] [-a attributes] [-o pe]"; + $rsp->{data}->[2]= " monshow [-h|--help|-v|--version]"; + $rsp->{data}->[3]= " name is the name of the monitoring plug-in module to be invoked."; + $rsp->{data}->[4]= " Only for rmcmon currently"; + $rsp->{data}->[5]= " noderange is a range of nodes to be showed for."; + $rsp->{data}->[6]= " -s option, shows the sum data only, default is not setting."; + $rsp->{data}->[7]= " -t option, specify to display data in a range of time, default is last 1h"; + $rsp->{data}->[8]= " -a option, specfiy the attribute of RMC resource, default is all specified in monsetting."; +# $cb->($rsp); + if($error){ + xCAT::MsgUtils->message("E", $rsp, $callback); + } else { + xCAT::MsgUtils->message("D", $rsp, $callback); + } + } + @ARGV=(); + if ($args) { @ARGV=@{$args} ; } + + # parse the options + if(!GetOptions( + 'h|help' => \$::HELP, + 'v|version' => \$::VERSION, + 's' => \$::SUMMARY, + 't=s' => \$::TIME, + 'a=s' => \$::ATTRS, + 'o=s' => \$::PE)) + { + &monshow_usage($callback, 1); + return (1, ""); + } + # display the usage if -h or --help is specified + if ($::HELP) { + &monshow_usage($callback, 0); + return (1, ""); + } + + # display the version statement if -v or --verison is specified + if ($::VERSION) + { + my $rsp={}; + $rsp->{data}->[0]= xCAT::Utils->Version(); + $callback->($rsp); + return (1, ""); + } + + if(@ARGV != 2) { + &monshow_usage($callback, 1); + return (1, ""); + } + + my $pname=""; + my $sum=0; + my $time = 60; + my @nodes=(); + my $attrs=undef; + my $pe = 0; + + if($::SUMMARY) {$sum=1;} + if($::TIME) {$time=$::TIME;} + if($::ATTRS) { + $attrs=$::ATTRS; + } else { + my $conftable = xCAT::Table->new('monsetting'); + my @metrixconf = $conftable->getAttribs({'name'=>'rmcmon'}, ('key','value')); + foreach (@metrixconf){ + my $key = $_->{key}; + my $value = $_->{value}; + my $temp = undef; + if($key =~ /^rmetrics/){ + if($value =~ /\]/){ + ($temp, $value) = split /\]/, $value; + } + ($value, $temp) = split /:/, $value; + if($attrs){ + $attrs = "$attrs,$value"; + } else { + $attrs = $value; + } + } + } + } + if($::PE) {$pe=$::PE;} + + $pname=$ARGV[0]; + my $noderange=$ARGV[1]; + @nodes = noderange($noderange); + my @temp = nodesmissed; + if(@temp && xCAT::Utils->isMN()){ + my $localhost = hostname(); + my $i = 0; + foreach (@temp) { + if($_ eq $localhost){ + $sum |= 0x2; + splice @temp, $i, 1; + last; + } + $i++; + } + } + if (xCAT::Utils->isMN() && @temp) { + my $rsp={}; + $rsp->{data}->[0]= "Invalid nodes in noderange:".join(',',@temp); + xCAT::MsgUtils->message("E", $rsp, $callback); + return (1, ""); + } + + my $file_name="$::XCATROOT/lib/perl/xCAT_monitoring/$pname.pm"; + if (!-e $file_name) { + my $rsp={}; + $rsp->{data}->[0]="File $file_name does not exist."; + xCAT::MsgUtils->message("E", $rsp, $callback); + return (1, ""); + } else { + #load the module in memory + eval {require($file_name)}; + if ($@) { + my $rsp={}; + $rsp->{data}->[0]="The file $file_name has compiling errors:\n$@\n"; + xCAT::MsgUtils->message("E", $rsp, $callback); + return (1, ""); + } + } + + my $table=xCAT::Table->new("monitoring", -create => 1,-autocommit => 1); + if ($table) { + my $found=0; + my $tmp1=$table->getAllEntries(); + if (defined($tmp1) && (@$tmp1 > 0)) { + foreach(@$tmp1) { + if ($pname eq $_->{name}) { + $found=1; + last; + } + } + } + + if (!$found) { + my $rsp={}; + $rsp->{data}->[0]="$pname cannot be found in the monitoring table."; + xCAT::MsgUtils->message("E", $rsp, $callback); + $table->close(); + return (1, ""); + } + + $table->close(); + } else { + my $rsp={}; + $rsp->{data}->[0]="Failed to open the monitoring table."; + xCAT::MsgUtils->message("E", $rsp, $callback); + return (1, ""); + } + + return (0, $pname, $sum, $time, \@nodes, $attrs, $pe); +} + +#-------------------------------------------------------------------------------- +=head3 monshow + This function configures the cluster performance for the given nodes. + Arguments: + request -- a hash table which contains the command name and the arguments. + callback -- the callback pointer for error and status displaying. It can be null. + Returns: + 0 for success. The output is returned through the callback pointer. + 1. for unsuccess. The error messages are returns through the callback pointer. +=cut +#-------------------------------------------------------------------------------- +sub monshow +{ + my $request=shift; + my $callback=shift; + + my $pname=$request->{module}->[0]; + my $nodeinfo=$request->{nodeinfo}->[0]; + my $sum=$request->{priv}->[0]; + my $time=$request->{priv}->[1]; + my $attrs=$request->{priv}->[2]; + my $pe=$request->{priv}->[3]; + + my @nodes=split(',', $nodeinfo); + + xCAT_monitoring::monitorctrl->show([$pname], \@nodes, $sum, $time, $attrs, $pe, $callback); + return 0; +}