From 72c2c0174e16cbe2e0318de0224fdeaa12b6b634 Mon Sep 17 00:00:00 2001 From: jbjohnso Date: Fri, 30 Jan 2009 14:27:32 +0000 Subject: [PATCH] -New conserver backends that coordinate better to avoid high startup loads and not penalize a single console being restarted git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@2672 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd --- xCAT-server/share/xcat/cons/blade | 50 ++++++++++++++++++++++++++--- xCAT-server/share/xcat/cons/ipmi | 53 ++++++++++++++++++++++++++++--- xCAT-server/share/xcat/cons/xen | 49 +++++++++++++++++++++++++--- 3 files changed, 137 insertions(+), 15 deletions(-) diff --git a/xCAT-server/share/xcat/cons/blade b/xCAT-server/share/xcat/cons/blade index b4db3e966..fa42a20cd 100755 --- a/xCAT-server/share/xcat/cons/blade +++ b/xCAT-server/share/xcat/cons/blade @@ -1,13 +1,41 @@ #!/usr/bin/env perl # IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html +use Fcntl qw(:DEFAULT :flock); +sub get_lock { + unless (flock(LOCKHANDLE,LOCK_EX|LOCK_NB)) { + $| = 1; + print "Acquiring startup lock..."; + flock(LOCKHANDLE,LOCK_EX) or die "Error trying to secure a startup lock"; + print "done\n"; + } + truncate(LOCKHANDLE,0); + print LOCKHANDLE $$."\n"; +} + +sub release_lock { + truncate(LOCKHANDLE,0); + flock(LOCKHANDLE,LOCK_UN); +} + BEGIN { use Time::HiRes qw(sleep); + use File::Path; + use Fcntl qw(:DEFAULT :flock); $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; - my $sleepint=int(rand(10)); #Stagger start to avoid overwhelming conserver/xCATd - print "Opening console in ".(2+(0.5*$sleepint))." seconds...\n"; - sleep $sleepint; + umask 0077; + mkpath("/tmp/xcat/"); + unless (sysopen(LOCKHANDLE,"/tmp/xcat/consolelock",O_WRONLY | O_CREAT)) { + sleep 15; + print "Unable to open lock file"; + exit 0; + } + get_lock(); + #my $sleepint=int(rand(10)); #Stagger start to avoid overwhelming conserver/xCATd + #print "Opening console in ".(2+(0.5*$sleepint))." seconds...\n"; + #sleep $sleepint; } +my $sleepint=int(rand(10)); #Stagger start to avoid overwhelming conserver/xCATd use lib "$::XCATROOT/lib/perl"; $ENV{HOME}='/root/'; require xCAT::Client; @@ -68,10 +96,22 @@ my $cmdref={ }; xCAT::Client::submit_request($cmdref,\&getans); until ($mm and $username and $slot) { - print "console not ready\n"; - sleep 5+int(rand(10)); + release_lock(); #Let other clients have a go + $sleepint=10+int(rand(20)); #Stagger to minimize lock collisions, but no big deal when it does happen + print "Console not ready, retrying in $sleepint seconds (Hit Ctrl-E,c,o to skip delay)\n"; + sleep $sleepint; + get_lock(); xCAT::Client::submit_request($cmdref,\&getans); } +release_lock(); #done with xcatd, can run with near impunity +my $solchkcmd = "ssh -t $username"."@"."$mm sol -T blade[$slot]"; +my $solstatus = `$solchkcmd`; +while ($solstatus !~ /SOL Session: Ready/ and $solstatus !~ /SOL Session: Active/) { + $sleepint=60+int(rand(30)); #Stagger sleep to take it easy on AMM/hosting server + print "SOL unavailable, retrying in $sleepint seconds (hit Ctrl-E,c,o to skip)\n"; + sleep $sleepint; + $solstatus = `$solchkcmd`; +} exec "ssh -t $username"."@"."$mm console -o -T blade[$slot]"; #my $pathtochild= dirname($scriptname). "/"; #exec $pathtochild."blade.expect $mm $slot $username $password"; diff --git a/xCAT-server/share/xcat/cons/ipmi b/xCAT-server/share/xcat/cons/ipmi index ebbc3691a..861486c4d 100755 --- a/xCAT-server/share/xcat/cons/ipmi +++ b/xCAT-server/share/xcat/cons/ipmi @@ -1,13 +1,40 @@ #!/usr/bin/env perl # IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html +use Fcntl qw(:DEFAULT :flock); +sub get_lock { + unless (flock(LOCKHANDLE,LOCK_EX|LOCK_NB)) { + $| = 1; + print "Acquiring startup lock..."; + flock(LOCKHANDLE,LOCK_EX) or die "Fatal Error securing startup lock"; + print "done\n"; + } + truncate(LOCKHANDLE,0); + print LOCKHANDLE $$."\n"; +} + +sub release_lock { + truncate(LOCKHANDLE,0); + flock(LOCKHANDLE,LOCK_UN); +} BEGIN { use Time::HiRes qw(sleep); + use File::Path; + use Fcntl qw(:DEFAULT :flock); $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; - my $sleepint=int(rand(10)); - print "Opening console in ".(2+(0.5*$sleepint))." seconds...\n"; - sleep $sleepint; + umask 0077; + mkpath("/tmp/xcat/"); + unless (sysopen(LOCKHANDLE,"/tmp/xcat/consolelock",O_WRONLY | O_CREAT)) { + sleep 15; + print "Unable to open lock file"; + exit 0; + } + get_lock(); + #my $sleepint=int(rand(10)); + #print "Opening console in ".(2+(0.5*$sleepint))." seconds...\n"; + #sleep $sleepint; } +my $sleepint=int(rand(10)); #Stagger start to avoid overwhelming conserver/xCATd $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; use lib "$::XCATROOT/lib/perl"; $ENV{HOME}='/root/'; @@ -31,10 +58,14 @@ my $cmdref={ }; xCAT::Client::submit_request($cmdref,\&getans); until ($username and $password and $bmc) { - print "console not ready\n"; - sleep 5+int(rand(10)); + release_lock(); #Let other clients have a go + $sleepint=10+int(rand(10)); + print "Console not ready, retrying in $sleepint seconds (Ctrl-e,c,o to skip delay) \n"; + sleep $sleepint; + get_lock(); xCAT::Client::submit_request($cmdref,\&getans); } +release_lock(); #require xCAT::Table; #require xCAT::Utils; #my $dba; @@ -58,7 +89,18 @@ until ($username and $password and $bmc) { #xCAT::Utils::close_all_dbhs; #my $isintel = system "ipmitool -I lanplus -U $username -P $password -H $bmc chassis status > /dev/null 2>&1"; my $isintel=0; +my $sleepint; +my $rc; my @mcinfo=`ipmitool -I lan -U $username -P $password -H $bmc mc info`;#| grep 'Manufacturer ID : 343' > /dev/null 2>&1"; +$rc = $?; +while ($rc != 0) { + $sleepint = 10+int(rand(20)); + print "Failure to reach IPMI device, retrying in $sleepint seconds (Hit Ctrl-E,c,o to skip)\n"; + sleep $sleepint; + @mcinfo=`ipmitool -I lan -U $username -P $password -H $bmc mc info`;#| grep 'Manufacturer ID : 343' > /dev/null 2>&1"; + $rc = $?; +} + my $solcom="sol"; my $iface="lanplus"; if (grep /IPMI Version : 1.5/,@mcinfo) { @@ -74,6 +116,7 @@ if ($isintel) { if ($iface eq "lanplus") { system "ipmitool -I lanplus $inteloption -U $username -P $password -H $bmc $solcom deactivate"; #Stop any active session } + exec "ipmitool -I $iface $inteloption -U $username -P $password -H $bmc $solcom activate"; diff --git a/xCAT-server/share/xcat/cons/xen b/xCAT-server/share/xcat/cons/xen index 283f8915e..3fabc7e0f 100755 --- a/xCAT-server/share/xcat/cons/xen +++ b/xCAT-server/share/xcat/cons/xen @@ -1,13 +1,48 @@ #!/usr/bin/env perl # IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html +use Fcntl qw(:DEFAULT :flock); +sub get_lock { + unless (flock(LOCKHANDLE,LOCK_EX|LOCK_NB)) { + $| = 1; + print "Acquiring startup lock..."; + flock(LOCKHANDLE,LOCK_EX) or die "Fatal error securing startup lock"; + print "done\n"; + } + truncate(LOCKHANDLE,0); + print LOCKHANDLE $$."\n"; +} + +sub release_lock { + truncate(LOCKHANDLE,0); + flock(LOCKHANDLE,LOCK_UN); +} BEGIN { use Time::HiRes qw(sleep); + use File::Path; + use Fcntl qw(:DEFAULT :flock); $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; - my $sleepint=int(rand(10)); - print "Opening console in ".(0.5*$sleepint)." seconds...\n"; - sleep (0.5*$sleepint); + umask 0077; + mkpath("/tmp/xcat/"); + unless (sysopen(LOCKHANDLE,"/tmp/xcat/consolelock",O_WRONLY | O_CREAT)) { + sleep 15; + print "Unable to open lock file"; + exit 0; + } + get_lock(); + #my $sleepint=int(rand(10)); + #print "Opening console in ".(2+(0.5*$sleepint))." seconds...\n"; + #sleep $sleepint; } +#BEGIN +#{ +# use Time::HiRes qw(sleep); +# $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; +# my $sleepint=int(rand(10)); +# print "Opening console in ".(0.5*$sleepint)." seconds...\n"; +# sleep (0.5*$sleepint); +#} +my $sleepint; use lib "$::XCATROOT/lib/perl"; $ENV{HOME}='/root/'; require xCAT::Client; @@ -35,10 +70,14 @@ sub getans { } xCAT::Client::submit_request($cmdref,\&getans); until ($dsthost and $speed and $dstty) { - print "console not ready\n"; - sleep 5+int(rand(10)); + release_lock(); + $sleepint=int(rand(30))+60; + print "Console not ready, retrying in $sleepint seconds (Ctrl-e,c,o to skip delay)\n"; + sleep $sleepint; + get_lock(); xCAT::Client::submit_request($cmdref,\&getans); } +release_lock(); exec "ssh -t $dsthost cu -s $speed -l $dstty"; #my $pathtochild= dirname($scriptname). "/";