From 707a73432224f93bce34c1e805dec0caf40fe218 Mon Sep 17 00:00:00 2001 From: datajerk Date: Tue, 24 Mar 2009 16:55:50 +0000 Subject: [PATCH] Auto GPFS for stateless (and stateful ;-) git-svn-id: https://svn.code.sf.net/p/xcat/code/xcat-core/trunk@2966 8638fb3e-16cb-4fca-ae20-7b5d299a9bcd --- .../xcat/netboot/add-on/autogpfs/autogpfsc | 99 +++++ .../xcat/netboot/add-on/autogpfs/autogpfsc.pl | 131 +++++++ .../netboot/add-on/autogpfs/autogpfsc.suse | 191 +++++++++ .../xcat/netboot/add-on/autogpfs/autogpfsd | 34 ++ .../xcat/netboot/add-on/autogpfs/autogpfsd.pl | 364 ++++++++++++++++++ .../netboot/add-on/autogpfs/autogpfsd.suse | 127 ++++++ 6 files changed, 946 insertions(+) create mode 100755 xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc create mode 100755 xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.pl create mode 100755 xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.suse create mode 100755 xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd create mode 100755 xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.pl create mode 100755 xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.suse diff --git a/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc new file mode 100755 index 000000000..24676e34c --- /dev/null +++ b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc @@ -0,0 +1,99 @@ +#!/bin/sh +# +# autogpfsc +# +# chkconfig: 345 56 24 +# description: action autogpfsc +# +# Source the library functions +. /etc/rc.d/init.d/functions + +. /etc/sysconfig/autogpfsc + +# let see how we were called +case "$1" in + start) + echo "Starting autoGPFS: " + + lastexit=2 + while [ "$lastexit" = "2" ] + do + for i in $(echo $SERVERS | tr ',' ' ') + do + /usr/sbin/autogpfsc.pl $i $PORT 2>/dev/null + lastexit=$? + if [ "$lastexit" = "0" ] + then + break + fi + if [ "$lastexit" = "1" ] + then + break + fi + done + sleep 1 + done + + if [ "$lastexit" != "0" ] + then + BLOCK=no + fi + + sleep 5 + + if [ "$BLOCK" = "yes" ] + then + echo -n "Waiting on GPFS mounts..." + DOBLOCK=yes + fi + + while [ "$DOBLOCK" = "yes" ] + do + DOBLOCK=no + + for i in $( + grep "\bgpfs\b" /etc/fstab | \ + awk '{print $2}' + ) + do + mount | grep "\b/gpfs1\b" >/dev/null 2>&1 + if [ "$?" != "0" ] + then + DOBLOCK=yes + fi + done + + sleep 1 + done + + if [ "$BLOCK" = "yes" ] + then + echo "done" + fi + + echo -n "autoGPFS: " + if [ "$lastexit" = "0" ] + then + echo_success + else + echo_failure + fi + echo + exit $lastexit + ;; + stop) + echo -n "Shutting down GPFS: " + /usr/lpp/mmfs/bin/mmshutdown + echo + ;; +# status) +# status autogpfsc.pl +# ;; + restart) + $0 stop + $0 start + ;; + *) + echo "Usage: autogpfsc {start|stop|restart}" + exit 1 +esac diff --git a/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.pl b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.pl new file mode 100755 index 000000000..3827ec207 --- /dev/null +++ b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.pl @@ -0,0 +1,131 @@ +#!/usr/bin/perl + +use strict; +use IO::Socket; +use IO::File; +use Sys::Syslog qw(:DEFAULT setlogsock); + +my $autogpfsdhost = shift; +my $autogpfsdport = shift; +my $line; + +my $sock = IO::Socket::INET->new( + PeerAddr => $autogpfsdhost, + PeerPort => $autogpfsdport, +); + +if(!defined($sock)) { + logger('info',"could not open socket"); + exit(2); +} + +#print "Starting autoGPFS...\n"; + +print $sock "hello\n"; + +while(<$sock>) { + chomp(); + $line = $_; + + if($line =~ /^busy/) { + exit(2); + } + if($line =~ /^new started/) { + next; + } + if($line =~ /^new ended/) { + system("/usr/lpp/mmfs/bin/mmstartup"); + $line = ""; + last; + } + if($line =~ /^new .*/) { + s/^new //; + $line = $_; + last; + } + if($line =~ /^old started/) { + next; + } + if($line =~ /^old ended/) { + system("/usr/lpp/mmfs/bin/mmstartup"); + $line = ""; + last; + } + if($line =~ /^old gpfs /) { + my $gotit = 0; + + s/^old gpfs //; + $line = $_; + + open(FSTAB,"/etc/fstab"); + while() { + chomp(); + if($_ eq $line) { + $gotit = 1; + last; + } + } + close(FSTAB); + + if($gotit == 1) { + next; + } + + open(FSTAB,">>/etc/fstab"); + print FSTAB "$line\n"; + close(FSTAB); + + next; + } + if($line =~ /^old dev /) { + my @a = split(/\s+/); + my $dev = $a[2]; + my $major = $a[3]; + my $minor = $a[4]; + + unlink($dev); + system("mknod $dev b $major $minor"); + system("chmod 644 $dev"); + + next; + } + if($line =~ /^old mmsdrfs start/) { + unlink("/var/mmfs/gen/mmsdrfs"); + system("mkdir -p /var/mmfs/gen"); + open(OUTPUT,">/var/mmfs/gen/mmsdrfs"); + while(<$sock>) { + if($_ =~ /^EOF/) { + last; + } + print OUTPUT $_; + } + close(OUTPUT); + next; + } + if($line =~ /^old .*/) { + s/^old //; + $line = $_; + last; + } +} + +if($line ne "") { + logger('err',$line); + print "$line\n"; + exit(1); +} + +#print "autoGPFS complete.\n"; + +exit(0); + +sub logger { + my $type = shift; + my $msg = shift; + + setlogsock('unix'); + openlog('autogpfsc','','local0'); + syslog($type,$msg); + closelog(); +} + diff --git a/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.suse b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.suse new file mode 100755 index 000000000..aaca58461 --- /dev/null +++ b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsc.suse @@ -0,0 +1,191 @@ +#! /bin/sh +# Copyright (c) 1995-2000 SuSE GmbH Nuernberg, Germany. +# Copyright (c) 2001 IBM +# +# Author: Kurt Garloff +# Author: Egan Ford set local and overall rc status to +# rc_reset clear local rc status (overall remains) +# rc_exit exit appropriate to overall rc status +. /etc/rc.status + +# First reset status of this service +rc_reset + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - generic or unspecified error +# 2 - invalid or excess argument(s) +# 3 - unimplemented feature (e.g. "reload") +# 4 - insufficient privilege +# 5 - program is not installed +# 6 - program is not configured +# 7 - program is not running +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signalling is not supported) are +# considered a success. + +case "$1" in + start) + echo "Starting autoGPFS" + ## Start daemon with startproc(8). If this fails + ## the echo return value is set appropriate. + + # NOTE: startproc return 0, even if service is + # already running to match LSB spec. + + + lastexit=2 + while [ "$lastexit" = "2" ] + do + for i in $(echo $SERVERS | tr ',' ' ') + do + /usr/sbin/autogpfsc.pl $i $PORT 2>/dev/null + lastexit=$? + if [ "$lastexit" = "0" ] + then + break + fi + if [ "$lastexit" = "1" ] + then + break + fi + done + sleep 1 + done + + if [ "$lastexit" != "0" ] + then + BLOCK=no + fi + + sleep 5 + + if [ "$BLOCK" = "yes" ] + then + echo -n "Waiting on GPFS mounts..." + DOBLOCK=yes + fi + + while [ "$DOBLOCK" = "yes" ] + do + DOBLOCK=no + + for i in $( + grep "\bgpfs\b" /etc/fstab | \ + awk '{print $2}' + ) + do + mount | grep "\b/gpfs1\b" >/dev/null 2>&1 + if [ "$?" != "0" ] + then + DOBLOCK=yes + fi + done + + sleep 1 + done + + if [ "$BLOCK" = "yes" ] + then + echo "done" + fi + + echo -n "autoGPFS" + if [ "$lastexit" = "0" ] + then + true + else + false + fi + + # Remember status and be verbose + rc_status -v + ;; + stop) + echo -n "Shutting down GPFS" + ## Stop daemon with killproc(8) and if this fails + ## set echo the echo return value. + + /usr/lpp/mmfs/bin/mmshutdown + + # Remember status and be verbose + rc_status -v + ;; + restart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + $0 stop + $0 start + + # Remember status and be quiet + rc_status + ;; +# status) +# echo -n "Checking for autogpfsc: " +# ## Check status with checkproc(8), if process is running +# ## checkproc will return with exit status 0. +# +# # Status has a slightly different for the status command: +# # 0 - service running +# # 1 - service dead, but /var/run/ pid file exists +# # 2 - service dead, but /var/lock/ lock file exists +# # 3 - service not running +# +# # NOTE: checkproc returns LSB compliant status values. +# checkproc $AUTOGPFSC +# rc_status -v +# ;; + *) + echo "Usage: $0 {start|stop|restart}" + exit 1 + ;; +esac +rc_exit diff --git a/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd new file mode 100755 index 000000000..ce135ec1c --- /dev/null +++ b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd @@ -0,0 +1,34 @@ +#!/bin/sh +# +# autogpfsd +# +# chkconfig: 345 86 84 +# description: autogpfsd +# + +# Source the library functions +. /etc/rc.d/init.d/functions + +# let see how we were called +case "$1" in + start) + echo -n "Starting autogpfsd daemon: " + /usr/sbin/autogpfsd.pl && echo_success || echo_failure + echo + ;; + stop) + echo -n "Shutting down autogpfsd daemon: " + killproc autogpfsd.pl + echo + ;; + status) + status autogpfsd.pl + ;; + restart) + $0 stop + $0 start + ;; + *) + echo "Usage: autogpfsd {start|stop|restart|status}" + exit 1 +esac diff --git a/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.pl b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.pl new file mode 100755 index 000000000..f8b631efd --- /dev/null +++ b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.pl @@ -0,0 +1,364 @@ +#!/usr/bin/perl + +use strict; +use IO::Socket; +use IO::File; +use POSIX qw(WNOHANG setsid); +use Sys::Syslog qw(:DEFAULT setlogsock); + +use constant PIDFILE => '/var/lock/autogpfsd/autogpfsd.pid'; +system("mkdir -p /var/lock/autogpfsd"); + +my $quit = 0; +my $node; +my $domain; +my $port; +my $max; +my $mode; +my $cc=0; +my $pid=$$; +my %kids; + +open(CONFIG,"/etc/sysconfig/autogpfsd") || \ + die "cannot open /etc/sysconfig/autogpfsd\n"; + +while() { + if(/^domain\b/) { + $_ =~ m/^domain\s+(.*)/; + $domain = $1; + } + if(/^autogpfsdport\b/) { + $_ =~ m/^autogpfsdport\s+(.*)/; + $port = $1; + } + if(/^autogpfsdmax\b/) { + $_ =~ m/^autogpfsdmax\s+(.*)/; + $max = $1; + } + if(/^autogpfsdmode\b/) { + $_ =~ m/^autogpfsdmode\s+(.*)/; + $mode = $1; + } +} + +close(CONFIG); + +if($domain eq "") { + logger('err',"domain not defined in /etc/sysconfig/autogpfsd"); + die "domain not defined in /etc/sysconfig/autogpfsd"; +} + +if($port eq "") { + logger('err',"autogpfsdport not defined in /etc/sysconfig/autogpfsd"); + die "autogpfsdport not defined in /etc/sysconfig/autogpfsd"; +} + +if($max eq "") { + logger('err',"autogpfsdmax not defined in /etc/sysconfig/autogpfsd"); + die "autogpfsdmax not defined in /etc/sysconfig/autogpfsd"; +} + +if($mode eq "") { + logger('err',"autogpfsdmode not defined in /etc/sysconfig/autogpfsd"); + die "autogpfsdmode not defined in /etc/sysconfig/autogpfsd"; +} + +#$SIG{USR1} = sub {$cc--;}; +$SIG{CHLD} = sub { while ( waitpid(-1,WNOHANG)>0 ) { } }; +$SIG{TERM} = $SIG{INT} = sub { $quit++ }; + +my $fh = lockfile(PIDFILE); +my $listen_socket = +IO::Socket::INET->new( + LocalPort => $port, + Listen => 20, + Proto => 'tcp', + Reuse => 1, + Timeout => 60*60, +); +die "Cannot create a listening socket: $@" unless $listen_socket; + +my $pid = daemonic(); +print $fh $pid; +close $fh; + +logger('info',"waiting for connection"); + +while (!$quit) { + next unless my $connection = $listen_socket->accept; + + my $child = fork(); + + if(!defined $child) { + logger('err',"cannot fork, exiting"); + die "cannot fork"; + } + + if ($child == 0) { + $listen_socket->close; + $node = gethostbyaddr(inet_aton($connection->peerhost),AF_INET); + $node =~ s/\.$domain$//; + $node =~ s/-eth\d$//; + $node =~ s/-myri\d$//; + logger('info',"connection from: $node"); + interact($connection,$node); + #kill(10,$pid); + exit 0; + } + + $connection->close; + + my $gotmax = 0; + do { + $cc=0; + $kids{$child} = 1; + my $key; + foreach $key (keys(%kids)) { + if(getpgrp($key) != -1) { + $cc++; + } + else { + delete $kids{$key}; + } + } + if($cc >= $max) { + $gotmax++; + sleep(1); + } + if($gotmax == 1) { + $gotmax++; + logger('info',"connection count ($cc) max ($max) reached"); + } + } while ($cc >= $max); + + if($gotmax > 0) { + logger('info',"connection count ($cc) dropped below max ($max)"); + } +} + +logger('info',"exiting clean"); + +sub interact { + my $sock = shift; + my $node = shift; + + STDIN->fdopen($sock,"<") or die "Can't reopen STDIN: $!"; + STDOUT->fdopen($sock,">") or die "Can't reopen STDOUT: $!"; + STDERR->fdopen($sock,">") or die "Can't reopen STDERR: $!"; + $| = 1; + +# if($cc > $max) { +# logger('info',"rejected connection from: $node ($cc) autogpfsdmax exceeded"); +# print "busy\n"; +# return; +# } + + while(<>) { + my $line = $_; + + if(!hostcheck($node)) { + print "unauthorized request\n"; + logger('info',"unauthorized request from $node"); + last; + } + + if(!open(OUTPUT,"/usr/lpp/mmfs/bin/mmlscluster 2>&1 |")) { + print "mmlscluster failed\n"; + logger('info',"mmlscluster failed"); + last; + } + + my $inlist = 0; + my $status = "new"; + while() { + my $line = $_; + + if($line =~ /Node number/ || $line =~ /Daemon node name/) { + $line = ; + $inlist = 1; + next; + } + + if($line =~ /-------/) { + $inlist = 1; + next; + } + + if($inlist == 0) { + next; + } + + s/^\s+//; + my @a = split(/[\.\s]+/); + my $lnode = $a[1]; + + if($lnode eq $node) { + $status = "old"; + last; + } + } + close(OUTPUT); + + logger('info',"$node detected as $status"); + + if($status eq "new") { + if($mode ne $status) { + print "$status run mmaddnode $node manually\n"; + logger('info',"run mmaddnode $node manually"); + last; + } + my $ok = 0; + print "$status started\n"; + if(!open(OUTPUT,"/usr/lpp/mmfs/bin/mmaddnode $node 2>&1 |")) { + print "$status mmaddnode failed\n"; + logger('info',"mmaddnode $node failed"); + last; + } + while() { + if(/Command successfully completed/) { + $ok = 1; + } + logger('info',"mmaddnode $node output: $_"); + } + close(OUTPUT); + if($ok == 0) { + print "$status mmaddnode failed\n"; + } + else { + print "$status ended\n"; + } + } + + if($status eq "old") { + if($mode ne $status) { + print "$status suspending autogpfs while adding nodes\n"; + logger('info',"suspending autogpfs while adding nodes"); + last; + } + my $ok = 0; + print "$status started\n"; + + if(!open(OUTPUT,"/etc/fstab")) { + print "$status failed\n"; + logger('info',"cannot read /etc/fstab"); + last; + } + while() { + chomp(); + s/^\s+//g; + if(/^#/) { + next; + } + my @a = split(/\s+/); + my $fstype = $a[2]; + if($fstype ne "gpfs") { + next; + } + print "$status gpfs $_\n"; + + s/.*dev=([^,]+).*/\1/; + my $devgpfs = "/dev/$_"; + my $mm = `ls -l $devgpfs | head -1`; + $mm =~ s/,//g; + @a = split(/\s+/,$mm); + my $major = $a[4]; + my $minor = $a[5]; + + print "$status dev $devgpfs $major $minor\n"; + } + close(OUTPUT); + + print "$status mmsdrfs start\n"; + if(!open(OUTPUT,"/var/mmfs/gen/mmsdrfs")) { + print "$status failed\n"; + logger('info',"cannot read /var/mmfs/gen/mmsdrfs"); + last; + } + while() { + print $_; + } + close(OUTPUT); + print "EOF\n"; + + $ok = 1; + + if($ok == 0) { + print "$status failed\n"; + } + else { + print "$status ended\n"; + } + } + + last; + + } + + logger('info',"connection from: $node ended"); +} + +sub daemonic { + my $child = fork(); + + if(!defined $child) { + die "Can't fork"; + } + if($child) { + exit 0; + } + setsid(); + open(STDIN, "/dev/null"); + open(STDERR,">&STDOUT"); + chdir('/'); + umask(0); + return $$; +} + +sub lockfile { + my $file = shift; + my $fh; + + if(-e $file) { + if(!($fh = IO::File->new($file))) { + return($fh); + } + my $pid = <$fh>; + if(kill 0 => $pid) { + die "Server already running with PID $pid"; + } + warn "Removing PID file for defunct server process $pid.\n"; + unless(-w $file && unlink $file) { + die "Can't unlink PID file $file" + } + } + if($fh = IO::File->new($file,O_WRONLY|O_CREAT|O_EXCL,0644)) { + return($fh); + } + else { + die "Can't create $file: $!\n"; + } +} + +sub hostcheck { + my $node = shift; + my $nodename; + my $attributes; + + #TBD, check DNS or xCAT for valid + + return(1); +} + +sub logger { + my $type = shift; + my $msg = shift; + + setlogsock('unix'); + openlog('autogpfsd','','local0'); + syslog($type,$msg); + closelog(); +} + +END { unlink PIDFILE if $$ == $pid; } + diff --git a/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.suse b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.suse new file mode 100755 index 000000000..bbf592c3a --- /dev/null +++ b/xCAT-server/share/xcat/netboot/add-on/autogpfs/autogpfsd.suse @@ -0,0 +1,127 @@ +#! /bin/sh +# Copyright (c) 1995-2000 SuSE GmbH Nuernberg, Germany. +# Copyright (c) 2001 IBM +# +# Author: Kurt Garloff +# Author: Egan Ford set local and overall rc status to +# rc_reset clear local rc status (overall remains) +# rc_exit exit appropriate to overall rc status +. /etc/rc.status + +# First reset status of this service +rc_reset + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - generic or unspecified error +# 2 - invalid or excess argument(s) +# 3 - unimplemented feature (e.g. "reload") +# 4 - insufficient privilege +# 5 - program is not installed +# 6 - program is not configured +# 7 - program is not running +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signalling is not supported) are +# considered a success. + +case "$1" in + start) + echo -n "Starting autogpfsd" + ## Start daemon with startproc(8). If this fails + ## the echo return value is set appropriate. + + # NOTE: startproc return 0, even if service is + # already running to match LSB spec. + startproc $AUTOGPFSD + + # Remember status and be verbose + rc_status -v + ;; + stop) + echo -n "Shutting down autogpfsd" + ## Stop daemon with killproc(8) and if this fails + ## set echo the echo return value. + + killproc -TERM $AUTOGPFSD + + # Remember status and be verbose + rc_status -v + ;; + restart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + $0 stop + $0 start + + # Remember status and be quiet + rc_status + ;; + status) + echo -n "Checking for autogpfsd: " + ## Check status with checkproc(8), if process is running + ## checkproc will return with exit status 0. + + # Status has a slightly different for the status command: + # 0 - service running + # 1 - service dead, but /var/run/ pid file exists + # 2 - service dead, but /var/lock/ lock file exists + # 3 - service not running + + # NOTE: checkproc returns LSB compliant status values. + checkproc $AUTOGPFSD + rc_status -v + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac +rc_exit