xcat-core/xCAT-server/lib/xcat/plugins/conserver.pm

# IBM(c) 2007 EPL license http://www.eclipse.org/legal/epl-v10.html
#TODO: delete entries not being refreshed if no noderange
package xCAT_plugin::conserver;
use strict;
use xCAT::Table;
use xCAT::Utils;
use xCAT::TableUtils;
use Getopt::Long;
use Sys::Hostname;
use xCAT::SvrUtils;
use xCAT::Scope;

use strict;
use Data::Dumper;
my @cservers = qw(mrv cyclades);
my %termservers;     #list of noted termservers
my $siteondemand;    # The site value for consoleondemand

my $usage_string =
  "   makeconservercf [-V|--verbose] [-d|--delete] noderange
   makeconservercf [-V|--verbose] [-l|--local] [noderange]
   makeconservercf [-V|--verbose] [-c|--conserver] [noderange]
   makeconservercf [-V|--verbose] noderange [-t|--trust] hosts
   makeconservercf [-h|--help|-v|--version]
    -c|--conserver   The conserver gets set up only on the conserver host.
                     The default goes down to all the conservers on
                     the server nodes and set them up
    -l|--local       The conserver gets set up only on the local host.
                     The default goes down to all the conservers on
                     the server nodes and set them up
    -d|--delete      Conserver has the relevant entries for the given noderange removed immediately from configuration
    -C|--cleanup     To remove the entries for the nodes that do not exist in xCAT db
    -t|--trust       Add additional trusted hosts.
    -h|--help        Display this usage statement.
    -V|--verbose     Verbose mode.
    -v|--version     Display the version number.";

my $version_string = xCAT::Utils->Version();

sub handled_commands {
    return {
        makeconservercf => "conserver"
      }
}

sub preprocess_request {
    my $request = shift;

    #if ($request->{_xcatdest}) { return [$request]; }    #exit if preprocessed
    if ($request->{_xcatpreprocessed}->[0] == 1) { return [$request]; }
    my $callback = shift;
    my @requests;
    my $noderange = $request->{node};    #Should be arrayref

    #display usage statement if -h
    my $extrargs = $request->{arg};
    my @exargs   = ($request->{arg});
    if (ref($extrargs)) {
        @exargs = @$extrargs;
    }
    @ARGV = @exargs;

    my $isSN     = xCAT::Utils->isServiceNode();
    my @hostinfo = xCAT::NetworkUtils->determinehostname();
    my %iphash   = ();
    foreach (@hostinfo) { $iphash{$_} = 1; }

    $Getopt::Long::ignorecase = 0;

    #$Getopt::Long::pass_through=1;
    if (!GetOptions(
            'c|conserver' => \$::CONSERVER,
            'l|local'     => \$::LOCAL,
            'h|help'      => \$::HELP,
            'D|debug'     => \$::DEBUG,
            'C|cleanup'   => \$::CLEANUP,
            'v|version'   => \$::VERSION,
            'V|verbose'   => \$::VERBOSE)) {
        $request = {};
        return;
    }
    if ($::HELP) {
        $callback->({ data => $usage_string });
        $request = {};
        return;
    }
    if ($::VERSION) {
        $callback->({ data => $version_string });
        $request = {};
        return;
    }
    if ($::LOCAL) {
        if ($noderange && @$noderange > 0) {
            $callback->({ data => "Invalid option -l or --local when there are nodes specified." });
            $request = {};
            return;
        }
    }
    if ($::CONSERVER && $::LOCAL) {
        $callback->({ data => "Can not specify -l or --local together with -c or --conserver." });
        $request = {};
        return;
    }
    if ($::CLEANUP && ($::CONSERVER or $::LOCAL)) {
        $callback->({ data => "Can not specify -l|--local or -c|--conserver together with -C|--cleanup." });
        $request = {};
        return;
    }
    # The cleanup shall run on both MN and all SNs
    if ($::CLEANUP) {
        if ($noderange && @$noderange > 0) {
            $callback->({ data => "Can not specify noderange together with -C|--cleanup." });
            $request = {};
            return;
        }
        my @sns = xCAT::ServiceNodeUtils->getSNList();
        unless ( @sns > 0 ) {
            return xCAT::Scope->get_parallel_scope($request);
        }
        return xCAT::Scope->get_broadcast_scope_with_parallel($request, \@sns);
    }
   
    # get site master
    my $master = xCAT::TableUtils->get_site_Master();
    if (!$master) { $master = hostname(); }

    # get conserver for each node
    my %cons_hash = ();
    my $hmtab     = xCAT::Table->new('nodehm');
    my @items;
    my $allnodes = 1;
    if ($noderange && @$noderange > 0) {
        $allnodes = 0;
        my $hmcache = $hmtab->getNodesAttribs($noderange, [ 'node', 'serialport', 'cons', 'conserver' ]);
        foreach my $node (@$noderange) {
            my $ent = $hmcache->{$node}->[0]; #$hmtab->getNodeAttribs($node,['node', 'serialport','cons', 'conserver']);
            push @items, $ent;
        }
    } else {
        $allnodes = 1;
        @items = $hmtab->getAllNodeAttribs([ 'node', 'serialport', 'cons', 'conserver' ]);
    }

    my @nodes = ();
    foreach (@items) {
        if (((!defined($_->{cons})) || ($_->{cons} eq "")) and !defined($_->{serialport})) { next; } #skip if 'cons' is not defined for this node, unless serialport suggests otherwise
        if (defined($_->{conserver})) { push @{ $cons_hash{ $_->{conserver} }{nodes} }, $_->{node}; }
        else { push @{ $cons_hash{$master}{nodes} }, $_->{node}; }
        push @nodes, $_->{node};
    }

    #send all nodes to the MN
    if (!$isSN && !$::CONSERVER) { #If -c flag is set, do not add the all nodes to the management node
        if ($::VERBOSE) {
            my $rsp;
            $rsp->{data}->[0] = "Setting the nodes into /etc/conserver.cf on the management node";
            xCAT::MsgUtils->message("I", $rsp, $callback);
        }
        my $reqcopy = {%$request};
        $reqcopy->{'_xcatdest'} = $master;
        $reqcopy->{_xcatpreprocessed}->[0] = 1;
        $reqcopy->{'_allnodes'} = $allnodes; # the original command comes with nodes or not
        if ($allnodes == 1) { @nodes = (); }
        $reqcopy->{node} = \@nodes;
        push @requests, $reqcopy;
        if ($::LOCAL) { return \@requests; }
    }

    # send to conserver hosts
    foreach my $cons (keys %cons_hash) {

        #print "cons=$cons\n";
        my $doit = 0;
        if ($isSN) {
            if (exists($iphash{$cons})) { $doit = 1; }
        } else {
            if (!exists($iphash{$cons}) || $::CONSERVER) { $doit = 1; }
        }

        if ($doit) {
            my $reqcopy = {%$request};
            $reqcopy->{'_xcatdest'} = $cons;
            $reqcopy->{_xcatpreprocessed}->[0] = 1;
            $reqcopy->{'_allnodes'} = [$allnodes]; # the original command comes with nodes or not
            $reqcopy->{node} = $cons_hash{$cons}{nodes};
            my $no = $reqcopy->{node};

            #print "node=@$no\n";
            push @requests, $reqcopy;
        }    #end if
    }    #end foreach

    if ($::DEBUG) {
        my $rsp;
        $rsp->{data}->[0] = "In preprocess_request, request is " . Dumper(@requests);
        xCAT::MsgUtils->message("I", $rsp, $callback);
    }
    return \@requests;
}

sub process_request {
    my $req = shift;
    my $cb  = shift;
    if ($req->{command}->[0] eq "makeconservercf") {
        if (-x "/usr/bin/goconserver") {
            my $cmd = "ps axf | grep -v grep | grep \/usr\/bin\/goconserver";
            xCAT::Utils->runcmd($cmd, 0);
            if ($::RUNCMD_RC == 0) {
                my $rsp->{data}->[0] = "goconserver is started, please stop it at first.";
                xCAT::MsgUtils->message("E", $rsp, $cb);
                return;
            }
        }
        makeconservercf($req, $cb);
    }
}

# Add the initial/global entries to the beginning of the file
sub docfheaders {

    # Put in standard headers common to all conserver.cf files
    my $content    = shift;
    my $cb         = shift;
    my @newheaders = ();
    my $numlines   = @$content;
    my $idx        = 0;
    my $skip       = 0;
    my @meat       = grep(!/^#/, @$content);
    unless (grep(/^config \* \{/, @meat)) {

        # do not add the ssl configurations
        # if conserver is not compiled with ssl support
        my $cmd = "console -h 2>&1";
        my $output = xCAT::Utils->runcmd($cmd, -1);
        if ($output !~ "encryption not compiled")
        {
            push @newheaders, "config * {\n";
            push @newheaders, "  sslrequired yes;\n";
            my $version = xCAT::Utils::get_conserver_version();
            if (!$version) {
                xCAT::SvrUtils::sendmsg([ 1, "Failed to get conserver version" ], $cb);
                return;
            }
            if (xCAT::Utils::calc_conserver_version($version) < xCAT::Utils::calc_conserver_version("8.1.19")) {
                push @newheaders, "  sslauthority /etc/xcat/cert/ca.pem;\n";
            } else {
                push @newheaders, "  sslcacertificatefile /etc/xcat/cert/ca.pem;\n";
            }
            push @newheaders, "  sslcredentials /etc/xcat/cert/server-cred.pem;\n";
            push @newheaders, "}\n";
        }
    }
    unless (grep(/^default cyclades/, @meat)) {
        push @newheaders, "default cyclades { type host; portbase 7000; portinc 1; }\n"
    }
    unless (grep(/^default mrv/, @meat)) {
        push @newheaders, "default mrv { type host; portbase 2000; portinc 100; }\n"
    }

    #Go through and delete that which would match access and default
    while ($idx < @$content) {
        if (($content->[$idx] =~ /^access \*/)
            || ($content->[$idx] =~ /^default \*/)) {
            $skip = 1;
        }
        if ($skip == 1) {
            splice(@$content, $idx, 1);
        } else {
            $idx++;
        }
        if ($skip and $content->[$idx] =~ /\}/) {
            splice(@$content, $idx, 1);
            $skip = 0;
        }
    }

    #push @$content,"#xCAT BEGIN ACCESS\n";
    push @newheaders, "access * {\n";
    push @newheaders, "  trusted 127.0.0.1;\n";
    my $master = xCAT::TableUtils->get_site_Master();
    push @newheaders, "  trusted $master;\n";

    # trust all the ip addresses configured on this node
    my @allips = xCAT::NetworkUtils->gethost_ips();
    my @ips    = ();

    #remove $xcatmaster and duplicate entries
    foreach my $ip (@allips) {
        if (($ip eq "127.0.0.1") || ($ip eq $master)) {
            next;
        }
        if (!grep(/^$ip$/, @ips)) {
            push @ips, $ip;
        }
    }
    if ($::TRUSTED_HOST)
    {
        my @trusted_host = (split /,/, $::TRUSTED_HOST);
        foreach my $tip (@trusted_host)
        {
            if (!grep(/^$tip$/, @ips)) {
                push @ips, $tip;
            }
        }
    }
    if (scalar(@ips) > 0) {
        my $ipstr = join(',', @ips);
        push @newheaders, "  trusted $ipstr;\n";
    }

    push @newheaders, "}\n";

    #push @$content,"#xCAT END ACCESS\n";

    push @newheaders, "default * {\n";
    push @newheaders, "  logfile /var/log/consoles/&;\n";
    push @newheaders, "  timestamp 1lab;\n";
    push @newheaders, "  rw *;\n";
    push @newheaders, "  master localhost;\n";

    #-- if option "conserverondemand" in site table is set to yes
    #-- then start all consoles on demand
    #-- this helps eliminate many ssh connections to blade AMM
    #-- which seems to kill AMMs occasionally
    my @entries    = xCAT::TableUtils->get_site_attribute("consoleondemand");
    my $site_entry = $entries[0];
    $siteondemand = 0;
    if (defined($site_entry)) {
        if (lc($site_entry) eq "yes") {
            push @newheaders, "  options ondemand;\n";
            $siteondemand = 1;
        }
        elsif (lc($site_entry) ne "no") {
            # consoleondemand attribute is set, but it is not "yes" or "no"
            xCAT::SvrUtils::sendmsg([ 1, "Unexpected value $site_entry for consoleondemand attribute in site table" ], $cb);
        }
    }
    push @newheaders, "}\n";
    unshift @$content, @newheaders;
}

# Read the file, get db info, update the file contents, and then write the file
sub makeconservercf {
    my $req = shift;
    %termservers = ();    #clear hash of existing entries
    my $cb       = shift;
    my $extrargs = $req->{arg};
    my @exargs   = ($req->{arg});
    if (ref($extrargs)) {
        @exargs = @$extrargs;
    }
    @ARGV                     = @exargs;
    $Getopt::Long::ignorecase = 0;

    #$Getopt::Long::pass_through=1;
    my $delmode;
    GetOptions('d|delete' => \$delmode,
        't|trust=s' => \$::TRUSTED_HOST,
        'C|cleanup' => \$::CLEANUP,
    );
    my $nodes  = $req->{node};
    my $svboot = 0;
    if (exists($req->{svboot})) { $svboot = 1; }
    my $cfile;
    my @filecontent;
    open $cfile, '/etc/conserver.cf';
    while (<$cfile>) {
        push @filecontent, $_;
    }
    close $cfile;
    docfheaders(\@filecontent,$cb);

    my $isSN     = xCAT::Utils->isServiceNode();
    my @hostinfo = xCAT::NetworkUtils->determinehostname();
    my %iphash   = ();
    foreach (@hostinfo) { $iphash{$_} = 1; }

    #print "process_request nodes=@$nodes\n";

    # Get db info for the nodes related to console
    my $hmtab = xCAT::Table->new('nodehm');
    my @cfgents1; # = $hmtab->getAllNodeAttribs(['cons','serialport','mgt','conserver','termserver','termport']);
    if (($nodes and @$nodes > 0) or $req->{noderange}->[0]) {
        @cfgents1 = $hmtab->getNodesAttribs($nodes, [ 'node', 'cons', 'serialport', 'mgt', 'conserver', 'termserver', 'termport', 'consoleondemand' ]);

        # Adjust the data structure to make the result consistent with the getAllNodeAttribs() call we make if a noderange was not specified
        my @tmpcfgents1;
        foreach my $ent (@cfgents1)
        {
            foreach my $nodeent (keys %$ent)
            {
                push @tmpcfgents1, $ent->{$nodeent}->[0];
            }
        }
        @cfgents1 = @tmpcfgents1

    } else {
        @cfgents1 = $hmtab->getAllNodeAttribs([ 'cons', 'serialport', 'mgt', 'conserver', 'termserver', 'termport', 'consoleondemand' ]);
    }


    #cfgents1 should now have all the nodes, so we can fill in the cfgents array and cfgenthash one at a time.
    # skip the nodes that do not have 'cons' defined, unless a serialport setting suggests otherwise
    my @cfgents = ();
    my %cfgenthash;
    foreach (@cfgents1) {
        if ($_->{cons} or defined($_->{'serialport'})) {
            unless ($_->{cons}) { $_->{cons} = $_->{mgt}; } #populate with fallback
            push @cfgents, $_;
            $cfgenthash{ $_->{node} } = $_; # also put the ref to the entry in a hash for quick look up
        }
    }

    if ($::DEBUG) {
        my $rsp;
        $rsp->{data}->[0] = "In makeconservercf, cfgents is " . Dumper(@cfgents);
        xCAT::MsgUtils->message("I", $rsp, $cb);
    }

    # if nodes defined, it is either on the service node or makeconserver was called with noderange on mn
    if (($nodes and @$nodes > 0) or $req->{noderange}->[0]) {

        # strip all xCAT configured nodes from config if the original command was for all nodes
        if (($req->{_allnodes}) && ($req->{_allnodes}->[0] == 1)) { zapcfg(\@filecontent); }

        # call donodeent to add all node entries into the file.  It will return the 1st node in error.
        my $node;
        if ($node = donodeent(\%cfgenthash, \@filecontent, $delmode)) {

            #$cb->({node=>[{name=>$node,error=>"Bad configuration, check attributes under the nodehm category",errorcode=>1}]});
            xCAT::SvrUtils::sendmsg([ 1, "Bad configuration, check attributes under the nodehm category" ], $cb, $node);
        }
    } elsif ($::CLEANUP) {
        my $nodelstab = xCAT::Table->new('nodelist');
        my @allnodeset = $nodelstab->getAllAttribs('node');
        my %allnodehash = map { $_->{node} => 1 } @allnodeset;
        my $rmnodes = delete_undefined_nodes_entry(\@filecontent, \%allnodehash);
        my $rsp;
        if (!defined($rmnodes)) {
            $rsp->{data}->[0] = "Nothing removed";
        } else{
            $rsp->{data}->[0] = "Remove console entry for the nodes:".join(',', @$rmnodes);
        }
        xCAT::MsgUtils->message("I", $rsp, $cb); 
    } else {    #no nodes specified, do em all up
        zapcfg(\@filecontent);    # strip all xCAT configured nodes from config

        # get nodetype so we can filter out node types without console support
        my $typetab = xCAT::Table->new('nodetype');
        my %type;

        if (defined($typetab)) {
            my @ents = $typetab->getAllNodeAttribs([qw(node nodetype)]);
            foreach (@ents) {
                $type{ $_->{node} } = $_->{nodetype};
            }
        }

        # remove nodes that arent for this SN or type of node doesnt have console
        foreach (@cfgents) {
            my $keepdoing = 0;
            if ($isSN && $_->{conserver} && exists($iphash{ $_->{conserver} })) {
                $keepdoing = 1; #only hanlde the nodes that use this SN as the conserver
            }
            if (!$isSN) { $keepdoing = 1; }    #handle all for MN
            if ($keepdoing) {
                if ($_->{termserver} and not $termservers{ $_->{termserver} }) {

                    # add a terminal server entry to file
                    dotsent($_, \@filecontent);
                    $termservers{ $_->{termserver} } = 1; # dont add this one again
                }
                if ($type{ $_->{node} } =~ /fsp|bpa|hmc|ivm/) {
                    $keepdoing = 0;    # these types dont have consoles
                }
            }
            if (!$keepdoing) { delete $cfgenthash{ $_->{node} }; } # remove this node from the hash so we dont process it later
        }

        # Now add into the file all the node entries that we kept
        my $node;
        if ($node = donodeent(\%cfgenthash, \@filecontent, $delmode)) {

            # donodeent will return the 1st node in error
            #$cb->({node=>[{name=>$node,error=>"Bad configuration, check attributes under the nodehm category",errorcode=>1}]});
            xCAT::SvrUtils::sendmsg([ 1, "Bad configuration, check attributes under the nodehm category" ], $cb, $node);
        }
    }

    # Write out the file contents
    open $cfile, '>', '/etc/conserver.cf';
    if ($::VERBOSE) {
        my $rsp;
        $rsp->{data}->[0] = "Setting the following lines into /etc/conserver.cf:\n @filecontent";
        xCAT::MsgUtils->message("I", $rsp, $cb);
    }
    foreach (@filecontent) {
        print $cfile $_;
    }
    close $cfile;

    # restart conserver
    if (!$svboot) {

        #restart conserver daemon
        my $cmd;
        if (xCAT::Utils->isAIX()) {
            $cmd = "stopsrc -s conserver";
            xCAT::Utils->runcmd($cmd, 0);
            $cmd = "startsrc -s conserver";
            xCAT::Utils->runcmd($cmd, 0);
        } else {
            $cmd = "/etc/init.d/conserver stop";
            xCAT::Utils->runcmd($cmd, 0);
            $cmd = "/etc/init.d/conserver start";
            xCAT::Utils->runcmd($cmd, 0);
        }
    }
}

# Put a terminal server entry in the file - not used much any more
sub dotsent {
    my $cfgent   = shift;
    my $tserv    = $cfgent->{termserver};
    my $content  = shift;
    my $idx      = 0;
    my $toidx    = -1;
    my $skip     = 0;
    my $skipnext = 0;

    while ($idx < $#$content) { # Go through and delete that which would match my entry
        if ($content->[$idx] =~ /^#xCAT BEGIN $tserv TS/) {
            $toidx    = $idx;    #TODO put it back right where I found it
            $skip     = 1;
            $skipnext = 1;
        } elsif ($content->[$idx] =~ /^#xCAT END $tserv TS/) {
            $skipnext = 0;
        }
        if ($skip) {
            splice(@$content, $idx, 1);
        } else {
            $idx++;
        }
        $skip = $skipnext;
    }
    push @$content, "#xCAT BEGIN $tserv TS\n";
    push @$content, "default $tserv {\n";
    push @$content, "  include " . $cfgent->{cons} . ";\n";
    push @$content, "  host $tserv;\n";
    push @$content, "}\n";
    push @$content, "#xCAT END $tserv TS\n";

}

# Add entries in the file for each node.  This function used to do 1 node at a time, but was changed to do
# all nodes at once for performance reasons.  If there is a problem with a nodes config, this
# function will return that node name as the one in error.
sub donodeent {
    my $cfgenthash = shift;
    my $content    = shift;
    my $delmode    = shift;
    my $idx        = 0;
    my $toidx      = -1;
    my $skip       = 0;
    my $skipnext   = 0;

    # Delete all the previous stanzas of the nodes specified
    my $isSN = xCAT::Utils->isServiceNode();
    my $curnode;

    # Loop till find the start of a node stanza and remove lines till get to the end of the stanza
    while ($idx <= $#$content) { # Go through and delete that which would match my entry
        my ($begorend, $node) = $content->[$idx] =~ /^#xCAT (\S+) (\S+) CONS/;
        if ($begorend eq 'BEGIN') {
            if ($cfgenthash->{$node}) {
                $toidx = $idx;    #TODO put it back right where I found it
                $skip  = 1;       # delete this line
                $skipnext = 1; # put us in skip mode until we find the end of the stanza
                $curnode = $node;
            }
        } elsif ($begorend eq 'END' && $node eq $curnode) {
            $skipnext = 0;
        }
        if ($skip) {
            splice(@$content, $idx, 1);
        } else {
            $idx++;
        }
        $skip = $skipnext;
    }
    if ($delmode) {

        # dont need to add node entries, so we are done
        return;
    }

    # Go thru all nodes specified to add them to the file
    foreach my $node (sort keys %$cfgenthash) {
        my $cfgent = $cfgenthash->{$node};
        my $cmeth  = $cfgent->{cons};
        if (not $cmeth or (grep(/^$cmeth$/, @cservers) and (not $cfgent->{termserver} or not $cfgent->{termport}))) {

            # either there is no console method (shouldnt happen) or not one of the supported terminal servers
            return $node;
        }
        push @$content, "#xCAT BEGIN $node CONS\n";
        push @$content, "console $node {\n";
        if (grep(/^$cmeth$/, @cservers)) {
            push @$content, " include " . $cfgent->{termserver} . ";\n";
            push @$content, " port " . $cfgent->{termport} . ";\n";
            if ((!$isSN) && ($cfgent->{conserver}) && xCAT::NetworkUtils->thishostisnot($cfgent->{conserver})) { # let the master handle it
                push @$content, "  master " . $cfgent->{conserver} . ";\n";
            }
        } else {    #a script method...
            push @$content, "  type exec;\n";
            if ((!$isSN) && ($cfgent->{conserver}) && xCAT::NetworkUtils->thishostisnot($cfgent->{conserver})) { # let the master handle it
                push @$content, "  master " . $cfgent->{conserver} . ";\n";
            } else {    # handle it here
                my $locerror = $isSN ? "PERL_BADLANG=0 " : ''; # on service nodes, often LC_ALL is not set and perl complains

                # add XCATSSLVER environment variable when it's set on sles11.x mn
                # for cons script to communicate with xcatd through tls
                my $env;
                if (defined($ENV{'XCATSSLVER'})) {
                    $env = "XCATSSLVER=$ENV{'XCATSSLVER'} ";
                }
                push @$content, "  exec $locerror$env" . $::XCATROOT . "/share/xcat/cons/" . $cmeth . " " . $node . ";\n"
            }
        }
        if (defined($cfgent->{consoleondemand})) {
            # consoleondemand attribute for node can be "1", "yes", "0" and "no"
            if ((($cfgent->{consoleondemand} eq "1") || lc($cfgent->{consoleondemand}) eq "yes") && !$siteondemand) {
                push @$content, "  options ondemand;\n";
            }
            elsif ((($cfgent->{consoleondemand} eq "0") || lc($cfgent->{consoleondemand}) eq "no") && $siteondemand) {
                push @$content, "  options !ondemand;\n";
            }
        }
        push @$content, "}\n";
        push @$content, "#xCAT END $node CONS\n";
    }
    return 0;
}
# Remove cons entries for the undefined nodes
sub delete_undefined_nodes_entry {
    my $content = shift;
    my $allnodeshash = shift;
    my $idx      = 0;
    my $toidx    = -1;
    my $skip     = 0;
    my $skipnext = 0;
    my @rmnodes = ();
    while ($idx <= $#$content) {
        if ($content->[$idx] =~ /^#xCAT BEGIN (\S+) CONS/) {
            $toidx   = $idx;
            my $node = $1;
            unless (exists($allnodeshash->{$node})) {
                $skip  = 1;
                $skipnext = 1; 
                push @rmnodes, $node;
                print __LINE__."===== push node: $node==\n";
            }
        } elsif ($content->[$idx] =~ /^#xCAT END/) {
            $skipnext = 0;
        }
        if ($skip) {
            splice(@$content, $idx, 1);
        } else {
            $idx++;
        }
        $skip = $skipnext;
    }
    if (scalar(@rmnodes) > 0) {
        return \@rmnodes;
    } else {
        return undef;
    }
}

# Delete any xcat added node entries from the file
sub zapcfg {
    my $content  = shift;
    my $idx      = 0;
    my $toidx    = -1;
    my $skip     = 0;
    my $skipnext = 0;
    while ($idx <= $#$content) { # Go through and delete that which would match my entry
        if ($content->[$idx] =~ /^#xCAT BEGIN/) {
            $toidx    = $idx;    #TODO put it back right where I found it
            $skip     = 1;
            $skipnext = 1;
        } elsif ($content->[$idx] =~ /^#xCAT END/) {
            $skipnext = 0;
        }
        if ($skip) {
            splice(@$content, $idx, 1);
        } else {
            $idx++;
        }
        $skip = $skipnext;
    }
}


1;