diff --git a/xCAT-server/lib/perl/xCAT/AGENT.pm b/xCAT-server/lib/perl/xCAT/AGENT.pm new file mode 100644 index 000000000..f62125615 --- /dev/null +++ b/xCAT-server/lib/perl/xCAT/AGENT.pm @@ -0,0 +1,198 @@ +#!/usr/bin/perl +## IBM(c) 2018 EPL license http://www.eclipse.org/legal/epl-v10.html + +package xCAT::AGENT; + +BEGIN +{ + $::XCATROOT = $ENV{'XCATROOT'} ? $ENV{'XCATROOT'} : '/opt/xcat'; +} +use lib "$::XCATROOT/lib/perl"; +use strict; +use warnings "all"; + +use JSON; +use Time::HiRes qw(sleep time); +use File::Path; +use Fcntl ":flock"; +use IO::Socket::UNIX qw( SOCK_STREAM ); +use xCAT_monitoring::monitorctrl; +use xCAT::TableUtils; + +my $LOCK_DIR = "/var/lock/xcat/"; +my $LOCK_PATH = "/var/lock/xcat/agent.lock"; +my $AGENT_SOCK_PATH = "/var/run/xcat/agent.sock"; +my $PYTHON_LOG_PATH = "/var/log/xcat/agent.log"; +my $PYTHON_AGENT_FILE = "/opt/xcat/lib/python/agent/agent.py"; +my $MSG_TYPE = "message"; +my $DB_TYPE = "db"; +my $lock_fd; + +my %module_type = ( + "openbmc" => "OpenBMC", + "redfish" => "Redfish", +); + +sub acquire_lock { + my $ppid = shift; + $ppid = shift if (($ppid) && ($ppid =~ /AGENT/)); + + mkpath($LOCK_DIR); + # always create a new lock file + if ($ppid) { + $LOCK_PATH = "$LOCK_PATH.$ppid"; + $AGENT_SOCK_PATH = "$AGENT_SOCK_PATH.$ppid"; + } + unlink($LOCK_PATH); + open($lock_fd, ">>", $LOCK_PATH) or return undef; + flock($lock_fd, LOCK_EX) or return undef; + return $lock_fd; +} + +sub exists_python_agent { + if ( -e $PYTHON_AGENT_FILE) { + return 1; + } + return 0; +} + +sub python_agent_reaper { + unlink($LOCK_PATH); + unlink($AGENT_SOCK_PATH); +} + +sub start_python_agent { + my $ppid = shift; + $ppid = shift if (($ppid) && ($ppid =~ /AGENT/)); + + if (!defined(acquire_lock($ppid))) { + xCAT::MsgUtils->message("S", "start_python_agent() Error: Failed to acquire lock"); + return undef; + } + + my $fd; + my $pid = fork; + if (!defined $pid) { + xCAT::MsgUtils->message("S", "start_python_agent() Error: Unable to fork process"); + return undef; + } elsif ($pid){ + open($fd, '>', $AGENT_SOCK_PATH) && close($fd); + $SIG{INT} = $SIG{TERM} = \&python_agent_reaper; + return $pid; + } + + $SIG{CHLD} = 'DEFAULT'; + if (!$pid) { + # child + open($fd, ">>", $PYTHON_LOG_PATH) && close($fd); + open(STDOUT, '>>', $PYTHON_LOG_PATH) or die("open: $!"); + open(STDERR, '>>&', \*STDOUT) or die("open: $!"); + my @args = ( "$PYTHON_AGENT_FILE --sock $AGENT_SOCK_PATH --lockfile $LOCK_PATH" ); + my $ret = exec @args; + if (!defined($ret)) { + xCAT::MsgUtils->message("S", "start_python_agent() Error: Failed to start the xCAT Python agent."); + exit(1); + } + } +} + +sub handle_message { + my ($data, $callback) = @_; + if($data->{type} eq $MSG_TYPE) { + my $msg = $data->{msg}; + if ($msg->{type} eq 'info') { + xCAT::MsgUtils->message("I", { data => [$msg->{data}] }, $callback); + } elsif ($msg->{type} eq 'warning') { + xCAT::MsgUtils->message("W", { data => [$msg->{data}] }, $callback); + } elsif ($msg->{type} eq 'error'){ + xCAT::SvrUtils::sendmsg([ 1, $msg->{data} ], $callback, $msg->{node}); + } elsif ($msg->{type} eq 'syslog'){ + xCAT::MsgUtils->message("S", $msg->{data}); + } + } elsif ($data->{type} eq $DB_TYPE) { + my $attribute = $data->{attribute}; + if ($attribute->{name} eq 'status' and $attribute->{method} eq 'set' and $attribute->{type} eq 'node') { + my %new_status = ($attribute->{value} => [$attribute->{node}]); + xCAT_monitoring::monitorctrl::setNodeStatusAttributes(\%new_status, 1) + } + } +} + +sub submit_agent_request { + my ($pid, $req, $module, $nodeinfo, $callback) = @_; + my $sock; + my $retry = 0; + while($retry < 30) { + $sock = IO::Socket::UNIX->new(Peer => $AGENT_SOCK_PATH, Type => SOCK_STREAM, Timeout => 10, Blocking => 1); + if (!defined($sock)) { + sleep(0.1); + } else { + last; + } + $retry++; + } + if (!defined($sock)) { + xCAT::MsgUtils->message("E", { data => ["$module_type{$module} management is using a Python framework. An error has occurred when trying to create socket $AGENT_SOCK_PATH."] }, $callback); + kill('TERM', $pid); + return; + } + my $xcatdebugmode = 0; + if ($::XCATSITEVALS{xcatdebugmode}) { $xcatdebugmode = $::XCATSITEVALS{xcatdebugmode} } + my %env_hash = (); + $env_hash{debugmode} = $xcatdebugmode; + my ($data, $sz, $ret, $buf); + $data->{module} = $module; + $data->{command} = $req->{command}->[0]; + $data->{args} = $req->{arg}; + $data->{cwd} = $req->{cwd}; + $data->{nodes} = $req->{node}; + $data->{nodeinfo} = $nodeinfo; + $data->{envs} = \%env_hash; + $buf = encode_json($data); + $sz = pack('i', length($buf)); + # send length of data first + $ret = $sock->send($sz); + if (!$ret) { + xCAT::MsgUtils->message("E", { data => ["Failed to send message to the agent"] }, $callback); + $sock->close(); + kill('TERM', $pid); + return; + } + # send data + $ret = $sock->send($buf); + if (!$ret) { + xCAT::MsgUtils->message("E", { data => ["Failed to send message to the agent"] }, $callback); + $sock->close(); + kill('TERM', $pid); + return; + } + while(1) { + $ret = $sock->recv($buf, 4); + if (!$ret) { + last; + } + # receive the length of data + $sz = unpack('i', $buf); + # read data with length is $sz + $ret = $sock->recv($buf, $sz); + if (!$ret) { + xCAT::MsgUtils->message("E", { data => ["receive data from python agent unexpectedly"] }, $callback); + last; + } + $data = decode_json($buf); + handle_message($data, $callback); + } + # no message received, the socket on the agent side should be closed. + $sock->close(); +} + +sub wait_agent { + my ($pid, $callback) = @_; + waitpid($pid, 0); + if ($? >> 8 != 0) { + xCAT::MsgUtils->message("E", { data => ["Agent exited unexpectedly. See $PYTHON_LOG_PATH for details."] }, $callback); + } + python_agent_reaper(); +} + +1; diff --git a/xCAT-server/lib/perl/xCAT/OPENBMC.pm b/xCAT-server/lib/perl/xCAT/OPENBMC.pm index 2061a191a..83ceafed2 100644 --- a/xCAT-server/lib/perl/xCAT/OPENBMC.pm +++ b/xCAT-server/lib/perl/xCAT/OPENBMC.pm @@ -19,19 +19,10 @@ use Data::Dumper; use Time::HiRes qw(sleep time); use JSON; use File::Path; -use Fcntl ":flock"; -use IO::Socket::UNIX qw( SOCK_STREAM ); use xCAT_monitoring::monitorctrl; use xCAT::TableUtils; -my $LOCK_DIR = "/var/lock/xcat/"; -my $LOCK_PATH = "/var/lock/xcat/agent.lock"; -my $AGENT_SOCK_PATH = "/var/run/xcat/agent.sock"; -my $PYTHON_LOG_PATH = "/var/log/xcat/agent.log"; my $PYTHON_AGENT_FILE = "/opt/xcat/lib/python/agent/agent.py"; -my $MSG_TYPE = "message"; -my $DB_TYPE = "db"; -my $lock_fd; my $header = HTTP::Headers->new('Content-Type' => 'application/json'); @@ -59,172 +50,6 @@ sub send_request { return $id; } -# if lock is released unexpectedly, python side would aware of the error after -# getting this lock -sub acquire_lock { - my $ppid = shift; - $ppid = shift if (($ppid) && ($ppid =~ /OPENBMC/)); - - mkpath($LOCK_DIR); - # always create a new lock file - if ($ppid) { - $LOCK_PATH = "$LOCK_PATH.$ppid"; - $AGENT_SOCK_PATH = "$AGENT_SOCK_PATH.$ppid"; - } - unlink($LOCK_PATH); - open($lock_fd, ">>", $LOCK_PATH) or return undef; - flock($lock_fd, LOCK_EX) or return undef; - return $lock_fd; -} - -sub exists_python_agent { - if ( -e $PYTHON_AGENT_FILE) { - return 1; - } - return 0; -} -sub python_agent_reaper { - unlink($LOCK_PATH); - unlink($AGENT_SOCK_PATH); -} -sub start_python_agent { - - my $ppid = shift; - $ppid = shift if (($ppid) && ($ppid =~ /OPENBMC/)); - - if (!defined(acquire_lock($ppid))) { - xCAT::MsgUtils->message("S", "start_python_agent() Error: Failed to acquire lock"); - return undef; - } - - my $fd; - my $pid = fork; - if (!defined $pid) { - xCAT::MsgUtils->message("S", "start_python_agent() Error: Unable to fork process"); - return undef; - } elsif ($pid){ - - open($fd, '>', $AGENT_SOCK_PATH) && close($fd); - $SIG{INT} = $SIG{TERM} = \&python_agent_reaper; - return $pid; - } - - $SIG{CHLD} = 'DEFAULT'; - if (!$pid) { - # child - open($fd, ">>", $PYTHON_LOG_PATH) && close($fd); - open(STDOUT, '>>', $PYTHON_LOG_PATH) or die("open: $!"); - open(STDERR, '>>&', \*STDOUT) or die("open: $!"); - my @args = ( "$PYTHON_AGENT_FILE --sock $AGENT_SOCK_PATH --lockfile $LOCK_PATH" ); - my $ret = exec @args; - if (!defined($ret)) { - xCAT::MsgUtils->message("S", "start_python_agent() Error: Failed to start the xCAT Python agent."); - exit(1); - } - } - -} - -sub handle_message { - my ($data, $callback) = @_; - if($data->{type} eq $MSG_TYPE) { - my $msg = $data->{msg}; - if ($msg->{type} eq 'info') { - xCAT::MsgUtils->message("I", { data => [$msg->{data}] }, $callback); - } elsif ($msg->{type} eq 'warning') { - xCAT::MsgUtils->message("W", { data => [$msg->{data}] }, $callback); - } elsif ($msg->{type} eq 'error'){ - xCAT::SvrUtils::sendmsg([ 1, $msg->{data} ], $callback, $msg->{node}); - } elsif ($msg->{type} eq 'syslog'){ - xCAT::MsgUtils->message("S", $msg->{data}); - } - } elsif ($data->{type} eq $DB_TYPE) { - my $attribute = $data->{attribute}; - if ($attribute->{name} eq 'status' and $attribute->{method} eq 'set' and $attribute->{type} eq 'node') { - my %new_status = ($attribute->{value} => [$attribute->{node}]); - xCAT_monitoring::monitorctrl::setNodeStatusAttributes(\%new_status, 1) - } - } -} - -sub submit_agent_request { - my ($pid, $req, $nodeinfo, $callback) = @_; - my $sock; - my $retry = 0; - while($retry < 30) { - $sock = IO::Socket::UNIX->new(Peer => $AGENT_SOCK_PATH, Type => SOCK_STREAM, Timeout => 10, Blocking => 1); - if (!defined($sock)) { - sleep(0.1); - } else { - last; - } - $retry++; - } - if (!defined($sock)) { - xCAT::MsgUtils->message("E", { data => ["OpenBMC management is using a Python framework. An error has occurred when trying to create socket $AGENT_SOCK_PATH."] }, $callback); - kill('TERM', $pid); - return; - } - my $xcatdebugmode = 0; - if ($::XCATSITEVALS{xcatdebugmode}) { $xcatdebugmode = $::XCATSITEVALS{xcatdebugmode} } - my %env_hash = (); - $env_hash{debugmode} = $xcatdebugmode; - my ($data, $sz, $ret, $buf); - $data->{module} = 'openbmc'; - $data->{command} = $req->{command}->[0]; - $data->{args} = $req->{arg}; - $data->{cwd} = $req->{cwd}; - $data->{nodes} = $req->{node}; - $data->{nodeinfo} = $nodeinfo; - $data->{envs} = \%env_hash; - $buf = encode_json($data); - $sz = pack('i', length($buf)); - # send length of data first - $ret = $sock->send($sz); - if (!$ret) { - xCAT::MsgUtils->message("E", { data => ["Failed to send message to the agent"] }, $callback); - $sock->close(); - kill('TERM', $pid); - return; - } - # send data - $ret = $sock->send($buf); - if (!$ret) { - xCAT::MsgUtils->message("E", { data => ["Failed to send message to the agent"] }, $callback); - $sock->close(); - kill('TERM', $pid); - return; - } - while(1) { - $ret = $sock->recv($buf, 4); - if (!$ret) { - last; - } - # receive the length of data - $sz = unpack('i', $buf); - # read data with length is $sz - $ret = $sock->recv($buf, $sz); - if (!$ret) { - xCAT::MsgUtils->message("E", { data => ["receive data from python agent unexpectedly"] }, $callback); - last; - } - $data = decode_json($buf); - handle_message($data, $callback); - } - # no message received, the socket on the agent side should be closed. - $sock->close(); -} - -sub wait_agent { - my ($pid, $callback) = @_; - waitpid($pid, 0); - if ($? >> 8 != 0) { - xCAT::MsgUtils->message("E", { data => ["Agent exited unexpectedly. See $PYTHON_LOG_PATH for details."] }, $callback); - xCAT::MsgUtils->message("I", { data => ["To revert to Perl framework: chdef -t site clustersite openbmcperl=ALL"] }, $callback); - } - python_agent_reaper(); -} - #-------------------------------------------------------------------------------- =head3 run_cmd_in_perl diff --git a/xCAT-server/lib/xcat/plugins/openbmc2.pm b/xCAT-server/lib/xcat/plugins/openbmc2.pm index 3f1e22886..66d955d6b 100644 --- a/xCAT-server/lib/xcat/plugins/openbmc2.pm +++ b/xCAT-server/lib/xcat/plugins/openbmc2.pm @@ -16,6 +16,7 @@ use Getopt::Long; use xCAT::Usage; use xCAT::SvrUtils; use xCAT::OPENBMC; +use xCAT::AGENT; #------------------------------------------------------- @@ -134,7 +135,7 @@ sub process_request { my $request = shift; $callback = shift; - if (!xCAT::OPENBMC::exists_python_agent()) { + if (!xCAT::AGENT::exists_python_agent()) { xCAT::MsgUtils->message("E", { data => ["The xCAT Python agent does not exist. Check if xCAT-openbmc-py package is installed on management node and service nodes."] }, $callback); return; } @@ -149,14 +150,14 @@ sub process_request { return unless(%node_info); # If we can't start the python agent, exit immediately - my $pid = xCAT::OPENBMC::start_python_agent($$); + my $pid = xCAT::AGENT::start_python_agent($$); if (!defined($pid)) { xCAT::MsgUtils->message("E", { data => ["Failed to start the xCAT Python agent. Check /var/log/xcat/cluster.log for more information."] }, $callback); return; } - xCAT::OPENBMC::submit_agent_request($pid, $request, \%node_info, $callback); - xCAT::OPENBMC::wait_agent($pid, $callback); + xCAT::AGENT::submit_agent_request($pid, $request, "openbmc", \%node_info, $callback); + xCAT::AGENT::wait_agent($pid, $callback); } my @rsp_common_options = qw/autoreboot bootmode powersupplyredundancy powerrestorepolicy timesyncmethod