From 5fc4e7fad57b550724100b6cc8f6805ab9b6897a Mon Sep 17 00:00:00 2001 From: Bin Xu Date: Tue, 21 Aug 2018 18:36:52 +0800 Subject: [PATCH] Enhancement for using site cache in plugin (#5535) * site cache when run plugin does not work very well - using cache from plugin when getNodesAttribs/getNodeAttribs (pass it into DB process from plugin process) - Site cache is a whole hash, so to use cache when by the hash is there, instead of the specified key is there. It is because that there might be no key defined in site table. * with XCATBYPASS, to populate site hash before scan_plugins. Then only 1 query for site table to do whole things. * cache site when init plugins on service nodes * missing to comment the old codes query from xCAT DB process --- perl-xCAT/xCAT/Client.pm | 3 +- perl-xCAT/xCAT/Table.pm | 56 ++++++++++++++++++++++-------------- perl-xCAT/xCAT/TableUtils.pm | 26 +++++++++-------- xCAT-server/sbin/xcatd | 4 +++ 4 files changed, 53 insertions(+), 36 deletions(-) diff --git a/perl-xCAT/xCAT/Client.pm b/perl-xCAT/xCAT/Client.pm index bc090c890..888c9242a 100644 --- a/perl-xCAT/xCAT/Client.pm +++ b/perl-xCAT/xCAT/Client.pm @@ -174,6 +174,7 @@ sub submit_request { # Load plugins from either specified or default dir require xCAT::Table; + populate_site_hash(); my %cmd_handlers; my @plugins_dirs = split('\:', $ENV{XCATBYPASS}); if (-d $plugins_dirs[0]) { @@ -192,8 +193,6 @@ sub submit_request { scan_plugins(); } - populate_site_hash(); - # don't do XML transformation -- assume request is well-formed # my $xmlreq=XMLout($request,RootName=>xcatrequest,NoAttr=>1,KeyAttr=>[]); # $request = XMLin($xmlreq,SuppressEmpty=>undef,ForceArray=>1) ; diff --git a/perl-xCAT/xCAT/Table.pm b/perl-xCAT/xCAT/Table.pm index 99bf2210c..103f8a09a 100644 --- a/perl-xCAT/xCAT/Table.pm +++ b/perl-xCAT/xCAT/Table.pm @@ -2312,10 +2312,6 @@ sub setNodesAttribs { #-------------------------------------------------------------------------------- sub getNodesAttribs { my $self = shift; - if ($dbworkerpid > 0) { - return dbc_call($self, 'getNodesAttribs', @_); - } - $self->trace_db(START_TYPE); my $nodelist = shift; unless ($nodelist) { $nodelist = []; } #common to be invoked with undef seemingly my %options = (); @@ -2326,6 +2322,21 @@ sub getNodesAttribs { } else { @attribs = @_; } + + if (!exists($options{hierarchy_attrs})) { + my @hierarchy_attrs = (); + my $hierarchy_field = xCAT::TableUtils->get_site_attribute("hierarchicalattrs"); + if ($hierarchy_field) { + @hierarchy_attrs = split(/,/, $hierarchy_field); + } + $options{hierarchy_attrs} = \@hierarchy_attrs; + } + + if ($dbworkerpid > 0) { + return dbc_call($self, 'getNodesAttribs', $nodelist, \@attribs, %options); + } + + $self->trace_db(START_TYPE); my @realattribs = @attribs; #store off the requester attribute list, the cached columns may end up being a superset and we shouldn't return more than asked #it should also be the case that cache will be used if it already is in play even if below cache threshold. This would be desired behavior if (scalar(@$nodelist) > $cachethreshold) { @@ -2348,12 +2359,12 @@ sub getNodesAttribs { $self->{nodelist}->{_use_cache} = 1; } my $rethash; - my @hierarchy_attrs = (); - my $hierarchy_field = xCAT::TableUtils->get_site_attribute("hierarchicalattrs"); - if ($hierarchy_field) { - @hierarchy_attrs = split(/,/, $hierarchy_field); - } - $options{hierarchy_attrs} = \@hierarchy_attrs; + #my @hierarchy_attrs = (); + #my $hierarchy_field = xCAT::TableUtils->get_site_attribute("hierarchicalattrs"); + #if ($hierarchy_field) { + # @hierarchy_attrs = split(/,/, $hierarchy_field); + #} + #$options{hierarchy_attrs} = \@hierarchy_attrs; foreach (@$nodelist) { my @nodeentries = $self->getNodeAttribs($_, \@realattribs, %options); $rethash->{$_} = \@nodeentries; #$self->getNodeAttribs($_,\@attribs); @@ -2732,20 +2743,10 @@ sub transRegexAttrs sub getNodeAttribs { my $self = shift; - if ($dbworkerpid > 0) { #TODO: should this be moved outside of the DB worker entirely? I'm thinking so, but I don't dare do so right now... - #the benefit would be the potentially computationally intensive substitution logic would be moved out and less time inside limited - #db worker scope - return dbc_call($self, 'getNodeAttribs', @_); - } - $self->trace_db(START_TYPE); - - if (!defined($self->{dbh})) { - xCAT::MsgUtils->message("S", "xcatd: DBI is missing, Please check the db access process."); - return undef; - } my $node = shift; - my @attribs; + my %options = (); + my @attribs; if (ref $_[0]) { @attribs = @{ shift() }; %options = @_; @@ -2761,6 +2762,17 @@ sub getNodeAttribs } $options{hierarchy_attrs} = \@hierarchy_attrs; } + if ($dbworkerpid > 0) { #TODO: should this be moved outside of the DB worker entirely? I'm thinking so, but I don't dare do so right now... + #the benefit would be the potentially computationally intensive substitution logic would be moved out and less time inside limited + #db worker scope + return dbc_call($self, 'getNodeAttribs', $node, \@attribs, %options); + } + $self->trace_db(START_TYPE); + + if (!defined($self->{dbh})) { + xCAT::MsgUtils->message("S", "xcatd: DBI is missing, Please check the db access process."); + return undef; + } my $datum; my $oldusecache; diff --git a/perl-xCAT/xCAT/TableUtils.pm b/perl-xCAT/xCAT/TableUtils.pm index 8fde01aa3..472a2aedc 100755 --- a/perl-xCAT/xCAT/TableUtils.pm +++ b/perl-xCAT/xCAT/TableUtils.pm @@ -1261,27 +1261,28 @@ sub getAppStatus get_site_attribute Arguments: - + $attribute -- the attribute you want to get + $dvalue -- Optional, the default string value if the attribute does not exist Returns: The value of the attribute requested from the site table - Globals: - none - Error: - undef - Example: - @attr=xCAT::TableUtils->get_site_attribute($attribute); - Comments: - none + Globals: + %::XCATSITEVALS + Error: + undef + Example: + @attr=xCAT::TableUtils->get_site_attribute($attribute); + Comments: + none =cut #------------------------------------------------------------------------ sub get_site_attribute { - my ($class, $attr) = @_; + my ($class, $attr, $dvalue) = @_; my $values; - if (defined($::XCATSITEVALS{$attr})) { + if (%::XCATSITEVALS) { $values = ($::XCATSITEVALS{$attr}); } else { my $sitetab = xCAT::Table->new('site'); @@ -1300,7 +1301,8 @@ sub get_site_attribute } } - return $values; + return $values if ( defined $values); + return $dvalue; } diff --git a/xCAT-server/sbin/xcatd b/xCAT-server/sbin/xcatd index 7a037e2e1..b9729d12c 100755 --- a/xCAT-server/sbin/xcatd +++ b/xCAT-server/sbin/xcatd @@ -970,6 +970,10 @@ if (defined $pid_init) { %cmd_handlers = %{ fd_retrieve($readpipe) }; } else { $$progname = "xcatd: plugin initialization"; + if (xCAT::Utils->isServiceNode()) { + # Cache the site hash to accelerate the speed of init_plugins on SN + populate_site_hash(); + } scan_plugins($writepipe); exit(0); }