mirror of
https://github.com/xcat2/xcat-core.git
synced 2025-11-04 05:12:30 +00:00
63a3efe8bb
RH7/Centos7 and recent versions of fedora relocate `/*bin/*` into `/usr/*bin/`. This causes conflicts for upgrades against xCAT-genesis-scripts-* packages which expect the `/bin` a to be a directory while `xCAT-genesis-base-*` packages provide a link. Relocating all files into `/usr` fixes that conflict and allows a clean upgrade from all old versions.
1552 lines
51 KiB
Bash
Executable File
1552 lines
51 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Usage:
|
|
#
|
|
# There are utils for diskdiscover and configraid.
|
|
#
|
|
|
|
log_file=$1
|
|
|
|
##############################
|
|
# declare all commands
|
|
##############################
|
|
awk="awk"
|
|
sed="sed"
|
|
cut="cut"
|
|
sleep="sleep"
|
|
sort="sort"
|
|
ps="ps"
|
|
head="head"
|
|
readlink="readlink"
|
|
basename="basename"
|
|
udevadm="udevadm"
|
|
touch="touch"
|
|
tail="tail"
|
|
dmesg="dmesg"
|
|
grep="grep"
|
|
lspci="lspci"
|
|
|
|
############################################
|
|
#
|
|
# source raidcmd
|
|
#
|
|
###########################################
|
|
str_dir_name=`dirname $0`
|
|
. $str_dir_name/raidcmd
|
|
|
|
################################################################
|
|
#
|
|
# Input PCI_ID to get PCI location
|
|
#
|
|
# input: pci_id
|
|
#
|
|
# output: pci locations
|
|
#
|
|
################################################################
|
|
function get_pciloc_by_id {
|
|
local __in_pciid=$1
|
|
if echo "$__in_pciid" | $grep -sq "_"; then
|
|
__in_pciid=`echo "$__in_pciid" | $sed -e 's/_/:/'`
|
|
fi
|
|
local sysdevdir=/sys/bus/pci/devices
|
|
local pcilocs=`cd $sysdevdir 2>/dev/null && for dev in *
|
|
do
|
|
lines=$($udevadm info --query=property --path=$sysdevdir/$dev)
|
|
if echo "$lines" | $grep -i -sq -E "^PCI_ID=$__in_pciid$|^PCI_SUBSYS_ID=$__in_pciid$"; then
|
|
echo $dev
|
|
fi
|
|
done `
|
|
[ -z "$pcilocs" ] && return 1
|
|
echo "$pcilocs"
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
#
|
|
# get pci_slot scsi device
|
|
#
|
|
# input: slocs, for example, 0:0:0:0 0:0:1:0
|
|
#
|
|
# output: <sloc>=<device_name> ...
|
|
# <sloc>=<device_name>
|
|
#
|
|
######################################################
|
|
function convert_sloc_to_sg {
|
|
local __slocs="$*"
|
|
for __sloc in $__slocs
|
|
do
|
|
if echo "$__sloc" | grep -sq "[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+"; then
|
|
__sg=`$readlink /sys/class/scsi_device/$__sloc/device/generic`
|
|
if [ "$__sg" ]; then
|
|
__sg=`$basename $__sg`
|
|
fi
|
|
elif echo "$__sloc" | grep -sq -E '^sg[0-9]+|^sd[a-z]+'; then
|
|
__sg="$__sloc"
|
|
__sloc=`convert_sg_to_sloc $__sg | $awk -F= '{print $2}'`
|
|
fi
|
|
echo "$__sloc=$__sg"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
#################################################################
|
|
#
|
|
# Through PCI/SCSI device to find PCI/SCSI location
|
|
#
|
|
# input: device names,
|
|
# sg0 ...sgn
|
|
#
|
|
# output: <dev>=<sloc>... <dev>=<sloc>
|
|
#
|
|
################################################################
|
|
function convert_sg_to_sloc {
|
|
local __sgs="$*"
|
|
local __sloc=""
|
|
for __sg in $__sgs
|
|
do
|
|
if echo "$__sg" | grep -sq "^sg[0-9]\+"; then
|
|
__sloc=`$readlink /sys/class/scsi_generic/$__sg/device`
|
|
if [ "$__sloc" ]; then
|
|
__sloc=`$basename $__sloc`
|
|
fi
|
|
elif echo "$__sg" | grep -sq "^sd[a-z]\+"; then
|
|
__sloc=`$readlink /sys/block/$__sg/device`
|
|
if [ "$__sloc" ]; then
|
|
__sloc=`$basename $__sloc`
|
|
fi
|
|
elif echo "$__sg" | grep -sq "[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+"; then
|
|
__sloc="$__sg"
|
|
__sg=`convert_sloc_to_sg $__sloc | $awk -F= '{print $2}'`
|
|
fi
|
|
echo "$__sg=$__sloc"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
############################################################################
|
|
#
|
|
# get devices which are qualified to be used to create raid
|
|
# it should equals to "query-raid-create" after all array had
|
|
# been deleted.
|
|
# Note: output format of this command is multilines
|
|
# <pciloc_of_ioa1>=<sloc_of_disk1>,<sloc_of_disk2>,...
|
|
# <pciloc_of_ioa2>=<sloc_of_disk1>,<sloc_of_disk2>,...
|
|
# ...
|
|
###########################################################################
|
|
function get_raid_create_devices_by_pciloc {
|
|
local lines=""
|
|
local pcilocs="$*"
|
|
[ -z "$pcilocs" ] && return 1
|
|
|
|
# reorder ipr ioa pcilocs by its Primary and Secondary state
|
|
local ioas=`get_ipr_ioas_by_pciloc $pcilocs`
|
|
pcilocs=`get_sg_pciloc $ioas | $awk -F= '{print $2}' `
|
|
lines=`cmd_show_config`
|
|
local slocs=""
|
|
local line=""
|
|
for pciloc in $pcilocs
|
|
do
|
|
# exclude:
|
|
# 1) scsi adapter(ioa);
|
|
# 2) scsi enclosure;
|
|
# 3) disk array;
|
|
slocs=`echo "$lines" \
|
|
| grep '^.*[ ]\+'$pciloc'\/[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+.*$' \
|
|
| grep -v -E "Adapter|Enclosure|Disk Array" \
|
|
| cut_sloc_from_iprconfig_line \
|
|
| $sort -V \
|
|
| $sed -e 's/ /,/g' \
|
|
| awk '{printf (NR>1)?","$0:$0}'`
|
|
if [ -n "$slocs" ]; then
|
|
line="$pciloc=$slocs"
|
|
echo "$line"
|
|
fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
########################################################################################
|
|
#
|
|
# return list of ioas on target pciloc with order of Primary --> Secondary --> Others
|
|
#
|
|
# input: pci locations
|
|
#
|
|
# output: sorted raid adapters location list
|
|
#
|
|
#########################################################################################
|
|
function get_ipr_ioas_by_pciloc {
|
|
local pcilocs="$*"
|
|
[ -z "$pcilocs" ] && return 1
|
|
|
|
# find out all ioas and its current adapter state
|
|
local ioas_all=`cmd_show_ioas | grep "^sg[0-9]\+.*Operational" | $awk '{print $1}'`
|
|
# group them into "Primary" and "Secondary" groups
|
|
local lines=`for ioa in $ioas_all
|
|
do
|
|
state=$(cmd_show_details $ioa | grep "Current Dual Adapter State" | $sed -e 's/^.* : \+\(.*\)$/\1/')
|
|
echo "$state=$ioa"
|
|
done`
|
|
local ioa_primary=`echo "$lines" | $awk -F= '($1 == "Primary") {print $2}'`
|
|
local ioa_secondary=`echo "$lines" | $awk -F= '($1 == "Secondary") {print $2}'`
|
|
local ioa_others=`echo "$lines" | $awk -F= '($1 != "Primary") && ($1 != "Secondary") {print $2}'`
|
|
ioas_all="$ioa_primary $ioa_secondary $ioa_others"
|
|
|
|
# pick up ioa on target pciloc
|
|
lines=`echo "$pcilocs" | $sed -e 's/[, ]/\n/g'`
|
|
local ioas_in=""
|
|
if [ "$lines" = "all" ]; then
|
|
ioas_in="$ioas_all"
|
|
else
|
|
ioas_in=`for ioa in $ioas_all
|
|
do
|
|
ioa_pciloc=$(get_sg_pciloc $ioa | awk -F= '{print $2}')
|
|
if echo "$lines" | grep -sq -i "^${ioa_pciloc}$"; then
|
|
echo $ioa
|
|
fi
|
|
done `
|
|
fi
|
|
[ -z "$ioas_in" ] && return 1
|
|
echo "$ioas_in"
|
|
return 0
|
|
}
|
|
|
|
###############################################
|
|
#
|
|
# use udev to determine pciloc of sg device
|
|
#
|
|
# input: disk name list
|
|
#
|
|
# output: <dev>=<sloc>...<dev>=<sloc>
|
|
#
|
|
###############################################
|
|
function get_sg_pciloc {
|
|
local sgs="$*"
|
|
[ -z "$sgs" ] && return 1
|
|
|
|
local sg=""
|
|
local pciloc=""
|
|
for item in $sgs
|
|
do
|
|
sg=`convert_sloc_to_sg $item | $awk -F= '{print $2}'`
|
|
[ -z "$sg" ] && continue
|
|
pciloc=`cmd_show_details $sg \
|
|
| grep "^PCI Address" \
|
|
| $sed -e 's/.*:[ ]\+\([0-9]\+:[0-9]\+:[0-9]\+\.[0-9]\+\).*$/\1/'`
|
|
|
|
if [ -n "$pciloc" ]; then
|
|
echo "$sg=$pciloc"
|
|
fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
#
|
|
# cut resouce_path
|
|
#
|
|
#####################################################
|
|
function cut_sloc_from_iprconfig_line {
|
|
$sed -e 's/^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$/\2/g'
|
|
}
|
|
|
|
#####################################################################
|
|
#
|
|
# find descriptions and status for device
|
|
#
|
|
# input: pci locations
|
|
#
|
|
# output: descriptions and status from "iprconfig -c show-config"
|
|
#
|
|
###################################################################
|
|
function find_desc_status_sg {
|
|
local lines=""
|
|
local pciscsilocs="$*"
|
|
[ -z "$pciscsilocs" ] && return 1
|
|
|
|
lines=`cmd_show_config`
|
|
local slocs=""
|
|
local line=""
|
|
for pciscsiloc in $pciscsilocs
|
|
do
|
|
slocs=`echo "$lines" \
|
|
| $grep "${pciscsilocs}" \
|
|
| awk '{for(i=3;i<=NF;++i) printf $i "\t";printf "\n"}'`
|
|
echo $slocs
|
|
done
|
|
return 0
|
|
|
|
}
|
|
|
|
#######################################################
|
|
#
|
|
# get disk devices through pci_id
|
|
#
|
|
# input: pci id
|
|
#
|
|
#######################################################
|
|
function get_devices_by_pciid {
|
|
|
|
local pciid="$*"
|
|
[ -z "$pciid" ] && return 1
|
|
|
|
pcilocs=`get_pciloc_by_id $pciid`
|
|
|
|
|
|
if [ -z "$pcilocs" ]; then
|
|
echo "There is no PCI_SLOT_NAME for PCI_ID:$pciid."
|
|
return 1
|
|
fi
|
|
slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs`
|
|
if [ -z "$slocs_grps" ]; then
|
|
echo "Could not find any disk on target pciloc ${pcilocs}!"
|
|
return 1
|
|
fi
|
|
arrage_output $pciid "$slocs_grps"
|
|
|
|
}
|
|
|
|
######################################################################
|
|
#
|
|
# get ipr raid arrays by PCI location
|
|
#
|
|
# input: pci locations
|
|
#
|
|
# output: raid arrays
|
|
#
|
|
#####################################################################
|
|
function get_ipr_arrays_by_pciloc {
|
|
local pcilocs="$*"
|
|
[ -z "$pcilocs" ] && return 1
|
|
|
|
# reorder ipr ioa pcilocs by its Primary and Secondary state
|
|
local ioas=`get_ipr_ioas_by_pciloc $pcilocs`
|
|
pcilocs=`get_sg_pciloc $ioas | $awk -F= '{print $2}' `
|
|
|
|
local lines=$(cmd_show_arrays)
|
|
local slocs=`for pciloc in $pcilocs
|
|
do
|
|
echo "$lines"
|
|
done `
|
|
[ -n "$slocs" ] && echo "$slocs"
|
|
return 0
|
|
}
|
|
|
|
#####################################################################
|
|
#
|
|
# get all af and jbod disks
|
|
#
|
|
# input: no
|
|
#
|
|
# output: all devices table
|
|
#
|
|
####################################################################
|
|
function get_all_devices_for_raid {
|
|
|
|
local showlines=`cmd_show_config`
|
|
|
|
pcilocs=`echo "$showlines" \
|
|
| $grep -v -E "Adapter|Enclosure|Disk Array" \
|
|
| $grep '^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$'|$sed 's/ /,/g'|cut -c 8-19`
|
|
|
|
[ -z "$pcilocs" ] && return 1
|
|
slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs`
|
|
if [ -z "$slocs_grps" ]; then
|
|
echo "Could not find any disk on target pciloc ${pcilocs}!"
|
|
fi
|
|
arrage_output "null" "$slocs_grps"
|
|
}
|
|
|
|
#####################################################################
|
|
#
|
|
# output of disks and arrays
|
|
#
|
|
# input: pci_id sloc_grps
|
|
#
|
|
# output: all devices table
|
|
#
|
|
####################################################################
|
|
function arrage_output {
|
|
pciid=$1
|
|
shift
|
|
slocs_grps=$*
|
|
|
|
# find the required member disks
|
|
echo "--------------------------------------------------------------------------"
|
|
echo "PCI_ID PCI_SLOT_NAME Resource_Path Device Description Status"
|
|
echo "------ ------------- ------------- ------ ----------- ----------------"
|
|
slocs=""
|
|
for item in $slocs_grps
|
|
do
|
|
pciloc=`echo "$item" | $awk -F= '{print $1}'`
|
|
slocs_grp=`echo "$item" | $awk -F= '{print $2;}'i \
|
|
| $sed 's/,/ /g'`
|
|
|
|
if [ x$pciid == "xnull" ]; then
|
|
pciid=`get_PCI_ID $pciloc`
|
|
fi
|
|
for sloc in $slocs_grp
|
|
do
|
|
pciscsiloc="$pciloc/$sloc"
|
|
desc=`find_desc_status_sg $pciscsiloc`
|
|
disk=`convert_sloc_to_sg $sloc | $awk -F= '{print $2}'`
|
|
echo "$pciid $pciloc $sloc $disk $desc "
|
|
done
|
|
echo "-------------------------------------------------------------------"
|
|
echo "Get ipr RAID arrays by PCI_SLOT_NAME: $pciloc"
|
|
echo "-------------------------------------------------------------------"
|
|
get_ipr_arrays_by_pciloc $pciloc
|
|
done
|
|
|
|
}
|
|
|
|
|
|
###############################################################
|
|
#
|
|
# get PCI_ID through lspci
|
|
#
|
|
# input: pci location
|
|
#
|
|
# output: pci id
|
|
#
|
|
###############################################################
|
|
function get_PCI_ID {
|
|
pcislot=$1
|
|
pciid=`$lspci | $grep ${pcislot} | $awk '{print $5}'`
|
|
echo $pciid
|
|
}
|
|
|
|
###########################################
|
|
#
|
|
# Identify if sloc is exist or not
|
|
#
|
|
# input: dev sloc
|
|
#
|
|
# output: 0 exist
|
|
# 1 not
|
|
#
|
|
##########################################
|
|
function sloc_exist {
|
|
local __sloc="$1"
|
|
__sloc=`convert_sg_to_sloc $__sloc | $awk -F= '{print $2}'`
|
|
rc=1
|
|
[ -n "$__sloc" ] && [ -d /sys/class/scsi_device/$__sloc/device ] && rc=0
|
|
return $rc
|
|
}
|
|
|
|
###########################################
|
|
#
|
|
# get all raid arrays
|
|
#
|
|
# output : array1#array2..
|
|
#
|
|
##########################################
|
|
function get_all_raid_arrays {
|
|
local arrays=`cmd_show_arrays \
|
|
| $awk '{print $1}'\
|
|
| $sed -n '3,$p'`
|
|
echo $arrays
|
|
}
|
|
|
|
#################################################
|
|
#
|
|
# Identify if sloc is raid or not
|
|
#
|
|
# input : dev sloc
|
|
#
|
|
# return : 1 ------not raid
|
|
# 0 ------is raid
|
|
#
|
|
##################################################
|
|
function is_ipr_array {
|
|
local __sloc="$1"
|
|
local __sg=`convert_sloc_to_sg $__sloc | $awk -F= '{print $2}'`
|
|
[ -n "$__sg" ] && is_array $__sg
|
|
}
|
|
|
|
#################################################
|
|
#
|
|
# get all ioas
|
|
#
|
|
##################################################
|
|
function get_ipr_ioas {
|
|
get_ipr_ioas_by_pciloc all
|
|
}
|
|
|
|
#################################################
|
|
#
|
|
# cut pci location from iprconfig line
|
|
#
|
|
##################################################
|
|
function cut_pciloc_from_iprconfig_line {
|
|
$sed -e 's/^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$/\1/g'
|
|
}
|
|
|
|
##############################################
|
|
#
|
|
# get member disk for ipr array
|
|
#
|
|
# input : array name
|
|
#
|
|
# output : member sloc list
|
|
# "member_sloc1 member_sloc2 ...."
|
|
#
|
|
#############################################
|
|
function get_ipr_array_members {
|
|
local array="$1"
|
|
local array_sloc=`convert_sg_to_sloc $array | $awk -F= '{print $2}'`
|
|
# array does not exist, error with rc=1
|
|
[ -z "$array_sloc" ] && return 1
|
|
|
|
local lines=`cmd_show_config`
|
|
# get scsi locations of disk array members
|
|
local member_slocs=`echo "$lines" \
|
|
| $awk '
|
|
BEGIN { hit=0; members=""; }
|
|
/^.*\/'$array_sloc'[ ]+.*Disk Array.*$/ { hit=1; next; }
|
|
{
|
|
if (hit==1) {
|
|
if ($0 ~ /RAID .* Member/) {
|
|
# ignore "Remote" array members
|
|
if ($0 !~ / *Remote *$/) {
|
|
print $0
|
|
}
|
|
} else {
|
|
hit=0
|
|
}
|
|
}
|
|
}' \
|
|
| cut_sloc_from_iprconfig_line `
|
|
# array does not have any member which should not, error with rc=1
|
|
[ -z "$member_slocs" ] && return 1
|
|
|
|
echo "$member_slocs"
|
|
log_details "array[$array] has member disks[$member_slocs]" >&2
|
|
return 0
|
|
}
|
|
|
|
#####################################################################
|
|
#
|
|
# get array by disk
|
|
#
|
|
# input : array disk member name
|
|
#
|
|
# output : array sloc
|
|
#
|
|
###################################################################
|
|
function get_ipr_array_by_member {
|
|
local sg="$1"
|
|
local member_sloc=`convert_sg_to_sloc $sg | $awk -F= '{print $2}'`
|
|
# member does not exists, error with rc=1
|
|
[ -z "$member_sloc" ] && return 1
|
|
|
|
local lines=`cmd_show_config`
|
|
local array_sloc=`echo "$lines" \
|
|
| $awk '
|
|
BEGIN { hit=0; line="na"; }
|
|
/Disk Array/ { hit=1; line=$0; next; }
|
|
{
|
|
if (hit==1) {
|
|
if ($0 ~ /RAID .* Member/) {
|
|
if ($0 ~ /\/'$member_sloc'[ ]+.*/) {
|
|
print line
|
|
exit 0
|
|
}
|
|
} else {
|
|
hit=0; line="na"
|
|
}
|
|
}
|
|
}' \
|
|
| cut_sloc_from_iprconfig_line`
|
|
# disk does not belongs to any array, error with rc=1
|
|
[ -z "$array_sloc" ] && return 1
|
|
|
|
echo "$array_sloc"
|
|
log_details "array[$array_sloc] owns disk[$sg]" >&2
|
|
return 0
|
|
}
|
|
|
|
#################################################
|
|
#
|
|
# get process
|
|
# input : pids
|
|
# output : child_pids pids
|
|
#
|
|
##################################################
|
|
function getpstree {
|
|
local pids="$*"
|
|
|
|
local pslines=`$ps -e -o pid,ppid`
|
|
while true
|
|
do
|
|
local has_expand=0
|
|
local lines=`echo "$pids" | $sed -e 's/ /\n/g'`
|
|
for pid in $pids
|
|
do
|
|
local child_pids=`echo "$pslines" | $awk -v pid=$pid '$2 == pid{print $1}'`
|
|
child_pids=`for pid2 in $child_pids
|
|
do
|
|
echo "$lines" | grep -sq "^$pid2$" || echo "$pid2"
|
|
done `
|
|
if [ -n "$child_pids" ]; then
|
|
pids="$child_pids $pids"
|
|
has_expand=1
|
|
fi
|
|
done
|
|
if [ $has_expand -eq 0 ]; then
|
|
break
|
|
fi
|
|
done
|
|
echo "$pids"
|
|
}
|
|
|
|
#####################################################################
|
|
#
|
|
# delete used disks from slocs
|
|
#
|
|
# input : disk slocs
|
|
#
|
|
# output : unused disk slocs
|
|
#
|
|
###################################################################
|
|
function delete_disks_belong_array_from_slocs {
|
|
local slocs=$*
|
|
validslocs=""
|
|
usedslocs=""
|
|
for sloc in $slocs
|
|
do
|
|
line=$(get_ipr_array_by_member $sloc)
|
|
if [ $? -eq 0 ]; then
|
|
if [ "x$usedslocs" != x ]; then
|
|
usedslocs=$usedslocs" "$sloc
|
|
else
|
|
usedslocs=$sloc
|
|
fi
|
|
fi
|
|
done
|
|
if [ -n "$usedslocs" ]; then
|
|
validslocs=`echo $slocs|sed "s/$usedslocs//g"`
|
|
if [ -n "$validslocs" ]; then
|
|
echo $validslocs
|
|
fi
|
|
else
|
|
echo $slocs
|
|
fi
|
|
}
|
|
|
|
####################################################################################
|
|
#
|
|
# create raid array
|
|
#
|
|
# input : $1 minutes,
|
|
# Times to wait for the raid creation finish, minutes
|
|
#
|
|
# $2 seconds
|
|
# Interval of minitoring raid creation, status check
|
|
#
|
|
# left: member disks , iprconfig_args
|
|
#
|
|
# output :
|
|
#
|
|
################################################################################
|
|
function create_ipr_array {
|
|
local global_rc=0
|
|
local lines=""
|
|
local monCnt=$1
|
|
shift
|
|
local monInt=$1
|
|
shift
|
|
local member_sgs=`echo "$1" | $sed -e 's/,/ /g'`
|
|
shift
|
|
|
|
local stage_1_timeout=1 # unit minutes
|
|
local stage_1_last_tick_count=`$awk -v monInt=$monInt -v timeout=$stage_1_timeout 'END{print timeout*60/monInt}' </dev/null`
|
|
|
|
[ -z "$member_sgs" ] && return 1
|
|
local member_slocs=`convert_sg_to_sloc $member_sgs | $awk -F= '{print $2}' `
|
|
local member_sgs=`convert_sloc_to_sg $member_sgs | $awk -F= '{print $2}' `
|
|
[ -z "$member_sgs" ] && return 1
|
|
|
|
local iprconfig_args="$*"
|
|
# log for details
|
|
{
|
|
echo "Attempt to create ipr array with command: iprconfig -c raid-create \"$iprconfig_args\" \"$member_sgs\""
|
|
for member_sg in $member_sgs
|
|
do
|
|
echo "-------------------------------------"
|
|
cmd_show_details $member_sg | $sed -e 's/^/details['$member_sg']: >> /g'
|
|
done
|
|
echo "-------------------------------------"
|
|
for member_sg in $member_sgs
|
|
do
|
|
cmd_show_status $member_sg | $sed -e 's/^/status['$member_sg']: >> /g'
|
|
done
|
|
echo "-------------------------------------"
|
|
} | log_lines details >&2
|
|
|
|
# prepare tmp log file for iprconfig worker process
|
|
local ftmp="/tmp/log_create.log.$$"
|
|
$touch $ftmp
|
|
|
|
# launch the iprconfig worker process
|
|
cmd_raid_create "$iprconfig_args" "$member_sgs" >$ftmp 2>&1 &
|
|
local pid_create=$!
|
|
# log for debug
|
|
local line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
|
|
log_details "ps1: $line"
|
|
|
|
# monitor the worker process and show status
|
|
# array_stage=0, monitor array member disk states only
|
|
# array_stage=1, wait for array showes up after array member disks are in termination states
|
|
# array_stage=2, monitor array state
|
|
local array_stage=0
|
|
local array_sloc=""
|
|
# is that possible sg names of member disks were changed after issued raid-create on top of them?
|
|
local slocs="$member_slocs"
|
|
local i=$monCnt
|
|
while [ $i -gt 0 -a $monCnt -gt 0 ];
|
|
do
|
|
# Get status of all sgs which might include array itself.
|
|
local status_lines=""
|
|
local rc=0
|
|
local sloc=""
|
|
[ $array_stage -ne 1 ] && \
|
|
for sloc in $slocs
|
|
do
|
|
local status=""
|
|
local sg=`convert_sloc_to_sg $sloc | $awk -F= '{print $2}'`
|
|
if [ -n "$sg" ]; then
|
|
status=`cmd_show_status $sg `
|
|
else
|
|
status="Missing-sg_name"
|
|
fi
|
|
[ -z "$status" ] && status="no_status"
|
|
|
|
local msg="status[$sloc/$sg]=\"$status\""
|
|
log_details "array create stage[$array_stage], $msg"
|
|
|
|
# list of "termination" status for member disks and target array
|
|
# Refer to "Disk array states" section in ipr doc.
|
|
local status_exp
|
|
if [ $array_stage -eq 0 -o $array_stage -eq 1 ]; then
|
|
# 1) If a physical disk fail to be formated as IPR AF, its status will be "Format Required" and be "Failed" later.
|
|
# Should we handle this?
|
|
status_exp="Active|Failed|Offline|Missing|R/W Protected|Format Required"
|
|
|
|
# I saw phy disk in following state which not in the doc.
|
|
status_exp+="|Optimized|Non-Optimized"
|
|
|
|
# my faked status for disk
|
|
status_exp+="|Missing-sg_name"
|
|
else
|
|
# Note:
|
|
# - Array state "Missing" is not a terminate state, exclude it by adding "#" as prefix
|
|
status_exp="Optimized|Active|Non-Optimized|Degraded|#Missing|Offline|Failed|R/W Protected|Rebuilding"
|
|
fi
|
|
# if not "termination" status, we record it and wait for next cycle.
|
|
if ! echo "$status" | grep -i -sq -E "$status_exp"; then
|
|
# aggregate status messages.
|
|
[ -n "$status_lines" ] && status_lines+=","
|
|
status_lines+="$msg"
|
|
rc=1
|
|
fi
|
|
done
|
|
# if all terminated, we fall back to check staus of array itself
|
|
if [ $rc -eq 0 ]; then
|
|
if [ $array_stage -eq 2 ]; then
|
|
log_info "Create array($array_sg=$array_sloc) successfully on top of disks: $member_slocs."
|
|
break
|
|
else # end if stage == 2
|
|
if [ $array_stage -eq 0 ]; then
|
|
# since all disks were in termination states, let's move to stage 2
|
|
# to wait for array to be created and start rebuild.
|
|
#
|
|
# It's possible that such array could not be created for some critical
|
|
# error, such in-sufficient disks for required raid level.
|
|
#
|
|
# Though there was smart way to handle this, here I only use a simple way
|
|
# by waiting for a fixed time(1 minute) before return error.
|
|
log_details "array create stage[1], entered, after finished formatting member disks $slocs"
|
|
lines=`{ cmd_show_config; cmd_show_alt_config; } | $sed -e 's/^/[0-->1] >> /g'`
|
|
log_lines details "$lines"
|
|
((array_stage=1))
|
|
((stage_1_last_tick_count=i-stage_1_last_tick_count))
|
|
fi
|
|
# IPR might had problem if some disks failed to response.
|
|
# These disks will disapear in the system with sloc and sg were all gone.
|
|
# in these case, we drop these disks in the operation and abort the operation
|
|
# if all member disks were gone.
|
|
local slocs_new=""
|
|
local __tmp_item=""
|
|
for __tmp_item in $slocs
|
|
do
|
|
if sloc_exist $__tmp_item; then
|
|
[ -n "$slocs_new" ] && slocs_new+=" "
|
|
slocs_new+="$__tmp_item"
|
|
else
|
|
log_error "$i: disk $__tmp_item disapear..."
|
|
fi
|
|
done
|
|
if [ -n "$slocs_new" ]; then
|
|
slocs="$slocs_new"
|
|
else
|
|
global_rc=1
|
|
log_error "All array member disks, $member_slocs, were gone, abort!"
|
|
break
|
|
fi
|
|
# get array owner of each member sg
|
|
local array_slocs=`for __tmp_item in $slocs
|
|
do
|
|
local sg_owner_sloc=$(get_ipr_array_by_member $__tmp_item)
|
|
[ -n "$sg_owner_sloc" ] && echo "$sg_owner_sloc"
|
|
done \
|
|
| $sort -u -V `
|
|
# we only allow one unique array owns all member disks.
|
|
# or, we we need to break out?
|
|
local array_cnt=`echo "$array_slocs" | $awk 'END {print NF; exit 0;}'`
|
|
log_details "All \"$array_cnt\" array(es), \"$array_slocs\", contain target disks, \"$slocs\"." >&2
|
|
if [ $array_cnt -gt 0 ]; then
|
|
|
|
array_sloc=`echo "$array_slocs" | $awk '{print $1}'`
|
|
local array_sg=`convert_sloc_to_sg $array_sloc | $awk -F= '{print $2}'`
|
|
if [ -n "$array_sg" ]; then
|
|
log_details "array create stage[2], entered, after array show up with sg name $array_sg and sloc $array_sloc" >&2
|
|
array_stage=2
|
|
slocs="$array_sloc"
|
|
else
|
|
log_warn "$i: >> Why new array $array_sloc($array_slocs) does not have a valid sg name?" >&2
|
|
log_warn "$i: >> Let wait for a while to see if it could show up or not until tick count equals to $stage_1_last_tick_count" >&2
|
|
fi
|
|
elif [ $array_stage -eq 1 -a $i -lt $stage_1_last_tick_count ]; then
|
|
log_error "Fail to wait array($array_slocs) showes up after its member disks, $member_slocs, were finished formatting." >&2
|
|
# log for debug
|
|
lines=`{
|
|
echo "$i: Current IPR configuration";
|
|
{
|
|
cmd_show_config;
|
|
cmd_show_alt_config;
|
|
} | $sed -e 's/^/'$i': >> /g';
|
|
}`
|
|
log_lines details "$lines"
|
|
|
|
global_rc=2
|
|
break
|
|
else
|
|
# see if we need to issue the raid-create again.
|
|
# for some reason, previous raid-create might be terminated unexpectedly with disks
|
|
# actually had been formated for raid and ready to be used. So, we try to issue
|
|
# the raid-create command again to see if that would help. This usually happened when
|
|
# some failures when formtting physical disks to ipr AF. In that situation, iprconfig
|
|
# might abort without have array created after it had actually formatted those disks.
|
|
local pid_create_still_alive=0
|
|
local pid_create_cur=`getpstree $pid_create`
|
|
local pid_cur
|
|
for pid_cur in $pid_create_cur
|
|
do
|
|
if $kill -0 $pid_cur 2>/dev/null; then
|
|
pid_create_still_alive=1
|
|
break
|
|
fi
|
|
done
|
|
if [ $pid_create_still_alive -eq 0 ]; then
|
|
log_info "$i: disks $slocs had finished formatting, but array was not shown up, and iprconfig process($pid_create) had terminated."
|
|
|
|
log_details "$i: Last several raid-create output:"
|
|
lines=`$tail -n10 $ftmp | $sed -e 's/^/'$i': >> /g'`
|
|
log_lines details "$lines"
|
|
# launch the iprconfig worker process again
|
|
member_sgs=`convert_sloc_to_sg $member_slocs | $awk -F= '{print $2}'`
|
|
if [ -n "$member_sgs" ]; then
|
|
$iprconfig -c raid-create $iprconfig_args $member_sgs >$ftmp 2>&1 &
|
|
pid_create=$!
|
|
|
|
log_info "$i: Restart \"iprconfig -c raid-create $iprconfig_args $member_sgs\" with new pid $pid_create." >&2
|
|
|
|
# log for debug
|
|
local line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
|
|
if [ -n "$line" ]; then
|
|
log_details "ps1.2: $line" >&2
|
|
else
|
|
log_error "ps1.2: new iprconfig($pid_create) was gone" >&2
|
|
fi
|
|
else
|
|
log_warn "$i: Fail to restart iprconfig since $member_slocs could not be mapped to sg." >&2
|
|
log_warn "$i: Let try again next loop until tick count equals to $stage_1_last_tick_count" >&2
|
|
fi
|
|
else
|
|
log_warn "$i: Wait for array($array_slocs) showes up after disk format were finished on $slocs until tick count equals to $stage_1_last_tick_count" >&2
|
|
log_details "$i: Last several raid-create output:" >&2
|
|
lines=`$tail -n2 $ftmp | $sed -e 's/^/'$i': >> /g'`
|
|
log_info debug "$lines" >&2
|
|
fi
|
|
fi
|
|
fi # end else if stage == 2
|
|
fi # end if $rc eq 0
|
|
|
|
# log status for monitor
|
|
# log for debug
|
|
if [ -n "$status_lines" ]; then
|
|
log_status "Creating IPR array with last info at time \""`date "+%Y-%m-%d %H:%M:%S"`"\": $status_lines" >&2
|
|
lines=`echo "$status_lines" | $sed -e 's/,/\n/g' | $sed -e "s/^/$i: >> /g"`
|
|
log_lines details "$lines" >&2
|
|
fi
|
|
|
|
$sleep $monInt
|
|
[ $monCnt -gt 0 ] && ((i-=1))
|
|
done
|
|
|
|
# log for debug
|
|
line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
|
|
[ -n "$line" ] && log_details "ps2: $line"
|
|
if $kill -0 $pid_create 2>/dev/null; then
|
|
# If succ out, array status should in "rebuilding" state while iprconfig is still running
|
|
# until the rebuild had finished. But we do not need to wait for its termination since the
|
|
# block device is actually ready to use though performance might be downgrade.
|
|
# so, by default, we will just kill the "iprconfig" command(? not sure if that would affect
|
|
# the rebuild or not) and let raid controller do its job at backend.
|
|
# in order to wait for the whole rebuilding, set IPR_REBUILD_WAIT=true explicitly.
|
|
if [ "x$IPR_REBUILD_WAIT" = "xtrue" -a $i -gt 0 ]; then
|
|
log_info "Wait for ipr raid creation command to finish. pid=$pid_create" >&2
|
|
wait $pid_create
|
|
else
|
|
log_warn "Terminate ipr raid creation process enforcely. pid=$pid_create" >&2
|
|
$kill -TERM $pid_create 2>/dev/null
|
|
$sleep 1
|
|
$kill -0 $pid_create 2>/dev/null \
|
|
&& $kill -KILL $pid_create 2>/dev/null
|
|
fi
|
|
fi
|
|
# log for debug
|
|
line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
|
|
[ -n "$line" ] && log_details "ps3: $line" >&2
|
|
|
|
log_details "Last several raid-create output:" >&2
|
|
lines=`$tail -n10 $ftmp | sed -e 's/^/[last]: >> /g'`
|
|
log_lines details "$lines" >&2
|
|
rm -f $ftmp
|
|
# Log "failed" status to PCM/xCAT with reason and instruction for future investigation.
|
|
if [ $global_rc -ne 0 ]; then
|
|
# set node "status" to "failed" to indicate that the installation was failed.
|
|
fstatus_app=/tmp/baz.py
|
|
if [ -x "$fstatus_app" ]; then
|
|
$fstatus_app "installstatus failed" >&2
|
|
fi
|
|
|
|
# set node state with brief reason why installation failed and future instructions.
|
|
# end user can get this message by running "nodestate" command to target nodes.
|
|
log_status "Fail to create IPR array with return code \"$global_rc\"! Refer to more details in syslog by checking \"rhPreDbg\" flag..." >&2
|
|
fi
|
|
return $global_rc
|
|
}
|
|
|
|
###########################################################################################
|
|
#
|
|
# main process to handle create raid array when using pci_id or pci_slot_name
|
|
#
|
|
##########################################################################################
|
|
function handle_create_raid_array {
|
|
local striple_size=$1
|
|
local raidlevel=$2
|
|
local disknum=$3
|
|
local pcilocs=$4
|
|
local disknames=$5
|
|
|
|
if [ "x$pcilocs" != "xnull" ]; then
|
|
|
|
# Find out all member disks in target IOAs
|
|
# pciloc=sloc,sloc,sloc...
|
|
slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs `
|
|
if [ -z "$slocs_grps" ]; then
|
|
log_error "Could not find any disk on target pciloc ${pcilocs}!" >&2
|
|
return 1
|
|
else
|
|
log_info "Disks in target IPR raid adapter: \"$slocs_grps\"" >&2
|
|
fi
|
|
|
|
# find the required member disks
|
|
slocs=""
|
|
validslocs=""
|
|
for item in $slocs_grps
|
|
do
|
|
pciloc=`echo "$item" | $awk -F= '{print $1}'`
|
|
slocs_grp=`echo "$item" | $awk -F= '{print $2;}'`
|
|
# order disks by its resource path
|
|
slocs_grp=`sort_devices_by_resource_path $slocs_grp`
|
|
|
|
slocs=`echo "$slocs_grp" | sed 's/,/ /g'`
|
|
if [ -n "$slocs" ]; then
|
|
break
|
|
else
|
|
log_warn "Could not find enough disk on target adapter $pciloc which only has following disks ${slocs_grp}!" >&2
|
|
fi
|
|
done
|
|
if [ -z "$slocs" ]; then
|
|
log_error "Could not find enough disk on target adapter ${pciloc}!" >&2
|
|
return 1
|
|
else
|
|
log_info "Find member disks $slocs on target IOA $pciloc" >&2
|
|
slocs=`echo "$slocs" | $sed -e 's/,/ /g'`
|
|
# if it is the first create_raid, do nothing, else:
|
|
# delete used disks belong to existed raid array
|
|
validslocs=`delete_disks_belong_array_from_slocs $slocs`
|
|
if [ -z "$validslocs" ]; then
|
|
log_error "Could not find enough unused disk on target adapter $pciloc."
|
|
return 1
|
|
else
|
|
slocs="$validslocs"
|
|
log_info "Find unused member disks $slocs on target IOA $pciloc" >&2
|
|
fi
|
|
|
|
fi
|
|
# handle disk_num
|
|
# if disknum is all, do nothing
|
|
# if disknum is a number, then it should be compared with actual_disknum
|
|
actual_disknum=`echo "$slocs"| $awk '{print NF}'`
|
|
slocs=`compare_disknum_actualnum $disknum $actual_disknum $slocs`
|
|
|
|
else
|
|
# find the slocs for disks in disknames
|
|
disklist=`echo $disknames|sed 's/#/ /g'`
|
|
slocs_grp=`convert_sg_to_sloc $disklist`
|
|
slocs=`echo "$slocs_grp"|awk -F= '{print $2}'`
|
|
fi
|
|
# check if target disks were already in a reusable array.
|
|
# if not, create one by calling block command "create_ipr_array".
|
|
# after than, check again to confirm if the target array was ready.
|
|
array_sloc=""
|
|
((__i=0))
|
|
while [ $__i -lt 2 ];
|
|
do
|
|
# get disk --> array association
|
|
# format 0:0:0:0=sg0
|
|
declare array_slocs_lines=`for sloc in $slocs
|
|
do
|
|
line=$(get_ipr_array_by_member $sloc)
|
|
echo "$sloc=$line"
|
|
done`
|
|
|
|
# extract arrays from associations
|
|
# get sg0 sg1
|
|
declare array_slocs=`echo "$array_slocs_lines" \
|
|
| $awk -F= '{print $2}' | $sort -u`
|
|
|
|
# count the number of arrays on target disks
|
|
# which might include "blank" array
|
|
declare n_arrays_lines=`echo "$array_slocs" | $awk 'END {print NR}'`
|
|
# count the number of arrays on target disks, again
|
|
# w/o "blank" array
|
|
declare n_arrays=`echo "$array_slocs" | grep -v "^ *$" | $awk 'END {print NR}'`
|
|
# get the sg names for owner arrays.
|
|
declare array_sgs=""
|
|
if [ $n_arrays -gt 0 ]; then
|
|
array_sgs=`convert_sloc_to_sg $array_slocs \
|
|
| $awk -F= '{print $2}' `
|
|
fi
|
|
# (cond_1): if ALL target disks are in ONE array, reuse it.
|
|
if [ $n_arrays_lines -eq 1 -a $n_arrays -eq 1 ]; then
|
|
if [ $__i -eq 0 ]; then
|
|
log_info "Target disks, $slocs, were already in an array, $array_slocs." >&2
|
|
else
|
|
log_info "An array, $array_slocs, had been created to include target disks, $slocs" >&2
|
|
fi
|
|
lines=`{ cmd_show_details $array_sgs;
|
|
cmd_show_status $array_sgs;
|
|
} | $sed -e 's/^/>> /g'`
|
|
log_lines details "$lines"
|
|
array_sloc="$array_slocs"
|
|
break
|
|
fi
|
|
# (cond_2): delete arrays which owe target disks first before any future operations.
|
|
# NOTE: we redo the loop after "delete" command was issued to ipr
|
|
if [ $n_arrays -gt 0 ]; then
|
|
log_info "Target disks, $slocs, were owned by multiple arrays, \"$array_slocs\"($array_sgs). Delete them first before creating a new array on top of them." >&2
|
|
delete_ipr_array enforce=1 tryCnt=360 tryInt=60 -- $array_sgs
|
|
continue
|
|
fi
|
|
# (cond_3): create an array to include all target disks
|
|
# we assume all these disks are ready to be included into an array.
|
|
if [ $__i -eq 0 ]; then
|
|
declare sgs=`convert_sloc_to_sg $slocs \
|
|
| $awk -F= '{print $2}' | $sed -e 's/ /,/g'`
|
|
# Times to wait for the raid creation finish
|
|
# The lager capacity of member disks, longer times to wait.
|
|
# :unit minutes
|
|
monTime=240
|
|
# Interval of minitoring raid creation, status check
|
|
# :unit seconds
|
|
monInt=30
|
|
monCnt=`$awk -v monInt=$monInt -v monTime=$monTime 'END{print 60*monTime/monInt}' </dev/null`
|
|
log_status "Creating RAID $raidlevel array on top of disks \""`echo "$slocs" | $sed -e 's/ /,/g'`"\"/\"$sgs\"..." >&2
|
|
# handle striple_size
|
|
if [ "x$striple_size" == "xdefault" ]; then
|
|
create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel
|
|
elif [ -z $striple_size ]; then
|
|
create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel
|
|
else
|
|
create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel -s $striple_size
|
|
fi
|
|
fi
|
|
((__i+=1))
|
|
done
|
|
lines=`cmd_show_config`
|
|
if [ $__i -lt 2 -a -n "$array_sloc" ]; then
|
|
array_sd=`echo "$lines" \
|
|
| grep '^sd[a-z]\+[ ]\+.*\/'$array_sloc'[ ]\+.*Disk Array' \
|
|
| $awk '{print $1}'`
|
|
if [ -n "$array_sd" ]; then
|
|
log_info "array_dev=$array_sd"
|
|
return 0
|
|
else
|
|
log_error "Fail to get scsi device name for new array $array_sloc" >&2
|
|
echo "$lines" | $sed -e 's/^/>> /g' | log_lines error >&2
|
|
return 1
|
|
fi
|
|
else
|
|
log_error "Fail to create array on top of disks $slocs" >&2
|
|
echo "$lines" | $sed -e 's/^/>> /g' | log_lines error >&2
|
|
return 1
|
|
fi
|
|
|
|
}
|
|
|
|
############################################################################
|
|
#
|
|
# handle disk_num
|
|
# if disknum is all, do nothing
|
|
# if disknum is a number, then it should be compared with actual_disknum
|
|
# return <disknum> of slocs
|
|
#
|
|
##############################################################################
|
|
function compare_disknum_actualnum {
|
|
local disknum=$1
|
|
local actualnum=$2
|
|
shift;
|
|
shift;
|
|
local slocs=$*
|
|
local actualslocs
|
|
if [ "x$disknum" != "xall" ]; then
|
|
if [ $actualnum -ge $disknum ]; then
|
|
actualslocs=`echo "$slocs"| \
|
|
awk -v anum=$disknum '{ for ( i=1; i<=anum; i++ ){print $i}}'`
|
|
echo $actualslocs
|
|
|
|
else
|
|
log_error "Could not find enough disk on target adapter ${pciloc}!" >&2
|
|
fi
|
|
else
|
|
echo "$slocs"
|
|
fi
|
|
|
|
}
|
|
|
|
############################################################################
|
|
#
|
|
# Delete raid array
|
|
#
|
|
# input format : enforce=1 tryCnt=2 tryInt=60 -- arrays_name list
|
|
#
|
|
##########################################################################
|
|
function delete_ipr_array {
|
|
# default to NO if enforcely wait for rebuilding array
|
|
local enforce=0
|
|
# default to wait 2 minutes for array rebuilding to finish
|
|
local tryCnt=2
|
|
# monitor rebuilding status every 60 seconds
|
|
local tryInt=60
|
|
|
|
# parser input arguments
|
|
while [ -n "$1" ];
|
|
do
|
|
local key=`echo "$1" | $cut -s -d= -f1`
|
|
if [ "$key" = "tryCnt" ] || \
|
|
[ "$key" = "tryInt" ] || \
|
|
[ "$key" = "enforce" ]; then
|
|
local val=`echo "$1" | $cut -s -d= -f2-`
|
|
eval "$key=\"$val\""
|
|
elif [ "$1" = "--" ]; then
|
|
shift
|
|
arrays="$*"
|
|
break
|
|
fi
|
|
shift
|
|
done
|
|
# should not with empty array list, if yes, error with rc=1
|
|
if [ -z "$arrays" ]; then
|
|
log_error "No array to delete, assume success!" >&2
|
|
return 0
|
|
fi
|
|
local array_slocs=`convert_sg_to_sloc $arrays | awk -F= '{print $2}'`
|
|
# if none of arrays were valid, error with rc=1
|
|
if [ -z "$array_slocs" ]; then
|
|
log_error "Fail to map arrayes, $arrays, to scsi location before deletion!"
|
|
return 1
|
|
fi
|
|
local ioas=`get_ipr_ioas`
|
|
|
|
local i=1
|
|
while [ $i -le $tryCnt ];
|
|
do
|
|
# verify if all target arrays are still exists
|
|
local tmp_array_slocs=`for _tmp_sloc in $array_slocs
|
|
do
|
|
is_ipr_array $_tmp_sloc && echo "$_tmp_sloc"
|
|
done `
|
|
if [ "$tmp_array_slocs" != "$array_slocs" ]; then
|
|
array_slocs="$tmp_array_slocs"
|
|
log_details "Renew available target arrays which will be deleting, \"$array_slocs\"" >&2
|
|
fi
|
|
if [ -z "$array_slocs" ]; then
|
|
log_details "Round $i: All target arrays were gone!" >&2
|
|
break
|
|
fi
|
|
# find out sg devs for all deletable arrays
|
|
local array_slocs_deletable=`for ioa in $ioas
|
|
do
|
|
# [root@dn01-dat ~]# iprconfig -c query-raid-delete sg5
|
|
# Name PCI/SCSI Location Description Status
|
|
# ------ ------------------------- ------------------------- -----------------
|
|
# sdk 0003:70:00.0/0:2:0:0 RAID 0 Disk Array Optimized
|
|
# sdl 0003:70:00.0/0:2:1:0 RAID 0 Disk Array Optimized
|
|
cmd_query_raid_delete $ioa \
|
|
| grep "Disk Array" \
|
|
| cut_sloc_from_iprconfig_line
|
|
done`
|
|
if [ -z "$array_slocs_deletable" -a $enforce -eq 0 ]; then
|
|
log_details "Round $i: No deletable array, nothing to do and finish." >&2
|
|
break
|
|
fi
|
|
# log for debug
|
|
{
|
|
echo "$array_slocs_deletable" | $sed -e 's/^/Round '$i': Deletable array >> /g'
|
|
cmd_show_config | $sed -e 's/^/Round '$i': ipr info before delete >> /g'
|
|
} | log_lines details >&2
|
|
log_info "Round $i: delete_ipr_array, \"$array_slocs\"" >&2
|
|
|
|
# if requested array is a deletable array, mark it to be deleted in this round.
|
|
local array_slocs_verified=`for array_sloc in $array_slocs
|
|
do
|
|
if echo "$array_slocs_deletable" | grep -sq "^$array_sloc$"; then
|
|
echo $array_sloc
|
|
else
|
|
log_warn "Round $i: Array $array_sloc is un-deletable at present." >&2
|
|
fi
|
|
done`
|
|
if [ -n "$array_slocs_verified" ]; then
|
|
log_details "Round $i: Verified array slocs for deletion are \"$array_slocs_verified\"." >&2
|
|
else
|
|
log_info "Round $i: All remaining target arrayes are un-deletable now." >&2
|
|
if [ $enforce -eq 1 ]; then
|
|
log_info "Round $i: Wait for these un-deletable arrayes deletable with tryCnt=$tryCnt, tryInt=$tryInt." >&2
|
|
wait_for_ipr_device_status \
|
|
tryCnt=$tryCnt tryInt=$tryInt cmd="grep -sq -E 'Optimized'" \
|
|
-- \
|
|
`echo "$array_slocs" | $sed -e "s/ /,/g"`
|
|
# if wait succ, let's try to delete again
|
|
if [ $? -eq 0 ]; then
|
|
((i+=1))
|
|
continue
|
|
else
|
|
break
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# issue delete command
|
|
local dcnt=0
|
|
for array_sloc in $array_slocs_verified
|
|
do
|
|
local array=`convert_sloc_to_sg $array_sloc | awk -F= '{print $2}'`
|
|
[ -z "$array" ] && continue
|
|
|
|
log_details "Round $i: Delete $dcnt array $array=$array_sloc" >&2
|
|
cmd__raid_delete $array
|
|
((dcnt+=1))
|
|
done
|
|
|
|
# break out the loop forcely if nothing deleted.
|
|
[ $dcnt -eq 0 ] && break
|
|
|
|
# wait for next round
|
|
$sleep $tryInt
|
|
((i+=1))
|
|
done
|
|
test $i -lt $tryCnt -a -z "$array_slocs"
|
|
return $?
|
|
}
|
|
|
|
#################################################
|
|
#
|
|
# sort devices by resource path,
|
|
#
|
|
# input: dev list, "sg0,..sgn"
|
|
#
|
|
##################################################
|
|
function sort_devices_by_resource_path {
|
|
local sgs="$*"
|
|
|
|
local del=","
|
|
if [ $# -gt 1 ]; then
|
|
del=" "
|
|
else
|
|
sgs=`echo "$sgs" | $sed -e "s/,/ /g"`
|
|
fi
|
|
|
|
# [root@dn01-dat ~]# iprconfig -c show-alt-config
|
|
# Name Resource Path/Address Vendor Product ID Status
|
|
# ------ -------------------------- -------- ---------------- -----------------
|
|
# sg15 FE IBM 57D7001SISIOA Operational
|
|
# sg10 00-0C IBM HUC101212CSS600 Active
|
|
# sg11 00-0D IBM HUC101212CSS600 Active
|
|
local lines=`$iprconfig -c show-alt-config | grep "^sg"`
|
|
local item
|
|
local lines_out=`for item in $sgs
|
|
do
|
|
local sg=$(convert_sloc_to_sg $item | $awk -F= '{print $2;}')
|
|
local rp=$(echo "$lines" | $awk -vsg=$sg '$1 == sg { print $2; }')
|
|
echo "$item $sg $rp"
|
|
done \
|
|
| $sort -t' ' -k3`
|
|
# log for debug
|
|
{
|
|
echo "sort devices \"$sgs\" by IPR resource path:"
|
|
echo "$lines_out" | $sed -e 's/^/sort by rp output: >> /g'
|
|
} | log_lines details >&2
|
|
|
|
# deliver ordered result
|
|
lines_out=`echo "$lines_out" | $awk '{print $1;}' `
|
|
if [ "$del" = "," ]; then
|
|
echo "$lines_out" | $sed -e "s/ /,/g"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
##########################################################################################################################
|
|
#
|
|
# wait to check ipr device status
|
|
# usage: wait_for_ipr_device_status tryCnt=<minitutes> tryInt=<interval_seconds> cmd="check_cmd_line" -- "sg0,sg1,...,sgn"
|
|
#
|
|
#########################################################################################################################
|
|
function wait_for_ipr_device_status {
|
|
local tryCnt=1
|
|
local tryInt=10
|
|
local cmd="true"
|
|
|
|
# parser input arguments
|
|
while [ -n "$1" ];
|
|
do
|
|
local key=`echo "$1" | $cut -s -d= -f1`
|
|
if [ "$key" = "tryCnt" ] || \
|
|
[ "$key" = "tryInt" ] || \
|
|
[ "$key" = "cmd" ]; then
|
|
local val=`echo "$1" | $cut -s -d= -f2-`
|
|
eval "$key=\"$val\""
|
|
elif [ "$1" = "--" ]; then
|
|
shift
|
|
sgs="$*"
|
|
break
|
|
fi
|
|
shift
|
|
done
|
|
sgs=`echo "$sgs" | $sed -e "s/,/ /g"`
|
|
local cnt=0
|
|
local lines
|
|
while [ $cnt -lt $tryCnt -o $tryCnt -le 0 ];
|
|
do
|
|
# sleep for specific interval for next cycle
|
|
[ $cnt -gt 0 ] && $sleep $tryInt
|
|
|
|
lines=`check_ipr_device_status $sgs`
|
|
local status_lines=""
|
|
local neg=0
|
|
local sg
|
|
for sg in $sgs
|
|
do
|
|
local _sg_status=`echo "$lines" | grep "^$sg=" | $cut -d= -f2- -s`
|
|
echo "$_sg_status" | eval $cmd >&2
|
|
if [ $? -ne 0 ]; then
|
|
neg=1
|
|
status_lines="${status_lines}${status_lines:+,}status[$sg]=\"$_sg_status\""
|
|
fi
|
|
done
|
|
|
|
# break out if NO negative matching
|
|
# or, try next loop
|
|
if [ $neg -eq 0 ]; then
|
|
break
|
|
else
|
|
log_status "Wait for device status at time \""`date "+%Y-%m-%d %H:%M:%S"`"\": $status_lines, expect: \"$cmd\"."
|
|
fi
|
|
|
|
((cnt+=1))
|
|
done
|
|
test $cnt -lt $tryCnt -o $tryCnt -le 0
|
|
local rc=$?
|
|
|
|
# log for debug
|
|
if [ $rc -eq 0 ]; then
|
|
log_info "Wait for status on devices: \"$sgs\" succeed! (expected: \"$cmd\")"
|
|
else
|
|
log_warn "Wait for status on devices: \"$sgs\" failed! (expected: \"$cmd\")"
|
|
echo "$lines" | $sed -e 's/^/last device status: >> /g' | log_lines debug
|
|
fi
|
|
|
|
return $rc
|
|
}
|
|
|
|
##############################################################
|
|
#
|
|
# check status of ipr devices
|
|
# input : slocs list, for example, 0:0:0:0 0:0:1:0
|
|
# usage : check_ipr_device_status <sloc1> <sloc2> ... <slocn>
|
|
#
|
|
##############################################################
|
|
function check_ipr_device_status {
|
|
local sgs="$*"
|
|
local lines=`convert_sloc_to_sg $sgs`
|
|
|
|
local item
|
|
for item in $sgs
|
|
do
|
|
declare sg=`echo "$lines" | $awk -F= -vkey=$item '$1 == key {print $2;}'`
|
|
declare _sg_status=`$iprconfig -c status $sg`
|
|
if [ "$_sg_status" = "Rebuilding" ]; then
|
|
declare _sg_status_alt=`cmd_alt_status $sg`
|
|
[ -n "$_sg_status_alt" ] && _sg_status="$_sg_status, $_sg_status_alt"
|
|
fi
|
|
echo "$item=$_sg_status"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
|
|
######################################################
|
|
#
|
|
# log lines
|
|
#
|
|
# input : details or status or error
|
|
#
|
|
######################################################
|
|
function log_lines {
|
|
local pcnt=$#
|
|
local __level=$1
|
|
shift
|
|
local cmd=log_${__level}
|
|
|
|
local hit=0
|
|
local OIFS=$IFS
|
|
local NIFS=$'\n'
|
|
IFS=$NIFS
|
|
local __msg
|
|
for __msg in $*
|
|
do
|
|
IFS=$OIFS
|
|
$cmd "$__msg"
|
|
hit=1
|
|
IFS=$NIFS
|
|
done
|
|
IFS=$OIFS
|
|
|
|
[ $hit -eq 0 -a $pcnt -le 1 ] && \
|
|
while read __msg;
|
|
do
|
|
$cmd "$__msg"
|
|
done
|
|
}
|
|
|
|
######################################################
|
|
#
|
|
# error information
|
|
#
|
|
######################################################
|
|
function log_error {
|
|
local __msg="$*"
|
|
$log_print_cmd $log_print_arg "[E]: $__msg" >&2
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
#
|
|
# warning information
|
|
#
|
|
# input : message
|
|
#
|
|
######################################################
|
|
function log_warn {
|
|
local __msg="$*"
|
|
$log_print_cmd $log_print_arg "[W]: $__msg" >&2
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
#
|
|
# log information
|
|
#
|
|
# input : message
|
|
#
|
|
######################################################
|
|
function log_info {
|
|
local __msg="$*"
|
|
$log_print_cmd $log_print_arg "[I]: $__msg" >&2
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
#
|
|
# more details information
|
|
#
|
|
######################################################
|
|
function log_details {
|
|
local __msg="$*"
|
|
$log_print_cmd $log_print_arg "[I]: $__msg" >&2
|
|
return 0
|
|
}
|
|
|
|
####################################################
|
|
#
|
|
# print command status
|
|
#
|
|
###################################################
|
|
__my_log_status=
|
|
function log_status {
|
|
local __msg="$*"
|
|
$log_print_cmd $log_print_arg "[S]: $__msg" >&2
|
|
|
|
# call my_log_status hook to triger more processing for status messages.
|
|
if [ -n "$__my_log_status" ]; then
|
|
$__my_log_status "$__msg"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
####################################################
|
|
#
|
|
# print output and logfile
|
|
#
|
|
###################################################
|
|
function log_print_default {
|
|
echo $*
|
|
echo $* >> $log_file
|
|
}
|
|
|
|
####################################################
|
|
#
|
|
# handle command
|
|
#
|
|
###################################################
|
|
function set_log_print {
|
|
local cmd="$1"
|
|
if [ -z "$cmd" ]; then
|
|
return 1
|
|
fi
|
|
shift
|
|
local args="$*"
|
|
|
|
eval "log_print_cmd=\"$cmd\""
|
|
eval "log_print_arg=\"$args\""
|
|
}
|
|
|
|
# let log work
|
|
[ -z "$log_print" ] && set_log_print "log_print_default"
|
|
|
|
|
|
|