diff --git a/xCAT-genesis-scripts/bin/configraid b/xCAT-genesis-scripts/bin/configraid new file mode 100755 index 000000000..90fc5015d --- /dev/null +++ b/xCAT-genesis-scripts/bin/configraid @@ -0,0 +1,283 @@ +#!/bin/bash +# +# This script is used to configure raid. +# +# +############################################## +# source raidutils +############################################## +# prepare tmp log file for configraid +touch="touch" +timestamp=`date +"%Y%m%d%H%M%S"` +log_file="/tmp/configraid.log.$timestamp" +$touch $log_file + +str_dir_name=`dirname $0` +. $str_dir_name/raidutils $log_file + + +################################################################################### +# +# Usage +# +################################################################################# +function usagesc { + echo 'Usage: configraid delete_raid=[all|""|null] ' + echo ' stripe_size=[16|64|256] ' + echo ' create_raid="rl!|[pci_id#|pci_slot_name#|disk_names##..#]|disk_num#" ... ' + echo ' 1. delete_raid: ' + echo ' List raid arrays which should be removed. If its value is all, all raid arrays detected should be deleted.' + echo ' If its value is a list of raid array names, these raid arrays will be deleted. Raid array names should be seperated by "#".' + echo ' If its value is null, no raid array will be deleted.' + echo ' If there is no delete_raid, the default value is null.' + echo ' The format is : delete_raid = [all|"#...#"|null]' + echo ' For example:' + echo ' delete_raid="sda#sdd" ' + echo ' ' + echo ' 2. stripe_size: ' + echo ' Currently supported stripe sizes in kb include 16, 64, and 256.' + echo ' If stripe size is not specified, it will default to the recommended stripe size for the selected RAID level.' + echo ' For example:' + echo ' stripe_size=256 ' + echo ' ' + echo ' 3. create_raid: ' + echo ' To create a raid array, add a line begginning with create_raid.' + echo ' The format is : ' + echo ' create_raid="rl#[0,10,5,6]|[PCI_ID#|PCI_SLOT_name#]|disk_num#"' + echo ' rl means RAID level, RAID level can be any supported RAID level for the given adapter, such as 0, 10, 5, 6;' + echo ' rl is a must for every create_raid. ' + echo ' pci_id is PCI vender and device ID; refer to http://pci-ids.ucw.cz/read/PC/1014/034a;' + echo ' disk_num is the number of disk this RAID will contain, default value is all unused disks in its pci slot; ' + echo ' pci_slot_name is the specified PCI location. If specify pci_slot_name, this raid will be created using disks from this pci_slot;' + echo ' disk_names is a list of advanced format disk names. If specify disk_names, this raid will be created using these disks;' + echo ' Example 1, create 2 raid0 arrays, one is using 3 disks, the other is using 2 disks, the disks PCI_ID is 1014:034a:' + echo ' create_raid="rl#0|pci_id#1014:034a|disk_num#1 create_raid="rl#0|pci_id#1014:034a|disk_num#2' + echo ' Example 2, pci_slot_name is 0001:08:00.0, create raid10 array, using 2 disks ' + echo ' create_raid="rl#10|pci_slot_name#0001:08:00.0|disk_num#2"' + echo ' Example 3, create 1 raid0 array using advanced format disk sg1' + echo ' create_raid="rl#0|disk_names#sg1|disk_num#1"' + +} + +############################################################################### +# +# handle each create raid +# +# input format: stripe_size=16 create_raid=rl#1|pci_id#1014:034a|disk_num#1 +# +############################################################################### +function handle_each_create_raid { + + local create_raid_list=$1 + local striple_pair=$2 + + # cut striple_size from striple_pair + # if striple_size is null, it will default to the recommended stripe size + local striple_size="" + if [ x$striple_pair != x ]; then + striple_size=`echo $striple_pair | $awk -F= '{print $2}'` + else + striple_size="default" + fi + + # cut create_raid_list into raid_level pci_id|pci_slot_name|disk_names disk_num + local create_raid_value="" + local raidlevel="" + local disknum="" + local pciid="" + local pcislot_name="" + local disk_names="" + + create_raid_value=`echo $create_raid_list| $awk -F= '{print $2}' \ + | $sed -e 's/|/ /g'` + # make sure create_raid_value is not null + firstpara=`echo $create_raid_value|awk '{print $1}'` + if [ x$firstpara != x ]; then + for item in $create_raid_value + do + key=`echo $item | $awk -F# '{print $1}'` + vla=`echo $item | $awk -F# '{if($1=="disk_names"){$1="";print $0} else { print $2}}' \ + | $sed -e 's/^\s*//g' -e 's/ /#/g'` + case $key in + rl ) + raidlevel=$vla + ;; + pci_id ) + pciid=$vla + ;; + pci_slot_name ) + pcislot_name=$vla + ;; + disk_num ) + disknum=$vla + ;; + disk_names ) + disk_names="$vla" + ;; + esac + done + fi + + if [ x$raidlevel == x ]; then + log_error "Can not create RAID for $create_raid_list. There is no raid level enter." + return 1 + fi + + # if disk_num is null, will pick up all the disks + if [ x$disknum == x ]; then + disknum=all + fi + + + log_info "handle create raid: $create_raid_value" + + local pcilocs="" + local slocs_grps="" + + # If there is disk_names, using them + if [ x$disk_names != x ]; then + handle_create_raid_array "$striple_size" "$raidlevel" "$disknum" "null" "$disk_names" + else + # If there is pci_slot_name, use it. + # If there is no pci_slot_name, there is pci_id, use pci_id. + if [ x$pciid != x -a x$pcislot_name == x ]; then + pcilocs=`get_pciloc_by_id $pciid` + elif [ x$pcislot_name != x ]; then + pcilocs=$pcislot_name + else + log_error "pci_id or pci_slot_name or pci_disk_names can not be empty." + return 1 + fi + + if [ -n "$pcilocs" ]; then + handle_create_raid_array "$striple_size" "$raidlevel" "$disknum" "$pcilocs" "null" + else + log_error "PCI adapter $pcilocs does not exists!" >&2 + return 1 + + fi + fi + +} + +############################################################################## +# +# main section to handle delete raid +# +# input format : delete_raid=[all|dev1#dev2...|null] +# +############################################################################## +function handle_delete_raid { + local input=$* + local raid_arrays="" + local valid_arrays="" + + del_value=`echo "$input"|awk -F= '{print $2}'` + # if delete_raid=all + if [ "x$del_value" == "xall" -o "x$del_value" == "xALL" ]; then + + raid_arrays=`get_all_raid_arrays` + + # if delete_raid=null or nothing + elif [ "x$del_value" == "xnull" -o "x$del_value" == "x" ]; then + + log_info "There is no need to delete RAID arrays." + return 1 + # if delete_raid=devlist + else + # delete # between raid names + raid_arrays=`echo ${del_value}|sed 's/#/ /g'` + fi + + for r_a in $raid_arrays + do + res=`is_array $r_a` + if [ $res -eq 0 ]; then + if [ "x$valid_arrays" != "x" ]; then + valid_arrays=$valid_arrays" "$r_a + else + valid_arrays=$r_a + fi + fi + done + if [ "x$valid_arrays" != "x" ]; then + # delete raid arrays + delete_ipr_array enforce=1 tryCnt=360 tryInt=60 -- $valid_arrays + else + log_info "There is no valid raid arrays." + return 1 + fi +} + + +############################################################################################# +# +# Main section : +# delete raid +# create raid +# +############################################################################################ + +input=$* + +# If input parameter number is less than 1, print usage +if [ $# -lt 1 ]; then + usagesc + exit 1 +fi + +# If "-h|--help", print usage +if [ "x$input" == "x--help" -o "x$input" == "x-h" ]; then + usagesc + exit 0 +fi + +# clear up delete_raid stripe_size and create_raid +del_raid_valid="" +cre_raid_list="" +stripe_pair="" +for args in "$@" +do + del_raid=`echo "$args" | grep "^delete_raid="` + cre_raid=`echo "$args" | grep "^create_raid="` + str_size=`echo "$args" | grep "^stripe_size="` + + if [ x$del_raid != x ]; then + del_raid_valid=$del_raid + fi + + if [ x$str_size != x ]; then + stripe_pair=$str_size + fi + + # This is for the first create_raid + if [ x$cre_raid_list == x -a x$cre_raid != x ]; then + + cre_raid_list=$cre_raid + + # if there are many create_raid + elif [ x$cre_raid_list != x -a x$cre_raid != x ]; then + + cre_raid_list=$cre_raid_list" "$cre_raid + fi +done + +# main process for deleting raid +if [ x$del_raid_valid != x ]; then + handle_delete_raid "$del_raid_valid" +fi + +# main process for creating raid +create_r1=`echo $cre_raid_list|awk '{print $1}'` +if [ "x${create_r1}" != "x" ]; then + + for cr in $cre_raid_list + do + # handle every create_raid + handle_each_create_raid $cr $stripe_pair $lable_num + done +else + log_info "Can not create RAID. There is no create_raid." +fi + + diff --git a/xCAT-genesis-scripts/bin/doxcat b/xCAT-genesis-scripts/bin/doxcat index e85bda22d..88560e058 100755 --- a/xCAT-genesis-scripts/bin/doxcat +++ b/xCAT-genesis-scripts/bin/doxcat @@ -3,7 +3,6 @@ # - Added slash in front of "var" in the NICSTOBRINGUP dhclient section. # Bug reported by Jeff Lang . Thanks, Jeff! # - modprobe acpi_cpufreq 2>/dev/null # on some machines this fails modprobe cpufreq_ondemand if ls /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor &>/dev/null; then @@ -240,10 +239,13 @@ if [ "$destiny" != "discover" ]; then #we aren't discoverying, we probably can a /bin/getcert $XCATMASTER:$XCATPORT fi while :; do - if [ -z "$destiny" ]; then - destiny=`getdestiny $XCATMASTER:$XCATPORT` + + grepconfigraid=`echo $destiny|grep "configraid"` + if [ -z "$destiny" -o -n "$grepconfigraid" ]; then + destiny=`getdestiny $XCATMASTER:$XCATPORT` fi - destparameter=`echo $destiny|awk -F= '{print $2}'` + + destparameter=`echo $destiny|cut -d '=' -f 2-` destiny=`echo $destiny|awk -F= '{print $1}'` dest=`echo $destiny|awk '{print $1}'` #could probably use bash but oh well if [ "$dest" = "discover" ]; then #skip a query to xCAT when /proc/cmdline will do @@ -259,7 +261,7 @@ while :; do elif [ "$dest" = runcmd ]; then destiny=`/bin/nextdestiny $XCATMASTER:$XCATPORT` dest=`echo $destiny|awk -F= '{print $1}'` - $destparameter + $destparameter elif [ "$dest" = runimage ]; then destiny=`/bin/nextdestiny $XCATMASTER:$XCATPORT` dest=`echo $destiny|awk -F= '{print $1}'` @@ -324,3 +326,4 @@ while :; do fi done +set +x diff --git a/xCAT-genesis-scripts/bin/raidcmd b/xCAT-genesis-scripts/bin/raidcmd index dcf85f287..e170b0760 100755 --- a/xCAT-genesis-scripts/bin/raidcmd +++ b/xCAT-genesis-scripts/bin/raidcmd @@ -49,3 +49,88 @@ function cmd_show_arrays { echo "$lines" } +################################################################### +# +# dev is raid array or not +# input : dev +# return : 1 ----- not raid array +# 0 ----- raid array +# +################################################################### +function is_array { + local lines="" + local dev=$* + lines=`$iprconfig -c show-details $dev | grep -sq "RAID Level"` + echo "$?" +} + +######################################## +# +# iprconfig -c alt-status +# +######################################## +function cmd_alt_status { + local lines="" + local dev=$* + lines=`$iprconfig -c alt-status $dev` + echo "$lines" +} + +######################################## +# +# iprconfig -c show-status +# +######################################## +function cmd_show_status { + local lines="" + local dev=$* + lines=`$iprconfig -c status $dev` + echo "$lines" +} + +######################################## +# +# iprconfig -c show-alt-config +# +######################################## +function cmd_show_alt_config { + local lines="" + local dev=$* + lines=`$iprconfig -c show-alt-config` + echo "$lines" +} + +######################################## +# +# iprconfig -c query-raid-delete +# +######################################## +function cmd_query_raid_delete { + local lines="" + local ioa=$* + lines=`$iprconfig -c query-raid-delete $ioa` + echo "$lines" +} + +######################################## +# +# iprconfig -c raid-delete +# +######################################## +function cmd__raid_delete { + local lines="" + local array=$* + lines=`$iprconfig -c raid-delete "$array"` +} + +################################################################## +# +# iprconfig -c raid-create $iprconfig_args $member_sgs +# +################################################################### +function cmd_raid_create { + local iprconfig_args=$1 + local member_sgs=$2 + lines=`$iprconfig -c raid-create $iprconfig_args $member_sgs` +} + diff --git a/xCAT-genesis-scripts/bin/raidutils b/xCAT-genesis-scripts/bin/raidutils index 572d781d7..45aeb61ed 100755 --- a/xCAT-genesis-scripts/bin/raidutils +++ b/xCAT-genesis-scripts/bin/raidutils @@ -5,8 +5,11 @@ # There are utils for diskdiscover and configraid. # +log_file=$1 +############################## # declare all commands +############################## awk="awk" sed="sed" cut="cut" @@ -121,6 +124,7 @@ function convert_sg_to_sloc { } ############################################################################ +# # get devices which are qualified to be used to create raid # it should equals to "query-raid-create" after all array had # been deleted. @@ -410,3 +414,1040 @@ function get_PCI_ID { pciid=`$lspci | $grep ${pcislot} | $awk '{print $5}'` echo $pciid } + +########################################### +# +# Identify if sloc is exist or not +# +# input: dev sloc +# +# output: 0 exist +# 1 not +# +########################################## +function sloc_exist { + local __sloc="$1" + __sloc=`convert_sg_to_sloc $__sloc | $awk -F= '{print $2}'` + rc=1 + [ -n "$__sloc" ] && [ -d /sys/class/scsi_device/$__sloc/device ] && rc=0 + return $rc +} + +########################################### +# +# get all raid arrays +# +# output : array1#array2.. +# +########################################## +function get_all_raid_arrays { + local arrays=`cmd_show_arrays \ + | $awk '{print $1}'\ + | $sed -n '3,$p'` + echo $arrays +} + +################################################# +# +# Identify if sloc is raid or not +# +# input : dev sloc +# +# return : 1 ------not raid +# 0 ------is raid +# +################################################## +function is_ipr_array { + local __sloc="$1" + local __sg=`convert_sloc_to_sg $__sloc | $awk -F= '{print $2}'` + [ -n "$__sg" ] && is_array $__sg +} + +################################################# +# +# get all ioas +# +################################################## +function get_ipr_ioas { + get_ipr_ioas_by_pciloc all +} + +################################################# +# +# cut pci location from iprconfig line +# +################################################## +function cut_pciloc_from_iprconfig_line { + $sed -e 's/^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$/\1/g' +} + +############################################## +# +# get member disk for ipr array +# +# input : array name +# +# output : member sloc list +# "member_sloc1 member_sloc2 ...." +# +############################################# +function get_ipr_array_members { + local array="$1" + local array_sloc=`convert_sg_to_sloc $array | $awk -F= '{print $2}'` + # array does not exist, error with rc=1 + [ -z "$array_sloc" ] && return 1 + + local lines=`cmd_show_config` + # get scsi locations of disk array members + local member_slocs=`echo "$lines" \ + | $awk ' + BEGIN { hit=0; members=""; } + /^.*\/'$array_sloc'[ ]+.*Disk Array.*$/ { hit=1; next; } + { + if (hit==1) { + if ($0 ~ /RAID .* Member/) { + # ignore "Remote" array members + if ($0 !~ / *Remote *$/) { + print $0 + } + } else { + hit=0 + } + } + }' \ + | cut_sloc_from_iprconfig_line ` + # array does not have any member which should not, error with rc=1 + [ -z "$member_slocs" ] && return 1 + + echo "$member_slocs" + log_details "array[$array] has member disks[$member_slocs]" >&2 + return 0 +} + +##################################################################### +# +# get array by disk +# +# input : array disk member name +# +# output : array sloc +# +################################################################### +function get_ipr_array_by_member { + local sg="$1" + local member_sloc=`convert_sg_to_sloc $sg | $awk -F= '{print $2}'` + # member does not exists, error with rc=1 + [ -z "$member_sloc" ] && return 1 + + local lines=`cmd_show_config` + local array_sloc=`echo "$lines" \ + | $awk ' + BEGIN { hit=0; line="na"; } + /Disk Array/ { hit=1; line=$0; next; } + { + if (hit==1) { + if ($0 ~ /RAID .* Member/) { + if ($0 ~ /\/'$member_sloc'[ ]+.*/) { + print line + exit 0 + } + } else { + hit=0; line="na" + } + } + }' \ + | cut_sloc_from_iprconfig_line` + # disk does not belongs to any array, error with rc=1 + [ -z "$array_sloc" ] && return 1 + + echo "$array_sloc" + log_details "array[$array_sloc] owns disk[$sg]" >&2 + return 0 +} + +################################################# +# +# get process +# input : pids +# output : child_pids pids +# +################################################## +function getpstree { + local pids="$*" + + local pslines=`$ps -e -o pid,ppid` + while true + do + local has_expand=0 + local lines=`echo "$pids" | $sed -e 's/ /\n/g'` + for pid in $pids + do + local child_pids=`echo "$pslines" | $awk -v pid=$pid '$2 == pid{print $1}'` + child_pids=`for pid2 in $child_pids + do + echo "$lines" | grep -sq "^$pid2$" || echo "$pid2" + done ` + if [ -n "$child_pids" ]; then + pids="$child_pids $pids" + has_expand=1 + fi + done + if [ $has_expand -eq 0 ]; then + break + fi + done + echo "$pids" +} + +##################################################################### +# +# delete used disks from slocs +# +# input : disk slocs +# +# output : unused disk slocs +# +################################################################### +function delete_disks_belong_array_from_slocs { + local slocs=$* + validslocs="" + usedslocs="" + for sloc in $slocs + do + line=$(get_ipr_array_by_member $sloc) + if [ $? -eq 0 ]; then + if [ "x$usedslocs" != x ]; then + usedslocs=$usedslocs" "$sloc + else + usedslocs=$sloc + fi + fi + done + if [ -n "$usedslocs" ]; then + validslocs=`echo $slocs|sed "s/$usedslocs//g"` + if [ -n "$validslocs" ]; then + echo $validslocs + fi + else + echo $slocs + fi +} + +#################################################################################### +# +# create raid array +# +# input : $1 minutes, +# Times to wait for the raid creation finish, minutes +# +# $2 seconds +# Interval of minitoring raid creation, status check +# +# left: member disks , iprconfig_args +# +# output : +# +################################################################################ +function create_ipr_array { + local global_rc=0 + local lines="" + local monCnt=$1 + shift + local monInt=$1 + shift + local member_sgs=`echo "$1" | $sed -e 's/,/ /g'` + shift + + local stage_1_timeout=1 # unit minutes + local stage_1_last_tick_count=`$awk -v monInt=$monInt -v timeout=$stage_1_timeout 'END{print timeout*60/monInt}' > /g' + done + echo "-------------------------------------" + for member_sg in $member_sgs + do + cmd_show_status $member_sg | $sed -e 's/^/status['$member_sg']: >> /g' + done + echo "-------------------------------------" + } | log_lines details >&2 + + # prepare tmp log file for iprconfig worker process + local ftmp="/tmp/log_create.log.$$" + $touch $ftmp + + # launch the iprconfig worker process + cmd_raid_create "$iprconfig_args" "$member_sgs" >$ftmp 2>&1 & + local pid_create=$! + # log for debug + local line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1` + log_details "ps1: $line" + + # monitor the worker process and show status + # array_stage=0, monitor array member disk states only + # array_stage=1, wait for array showes up after array member disks are in termination states + # array_stage=2, monitor array state + local array_stage=0 + local array_sloc="" + # is that possible sg names of member disks were changed after issued raid-create on top of them? + local slocs="$member_slocs" + local i=$monCnt + while [ $i -gt 0 -a $monCnt -gt 0 ]; + do + # Get status of all sgs which might include array itself. + local status_lines="" + local rc=0 + local sloc="" + [ $array_stage -ne 1 ] && \ + for sloc in $slocs + do + local status="" + local sg=`convert_sloc_to_sg $sloc | $awk -F= '{print $2}'` + if [ -n "$sg" ]; then + status=`cmd_show_status $sg ` + else + status="Missing-sg_name" + fi + [ -z "$status" ] && status="no_status" + + local msg="status[$sloc/$sg]=\"$status\"" + log_details "array create stage[$array_stage], $msg" + + # list of "termination" status for member disks and target array + # Refer to "Disk array states" section in ipr doc. + local status_exp + if [ $array_stage -eq 0 -o $array_stage -eq 1 ]; then + # 1) If a physical disk fail to be formated as IPR AF, its status will be "Format Required" and be "Failed" later. + # Should we handle this? + status_exp="Active|Failed|Offline|Missing|R/W Protected|Format Required" + + # I saw phy disk in following state which not in the doc. + status_exp+="|Optimized|Non-Optimized" + + # my faked status for disk + status_exp+="|Missing-sg_name" + else + # Note: + # - Array state "Missing" is not a terminate state, exclude it by adding "#" as prefix + status_exp="Optimized|Active|Non-Optimized|Degraded|#Missing|Offline|Failed|R/W Protected|Rebuilding" + fi + # if not "termination" status, we record it and wait for next cycle. + if ! echo "$status" | grep -i -sq -E "$status_exp"; then + # aggregate status messages. + [ -n "$status_lines" ] && status_lines+="," + status_lines+="$msg" + rc=1 + fi + done + # if all terminated, we fall back to check staus of array itself + if [ $rc -eq 0 ]; then + if [ $array_stage -eq 2 ]; then + log_info "Create array($array_sg=$array_sloc) successfully on top of disks: $member_slocs." + break + else # end if stage == 2 + if [ $array_stage -eq 0 ]; then + # since all disks were in termination states, let's move to stage 2 + # to wait for array to be created and start rebuild. + # + # It's possible that such array could not be created for some critical + # error, such in-sufficient disks for required raid level. + # + # Though there was smart way to handle this, here I only use a simple way + # by waiting for a fixed time(1 minute) before return error. + log_details "array create stage[1], entered, after finished formatting member disks $slocs" + lines=`{ cmd_show_config; cmd_show_alt_config; } | $sed -e 's/^/[0-->1] >> /g'` + log_lines details "$lines" + ((array_stage=1)) + ((stage_1_last_tick_count=i-stage_1_last_tick_count)) + fi + # IPR might had problem if some disks failed to response. + # These disks will disapear in the system with sloc and sg were all gone. + # in these case, we drop these disks in the operation and abort the operation + # if all member disks were gone. + local slocs_new="" + local __tmp_item="" + for __tmp_item in $slocs + do + if sloc_exist $__tmp_item; then + [ -n "$slocs_new" ] && slocs_new+=" " + slocs_new+="$__tmp_item" + else + log_error "$i: disk $__tmp_item disapear..." + fi + done + if [ -n "$slocs_new" ]; then + slocs="$slocs_new" + else + global_rc=1 + log_error "All array member disks, $member_slocs, were gone, abort!" + break + fi + # get array owner of each member sg + local array_slocs=`for __tmp_item in $slocs + do + local sg_owner_sloc=$(get_ipr_array_by_member $__tmp_item) + [ -n "$sg_owner_sloc" ] && echo "$sg_owner_sloc" + done \ + | $sort -u -V ` + # we only allow one unique array owns all member disks. + # or, we we need to break out? + local array_cnt=`echo "$array_slocs" | $awk 'END {print NF; exit 0;}'` + log_details "All \"$array_cnt\" array(es), \"$array_slocs\", contain target disks, \"$slocs\"." >&2 + if [ $array_cnt -gt 0 ]; then + + array_sloc=`echo "$array_slocs" | $awk '{print $1}'` + local array_sg=`convert_sloc_to_sg $array_sloc | $awk -F= '{print $2}'` + if [ -n "$array_sg" ]; then + log_details "array create stage[2], entered, after array show up with sg name $array_sg and sloc $array_sloc" >&2 + array_stage=2 + slocs="$array_sloc" + else + log_warn "$i: >> Why new array $array_sloc($array_slocs) does not have a valid sg name?" >&2 + log_warn "$i: >> Let wait for a while to see if it could show up or not until tick count equals to $stage_1_last_tick_count" >&2 + fi + elif [ $array_stage -eq 1 -a $i -lt $stage_1_last_tick_count ]; then + log_error "Fail to wait array($array_slocs) showes up after its member disks, $member_slocs, were finished formatting." >&2 + # log for debug + lines=`{ + echo "$i: Current IPR configuration"; + { + cmd_show_config; + cmd_show_alt_config; + } | $sed -e 's/^/'$i': >> /g'; + }` + log_lines details "$lines" + + global_rc=2 + break + else + # see if we need to issue the raid-create again. + # for some reason, previous raid-create might be terminated unexpectedly with disks + # actually had been formated for raid and ready to be used. So, we try to issue + # the raid-create command again to see if that would help. This usually happened when + # some failures when formtting physical disks to ipr AF. In that situation, iprconfig + # might abort without have array created after it had actually formatted those disks. + local pid_create_still_alive=0 + local pid_create_cur=`getpstree $pid_create` + local pid_cur + for pid_cur in $pid_create_cur + do + if $kill -0 $pid_cur 2>/dev/null; then + pid_create_still_alive=1 + break + fi + done + if [ $pid_create_still_alive -eq 0 ]; then + log_info "$i: disks $slocs had finished formatting, but array was not shown up, and iprconfig process($pid_create) had terminated." + + log_details "$i: Last several raid-create output:" + lines=`$tail -n10 $ftmp | $sed -e 's/^/'$i': >> /g'` + log_lines details "$lines" + # launch the iprconfig worker process again + member_sgs=`convert_sloc_to_sg $member_slocs | $awk -F= '{print $2}'` + if [ -n "$member_sgs" ]; then + $iprconfig -c raid-create $iprconfig_args $member_sgs >$ftmp 2>&1 & + pid_create=$! + + log_info "$i: Restart \"iprconfig -c raid-create $iprconfig_args $member_sgs\" with new pid $pid_create." >&2 + + # log for debug + local line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1` + if [ -n "$line" ]; then + log_details "ps1.2: $line" >&2 + else + log_error "ps1.2: new iprconfig($pid_create) was gone" >&2 + fi + else + log_warn "$i: Fail to restart iprconfig since $member_slocs could not be mapped to sg." >&2 + log_warn "$i: Let try again next loop until tick count equals to $stage_1_last_tick_count" >&2 + fi + else + log_warn "$i: Wait for array($array_slocs) showes up after disk format were finished on $slocs until tick count equals to $stage_1_last_tick_count" >&2 + log_details "$i: Last several raid-create output:" >&2 + lines=`$tail -n2 $ftmp | $sed -e 's/^/'$i': >> /g'` + log_info debug "$lines" >&2 + fi + fi + fi # end else if stage == 2 + fi # end if $rc eq 0 + + # log status for monitor + # log for debug + if [ -n "$status_lines" ]; then + log_status "Creating IPR array with last info at time \""`date "+%Y-%m-%d %H:%M:%S"`"\": $status_lines" >&2 + lines=`echo "$status_lines" | $sed -e 's/,/\n/g' | $sed -e "s/^/$i: >> /g"` + log_lines details "$lines" >&2 + fi + + $sleep $monInt + [ $monCnt -gt 0 ] && ((i-=1)) + done + + # log for debug + line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1` + [ -n "$line" ] && log_details "ps2: $line" + if $kill -0 $pid_create 2>/dev/null; then + # If succ out, array status should in "rebuilding" state while iprconfig is still running + # until the rebuild had finished. But we do not need to wait for its termination since the + # block device is actually ready to use though performance might be downgrade. + # so, by default, we will just kill the "iprconfig" command(? not sure if that would affect + # the rebuild or not) and let raid controller do its job at backend. + # in order to wait for the whole rebuilding, set IPR_REBUILD_WAIT=true explicitly. + if [ "x$IPR_REBUILD_WAIT" = "xtrue" -a $i -gt 0 ]; then + log_info "Wait for ipr raid creation command to finish. pid=$pid_create" >&2 + wait $pid_create + else + log_warn "Terminate ipr raid creation process enforcely. pid=$pid_create" >&2 + $kill -TERM $pid_create 2>/dev/null + $sleep 1 + $kill -0 $pid_create 2>/dev/null \ + && $kill -KILL $pid_create 2>/dev/null + fi + fi + # log for debug + line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1` + [ -n "$line" ] && log_details "ps3: $line" >&2 + + log_details "Last several raid-create output:" >&2 + lines=`$tail -n10 $ftmp | sed -e 's/^/[last]: >> /g'` + log_lines details "$lines" >&2 + rm -f $ftmp + # Log "failed" status to PCM/xCAT with reason and instruction for future investigation. + if [ $global_rc -ne 0 ]; then + # set node "status" to "failed" to indicate that the installation was failed. + fstatus_app=/tmp/baz.py + if [ -x "$fstatus_app" ]; then + $fstatus_app "installstatus failed" >&2 + fi + + # set node state with brief reason why installation failed and future instructions. + # end user can get this message by running "nodestate" command to target nodes. + log_status "Fail to create IPR array with return code \"$global_rc\"! Refer to more details in syslog by checking \"rhPreDbg\" flag..." >&2 + fi + return $global_rc +} + +########################################################################################### +# +# main process to handle create raid array when using pci_id or pci_slot_name +# +########################################################################################## +function handle_create_raid_array { + local striple_size=$1 + local raidlevel=$2 + local disknum=$3 + local pcilocs=$4 + local disknames=$5 + + if [ "x$pcilocs" != "xnull" ]; then + + # Find out all member disks in target IOAs + # pciloc=sloc,sloc,sloc... + slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs ` + if [ -z "$slocs_grps" ]; then + log_error "Could not find any disk on target pciloc ${pcilocs}!" >&2 + return 1 + else + log_info "Disks in target IPR raid adapter: \"$slocs_grps\"" >&2 + fi + + # find the required member disks + slocs="" + validslocs="" + for item in $slocs_grps + do + pciloc=`echo "$item" | $awk -F= '{print $1}'` + slocs_grp=`echo "$item" | $awk -F= '{print $2;}'` + # order disks by its resource path + slocs_grp=`sort_devices_by_resource_path $slocs_grp` + + slocs=`echo "$slocs_grp" | sed 's/,/ /g'` + if [ -n "$slocs" ]; then + break + else + log_warn "Could not find enough disk on target adapter $pciloc which only has following disks ${slocs_grp}!" >&2 + fi + done + if [ -z "$slocs" ]; then + log_error "Could not find enough disk on target adapter ${pciloc}!" >&2 + return 1 + else + log_info "Find member disks $slocs on target IOA $pciloc" >&2 + slocs=`echo "$slocs" | $sed -e 's/,/ /g'` + # if it is the first create_raid, do nothing, else: + # delete used disks belong to existed raid array + validslocs=`delete_disks_belong_array_from_slocs $slocs` + if [ -z "$validslocs" ]; then + log_error "Could not find enough unused disk on target adapter $pciloc." + return 1 + else + slocs="$validslocs" + log_info "Find unused member disks $slocs on target IOA $pciloc" >&2 + fi + + fi + # handle disk_num + # if disknum is all, do nothing + # if disknum is a number, then it should be compared with actual_disknum + actual_disknum=`echo "$slocs"| $awk '{print NF}'` + slocs=`compare_disknum_actualnum $disknum $actual_disknum $slocs` + + else + # find the slocs for disks in disknames + disklist=`echo $disknames|sed 's/#/ /g'` + slocs_grp=`convert_sg_to_sloc $disklist` + slocs=`echo "$slocs_grp"|awk -F= '{print $2}'` + fi + # check if target disks were already in a reusable array. + # if not, create one by calling block command "create_ipr_array". + # after than, check again to confirm if the target array was ready. + array_sloc="" + ((__i=0)) + while [ $__i -lt 2 ]; + do + # get disk --> array association + # format 0:0:0:0=sg0 + declare array_slocs_lines=`for sloc in $slocs + do + line=$(get_ipr_array_by_member $sloc) + echo "$sloc=$line" + done` + + # extract arrays from associations + # get sg0 sg1 + declare array_slocs=`echo "$array_slocs_lines" \ + | $awk -F= '{print $2}' | $sort -u` + + # count the number of arrays on target disks + # which might include "blank" array + declare n_arrays_lines=`echo "$array_slocs" | $awk 'END {print NR}'` + # count the number of arrays on target disks, again + # w/o "blank" array + declare n_arrays=`echo "$array_slocs" | grep -v "^ *$" | $awk 'END {print NR}'` + # get the sg names for owner arrays. + declare array_sgs="" + if [ $n_arrays -gt 0 ]; then + array_sgs=`convert_sloc_to_sg $array_slocs \ + | $awk -F= '{print $2}' ` + fi + # (cond_1): if ALL target disks are in ONE array, reuse it. + if [ $n_arrays_lines -eq 1 -a $n_arrays -eq 1 ]; then + if [ $__i -eq 0 ]; then + log_info "Target disks, $slocs, were already in an array, $array_slocs." >&2 + else + log_info "An array, $array_slocs, had been created to include target disks, $slocs" >&2 + fi + lines=`{ cmd_show_details $array_sgs; + cmd_show_status $array_sgs; + } | $sed -e 's/^/>> /g'` + log_lines details "$lines" + array_sloc="$array_slocs" + break + fi + # (cond_2): delete arrays which owe target disks first before any future operations. + # NOTE: we redo the loop after "delete" command was issued to ipr + if [ $n_arrays -gt 0 ]; then + log_info "Target disks, $slocs, were owned by multiple arrays, \"$array_slocs\"($array_sgs). Delete them first before creating a new array on top of them." >&2 + delete_ipr_array enforce=1 tryCnt=360 tryInt=60 -- $array_sgs + continue + fi + # (cond_3): create an array to include all target disks + # we assume all these disks are ready to be included into an array. + if [ $__i -eq 0 ]; then + declare sgs=`convert_sloc_to_sg $slocs \ + | $awk -F= '{print $2}' | $sed -e 's/ /,/g'` + # Times to wait for the raid creation finish + # The lager capacity of member disks, longer times to wait. + # :unit minutes + monTime=240 + # Interval of minitoring raid creation, status check + # :unit seconds + monInt=30 + monCnt=`$awk -v monInt=$monInt -v monTime=$monTime 'END{print 60*monTime/monInt}' &2 + # handle striple_size + if [ "x$striple_size" == "xdefault" ]; then + create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel + elif [ -z $striple_size ]; then + create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel + else + create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel -s $striple_size + fi + fi + ((__i+=1)) + done + lines=`cmd_show_config` + if [ $__i -lt 2 -a -n "$array_sloc" ]; then + array_sd=`echo "$lines" \ + | grep '^sd[a-z]\+[ ]\+.*\/'$array_sloc'[ ]\+.*Disk Array' \ + | $awk '{print $1}'` + if [ -n "$array_sd" ]; then + log_info "array_dev=$array_sd" + return 0 + else + log_error "Fail to get scsi device name for new array $array_sloc" >&2 + echo "$lines" | $sed -e 's/^/>> /g' | log_lines error >&2 + return 1 + fi + else + log_error "Fail to create array on top of disks $slocs" >&2 + echo "$lines" | $sed -e 's/^/>> /g' | log_lines error >&2 + return 1 + fi + +} + +############################################################################ +# +# handle disk_num +# if disknum is all, do nothing +# if disknum is a number, then it should be compared with actual_disknum +# return of slocs +# +############################################################################## +function compare_disknum_actualnum { + local disknum=$1 + local actualnum=$2 + shift; + shift; + local slocs=$* + local actualslocs + if [ "x$disknum" != "xall" ]; then + if [ $actualnum -ge $disknum ]; then + actualslocs=`echo "$slocs"| \ + awk -v anum=$disknum '{ for ( i=1; i<=anum; i++ ){print $i}}'` + echo $actualslocs + + else + log_error "Could not find enough disk on target adapter ${pciloc}!" >&2 + fi + else + echo "$slocs" + fi + +} + +############################################################################ +# +# Delete raid array +# +# input format : enforce=1 tryCnt=360 tryInt=60 -- arrays_name list +# +########################################################################## +function delete_ipr_array { + # default to NO if enforcely wait for rebuilding array + local enforce=0 + # default to wait 6 hours for array rebuilding to finish + local tryCnt=360 + # monitor rebuilding status every 60 seconds + local tryInt=60 + + # parser input arguments + while [ -n "$1" ]; + do + local key=`echo "$1" | $cut -s -d= -f1` + if [ "$key" = "tryCnt" ] || \ + [ "$key" = "tryInt" ] || \ + [ "$key" = "enforce" ]; then + local val=`echo "$1" | $cut -s -d= -f2-` + eval "$key=\"$val\"" + elif [ "$1" = "--" ]; then + shift + arrays="$*" + break + fi + shift + done + # should not with empty array list, if yes, error with rc=1 + if [ -z "$arrays" ]; then + log_error "No array to delete, assume success!" >&2 + return 0 + fi + local array_slocs=`convert_sg_to_sloc $arrays | awk -F= '{print $2}'` + # if none of arrays were valid, error with rc=1 + if [ -z "$array_slocs" ]; then + log_error "Fail to map arrayes, $arrays, to scsi location before deletion!" + return 1 + fi + local ioas=`get_ipr_ioas` + + local i=1 + while [ $i -le $tryCnt ]; + do + # verify if all target arrays are still exists + local tmp_array_slocs=`for _tmp_sloc in $array_slocs + do + is_ipr_array $_tmp_sloc && echo "$_tmp_sloc" + done ` + if [ "$tmp_array_slocs" != "$array_slocs" ]; then + array_slocs="$tmp_array_slocs" + log_details "Renew available target arrays which will be deleting, \"$array_slocs\"" >&2 + fi + if [ -z "$array_slocs" ]; then + log_details "Round $i: All target arrays were gone!" >&2 + break + fi + # find out sg devs for all deletable arrays + local array_slocs_deletable=`for ioa in $ioas + do + # [root@dn01-dat ~]# iprconfig -c query-raid-delete sg5 + # Name PCI/SCSI Location Description Status + # ------ ------------------------- ------------------------- ----------------- + # sdk 0003:70:00.0/0:2:0:0 RAID 0 Disk Array Optimized + # sdl 0003:70:00.0/0:2:1:0 RAID 0 Disk Array Optimized + cmd_query_raid_delete $ioa \ + | grep "Disk Array" \ + | cut_sloc_from_iprconfig_line + done` + if [ -z "$array_slocs_deletable" -a $enforce -eq 0 ]; then + log_details "Round $i: No deletable array, nothing to do and finish." >&2 + break + fi + # log for debug + { + echo "$array_slocs_deletable" | $sed -e 's/^/Round '$i': Deletable array >> /g' + cmd_show_config | $sed -e 's/^/Round '$i': ipr info before delete >> /g' + } | log_lines details >&2 + log_info "Round $i: delete_ipr_array, \"$array_slocs\"" >&2 + + # if requested array is a deletable array, mark it to be deleted in this round. + local array_slocs_verified=`for array_sloc in $array_slocs + do + if echo "$array_slocs_deletable" | grep -sq "^$array_sloc$"; then + echo $array_sloc + else + log_warn "Round $i: Array $array_sloc is un-deletable at present." >&2 + fi + done` + if [ -n "$array_slocs_verified" ]; then + log_details "Round $i: Verified array slocs for deletion are \"$array_slocs_verified\"." >&2 + else + log_info "Round $i: All remaining target arrayes are un-deletable now." >&2 + if [ $enforce -eq 1 ]; then + log_info "Round $i: Wait for these un-deletable arrayes deletable with tryCnt=$tryCnt, tryInt=$tryInt." >&2 + wait_for_ipr_device_status \ + tryCnt=$tryCnt tryInt=$tryInt cmd="grep -sq -E 'Optimized'" \ + -- \ + `echo "$array_slocs" | $sed -e "s/ /,/g"` + # if wait succ, let's try to delete again + if [ $? -eq 0 ]; then + ((i+=1)) + continue + else + break + fi + fi + fi + + # issue delete command + local dcnt=0 + for array_sloc in $array_slocs_verified + do + local array=`convert_sloc_to_sg $array_sloc | awk -F= '{print $2}'` + [ -z "$array" ] && continue + + log_details "Round $i: Delete $dcnt array $array=$array_sloc" >&2 + cmd__raid_delete $array + ((dcnt+=1)) + done + + # break out the loop forcely if nothing deleted. + [ $dcnt -eq 0 ] && break + + # wait for next round + $sleep $tryInt + ((i+=1)) + done + test $i -lt $tryCnt -a -z "$array_slocs" + return $? +} + +################################################# +# +# sort devices by resource path, +# +# input: dev list, "sg0,..sgn" +# +################################################## +function sort_devices_by_resource_path { + local sgs="$*" + + local del="," + if [ $# -gt 1 ]; then + del=" " + else + sgs=`echo "$sgs" | $sed -e "s/,/ /g"` + fi + + # [root@dn01-dat ~]# iprconfig -c show-alt-config + # Name Resource Path/Address Vendor Product ID Status + # ------ -------------------------- -------- ---------------- ----------------- + # sg15 FE IBM 57D7001SISIOA Operational + # sg10 00-0C IBM HUC101212CSS600 Active + # sg11 00-0D IBM HUC101212CSS600 Active + local lines=`$iprconfig -c show-alt-config | grep "^sg"` + local item + local lines_out=`for item in $sgs + do + local sg=$(convert_sloc_to_sg $item | $awk -F= '{print $2;}') + local rp=$(echo "$lines" | $awk -vsg=$sg '$1 == sg { print $2; }') + echo "$item $sg $rp" + done \ + | $sort -t' ' -k3` + # log for debug + { + echo "sort devices \"$sgs\" by IPR resource path:" + echo "$lines_out" | $sed -e 's/^/sort by rp output: >> /g' + } | log_lines details >&2 + + # deliver ordered result + lines_out=`echo "$lines_out" | $awk '{print $1;}' ` + if [ "$del" = "," ]; then + echo "$lines_out" | $sed -e "s/ /,/g" + fi + return 0 +} + + +###################################################### +# +# log lines +# +# input : details or status or error +# +###################################################### +function log_lines { + local pcnt=$# + local __level=$1 + shift + local cmd=log_${__level} + + local hit=0 + local OIFS=$IFS + local NIFS=$'\n' + IFS=$NIFS + local __msg + for __msg in $* + do + IFS=$OIFS + $cmd "$__msg" + hit=1 + IFS=$NIFS + done + IFS=$OIFS + + [ $hit -eq 0 -a $pcnt -le 1 ] && \ + while read __msg; + do + $cmd "$__msg" + done +} + +###################################################### +# +# error information +# +###################################################### +function log_error { + local __msg="$*" + $log_print_cmd $log_print_arg "[E]: $__msg" >&2 + return 0 +} + +###################################################### +# +# warning information +# +# input : message +# +###################################################### +function log_warn { + local __msg="$*" + $log_print_cmd $log_print_arg "[W]: $__msg" >&2 + return 0 +} + +###################################################### +# +# log information +# +# input : message +# +###################################################### +function log_info { + local __msg="$*" + $log_print_cmd $log_print_arg "[I]: $__msg" >&2 + return 0 +} + +###################################################### +# +# more details information +# +###################################################### +function log_details { + local __msg="$*" + $log_print_cmd $log_print_arg "[I]: $__msg" >&2 + return 0 +} + +#################################################### +# +# print command status +# +################################################### +__my_log_status= +function log_status { + local __msg="$*" + $log_print_cmd $log_print_arg "[S]: $__msg" >&2 + + # call my_log_status hook to triger more processing for status messages. + if [ -n "$__my_log_status" ]; then + $__my_log_status "$__msg" + fi + return 0 +} + +#################################################### +# +# print output and logfile +# +################################################### +function log_print_default { + echo $* + echo $* >> $log_file +} + +#################################################### +# +# handle command +# +################################################### +function set_log_print { + local cmd="$1" + if [ -z "$cmd" ]; then + return 1 + fi + shift + local args="$*" + + eval "log_print_cmd=\"$cmd\"" + eval "log_print_arg=\"$args\"" +} + +# let log work +[ -z "$log_print" ] && set_log_print "log_print_default" + + + diff --git a/xCAT-genesis-scripts/xCAT-genesis-scripts.spec b/xCAT-genesis-scripts/xCAT-genesis-scripts.spec index 06951335f..ba883aa3f 100755 --- a/xCAT-genesis-scripts/xCAT-genesis-scripts.spec +++ b/xCAT-genesis-scripts/xCAT-genesis-scripts.spec @@ -81,6 +81,7 @@ touch /etc/xcat/genesis-scripts-updated %{rpminstallroot}/bin/raidcmd %{rpminstallroot}/bin/raidutils %{rpminstallroot}/bin/diskdiscover +%{rpminstallroot}/bin/configraid %{rpminstallroot}/bin/dodiscovery %{rpminstallroot}/bin/dosysclone %{rpminstallroot}/bin/doxcat