#!/bin/bash
#
# Usage:
#
# There are utils for diskdiscover and configraid.
#

log_file=$1

##############################
# declare all commands
##############################
awk="awk"
sed="sed"
cut="cut"
sleep="sleep"
sort="sort"
ps="ps"
head="head"
readlink="readlink"
basename="basename"
udevadm="udevadm"
touch="touch"
tail="tail"
dmesg="dmesg"
grep="grep"
lspci="lspci"

############################################
#
# source raidcmd
#
###########################################
str_dir_name=`dirname $0`
. $str_dir_name/raidcmd

################################################################
#
#  Input PCI_ID to get PCI location
#
#  input: pci_id
#
#  output: pci locations 
#
################################################################
function get_pciloc_by_id {
    local __in_pciid=$1
    if echo "$__in_pciid" | $grep -sq "_"; then
        __in_pciid=`echo "$__in_pciid" | $sed -e 's/_/:/'`
    fi
    local sysdevdir=/sys/bus/pci/devices
    local pcilocs=`cd $sysdevdir 2>/dev/null && for dev in *
    do
        lines=$($udevadm info --query=property --path=$sysdevdir/$dev)
        if echo "$lines" | $grep -i -sq -E "^PCI_ID=$__in_pciid$|^PCI_SUBSYS_ID=$__in_pciid$"; then
            echo $dev
        fi
    done `
    [ -z "$pcilocs" ] && return 1
    echo "$pcilocs"
    return 0
}

######################################################
#
# get pci_slot scsi device
#
# input: slocs, for example, 0:0:0:0 0:0:1:0
#
# output: <sloc>=<device_name> ... 
#         <sloc>=<device_name>
#
######################################################
function convert_sloc_to_sg {
    local __slocs="$*"
    for __sloc in $__slocs
    do
        if echo "$__sloc" | grep -sq "[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+"; then
            __sg=`$readlink /sys/class/scsi_device/$__sloc/device/generic`
            if [ "$__sg" ]; then
                __sg=`$basename $__sg`
            fi
        elif echo "$__sloc" | grep -sq -E '^sg[0-9]+|^sd[a-z]+'; then
            __sg="$__sloc"
            __sloc=`convert_sg_to_sloc $__sg | $awk -F= '{print $2}'`
        fi
        echo "$__sloc=$__sg"
    done
    return 0
}

#################################################################
#
#  Through PCI/SCSI device to find PCI/SCSI location
#
#  input: device names, 
#         sg0 ...sgn
#
#  output: <dev>=<sloc>... <dev>=<sloc>
#
################################################################
function convert_sg_to_sloc {
    local __sgs="$*"
    local __sloc=""
    for __sg in $__sgs
    do
        if echo "$__sg" | grep -sq "^sg[0-9]\+"; then
            __sloc=`$readlink /sys/class/scsi_generic/$__sg/device`
            if [ "$__sloc" ]; then
                __sloc=`$basename $__sloc`
            fi
        elif echo "$__sg" | grep -sq "^sd[a-z]\+"; then
            __sloc=`$readlink /sys/block/$__sg/device`
            if [ "$__sloc" ]; then
                __sloc=`$basename $__sloc`
            fi
        elif echo "$__sg" | grep -sq "[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+"; then
            __sloc="$__sg"
            __sg=`convert_sloc_to_sg $__sloc | $awk -F= '{print $2}'`
        fi
        echo "$__sg=$__sloc"
    done
    return 0
}

############################################################################
#
# get devices which are qualified to be used to create raid
# it should equals to "query-raid-create" after all array had
# been deleted.
# Note: output format of this command is multilines
# <pciloc_of_ioa1>=<sloc_of_disk1>,<sloc_of_disk2>,...
# <pciloc_of_ioa2>=<sloc_of_disk1>,<sloc_of_disk2>,...
# ...
###########################################################################
function get_raid_create_devices_by_pciloc {
    local lines=""
    local pcilocs="$*"
    [ -z "$pcilocs" ] && return 1

    # reorder ipr ioa pcilocs by its Primary and Secondary state
    local ioas=`get_ipr_ioas_by_pciloc $pcilocs`
    pcilocs=`get_sg_pciloc $ioas | $awk -F= '{print $2}' `
    lines=`cmd_show_config`
    local slocs=""
    local line=""
    for pciloc in $pcilocs
    do
        # exclude:
        # 1) scsi adapter(ioa);
        # 2) scsi enclosure;
        # 3) disk array;
        slocs=`echo "$lines" \
        | grep '^.*[ ]\+'$pciloc'\/[0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+.*$' \
        | grep -v -E "Adapter|Enclosure|Disk Array" \
        | cut_sloc_from_iprconfig_line \
        | $sort -V \
        | $sed -e 's/ /,/g' \
        | awk '{printf (NR>1)?","$0:$0}'`
        if [ -n "$slocs" ]; then
            line="$pciloc=$slocs"
            echo "$line"
        fi
    done
    return 0
}

########################################################################################
#
# return list of ioas on target pciloc with order of Primary --> Secondary --> Others
#
# input: pci locations
# 
# output: sorted raid adapters location list
#
#########################################################################################
function get_ipr_ioas_by_pciloc {
    local pcilocs="$*"
    [ -z "$pcilocs" ] && return 1

    # find out all ioas and its current adapter state
    local ioas_all=`cmd_show_ioas | grep "^sg[0-9]\+.*Operational" | $awk '{print $1}'`
    # group them into "Primary" and "Secondary" groups
    local lines=`for ioa in $ioas_all
    do
        state=$(cmd_show_details $ioa | grep "Current Dual Adapter State" | $sed -e 's/^.* : \+\(.*\)$/\1/')
        echo "$state=$ioa"
    done`
    local ioa_primary=`echo "$lines" | $awk -F= '($1 == "Primary") {print $2}'`
    local ioa_secondary=`echo "$lines" | $awk -F= '($1 == "Secondary") {print $2}'`
    local ioa_others=`echo "$lines" | $awk -F= '($1 != "Primary") && ($1 != "Secondary") {print $2}'`
    ioas_all="$ioa_primary $ioa_secondary $ioa_others"

    # pick up ioa on target pciloc
    lines=`echo "$pcilocs" | $sed -e 's/[, ]/\n/g'`
    local ioas_in=""
    if [ "$lines" = "all" ]; then
        ioas_in="$ioas_all"
    else
        ioas_in=`for ioa in $ioas_all
        do
            ioa_pciloc=$(get_sg_pciloc $ioa | awk -F= '{print $2}')
            if echo "$lines" | grep -sq -i "^${ioa_pciloc}$"; then
                echo $ioa
            fi
        done `
    fi
    [ -z "$ioas_in" ] && return 1
    echo "$ioas_in"
    return 0
}

###############################################
#
# use udev to determine pciloc of sg device
#
# input: disk name list
#
# output: <dev>=<sloc>...<dev>=<sloc>
#
###############################################
function get_sg_pciloc {
    local sgs="$*"
    [ -z "$sgs" ] && return 1

    local sg=""
    local pciloc=""
    for item in $sgs
    do
        sg=`convert_sloc_to_sg $item | $awk -F= '{print $2}'`
        [ -z "$sg" ] && continue
        pciloc=`cmd_show_details $sg \
                | grep "^PCI Address" \
                | $sed -e 's/.*:[ ]\+\([0-9]\+:[0-9]\+:[0-9]\+\.[0-9]\+\).*$/\1/'`

        if [ -n "$pciloc" ]; then
            echo "$sg=$pciloc"
        fi
    done
    return 0
}

######################################################
#
# cut resouce_path
#
#####################################################
function cut_sloc_from_iprconfig_line {
    $sed -e 's/^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$/\2/g'
}

#####################################################################
#
# find descriptions and status for device
#
# input: pci locations
#
# output:  descriptions and status from "iprconfig -c show-config" 
#
###################################################################
function find_desc_status_sg {
    local lines=""
    local pciscsilocs="$*"
    [ -z "$pciscsilocs" ] && return 1

    lines=`cmd_show_config`
    local slocs=""
    local line=""
    for pciscsiloc in $pciscsilocs
    do
        slocs=`echo "$lines" \
        | $grep "${pciscsilocs}" \
        | awk '{for(i=3;i<=NF;++i) printf $i "\t";printf "\n"}'`
        echo $slocs
    done
    return 0
    
}

#######################################################
#
#  get disk devices through pci_id
#
#  input: pci id
#
#######################################################
function get_devices_by_pciid {

    local pciid="$*"
    [ -z "$pciid" ] && return 1

    pcilocs=`get_pciloc_by_id $pciid`


    if [ -z "$pcilocs" ]; then
        echo "There is no PCI_SLOT_NAME for PCI_ID:$pciid."
        return 1
    fi
    slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs`
    if [ -z "$slocs_grps" ]; then
        echo "Could not find any disk on target pciloc ${pcilocs}!"
        return 1
    fi
    arrage_output $pciid "$slocs_grps"

}

######################################################################
#
# get ipr raid arrays by PCI location
#
# input: pci locations
#
# output: raid arrays
#
#####################################################################
function get_ipr_arrays_by_pciloc {
    local pcilocs="$*"
    [ -z "$pcilocs" ] && return 1

    # reorder ipr ioa pcilocs by its Primary and Secondary state
    local ioas=`get_ipr_ioas_by_pciloc $pcilocs`
    pcilocs=`get_sg_pciloc $ioas | $awk -F= '{print $2}' `

    local lines=$(cmd_show_arrays)
    local slocs=`for pciloc in $pcilocs
    do
        echo "$lines" 
    done `
    [ -n "$slocs" ] && echo "$slocs"
    return 0
}

#####################################################################
#
# get all af and jbod disks 
#
# input: no
#
# output: all devices table
#
####################################################################
function get_all_devices_for_raid {

    local showlines=`cmd_show_config`

    pcilocs=`echo "$showlines" \
             | $grep -v -E "Adapter|Enclosure|Disk Array" \
             | $grep '^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$'|$sed 's/ /,/g'|cut -c 8-19`

    [ -z "$pcilocs" ] && return 1
    slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs`
    if [ -z "$slocs_grps" ]; then
        echo "Could not find any disk on target pciloc ${pcilocs}!"
    fi
    arrage_output "null" "$slocs_grps"
}

#####################################################################
#
# output of disks and arrays
#
# input: pci_id sloc_grps
#
# output: all devices table
#
####################################################################
function arrage_output {
    pciid=$1
    shift
    slocs_grps=$*

    # find the required member disks
    echo "--------------------------------------------------------------------------"
    echo "PCI_ID     PCI_SLOT_NAME  Resource_Path  Device  Description   Status"
    echo "------     -------------  -------------  ------  -----------   ----------------"
    slocs=""
    for item in $slocs_grps
    do
         pciloc=`echo "$item" | $awk -F= '{print $1}'`
         slocs_grp=`echo "$item" | $awk -F= '{print $2;}'i \
                                 | $sed 's/,/ /g'`

         if [ x$pciid == "xnull" ]; then
             pciid=`get_PCI_ID $pciloc`
         fi
         for sloc in $slocs_grp
         do
              pciscsiloc="$pciloc/$sloc"
              desc=`find_desc_status_sg $pciscsiloc`
              disk=`convert_sloc_to_sg $sloc | $awk -F= '{print $2}'`
              echo "$pciid  $pciloc   $sloc        $disk     $desc "
         done
         echo "-------------------------------------------------------------------"
         echo "Get ipr RAID arrays by PCI_SLOT_NAME: $pciloc"
         echo "-------------------------------------------------------------------"
         get_ipr_arrays_by_pciloc $pciloc
    done

}


###############################################################
#
# get PCI_ID through lspci
#
# input: pci location
#
# output: pci id
#
###############################################################
function get_PCI_ID {
    pcislot=$1
    pciid=`$lspci | $grep ${pcislot} | $awk '{print $5}'`
    echo $pciid
}

###########################################
#
# Identify if sloc is exist or not
#
# input: dev sloc
#
# output: 0  exist
#         1  not
# 
##########################################
function sloc_exist {
    local __sloc="$1"
    __sloc=`convert_sg_to_sloc $__sloc | $awk -F= '{print $2}'`
    rc=1
    [ -n "$__sloc" ] && [ -d /sys/class/scsi_device/$__sloc/device ] && rc=0
    return $rc
}

###########################################
#
# get all raid arrays
#
# output : array1#array2..
#
##########################################
function get_all_raid_arrays {
    local arrays=`cmd_show_arrays \
                  | $awk '{print $1}'\
                  | $sed -n '3,$p'`
    echo $arrays
}

#################################################
#
# Identify if sloc is raid or not
#
# input : dev sloc
#
# return : 1  ------not raid
#          0  ------is raid
#
##################################################
function is_ipr_array {
    local __sloc="$1"
    local __sg=`convert_sloc_to_sg $__sloc | $awk -F= '{print $2}'`
    [ -n "$__sg" ] && is_array $__sg
}

#################################################
#
# get all ioas
#
##################################################
function get_ipr_ioas {
    get_ipr_ioas_by_pciloc all
}

#################################################
#
# cut pci location from iprconfig line
#
##################################################
function cut_pciloc_from_iprconfig_line {
    $sed -e 's/^.*[ ]\+\(.*\)\/\([0-9]\+:[0-9]\+:[0-9]\+:[0-9]\+\).*$/\1/g'
}

##############################################
#
# get member disk for ipr array
#
# input : array name
#
# output : member sloc list
#          "member_sloc1 member_sloc2 ...." 
#
#############################################
function get_ipr_array_members {
    local array="$1"
    local array_sloc=`convert_sg_to_sloc $array | $awk -F= '{print $2}'`
    # array does not exist, error with rc=1
    [ -z "$array_sloc" ] && return 1

    local lines=`cmd_show_config`
    # get scsi locations of disk array members
    local member_slocs=`echo "$lines" \
    | $awk '
    BEGIN { hit=0; members=""; }
    /^.*\/'$array_sloc'[ ]+.*Disk Array.*$/ { hit=1; next; }
    {
        if (hit==1) {
            if ($0 ~ /RAID .* Member/) {
                # ignore "Remote" array members
                if ($0 !~ / *Remote *$/) {
                    print $0
                }
            } else {
                hit=0
            }
        }
    }' \
    | cut_sloc_from_iprconfig_line `
    # array does not have any member which should not, error with rc=1
    [ -z "$member_slocs" ] && return 1

    echo "$member_slocs"
    log_details "array[$array] has member disks[$member_slocs]" >&2
    return 0
}

#####################################################################
#
# get array by disk
#
# input : array disk member name
#
# output : array sloc
#
###################################################################
function get_ipr_array_by_member {
    local sg="$1"
    local member_sloc=`convert_sg_to_sloc $sg | $awk -F= '{print $2}'`
    # member does not exists, error with rc=1
    [ -z "$member_sloc" ] && return 1

    local lines=`cmd_show_config`
    local array_sloc=`echo "$lines" \
    | $awk '
    BEGIN { hit=0; line="na"; }
    /Disk Array/ { hit=1; line=$0; next; }
    {
        if (hit==1) {
            if ($0 ~ /RAID .* Member/) {
                if ($0 ~ /\/'$member_sloc'[ ]+.*/) {
                    print line
                    exit 0
                }
            } else {
              hit=0; line="na"
            }
        }
    }' \
    | cut_sloc_from_iprconfig_line`
    # disk does not belongs to any array, error with rc=1
    [ -z "$array_sloc" ] && return 1

    echo "$array_sloc"
    log_details "array[$array_sloc] owns disk[$sg]" >&2
    return 0
}

#################################################
#
# get process
# input : pids
# output : child_pids pids
#
##################################################
function getpstree {
    local pids="$*"

    local pslines=`$ps -e -o pid,ppid`
    while true
    do
        local has_expand=0
        local lines=`echo "$pids" | $sed -e 's/ /\n/g'`
        for pid in $pids
        do
            local child_pids=`echo "$pslines" | $awk -v pid=$pid '$2 == pid{print $1}'`
            child_pids=`for pid2 in $child_pids
            do
                echo "$lines" | grep -sq "^$pid2$" || echo "$pid2"
            done `
            if [ -n "$child_pids" ]; then
                pids="$child_pids $pids"
                has_expand=1
            fi
        done
        if [ $has_expand -eq 0 ]; then
            break
        fi
    done
    echo "$pids"
}

#####################################################################
#
# delete used disks from slocs
#
# input : disk slocs
#
# output : unused disk slocs
#
###################################################################
function delete_disks_belong_array_from_slocs {
    local slocs=$*
    validslocs=""
    usedslocs=""
    for sloc in $slocs
    do
        line=$(get_ipr_array_by_member $sloc)
        if [ $? -eq 0 ]; then
            if [ "x$usedslocs" != x ]; then
                usedslocs=$usedslocs" "$sloc
            else
                usedslocs=$sloc
            fi
        fi
    done
    if [ -n "$usedslocs" ]; then
        validslocs=`echo $slocs|sed "s/$usedslocs//g"`
        if [ -n "$validslocs" ]; then
            echo $validslocs
        fi
    else
        echo $slocs
    fi
}

####################################################################################
#
# create raid array
#
# input : $1 minutes,
#            Times to wait for the raid creation finish, minutes
#
#         $2 seconds
#            Interval of minitoring raid creation, status check
#
#         left: member disks , iprconfig_args
#
# output : 
#
################################################################################
function create_ipr_array {
    local global_rc=0
    local lines=""
    local monCnt=$1
    shift
    local monInt=$1
    shift
    local member_sgs=`echo "$1" | $sed -e 's/,/ /g'`
    shift

    local stage_1_timeout=1        # unit minutes
    local stage_1_last_tick_count=`$awk -v monInt=$monInt -v timeout=$stage_1_timeout 'END{print timeout*60/monInt}' </dev/null`

    [ -z "$member_sgs" ] && return 1
    local member_slocs=`convert_sg_to_sloc $member_sgs | $awk -F= '{print $2}' `
    local member_sgs=`convert_sloc_to_sg $member_sgs | $awk -F= '{print $2}' `
    [ -z "$member_sgs" ] && return 1

    local iprconfig_args="$*"
    # log for details
    {
        echo "Attempt to create ipr array with command: iprconfig -c raid-create \"$iprconfig_args\" \"$member_sgs\""
        for member_sg in $member_sgs
        do
            echo "-------------------------------------"
            cmd_show_details $member_sg | $sed -e 's/^/details['$member_sg']: >> /g'
        done
        echo "-------------------------------------"
        for member_sg in $member_sgs
        do
            cmd_show_status $member_sg | $sed -e 's/^/status['$member_sg']: >> /g'
        done
        echo "-------------------------------------"
    } | log_lines details >&2

    # prepare tmp log file for iprconfig worker process
    local ftmp="/tmp/log_create.log.$$"
    $touch $ftmp

    # launch the iprconfig worker process
    cmd_raid_create "$iprconfig_args" "$member_sgs" >$ftmp 2>&1 &
    local pid_create=$!
    # log for debug
    local line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
    log_details "ps1: $line"

    # monitor the worker process and show status
    # array_stage=0, monitor array member disk states only
    # array_stage=1, wait for array showes up after array member disks are in termination states
    # array_stage=2, monitor array state
    local array_stage=0
    local array_sloc=""
    # is that possible sg names of member disks were changed after issued raid-create on top of them?
    local slocs="$member_slocs"
    local i=$monCnt
    while [ $i -gt 0 -a $monCnt -gt 0 ];
    do
        # Get status of all sgs which might include array itself.
        local status_lines=""
        local rc=0
        local sloc=""
        [ $array_stage -ne 1 ] && \
        for sloc in $slocs
        do
            local status=""
            local sg=`convert_sloc_to_sg $sloc | $awk -F= '{print $2}'`
            if [ -n "$sg" ]; then
                status=`cmd_show_status $sg `
            else
                status="Missing-sg_name"
            fi
            [ -z "$status" ] && status="no_status"

            local msg="status[$sloc/$sg]=\"$status\""
            log_details "array create stage[$array_stage], $msg"

            # list of "termination" status for member disks and target array
            # Refer to "Disk array states" section in ipr doc.
            local status_exp
            if [ $array_stage -eq 0 -o $array_stage -eq 1 ]; then
                # 1) If a physical disk fail to be formated as IPR AF, its status will be "Format Required" and be "Failed" later.
                # Should we handle this?
                status_exp="Active|Failed|Offline|Missing|R/W Protected|Format Required"

                # I saw phy disk in following state which not in the doc.
                status_exp+="|Optimized|Non-Optimized"

                # my faked status for disk
                status_exp+="|Missing-sg_name"
            else
                # Note:
                # - Array state "Missing" is not a terminate state, exclude it by adding "#" as prefix
                status_exp="Optimized|Active|Non-Optimized|Degraded|#Missing|Offline|Failed|R/W Protected|Rebuilding"
            fi
            # if not "termination" status, we record it and wait for next cycle.
            if ! echo "$status" | grep -i -sq -E "$status_exp"; then
                # aggregate status messages.
                [ -n "$status_lines" ] && status_lines+=","
                status_lines+="$msg"
                rc=1
            fi
        done
        # if all terminated, we fall back to check staus of array itself
        if [ $rc -eq 0 ]; then
            if [ $array_stage -eq 2 ]; then
                log_info "Create array($array_sg=$array_sloc) successfully on top of disks: $member_slocs."
                break
            else # end if stage == 2
                if [ $array_stage -eq 0 ]; then
                    # since all disks were in termination states, let's move to stage 2
                    # to wait for array to be created and start rebuild.
                    #
                    # It's possible that such array could not be created for some critical
                    # error, such in-sufficient disks for required raid level.
                    #
                    # Though there was smart way to handle this, here I only use a simple way
                    # by waiting for a fixed time(1 minute) before return error.
                    log_details "array create stage[1], entered, after finished formatting member disks $slocs"
                    lines=`{ cmd_show_config; cmd_show_alt_config; } | $sed -e 's/^/[0-->1] >> /g'`
                    log_lines details "$lines"
                    ((array_stage=1))
                    ((stage_1_last_tick_count=i-stage_1_last_tick_count))
                fi
                # IPR might had problem if some disks failed to response.
                # These disks will disapear in the system with sloc and sg were all gone.
                # in these case, we drop these disks in the operation and abort the operation
                # if all member disks were gone.
                local slocs_new=""
                local __tmp_item=""
                for __tmp_item in $slocs
                do
                    if sloc_exist $__tmp_item; then
                        [ -n "$slocs_new" ] && slocs_new+=" "
                        slocs_new+="$__tmp_item"
                    else
                        log_error "$i: disk $__tmp_item disapear..."
                    fi
                done
                if [ -n "$slocs_new" ]; then
                    slocs="$slocs_new"
                else
                    global_rc=1
                    log_error "All array member disks, $member_slocs, were gone, abort!"
                    break
                fi
                # get array owner of each member sg
                local array_slocs=`for __tmp_item in $slocs
                do
                    local sg_owner_sloc=$(get_ipr_array_by_member $__tmp_item)
                    [ -n "$sg_owner_sloc" ] && echo "$sg_owner_sloc"
                done \
                | $sort -u -V `
                # we only allow one unique array owns all member disks.
                # or, we we need to break out?
                local array_cnt=`echo "$array_slocs" | $awk 'END {print NF; exit 0;}'`
                log_details "All \"$array_cnt\" array(es), \"$array_slocs\", contain target disks, \"$slocs\"." >&2
                if [ $array_cnt -gt 0 ]; then

                    array_sloc=`echo "$array_slocs" | $awk '{print $1}'`
                    local array_sg=`convert_sloc_to_sg $array_sloc | $awk -F= '{print $2}'`
                    if [ -n "$array_sg" ]; then
                        log_details "array create stage[2], entered, after array show up with sg name $array_sg and sloc $array_sloc" >&2
                        array_stage=2
                        slocs="$array_sloc"
                    else
                        log_warn "$i: >> Why new array $array_sloc($array_slocs) does not have a valid sg name?" >&2
                        log_warn "$i: >> Let wait for a while to see if it could show up or not until tick count equals to $stage_1_last_tick_count" >&2
                    fi
                elif [ $array_stage -eq 1 -a $i -lt $stage_1_last_tick_count ]; then
                    log_error "Fail to wait array($array_slocs) showes up after its member disks, $member_slocs, were finished formatting." >&2
                    # log for debug
                    lines=`{
                        echo "$i: Current IPR configuration";
                        {
                            cmd_show_config;
                            cmd_show_alt_config;
                        } | $sed -e 's/^/'$i': >> /g';
                    }`
                    log_lines details "$lines"

                    global_rc=2
                    break
                else
                    # see if we need to issue the raid-create again.
                    # for some reason, previous raid-create might be terminated unexpectedly with disks
                    # actually had been formated for raid and ready to be used. So, we try to issue
                    # the raid-create command again to see if that would help. This usually happened when
                    # some failures when formtting physical disks to ipr AF. In that situation, iprconfig
                    # might abort without have array created after it had actually formatted those disks.
                    local pid_create_still_alive=0
                    local pid_create_cur=`getpstree $pid_create`
                    local pid_cur
                    for pid_cur in $pid_create_cur
                    do
                        if $kill -0 $pid_cur 2>/dev/null; then
                            pid_create_still_alive=1
                            break
                        fi
                    done
                    if [ $pid_create_still_alive -eq 0 ]; then
                        log_info "$i: disks $slocs had finished formatting, but array was not shown up, and iprconfig process($pid_create) had terminated."

                        log_details "$i: Last several raid-create output:"
                        lines=`$tail -n10 $ftmp | $sed -e 's/^/'$i': >> /g'`
                        log_lines details "$lines"
                        # launch the iprconfig worker process again
                        member_sgs=`convert_sloc_to_sg $member_slocs | $awk -F= '{print $2}'`
                        if [ -n "$member_sgs" ]; then
                            $iprconfig -c raid-create $iprconfig_args $member_sgs >$ftmp 2>&1 &
                            pid_create=$!

                            log_info "$i: Restart \"iprconfig -c raid-create $iprconfig_args $member_sgs\" with new pid $pid_create." >&2

                            # log for debug
                            local line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
                            if [ -n "$line" ]; then
                                log_details "ps1.2: $line" >&2
                            else
                                log_error "ps1.2: new iprconfig($pid_create) was gone" >&2
                            fi
                        else
                            log_warn "$i: Fail to restart iprconfig since $member_slocs could not be mapped to sg." >&2
                            log_warn "$i: Let try again next loop until tick count equals to $stage_1_last_tick_count" >&2
                        fi
                    else
                        log_warn "$i: Wait for array($array_slocs) showes up after disk format were finished on $slocs until tick count equals to $stage_1_last_tick_count" >&2
                        log_details "$i: Last several raid-create output:" >&2
                        lines=`$tail -n2 $ftmp | $sed -e 's/^/'$i': >> /g'`
                        log_info debug "$lines" >&2
                    fi
                fi
            fi # end else if stage == 2
        fi # end if $rc eq 0

        # log status for monitor
        # log for debug
        if [ -n "$status_lines" ]; then
            log_status "Creating IPR array with last info at time \""`date "+%Y-%m-%d %H:%M:%S"`"\": $status_lines" >&2
            lines=`echo "$status_lines" | $sed -e 's/,/\n/g' | $sed -e "s/^/$i: >> /g"`
            log_lines details "$lines" >&2
        fi

        $sleep $monInt
        [ $monCnt -gt 0 ] && ((i-=1))
    done

    # log for debug
    line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
    [ -n "$line" ] && log_details "ps2: $line"
    if $kill -0 $pid_create 2>/dev/null; then
        # If succ out, array status should in "rebuilding" state while iprconfig is still running
        # until the rebuild had finished. But we do not need to wait for its termination since the
        # block device is actually ready to use though performance might be downgrade.
        # so, by default, we will just kill the "iprconfig" command(? not sure if that would affect
        # the rebuild or not) and let raid controller do its job at backend.
        # in order to wait for the whole rebuilding, set IPR_REBUILD_WAIT=true explicitly.
        if [ "x$IPR_REBUILD_WAIT" = "xtrue" -a $i -gt 0 ]; then
            log_info "Wait for ipr raid creation command to finish. pid=$pid_create" >&2
            wait $pid_create
        else
            log_warn "Terminate ipr raid creation process enforcely. pid=$pid_create" >&2
            $kill -TERM $pid_create 2>/dev/null
            $sleep 1
            $kill -0 $pid_create 2>/dev/null \
            && $kill -KILL $pid_create 2>/dev/null
        fi
    fi
    # log for debug
    line=`$ps -f -p $pid_create | $awk -v pid=$pid_create '$2==pid{print $0}' | $head -n1`
    [ -n "$line" ] && log_details "ps3: $line" >&2

    log_details "Last several raid-create output:" >&2
    lines=`$tail -n10 $ftmp | sed -e 's/^/[last]: >> /g'`
    log_lines details "$lines" >&2
    rm -f $ftmp
    # Log "failed" status to PCM/xCAT with reason and instruction for future investigation.
    if [ $global_rc -ne 0 ]; then
        # set node "status" to "failed" to indicate that the installation was failed.
        fstatus_app=/tmp/baz.py
        if [ -x "$fstatus_app" ]; then
            $fstatus_app "installstatus failed" >&2
        fi

        # set node state with brief reason why installation failed and future instructions.
        # end user can get this message by running "nodestate" command to target nodes.
        log_status "Fail to create IPR array with return code \"$global_rc\"! Refer to more details in syslog by checking \"rhPreDbg\" flag..." >&2
    fi
    return $global_rc
}

###########################################################################################
#
# main process to handle create raid array when using pci_id or pci_slot_name
#
##########################################################################################
function handle_create_raid_array {
    local striple_size=$1
    local raidlevel=$2
    local disknum=$3
    local pcilocs=$4
    local disknames=$5

    if [ "x$pcilocs" != "xnull" ]; then

        # Find out all member disks in target IOAs
        # pciloc=sloc,sloc,sloc...
        slocs_grps=`get_raid_create_devices_by_pciloc $pcilocs `
        if [ -z "$slocs_grps" ]; then
            log_error "Could not find any disk on target pciloc ${pcilocs}!" >&2
            return 1
        else
            log_info "Disks in target IPR raid adapter: \"$slocs_grps\"" >&2
        fi

        # find the required member disks
        slocs=""
        validslocs=""
        for item in $slocs_grps
        do
            pciloc=`echo "$item" | $awk -F= '{print $1}'`
            slocs_grp=`echo "$item" | $awk -F= '{print $2;}'`
            # order disks by its resource path
            slocs_grp=`sort_devices_by_resource_path $slocs_grp`

            slocs=`echo "$slocs_grp" | sed 's/,/ /g'`
            if [ -n "$slocs" ]; then
                break
            else
                log_warn "Could not find enough disk on target adapter $pciloc which only has following disks ${slocs_grp}!" >&2
            fi
        done
        if [ -z "$slocs" ]; then
            log_error "Could not find enough disk on target adapter ${pciloc}!" >&2
            return 1
        else
            log_info "Find member disks $slocs on target IOA $pciloc" >&2
            slocs=`echo "$slocs" | $sed -e 's/,/ /g'`
            # if it is the first create_raid, do nothing, else:
            # delete used disks belong to existed raid array
            validslocs=`delete_disks_belong_array_from_slocs $slocs`
            if [ -z "$validslocs" ]; then
                log_error "Could not find enough unused disk on target adapter $pciloc."
                return 1
            else
                slocs="$validslocs"
                log_info "Find unused member disks $slocs on target IOA $pciloc" >&2
            fi

        fi
        # handle disk_num
        # if disknum is all, do nothing
        # if disknum is a number, then it should be compared with actual_disknum
        actual_disknum=`echo "$slocs"| $awk '{print NF}'`
        slocs=`compare_disknum_actualnum $disknum $actual_disknum $slocs`

    else
        # find the slocs for disks in disknames
        disklist=`echo $disknames|sed 's/#/ /g'`
        slocs_grp=`convert_sg_to_sloc $disklist`
        slocs=`echo "$slocs_grp"|awk -F= '{print $2}'`
    fi
    # check if target disks were already in a reusable array.
    # if not, create one by calling block command "create_ipr_array".
    # after than, check again to confirm if the target array was ready.
    array_sloc=""
    ((__i=0))
    while [ $__i -lt 2 ];
    do
        # get disk --> array association
        # format 0:0:0:0=sg0
        declare array_slocs_lines=`for sloc in $slocs
            do
                line=$(get_ipr_array_by_member $sloc)
                echo "$sloc=$line"
            done`

        # extract arrays from associations
        # get sg0 sg1
        declare array_slocs=`echo "$array_slocs_lines" \
                             | $awk -F= '{print $2}' | $sort -u`

        # count the number of arrays on target disks
        # which might include "blank" array
        declare n_arrays_lines=`echo "$array_slocs" | $awk 'END {print NR}'`
        # count the number of arrays on target disks, again
        # w/o "blank" array
        declare n_arrays=`echo "$array_slocs" | grep -v "^ *$" | $awk 'END {print NR}'`
        # get the sg names for owner arrays.
        declare array_sgs=""
        if [ $n_arrays -gt 0 ]; then
             array_sgs=`convert_sloc_to_sg $array_slocs \
                        | $awk -F= '{print $2}' `
        fi
        # (cond_1): if ALL target disks are in ONE array, reuse it.
        if [ $n_arrays_lines -eq 1 -a $n_arrays -eq 1 ]; then
            if [ $__i -eq 0 ]; then
                log_info "Target disks, $slocs, were already in an array, $array_slocs." >&2
            else
                log_info "An array, $array_slocs, had been created to include target disks, $slocs" >&2
            fi
            lines=`{   cmd_show_details $array_sgs;
                       cmd_show_status $array_sgs;
                    } | $sed -e 's/^/>> /g'`
            log_lines details "$lines"
            array_sloc="$array_slocs"
            break
        fi
        # (cond_2): delete arrays which owe target disks first before any future operations.
        #           NOTE: we redo the loop after "delete" command was issued to ipr
        if [ $n_arrays -gt 0 ]; then
            log_info "Target disks, $slocs, were owned by multiple arrays, \"$array_slocs\"($array_sgs). Delete them first before creating a new array on top of them." >&2
            delete_ipr_array enforce=1 tryCnt=360 tryInt=60 -- $array_sgs
            continue
        fi
        # (cond_3): create an array to include all target disks
        #           we assume all these disks are ready to be included into an array.
        if [ $__i -eq 0 ]; then
            declare sgs=`convert_sloc_to_sg $slocs \
                         | $awk -F= '{print $2}' | $sed -e 's/ /,/g'`
            # Times to wait for the raid creation finish
            # The lager capacity of member disks, longer times to wait.
            # :unit minutes
            monTime=240
            # Interval of minitoring raid creation, status check
            # :unit seconds
            monInt=30
            monCnt=`$awk -v monInt=$monInt -v monTime=$monTime 'END{print 60*monTime/monInt}' </dev/null`
            log_status "Creating RAID $raidlevel array on top of disks \""`echo "$slocs" | $sed -e 's/ /,/g'`"\"/\"$sgs\"..." >&2
            # handle striple_size
            if [ "x$striple_size" == "xdefault" ]; then
                create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel
            elif [ -z $striple_size ]; then
                create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel
            else
                create_ipr_array $monCnt $monInt "$sgs" -r $raidlevel -s $striple_size
            fi
        fi
    ((__i+=1))
    done
    lines=`cmd_show_config`
    if [ $__i -lt 2 -a -n "$array_sloc" ]; then
        array_sd=`echo "$lines" \
                | grep '^sd[a-z]\+[ ]\+.*\/'$array_sloc'[ ]\+.*Disk Array' \
                | $awk '{print $1}'`
        if [ -n "$array_sd" ]; then
            log_info "array_dev=$array_sd"
            return 0
        else
            log_error "Fail to get scsi device name for new array $array_sloc" >&2
            echo "$lines" | $sed -e 's/^/>> /g' | log_lines error >&2
            return 1
        fi
    else
        log_error "Fail to create array on top of disks $slocs" >&2
        echo "$lines" | $sed -e 's/^/>> /g' | log_lines error >&2
        return 1
    fi

}

############################################################################
#
# handle disk_num
# if disknum is all, do nothing
# if disknum is a number, then it should be compared with actual_disknum
# return <disknum> of slocs
#
##############################################################################
function compare_disknum_actualnum {
    local disknum=$1
    local actualnum=$2
    shift;
    shift;
    local slocs=$*
    local actualslocs
    if [ "x$disknum" != "xall" ]; then
        if [ $actualnum -ge $disknum ]; then
            actualslocs=`echo "$slocs"| \
                         awk -v anum=$disknum '{ for ( i=1; i<=anum; i++ ){print $i}}'`
            echo $actualslocs

        else
            log_error "Could not find enough disk on target adapter ${pciloc}!" >&2
        fi
    else
        echo "$slocs"
    fi

}

############################################################################
#
# Delete raid array
#
# input format : enforce=1 tryCnt=360 tryInt=60 -- arrays_name list
#
##########################################################################
function delete_ipr_array {
    # default to NO if enforcely wait for rebuilding array
    local enforce=0
    # default to wait 6 hours for array rebuilding to finish
    local tryCnt=360
    # monitor rebuilding status every 60 seconds
    local tryInt=60

    # parser input arguments
    while [ -n "$1" ];
    do
        local key=`echo "$1" | $cut -s -d= -f1`
        if [ "$key" = "tryCnt" ] || \
           [ "$key" = "tryInt" ] || \
           [ "$key" = "enforce" ]; then
            local val=`echo "$1" | $cut -s -d= -f2-`
            eval "$key=\"$val\""
        elif [ "$1" = "--" ]; then
            shift
            arrays="$*"
            break
        fi
        shift
    done
    # should not with empty array list, if yes, error with rc=1
    if [ -z "$arrays" ]; then
        log_error "No array to delete, assume success!" >&2
        return 0
    fi
    local array_slocs=`convert_sg_to_sloc $arrays | awk -F= '{print $2}'`
    # if none of arrays were valid, error with rc=1
    if [ -z "$array_slocs" ]; then
        log_error "Fail to map arrayes, $arrays, to scsi location before deletion!"
        return 1
    fi
    local ioas=`get_ipr_ioas`

    local i=1
    while [ $i -le $tryCnt ];
    do
        # verify if all target arrays are still exists
        local tmp_array_slocs=`for _tmp_sloc in $array_slocs
        do
            is_ipr_array $_tmp_sloc && echo "$_tmp_sloc"
        done `
        if [ "$tmp_array_slocs" != "$array_slocs" ]; then
            array_slocs="$tmp_array_slocs"
            log_details "Renew available target arrays which will be deleting, \"$array_slocs\"" >&2
        fi
        if [ -z "$array_slocs" ]; then
            log_details "Round $i: All target arrays were gone!" >&2
            break
        fi
        # find out sg devs for all deletable arrays
        local array_slocs_deletable=`for ioa in $ioas
        do
            # [root@dn01-dat ~]# iprconfig -c query-raid-delete sg5
            # Name   PCI/SCSI Location         Description               Status
            # ------ ------------------------- ------------------------- -----------------
            # sdk    0003:70:00.0/0:2:0:0       RAID 0 Disk Array         Optimized
            # sdl    0003:70:00.0/0:2:1:0       RAID 0 Disk Array         Optimized
            cmd_query_raid_delete $ioa \
            | grep "Disk Array" \
            | cut_sloc_from_iprconfig_line
        done`
        if [ -z "$array_slocs_deletable" -a $enforce -eq 0 ]; then
            log_details "Round $i: No deletable array, nothing to do and finish." >&2
            break
        fi
        # log for debug
        {
            echo "$array_slocs_deletable" | $sed -e 's/^/Round '$i': Deletable array >> /g'
            cmd_show_config | $sed -e 's/^/Round '$i': ipr info before delete >> /g'
        } | log_lines details >&2
        log_info "Round $i: delete_ipr_array, \"$array_slocs\"" >&2

        # if requested array is a deletable array, mark it to be deleted in this round.
        local array_slocs_verified=`for array_sloc in $array_slocs
        do
            if echo "$array_slocs_deletable" | grep -sq "^$array_sloc$"; then
                echo $array_sloc
            else
                log_warn "Round $i: Array $array_sloc is un-deletable at present." >&2
            fi
        done`
        if [ -n "$array_slocs_verified" ]; then
            log_details "Round $i: Verified array slocs for deletion are \"$array_slocs_verified\"." >&2
        else
            log_info "Round $i: All remaining target arrayes are un-deletable now." >&2
            if [ $enforce -eq 1 ]; then
                log_info "Round $i: Wait for these un-deletable arrayes deletable with tryCnt=$tryCnt, tryInt=$tryInt." >&2
                wait_for_ipr_device_status \
                    tryCnt=$tryCnt tryInt=$tryInt cmd="grep -sq -E 'Optimized'" \
                    -- \
                    `echo "$array_slocs" | $sed -e "s/ /,/g"`
                # if wait succ, let's try to delete again
                if [ $? -eq 0 ]; then
                    ((i+=1))
                    continue
                else
                    break
                fi
            fi
        fi

        # issue delete command
        local dcnt=0
        for array_sloc in $array_slocs_verified
        do
            local array=`convert_sloc_to_sg $array_sloc | awk -F= '{print $2}'`
            [ -z "$array" ] && continue

            log_details "Round $i: Delete $dcnt array $array=$array_sloc" >&2
            cmd__raid_delete $array
            ((dcnt+=1))
        done

        # break out the loop forcely if nothing deleted.
        [ $dcnt -eq 0 ] && break

        # wait for next round
        $sleep $tryInt
        ((i+=1))
    done
    test $i -lt $tryCnt -a -z "$array_slocs"
    return $?
}

#################################################
#
# sort devices by resource path,
#
# input: dev list, "sg0,..sgn"
#
##################################################
function sort_devices_by_resource_path {
    local sgs="$*"

    local del=","
    if [ $# -gt 1 ]; then
        del=" "
    else
        sgs=`echo "$sgs" | $sed -e "s/,/ /g"`
    fi

    # [root@dn01-dat ~]# iprconfig -c show-alt-config
    # Name   Resource Path/Address      Vendor   Product ID       Status
    # ------ -------------------------- -------- ---------------- -----------------
    # sg15   FE                         IBM      57D7001SISIOA    Operational
    # sg10   00-0C                      IBM      HUC101212CSS600  Active
    # sg11   00-0D                      IBM      HUC101212CSS600  Active
    local lines=`$iprconfig -c show-alt-config | grep "^sg"`
    local item
    local lines_out=`for item in $sgs
    do
        local sg=$(convert_sloc_to_sg $item | $awk -F= '{print $2;}')
        local rp=$(echo "$lines" | $awk -vsg=$sg '$1 == sg { print $2; }')
        echo "$item $sg $rp"
    done \
    | $sort -t' ' -k3`
    # log for debug
    {
        echo "sort devices \"$sgs\" by IPR resource path:"
        echo "$lines_out" | $sed -e 's/^/sort by rp output: >> /g'
    } | log_lines details >&2

    # deliver ordered result
    lines_out=`echo "$lines_out" | $awk '{print $1;}' `
    if [ "$del" = "," ]; then
        echo "$lines_out" | $sed -e "s/ /,/g"
    fi
    return 0
}


######################################################
#
# log lines
#
# input : details or status or error
#
######################################################
function log_lines {
    local pcnt=$#
    local __level=$1
    shift
    local cmd=log_${__level}

    local hit=0
    local OIFS=$IFS
    local NIFS=$'\n'
    IFS=$NIFS
    local __msg
    for __msg in $*
    do
        IFS=$OIFS
        $cmd "$__msg"
        hit=1
        IFS=$NIFS
    done
    IFS=$OIFS

    [ $hit -eq 0 -a $pcnt -le 1 ] && \
    while read __msg;
    do
        $cmd "$__msg"
    done
}

######################################################
#
# error information
#
######################################################
function log_error {
    local __msg="$*"
    $log_print_cmd $log_print_arg "[E]: $__msg" >&2
    return 0
}

######################################################
#
# warning information
#
# input : message
#
######################################################
function log_warn {
    local __msg="$*"
    $log_print_cmd $log_print_arg "[W]: $__msg" >&2
    return 0
}

######################################################
#
# log information
#
# input : message
#
######################################################
function log_info {
    local __msg="$*"
    $log_print_cmd $log_print_arg "[I]: $__msg" >&2
    return 0
}

######################################################
#
# more details information
#
######################################################
function log_details {
    local __msg="$*"
    $log_print_cmd $log_print_arg "[I]: $__msg" >&2
    return 0
}

####################################################
#
# print command status
#
###################################################
__my_log_status=
function log_status {
    local __msg="$*"
    $log_print_cmd $log_print_arg "[S]: $__msg" >&2

    # call my_log_status hook to triger more processing for status messages.
    if [ -n "$__my_log_status" ]; then
        $__my_log_status "$__msg"
    fi
    return 0
}

####################################################
#
# print output and logfile
#
###################################################
function log_print_default {
    echo $*
    echo $* >> $log_file
}

####################################################
#
# handle command
#
###################################################
function set_log_print {
    local cmd="$1"
    if [ -z "$cmd" ]; then
        return 1
    fi
    shift
    local args="$*"

    eval "log_print_cmd=\"$cmd\""
    eval "log_print_arg=\"$args\""
}

# let log work
[ -z "$log_print" ] && set_log_print "log_print_default"