diff --git a/perl-xCAT/xCAT/Schema.pm b/perl-xCAT/xCAT/Schema.pm
index 6f161c83a..94f0c1933 100755
--- a/perl-xCAT/xCAT/Schema.pm
+++ b/perl-xCAT/xCAT/Schema.pm
@@ -1437,7 +1437,7 @@ firmware => {
 },
 
 nics => {
-        cols => [qw(node nicips  nichostnamesuffixes nichostnameprefixes nictypes niccustomscripts nicnetworks nicaliases nicextraparams comments disable)], 
+        cols => [qw(node nicips  nichostnamesuffixes nichostnameprefixes nictypes niccustomscripts nicnetworks nicaliases nicextraparams nicdevice comments disable)], 
         keys => [qw(node)],
         tablespace =>'XCATTBS16K',
         table_desc => 'Stores NIC details.',
@@ -1480,6 +1480,7 @@ nics => {
                 If multiple ip addresses are associated with each NIC:
                     <nic1>!<param1=value1 param2=value2>|<param3=value3>,<nic2>!<param4=value4 param5=value5>|<param6=value6>, for example, eth0!MTU=1500|MTU=1460,ib0!MTU=65520 CONNECTED_MODE=yes.
             The xCAT object definition commands support to use nicextraparams.<nicname> as the sub attributes.',
+            nicdevice => 'Comma-separated list of NIC device per NIC. <nic1>!<dev1>,<nic2>!<dev2>, e.g. eth0!bond0,bond0!br0. The xCAT object definition commands support to use nicdevice.<nicname> as the sub attributes.',
             comments => 'Any user-written notes.',
             disable => "Set to 'yes' or '1' to comment out this row.",
         },
@@ -2410,6 +2411,10 @@ my @nodeattrs = (
 				tabentry => 'nics.nicextraparams',
 				access_tabentry => 'nics.node=attr:node',
 		},
+                {attr_name => 'nicdevice',
+                                tabentry => 'nics.nicdevice',
+                                access_tabentry => 'nics.node=attr:node',
+                },
 #######################
 #  prodkey table     #
 ######################
diff --git a/xCAT-server/lib/xcat/plugins/DBobjectdefs.pm b/xCAT-server/lib/xcat/plugins/DBobjectdefs.pm
index 762712123..5bd4d37bd 100755
--- a/xCAT-server/lib/xcat/plugins/DBobjectdefs.pm
+++ b/xCAT-server/lib/xcat/plugins/DBobjectdefs.pm
@@ -575,7 +575,7 @@ sub processArgs
 
     # --nics is the equivalent of -i nicips,nichostnamesuffixes...
     if ($::opt_nics) {
-        $::opt_i="nicips,nichostnamesuffixes,nichostnameprefixes,nictypes,niccustomscripts,nicnetworks,nicaliases,nicextraparams";
+        $::opt_i="nicips,nichostnamesuffixes,nichostnameprefixes,nictypes,niccustomscripts,nicnetworks,nicaliases,nicextraparams,nicdevice";
     }
 
     # -i and -s cannot be used together
diff --git a/xCAT/postscripts/confignetwork b/xCAT/postscripts/confignetwork
new file mode 100755
index 000000000..f45f45621
--- /dev/null
+++ b/xCAT/postscripts/confignetwork
@@ -0,0 +1,358 @@
+#!/bin/bash
+#-------------------------------------------------------------------------------
+#=head1  confignetwork
+#=head2  Used on Redhat only. Configure bond/vlan/linux bridge/ on the nodes
+#
+#  You can configure nicdevice,nictypes,nicips,nicnetworks in nics table
+#  Then you can run the following commands on MN:
+#       updatenode noderange confignetwork
+#
+#  
+#
+#=cut
+#-------------------------------------------------------------------------------
+
+
+# load library 
+if [ "$(uname -s|tr 'A-Z' 'a-z')" = "linux" ];then
+   str_dir_name=`dirname $0`
+   . $str_dir_name/xcatlib.sh
+   . $str_dir_name/nicsutils
+fi
+
+
+######################################################################
+#
+# OS support
+# Check OS version and get the directory of network configuration file
+#
+#####################################################################
+str_os_type=`uname | tr 'A-Z' 'a-z'`
+nwdir=''
+isRedhat=0
+isDebian=0
+isSLES=0
+if [ "$str_os_type" = "linux" ];then
+    str_temp=`echo $OSVER | grep -E '(sles|suse)'`
+    if [ -f "/etc/redhat-release" ];then
+        isRedhat=1
+        nwdir="/etc/sysconfig/network-scripts"
+    elif [ -f "/etc/SuSE-release" -o -n "$str_temp" ];then
+        isSLES=1
+        nwdir="/etc/sysconfig/network"
+        log_error "Only supports RHEL" "error"
+    elif [ -f "/etc/debian_version" ];then
+        nwdir="/etc/network/interfaces.d"
+        isDebian=1
+    else
+        log_error "Only supports RHEL" "error"
+        exit -1
+    fi
+else
+    log_error "Does NOT support non-Linux Operating System" "error"
+    exit -1
+fi
+
+#############################################################
+#
+# parser nicdevices from mypostscripts
+#
+# input : NICDEVICE from mypostsctipts
+#
+#################################################################
+function splitconfig(){
+    if [ ! "$1" ];then
+        return
+    fi
+    old_ifs=$IFS
+    IFS=$','
+    array_conf_temp=($1)
+    IFS=$old_ifs
+    i=0
+    while [ $i -lt ${#array_conf_temp[@]} ]
+    do
+        token="${array_conf_temp[$i]}"
+        D=
+        if echo "$token" | grep "!" >/dev/null; then
+            D="!"
+        else
+            D=":"
+        fi
+        key=`echo "$token" | cut -d"$D" -f 1`
+        str_temp_value=`echo "$token" | cut -d"$D" -f 2`
+
+        str_temp=$(hashget hash_defined_nics $key)
+        if [ -n "$str_temp" ];then
+           str_temp=$str_temp",${str_temp_value}"
+        else
+           str_temp="$str_temp_value"
+           str_all_nics=$str_all_nics"$key "
+        fi
+        hashset hash_defined_nics $key "$str_temp"
+        i=$((i+1))
+    done
+}
+
+
+
+#############################################################
+#
+# parser nictypes from mypostscripts
+# input : NICTYPES from mypostscripts
+#
+#################################################################
+function splittypes {
+    if [ ! "$1" ];then
+        return
+    fi
+    old_ifs=$IFS
+    IFS=$','
+    array_dev_temp=($1)
+    IFS=$old_ifs
+    i=0
+    while [ $i -lt ${#array_dev_temp[@]} ]
+    do
+        token="${array_dev_temp[$i]}"
+        D=
+        if echo "$token" | grep "!" >/dev/null; then
+            D="!"
+        else
+            D=":"
+        fi
+        key=`echo "$token" | cut -d"$D" -f 1`
+        str_temp_value=`echo "$token" | cut -d"$D" -f 2`
+
+        hashset hash_new_nics $key "$str_temp_value"
+        i=$((i+1))
+    done
+
+}
+
+
+################################################################
+#
+# find nic type
+# input : nic
+#
+###############################################################
+function find_nic_type {
+    if [ ! "$1" ];then
+        return
+    fi
+    nic=($1)
+    echo $(hashget hash_new_nics $nic)
+}
+
+################################################################
+#
+# parser nicdevices into nic device list
+# input : nicdevice hash
+#
+##############################################################
+function parser_nicdevices {
+
+    array_nics_temp=$*
+    for key in ${array_nics_temp[@]}
+    do
+         nic_dev=`echo $(hashget hash_defined_nics $key)|awk -F, '{print $1}'`
+         echo "$key $nic_dev"
+
+    done
+
+}
+
+#####################################################################
+#
+# nics list is nic and its device
+# sort the nicdevice raw logic according to nicdevice type
+# nicdevice type order is ethernet bond and vlan.
+# for example, after sorted, the order should be like:
+#     nicX  ethX
+#     nicY  bondU
+#     nicZ  vlanW
+# input : nic and nicdevice pair
+#     for example: nic1 nic2
+#                  nic3 nic4
+#
+#####################################################################
+function sort_nics_device_order {
+    nics_list=$*
+    
+    eth_slot=""
+    bond_slot=""
+    vlan_slot=""
+    num=1
+    max=`echo "$nics_list"|wc -l`
+    ((max+=1))
+    while [ $num -lt $max ];
+    do
+        right_nic=`echo "$nics_list" |sed -n "${num}p"|awk '{print $2}'`
+        if echo "$right_nic"|grep "@" >/dev/null; then
+            right_temp_nic=`echo $right_nic|awk -F@ '{print $1}'`
+        else
+            right_temp_nic=$right_nic
+        fi
+        right_type=`find_nic_type "$right_temp_nic"`
+        if [ x"$right_type" == "xethernet" ]; then
+            if [ x"$eth_slot" == x ]; then
+                eth_slot=$num
+            else
+                eth_slot=$eth_slot" "$num
+            fi
+        elif [ x"$right_type" == "xbond" ]; then
+            if [ x"$bond_slot" == x ]; then
+                bond_slot=$num
+            else
+                bond_slot=$bond_slot" "$num
+            fi
+        elif [ x"$right_type" == "xvlan" ]; then
+            if [ x"$vlan_slot" == x ]; then
+                vlan_slot=$num
+            else
+                vlan_slot=$vlan_slot" "$num
+            fi
+
+        fi
+        ((num+=1))
+    done
+    new_order=$eth_slot" "$bond_slot" "$vlan_slot
+    new_order_list=""
+    if [ -n "$new_order" ]; then
+    new_order_list=`for i in $new_order
+        do
+            echo "$nics_list" |sed -n "${i}p"
+        done`
+    fi
+    echo "$new_order_list"
+}
+
+#####################################################################################
+#
+# framework to configure bond/vlan/bridge
+#
+# input : orderd nic and its device
+# for example: nicx nicy
+#
+###################################################################################
+function parse_nics {
+    nics_pair=$*
+   
+    num=1
+    max=`echo "$nics_pair"|wc -l`
+    right_temp_nic=""
+    right_nic_t=""
+    line_num=""
+    ((max+=1))
+    #configure nic and its device pair one by one
+    while [ $num -lt $max ];
+    do
+        #"|" has problem in "ethX|ethY"
+        #replace "|" with "@" 
+        right_nic=`echo "$nics_pair" |sed -n "${num}p"|awk '{print $2}'`
+        if echo "$right_nic"|grep "@" >/dev/null; then
+            right_temp_nic=`echo $right_nic|awk -F@ '{print $1}'`
+            right_nic_t=`echo $right_nic|sed 's/@/%2c/g'`
+        else
+            right_temp_nic=$right_nic
+        fi
+        right_type=`find_nic_type "$right_temp_nic"`
+        left_nic=`echo "$nics_pair" |sed -n "${num}p"|awk '{print $1}'`
+        left_type=`find_nic_type "$left_nic"`
+        echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+        nic_pair=`echo "$nics_pair" |sed -n "${num}p"`
+        echo "configure nic and its device : $nic_pair"
+        #configure bridge
+        #linux bridge type is bridge
+        #openvswitch bridge type is bridge_ovs
+        if [ x"$right_type" == "xethernet" -a x"$left_type" == "xbridge" ] || \
+           [ x"$right_type" == "xethernet" -a x"$left_type" == "xbridge_ovs" ] || \
+           [ x"$right_type" == "xvlan" -a x"$left_type" == "xbridge" ] || \
+           [ x"$right_type" == "xvlan" -a x"$left_type" == "xbridge_ovs" ]; then
+            create_bridge_interface ifname=$left_nic _brtype=$left_type _port=$right_nic _pretype=$right_type
+
+        #configure vlan 
+        elif [ x"$right_type" == "xethernet" -a x"$left_type" == "xvlan" ] || \
+             [ x"$right_type" == "xbond" -a x"$left_type" == "xvlan" ]; then
+            if echo "$left_nic" | grep -sq ".*\.[0-9]\+"; then
+                vlanid=`echo "$left_nic" | $cut -s -d. -f2-`
+                vlanname=`echo "$left_nic" | $cut -s -d. -f1`
+            elif echo "$left_nic" | grep -sq ".*vla\?n\?[0-9]\+"; then
+                vlanid=`echo "$left_nic" | $sed -e 's/^\(.*\)vla\?n\?\([0-9]\+\)$/\2/'`
+                vlanname=`echo "$left_nic" | $sed -e 's/^\(.*\)vla\?n\?\([0-9]\+\)$/\1/'`
+            fi
+            create_vlan_interface ifname=$vlanname vlanid=$vlanid
+ 
+        #configure bond 
+        elif [ x"$right_type" == "xethernet" -a x"$left_type" == "xbond" ]; then
+            config_bond_vlan ifname=$left_nic mports=$right_nic_t ifcmd=$left_type
+       
+        #configure bridge/bridge_ovs     
+        elif [ x"$right_type" == "xbond" -a x"$left_type" == "xbridge" ] || \
+             [ x"$right_type" == "xbond" -a x"$left_type" == "xbridge_ovs" ]; then
+            create_bridge_interface ifname=${left_nic} _brtype=$left_type _port=${right_nic} _pretype=$right_type _bonding_opts="mode=802.3ad miimon=100"
+        else
+            log_error "Error : please check nictypes for $nic_pair."
+        fi 
+
+        ((num+=1))
+    done
+}
+
+############################################################################
+#
+# disable NetworkManager and start network
+#
+###########################################################################
+function enable_network_service {
+    checkservicestatus NetworkManager > /dev/null
+    if [ $? -eq 0 ]; then
+        log_info "NetworkManager is active. start to stop it ..."
+        stopservice NetworkManager | log_lines debug
+        disableservice NetworkManager
+        enableservice network
+        startservice network
+    else
+        log_info "NetworkManager is inactive."
+    fi 
+
+}
+
+############################################################################
+#
+# Main process
+#
+############################################################################
+
+#replace | with @, for example, eth1|eth2  ---->   eth1@eth2
+nicdevice=`echo "$NICDEVICE" | sed 's/|/@/g'`
+
+#make hash for nicdevice
+splitconfig "$nicdevice"
+
+#make hash for nic and its type
+splittypes "$NICTYPES"
+
+#get nic and its device pair, for example
+#eth0.6 eth0
+#eth0.7 eth0
+#br1 eth1
+#bond0 eth2@eth3
+new_nicdevice=`parser_nicdevices $str_all_nics`
+
+if [ -z "$new_nicdevice" ]; then
+    log_info "There is no nic device to configure."
+    exit 
+fi
+
+#enable network service
+enable_network_service
+
+#sort nics device pair based on device type
+#ethernet -> bond -> vlan
+sorted_nicdevice_list=`sort_nics_device_order "$new_nicdevice"`
+log_info "All nics and  device list:"
+echo "$sorted_nicdevice_list" |log_lines debug
+
+
+#config nics and ifcfg files
+parse_nics "$sorted_nicdevice_list"
diff --git a/xCAT/postscripts/nicsutils b/xCAT/postscripts/nicsutils
new file mode 100755
index 000000000..15242822c
--- /dev/null
+++ b/xCAT/postscripts/nicsutils
@@ -0,0 +1,1413 @@
+#!/bin/bash
+#
+#This is network lib for confignetwork to configure bond/vlan/bridge
+
+awk="awk"
+sed="sed"
+cut="cut"
+sleep="sleep"
+sort="sort"
+ps="ps"
+head="head"
+readlink="readlink"
+basename="basename"
+udevadm="udevadm"
+touch="touch"
+tail="tail"
+dmesg="dmesg"
+grep="grep"
+lspci="lspci"
+ifup="ifup"
+ifdown="ifdown"
+nmcli="nmcli"
+dirname="dirname"
+ip="ip"
+ifconfig="ifconfig"
+brctl="brctl"
+uniq="uniq"
+xargs="xargs"
+declare myawk=${awk:-awk}
+
+
+######################################################
+#
+# log lines
+#
+# input : details or status or error
+#
+######################################################
+function log_lines {
+    local pcnt=$#
+    local __level=$1
+    shift
+    local cmd=log_${__level}
+
+    local hit=0
+    local OIFS=$IFS
+    local NIFS=$'\n'
+    IFS=$NIFS
+    local __msg
+    for __msg in $*
+    do
+        IFS=$OIFS
+        $cmd "$__msg"
+        hit=1
+        IFS=$NIFS
+    done
+    IFS=$OIFS
+
+    [ $hit -eq 0 -a $pcnt -le 1 ] && \
+    while read __msg;
+    do
+        $cmd "$__msg"
+    done
+}
+######################################################
+#
+# error information
+#
+# input : message
+#
+######################################################
+function log_error {
+    local __msg="$*"
+    $log_print_cmd $log_print_arg "[E]: $__msg" >&2
+    return 0
+}
+
+######################################################
+#
+# warning information
+#
+# input : message
+#
+######################################################
+function log_warn {
+    local __msg="$*"
+    $log_print_cmd $log_print_arg "[W]: $__msg" >&2
+    return 0
+}
+
+######################################################
+#
+# log information
+#
+# input : message
+#
+######################################################
+function log_info {
+    local __msg="$*"
+    $log_print_cmd $log_print_arg "[I]: $__msg" >&2
+    return 0
+}
+######################################################
+#
+# more details information
+#
+# input : message
+#
+######################################################
+function log_debug {
+    local __msg="$*"
+    $log_print_cmd $log_print_arg "[I]: $__msg" >&2
+    return 0
+}
+
+####################################################
+#
+# print command status
+#
+# input : message
+#
+###################################################
+__my_log_status=
+function log_status {
+    local __msg="$*"
+    $log_print_cmd $log_print_arg "[S]: $__msg" >&2
+
+    # call my_log_status hook to triger more processing for status messages.
+    if [ -n "$__my_log_status" ]; then
+        $__my_log_status "$__msg"
+    fi
+    return 0
+}
+
+####################################################
+#
+# print output 
+#
+###################################################
+function log_print_default {
+    echo $*
+}
+
+####################################################
+#
+# handle command
+#
+###################################################
+function set_log_print {
+    local cmd="$1"
+    if [ -z "$cmd" ]; then
+        return 1
+    fi
+    shift
+    local args="$*"
+
+    eval "log_print_cmd=\"$cmd\""
+    eval "log_print_arg=\"$args\""
+}
+
+# let log work
+[ -z "$log_print" ] && set_log_print "log_print_default"
+
+#####################################################
+#
+# uniq line in cfg files 
+#
+# input : -t"str_for_FS" -k"num"
+#
+#####################################################
+function uniq_per_key {
+    local fs=""
+    local keyno=0
+
+    local opt
+    while getopts "t:k:" opt;
+    do
+        case $opt in
+            t) fs="$OPTARG";;
+            k) keyno="$OPTARG";;
+        esac
+    done
+    shift $(($OPTIND - 1))
+
+    $awk ${fs:+"-F"}"$fs" -v keyno=$keyno '
+    BEGIN { cnt=0; }
+    {
+        if(!($keyno in keya)) {
+            keya[$keyno]=cnt;
+            cnt+=1;
+        };
+        idx=keya[$keyno];
+        vala[idx]=$0;
+    }
+    END {
+        for(i = 0; i < cnt; i++) {
+            print vala[i];
+        };
+    }'
+}
+
+##################################################################################
+#
+# load kernel module
+#
+# input : module=<type> retry=<retry_times> interval=<time> -- <module_parameters>
+#
+################################################################################
+function load_kmod {
+    local lines=""
+
+    local retry=0
+    local interval=0
+    local module=""
+    local module_parameters=""
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | cut -s -d= -f1`
+        if [ "$key" = "retry" ] || \
+           [ "$key" = "interval" ] || \
+           [ "$key" = "module" ]; then
+            eval "$1"
+        elif [ "$1" = "--" ]; then
+            shift
+            module_parameters="$*"
+            break
+        fi
+        shift
+    done
+
+    if [ -z "$module" ]; then
+        log_error "Empty kernel module name to be loaded!"
+    fi
+
+    # load the module
+    ((i=0))
+    while [ ! -d /sys/module/$module ];
+    do
+        [ $i -eq 0 ] && lines=`$modprobe $module $module_parameters 2>&1`
+        $sleep $interval
+        ((i+=1))
+        [ $i -ge $retry ] && break
+    done
+    if [ $i -ge $retry ]; then
+        log_error "Fail to load kernel module \"$module\""
+        echo "$lines" \
+        | $sed -e 's/^/>> /g' \
+        | log_lines debug
+        $false
+    fi
+}
+
+#################################################################
+#
+# query networks  NICNETWORKS='eth1!provision,bond0!data'
+#
+# input : fkey=<number> vkey=<nic> fval=<number>
+#
+###############################################################
+function query_nicnetworks {
+    local fkey=2
+    local vkey=$1
+    local fval=1
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | cut -s -d= -f1`
+        if [ "$key" = "fkey" ] || \
+           [ "$key" = "vkey" ] || \
+           [ "$key" = "fval" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    local vval=`echo "$NICNETWORKS" | $sed -e 's/,/\n/g' \
+    | $awk -v fkey=$fkey -v vkey=$vkey -v fval=$fval -F'!' \
+        '$fkey == vkey {r=$fval;} END{print r;}'`
+    [ -n "$vval" ] && echo "$vval"
+}
+
+###############################################################
+#
+# query networks nic
+#
+# input : nic
+#
+###########################################################
+function query_nicnetworks_nic {
+    query_nicnetworks fkey=2 vkey=$1 fval=1
+}
+
+#############################################################
+#
+# query networks 
+#
+# input : nic
+#
+#############################################################
+function query_nicnetworks_net {
+    query_nicnetworks fkey=1 vkey=$1 fval=2
+}
+#######################################################################################
+#
+# get network attribute from NETWORKS_LINEX
+#
+# NETWORKS_LINES=2
+# NETWORKS_LINE1='netname=10_0_0_0-255_255_255_0||net=10.0.0.0||mask=255.255.255.0||mgtifname=eth2||gateway=&lt;xcatmaster&gt;||dhcpserver=||tftpserver=10.0.0.153||nameservers=||ntpservers=||logservers=||dynamicrange=10.0.0.1-10.0.0.254||staticrange=||staticrangeincrement=||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments=__BEAT_IPRANGE_10.0.0.1-10.0.0.254'
+# NETWORKS_LINE2='netname=10_9_10_0-255_255_255_0||net=10.9.10.0||mask=255.255.255.0||mgtifname=eth1:1||gateway=&lt;xcatmaster&gt;||dhcpserver=||tftpserver=10.9.10.1||nameservers=||ntpservers=||logservers=||dynamicrange=||staticrange=10.9.10.11-10.9.10.30||staticrangeincrement=1||nodehostname=||ddnsdomain=||vlanid=||domain=||disable=||comments=__BEAT_IPRANGE_10.9.10.11-10.9.10.30'
+#
+# input : network_name attribute_name
+#
+########################################################################################
+function get_network_attr {
+   local netname=$1
+   local attrname=$2
+
+   local netline=""
+   local index=1
+   while [ $index -le $NETWORKS_LINES ]
+   do
+       eval netline=\$NETWORKS_LINE$index
+       echo "$netline" | grep -sq ".*netname=$netname" && break;
+       #echo "$netline" | grep "netname=$netname" && break;
+       ((index+=1))
+   done
+
+   if [ $index -le $NETWORKS_LINES ]; then
+       echo "$netline" | $sed -e 's/||/\n/g' | $awk -F'=' '$1 == "'$attrname'" {print $2}'
+   else
+       log_error "Fail to get \"$attrname\" for network \"$netname\""
+       exit 1
+   fi
+}
+
+#######################################################################
+#
+# get mac
+#
+# input : nic name
+#
+######################################################################
+function get_mac {
+    declare ifname=$1
+    # if bond slave interface, get its real mac
+    if [ -L /sys/class/net/$ifname/master ]; then
+        declare ifmaster=`ls -l /sys/class/net/$ifname/master | $sed -e 's/^.*virtual\/net\///g'`
+        grep -E "^Slave Interface:|^Permanent HW addr:" /proc/net/bonding/$ifmaster \
+        | grep -A1 ": $ifname" | $tail -n1 | $awk '{print $4}'
+
+    # confirm the interface does exist before running "ip link show" command
+    elif [ -L /sys/class/net/$ifname ]; then
+        $ip link show $ifname | grep "link\/ether" | $awk '{print $2}'
+    fi
+}
+
+####################################################################
+#
+# wait for nic state
+#
+# input : <nic> <expect_state> <try_count> <sleep_time> 
+#
+####################################################################
+function wait_for_ifstate {
+    local ifname=$1
+    local ifstate=$2
+    local tryCnt=$3
+    local tryInt=$4
+    local state=""
+    local i
+    ((i=0))
+    while [ $i -lt $tryCnt ]
+    do
+        lines=`$ip link show $ifname`
+        echo "$lines" | grep -sq "state $ifstate"
+        rc=$?
+        [ $rc -eq 0 ] && break
+
+        if [ $tryInt -ne 0 ]; then
+            state=`echo "$lines" | grep "state" | $sed -e 's/^.*state \([a-zA-Z]\+\) .*$/\1/g'`
+            log_debug "State of \"$ifname\" was \"$state\" instead of expected \"$ifstate\". Wait $i of $tryCnt with interval $tryInt."
+        fi
+        $sleep $tryInt
+        ((i+=1))
+    done
+    test $i -lt $tryCnt
+}
+
+##################################################################
+#
+# create ifcfg-* files
+#
+# input : ifname=<ifname> nwdir="cfg_file_dir" xcatnet=<xcatnet> _ipaddr=<ip> _netmask=<mask> inattrs=<attrs>
+#         
+#
+#################################################################
+function create_persistent_ifcfg {
+    log_debug "create_persistent_ifcfg $@" >&2
+
+    local nwdir="/etc/sysconfig/network-scripts"
+
+    local ifname=""
+    local xcatnet=""
+    local _ipaddr=""
+    local _netmask=""
+    local inattrs=""
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "nwdir" ] || \
+           [ "$key" = "xcatnet" ] || \
+           [ "$key" = "_ipaddr" ] || \
+           [ "$key" = "_netmask" ] || \
+           [ "$key" = "inattrs" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    local fcfg=$nwdir/ifcfg-$ifname
+    # if no ip addr/mask specified explicitely, search in xCAT environment.
+    if [ -n "$xcatnet" ]; then
+        if [ -z "$_ipaddr" ]; then
+            ifname_exp=`query_nicnetworks_nic $xcatnet`
+            # search NICIPS for static config
+            if [ x"$ifname_exp" == x ]; then
+                ifname_exp=$ifname
+            fi
+            _ipaddr=`echo "$NICIPS" | $sed -e 's/,/\n/g' \
+                     | $awk -vifname=$ifname_exp -F'!' '$1 == ifname{print $2}' \
+                     | $awk -F'|' '{print $1}'`
+        fi
+        if [ -z "$_netmask" ]; then
+            _netmask=`get_network_attr $xcatnet mask`
+        fi
+    fi
+    local attrs=""
+    attrs=${attrs}${attrs:+,}"DEVICE=$ifname"
+    attrs=${attrs}${attrs:+,}"BOOTPROTO=static"
+    [ -n "$_ipaddr" ] && \
+    attrs=${attrs}${attrs:+,}"IPADDR=$_ipaddr"
+    [ -n "$_netmask" ] && \
+    attrs=${attrs}${attrs:+,}"NETMASK=$_netmask"
+
+    # NetworkManager attributes
+    attrs=${attrs}${attrs:+,}"NAME=$ifname"
+
+    # some auto-detected attributes
+    # - mark vlan interfac3
+    if [ -f /proc/net/vlan/$ifname ]; then
+        attrs=${attrs}${attrs:+,}"VLAN=yes"
+    fi
+
+    # - mark bond interfac3
+    # https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Networking_Guide/sec-Network_Bonding_Using_the_Command_Line_Interface.html#sec-Create_a_Channel_Bonding_Interface
+    if [ -f /proc/net/bonding/$ifname ]; then
+        attrs=${attrs}${attrs:+,}"BONDING_MASTER=yes"
+    fi
+
+    # - mark mac address for non-virtual interface.
+    # Note: ignore HWADDR attribute if it's a bond slave.
+    if ! echo "$inattrs" | grep -sq 'SLAVE="\?yes"\?'; then
+        mac=`get_mac $ifname`
+        if [ -n "$mac" -a ! -d /sys/devices/virtual/net/$ifname ]; then
+            attrs=${attrs}${attrs:+,}"HWADDR=$mac"
+        fi
+    fi
+
+    # record manual and auto attributes first
+    # since input attributes might overwrite them.
+    #
+    # record input attributes later. They will overwrite
+    # previous generated attributes if duplicate.
+    [ -f $fcfg ] && mv -f $fcfg `dirname $fcfg`/.`basename $fcfg`.bak
+    echo "$attrs,$inattrs" \ | $sed -e 's/,/\n/g' | grep -v "^$" \
+    | $sed -e 's/=/="/' -e 's/ *$/"/' \
+    | uniq_per_key -t'=' -k1 >$fcfg
+    local rc=$?
+
+    # log for debug
+    echo "['ifcfg-${ifname}']" >&2
+    cat $fcfg | $sed -e 's/^/ >> /g' | log_lines debug >&2
+    return $rc
+}
+
+################################################################################
+#
+# get all physical network devices
+# remove duplicate entries while keep their order.
+#
+# input : <cat1>=<filter1_params>,<cat2>=<filter2_params>
+#
+###############################################################################
+function expand_ports {
+    log_debug "expand_ports $@" >&2
+
+    local i
+
+    local ports=`echo "$1" | $sed -e 's/,/ /g'`
+
+    # get all interfaces
+    local allifs=""
+
+    # get all physical network devices
+    local alldevs=""
+
+    local rports=""
+    local p
+    for p in $ports
+    do
+        if echo "$p" | grep -sq ".*="; then
+            key=`echo "$p" | cut -s -d= -f1`
+            val=`echo "$p" | cut -s -d= -f2-`
+
+        # include direct non-virtual interface
+        elif [ -L /sys/class/net/$p -a ! -d /sys/devices/virtual/net/$p ]; then
+            rports=${rports}${rports:+,}$p
+
+        # warn other direct interface, such as virtual
+        else
+            log_warn "Invalid member port \"$p\". Ignore it!"
+        fi
+    done
+
+    # remove duplicate entries while keep their order.
+    [ -n "$rports" ] && echo "$rports" | \
+    $sed -e 's/,\+/,/g' -e 's/^,//' -e 's/,/\n/g' \
+    | uniq_per_key -t" " -k0 \
+    | $xargs | $sed -e 's/ /,/g'
+}
+
+###########################################################################
+#
+# migrate ip from sports to ifname
+#
+# input : ifname=<current_nic> sports=<pre_ports>
+#
+#########################################################################
+function migrate_ip {
+    log_debug "migrate_ip $@" >&2
+
+    local ifname=""
+    local sports=""
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "sports" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    [ -n "$sports" ] && sports=`echo "$sports" | $sed -e 's/,/ /g'`
+
+
+    # loop for every source port and migrate ips and routes
+    local p=""
+    for p in $sports
+    do
+        [ -L /sys/class/net/$p ] || continue
+
+
+        #route for now
+        saveroutes=`$ip route | grep default | grep "dev $p"| grep via | $sed -e 's/dev .*//'`
+
+        saveips=`$ip addr show dev $p scope global | grep inet | $sed -e 's/inet.//' | $sed -e 's/[^ ]*$//'`
+        if [ -n "$saveips" ]; then
+            # Migrate ip address from source ports to target port
+            OIFS=$IFS
+            IFS=$'\n'
+            for line in $saveips
+            do
+                newline=`echo $line|sed 's/dynamic//g'`
+                eval "$ip addr del dev $p $newline"
+                eval "$ip addr add dev $ifname $newline"
+            done
+            IFS=$OIFS
+        fi
+
+        # restore saved routes which assume to be applied to the target interface
+        if [ -n "$saveroutes" ]; then
+            eval "$ip route add $saveroutes"
+        fi
+    done
+
+}
+
+###############################################################################
+#
+# create bridge
+#
+# input : bridge name
+#
+##############################################################################
+function add_br() {
+
+     BNAME=$1
+     BRIDGE=$2
+
+     if [[ $BRIDGE == "bridge_ovs" ]]; then
+
+         log_debug "ovs-vsctl add-br $BNAME" >&2
+         ovs-vsctl add-br $BNAME
+     elif [[ $BRIDGE == "bridge" ]]; then
+         log_debug "brctl addbr $BNAME" >&2
+         brctl addbr $BNAME
+     fi
+}
+
+###############################################################################
+#
+# create add port for bridge
+#
+# input : bridge name
+#         port name
+#
+##############################################################################
+function add_if() {
+    BNAME=$1
+    PORT=$2
+    BRIDGE=$3
+
+    if [[ $BRIDGE == "bridge_ovs" ]]; then
+        log_debug "ovs-vsctl add-port $BNAME $PORT" >&2
+        ovs-vsctl add-port $BNAME $PORT
+    elif [[ $BRIDGE == "bridge" ]]; then
+        log_debug "brctl addif $BNAME $PORT" >&2
+        brctl addif $BNAME $PORT
+    fi
+}
+
+###############################################################################
+#
+# create raw vlan for bridge
+#
+# input : ifname=<ifname> _mtu=<mtu> _bridge=<bridge_name>
+#
+###############################################################################
+function create_raw_vlan_for_br {
+  
+    log_debug "create_raw_vlan_interface $@" >&2
+    local lines=""
+
+    local ifname=""
+    local _mtu=""
+    local _bridge=""
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "_mtu" ] || \
+           [ "$key" = "_bridge" ]; then 
+            eval "$1"
+        fi
+        shift
+    done
+
+    local vlanid=""
+    if echo "$ifname" | grep -sq ".*\.[0-9]\+"; then
+        vlanid=`echo "$ifname" | $cut -s -d. -f2-`
+        ifname=`echo "$ifname" | $cut -s -d. -f1`
+    elif echo "$ifname" | grep -sq ".*vla\?n\?[0-9]\+"; then
+        vlanid=`echo "$ifname" | $sed -e 's/^\(.*\)vla\?n\?\([0-9]\+\)$/\2/'`
+        ifname=`echo "$ifname" | $sed -e 's/^\(.*\)vla\?n\?\([0-9]\+\)$/\1/'`
+    fi
+
+    # generate raw vlan interface definition
+    cfg=""
+    cfg="${cfg}${cfg:+,}ONBOOT=yes"
+
+    if grep -q -i "release 6" /etc/redhat-release ; then
+       cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+    fi
+
+    cfg="${cfg}${cfg:+,}USERCTL=no"
+    cfg="${cfg}${cfg:+,}TYPE=Ethernet"
+    cfg="${cfg}${cfg:+,}VLAN=yes"
+    cfg="${cfg}${cfg:+,}BRIDGE=$_bridge"
+    [ -n "$_mtu" ] && \
+    cfg="${cfg}${cfg:+,}MTU=$_mtu"
+    create_persistent_ifcfg \
+        ifname=$ifname.$vlanid \
+        inattrs="$cfg"
+
+}
+
+###############################################################################
+#
+# create raw bond for bridge
+#
+# input : ifname=<ifname> _mtu=<mtu> _bonding_opts=<string> _bridge=<bridge_name>
+#
+###############################################################################
+function create_raw_bond_for_br {
+
+    log_debug "create_raw_bond_interface $@" >&2
+
+    local lines=""
+
+    local ifname=""
+    local _mtu=""
+    local _bridge=""
+    local _bonding_opts=""
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "_mtu" ] || \
+           [ "$key" = "_bonding_opts" ] || \
+           [ "$key" = "_bridge" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    # migrate bond ports ip and route to bridge
+    #migrate_ip ifname=$ifname sports="$_bridge"
+
+    # define and bring up raw bond interface
+    cfg=""
+    cfg="${cfg}${cfg:+,}ONBOOT=yes"
+
+    if grep -q -i "release 6" /etc/redhat-release ; then
+        cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+    fi
+
+    cfg="${cfg}${cfg:+,}USERCTL=no"
+    cfg="${cfg}${cfg:+,}TYPE=Bond"
+    cfg="${cfg}${cfg:+,}BONDING_MASTER=yes"
+    cfg="${cfg}${cfg:+,}BONDING_OPTS='$_bonding_opts'"
+    cfg="${cfg}${cfg:+,}BOOTPROTO=none"
+    cfg="${cfg}${cfg:+,}DHCLIENTARGS='-timeout 200'"
+    cfg="${cfg}${cfg:+,}BRIDGE=$_bridge"
+    [ -n "$_mtu" ] && \
+    cfg="${cfg}${cfg:+,}MTU=$_mtu"
+    create_persistent_ifcfg \
+        ifname=$ifname \
+        inattrs="$cfg"
+}
+
+###############################################################################
+#
+# create  bridge
+#
+# input : ifname=<ifname> xcatnet=<xcat_network> _ipaddr=<ip> _netmask=<netmask> _port=<port> _pretype=<nic_type> _brtype=<bridge|bridge_ovs> _mtu=<mtu> _bridge=<bridge_name>
+#
+###############################################################################
+function create_bridge_interface {
+
+    log_debug "create_bridge_interface $@" >&2
+    local lines=""
+    local ifname="" #current bridge
+    local xcatnet=""
+    local _ipaddr=""
+    local _netmask=""
+    local _brtype=""
+    local _pretype=""
+    local _port=""  #pre nic
+    local _mtu=""
+    local _bonding_opts="" #for bond
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "xcatnet" ] || \
+           [ "$key" = "_ipaddr" ] || \
+           [ "$key" = "_netmask" ] || \
+           [ "$key" = "_brtype" ] || \
+           [ "$key" = "_pretype" ] || \
+           [ "$key" = "_port" ] || \
+           [ "$key" = "_bonding_opts" ] || \
+           [ "$key" = "_mtu" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    # let's query "nicnetworks" table about its target "xcatnet"
+    if [ -n "$ifname" -a -z "$xcatnet" -a -z "$_ipaddr" ]; then
+        xcatnet=`query_nicnetworks_net $ifname`
+        log_debug "Pickup xcatnet, \"$xcatnet\", from NICNETWORKS for interface \"$ifname\"." >&2
+    fi
+
+
+    if [ x$_pretype == "xethernet" ]; then 
+        create_raw_ethernet_for_br \
+            ifname=$_port \
+            _bridge=$ifname \
+            _mtu=$_mtu
+    elif [ x$_pretype == "xvlan" ]; then
+        create_raw_vlan_for_br \
+            ifname=$_port \
+            _bridge=$ifname \
+            _mtu=$_mtu
+            
+    elif [ x$_pretype == "xbond" ]; then
+        create_raw_bond_for_br \
+            ifname=$_port \
+            _bridge=$ifname \
+            _mtu=$_mtu \
+            _bonding_opts=$_bonding_opts   
+    fi  
+
+    add_br $ifname $_brtype
+    add_if $ifname $_port $_brtype
+    # setup interface on the fly
+    [ -n "$_mtu" ] && $ip link set $ifname mtu $_mtu
+
+    # log for debug
+   
+    migrate_ip ifname=$ifname sports="$_port"
+
+    # define and bring up bridge interface, if required.
+    # generate bridge interface definition
+    cfg=""
+    cfg="${cfg}${cfg:+,}ONBOOT=yes"
+
+    if grep -q -i "release 6" /etc/redhat-release ; then
+        cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+    fi
+
+    if [ x$_brtype == x"bridge" ]; then
+    
+        cfg="${cfg}${cfg:+,}TYPE=Bridge"
+    elif [ x$_brtype == x"bridge_ovs" ]; then
+    
+        cfg="${cfg}${cfg:+,}TYPE=OVSBridge"
+
+    fi
+    [ -n "$_mtu" ] && \
+         cfg="${cfg}${cfg:+,}MTU=$_mtu"
+         create_persistent_ifcfg \
+             ifname=$ifname \
+             xcatnet=$xcatnet \
+         inattrs="$cfg"
+
+    # bring up interface formally
+    lines=`$ifdown $ifname; $ifup $ifname`
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        log_warn "ifup $ifname failed with return code equals to $rc"
+        echo "$lines" \
+        | $sed -e 's/^/>> /g' \
+        | log_lines debug
+    fi
+    return $rc
+
+}
+
+###############################################################################
+#
+# create vlan 
+#
+# input : ifname=<ifname> slave_ports=<ports> xcatnet=<xcatnetwork> _ipaddr=<ip> _netmask=<netmask> _mtu=<mtu> _bridge=<bridge_name> vlanid=<vlanid>
+#
+###############################################################################
+function create_vlan_interface {
+    log_debug "create_vlan_interface $@" >&2
+
+    local lines=""
+
+    local ifname=""
+    local vlanid=""
+    local xcatnet=""
+    local _ipaddr=""
+    local _netmask=""
+    local _mtu=""
+    local _bridge=""
+    # in case it's on top of bond, we need to migrate ip from its
+    # member vlan ports.
+    local slave_ports=""
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "slave_ports" ] || \
+           [ "$key" = "xcatnet" ] || \
+           [ "$key" = "_ipaddr" ] || \
+           [ "$key" = "_netmask" ] || \
+           [ "$key" = "_mtu" ] || \
+           [ "$key" = "_bridge" ] || \
+           [ "$key" = "vlanid" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    if [ -z "$vlanid" ]; then
+        log_error "No \"vlanid\" specificd for vlan interface. Abort!" >&2
+        return 1
+    fi
+
+    # let's query "nicnetworks" table about its target "xcatnet"
+    if [ -n "$ifname" -a -z "$xcatnet" -a -z "$_ipaddr" -a -n "$vlanid" ]; then
+        xcatnet=`query_nicnetworks_net $ifname.$vlanid`
+        log_debug "Pickup xcatnet, \"$xcatnet\", from NICNETWORKS for interface \"$ifname\"." >&2
+    fi
+
+
+    #load the 8021q module if not loaded.
+    #load_kmod module=8021q retry=10 interval=0.5
+
+    # create vlan on top of target interface if that's required.
+    ((i=0))
+    while [ ! -f /proc/net/vlan/$ifname.$vlanid ];
+    do
+        if [ $i -eq 0 ]; then
+            # alternative cmd to "vconfig add $ifname $vlanid"
+            $ip link add link $ifname name $ifname.$vlanid type vlan id $vlanid
+        fi
+        $sleep 0.5
+        ((i+=1))
+        [ $i -ge 10 ] && break
+    done
+    if [ $i -ge 10 ]; then
+        log_error "Fail to create vlan interface \"$ifname.$vlanid\""
+        return 1
+    fi
+
+    # setup interface 
+    [ -n "$_mtu" ] && $ip link set $ifname.$vlanid mtu $_mtu
+    $ip link set $ifname.$vlanid up
+    wait_for_ifstate $ifname.$vlanid UP 200 1
+    rc=$?
+
+    _g_migrate_ip=1
+
+
+    [ $_g_migrate_ip -eq 1 ] && \
+    [ -n "$slave_ports" ] && \
+    migrate_ip ifname=$ifname.$vlanid sports="$slave_ports"
+
+    # define and bring up vlan interface on top of raw bond interface, if required.
+    # generate vlan interface definition
+    cfg=""
+    cfg="${cfg}${cfg:+,}ONBOOT=yes"
+    
+    if grep -q -i "release 6" /etc/redhat-release ; then
+       cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+    fi
+    
+    cfg="${cfg}${cfg:+,}USERCTL=no"
+    cfg="${cfg}${cfg:+,}TYPE=Ethernet"
+    cfg="${cfg}${cfg:+,}VLAN=yes"
+    [ -n "$_mtu" ] && \
+    cfg="${cfg}${cfg:+,}MTU=$_mtu"
+    create_persistent_ifcfg \
+        ifname=$ifname.$vlanid \
+        xcatnet=$xcatnet \
+        inattrs="$cfg"
+
+    # bring up interface formally
+    lines=`$ifdown $ifname.$vlanid; $ifup $ifname.$vlanid`
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        log_warn "ifup $ifname.$vlanid failed with return code equals to $rc"
+        echo "$lines" \
+        | $sed -e 's/^/>> /g' \
+        | log_lines debug
+    fi
+
+    return $rc
+}
+###############################################################################
+#
+# create raw ethernet cfg file for bridge
+# This is for eth-> br
+#
+# input : ifname=<ifname> _mtu=<mtu> _bridge=<bridge_name>
+#
+###############################################################################
+function create_raw_ethernet_for_br {
+
+    log_debug "create_raw_eth_interface_for_br $@" >&2
+
+    local lines=""
+
+    local ifname=""
+    local _bridge=""
+    local _mtu=""
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "_mtu" ] || \
+           [ "$key" = "_bridge" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+
+    # create raw ethnet ifcfg file for bridge.
+    cfg=""
+    cfg="${cfg}${cfg:+,}ONBOOT=yes"
+
+    if grep -q -i "release 6" /etc/redhat-release ; then
+        cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+    fi
+
+    cfg="${cfg}${cfg:+,}TYPE=Ethernet"
+    cfg="${cfg}${cfg:+,}BRIDGE=$_bridge"
+    cfg="${cfg}${cfg:+,}BOOTPROTO=none"
+    [ -n "$_mtu" ] && \
+    cfg="${cfg}${cfg:+,}MTU=$_mtu"
+    create_persistent_ifcfg \
+    ifname=$ifname \
+    inattrs="$cfg"
+
+}
+
+#############################################################################################################################
+#
+# create bond or bond->vlan interface
+# https://www.kernel.org/doc/Documentation/networking/bonding.txt
+# https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Networking_Guide/sec-Using_Channel_Bonding.html
+#
+# input : ifname=<nic> xcatnet=<xcatnetwork> _ipaddr=<ip> _netmask=<netmask> _bonding_opts=<bonding_opts> _mtu=<mtu> slave_ports=<port1,port2> 
+#
+############################################################################################################################
+function create_bond_interface {
+    log_debug "create_bond_interface $@" >&2
+
+    local lines=""
+
+    local ifname=""
+    local xcatnet=""
+    local _ipaddr=""
+    local _netmask=""
+    # note:
+    # - "miimon" requires drivers for each slave nic support MII tool.
+    #   $ ethtool <interface_name> | grep "Link detected:"
+    # - "802.3ad" mode requires a switch that is 802.3ad compliant.
+    local _bonding_opts="mode=802.3ad miimon=100"
+    local _mtu=""
+    local slave_ports=""
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | $cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "xcatnet" ] || \
+           [ "$key" = "_ipaddr" ] || \
+           [ "$key" = "_netmask" ] || \
+           [ "$key" = "_bonding_opts" ] || \
+           [ "$key" = "_mtu" ] || \
+           [ "$key" = "slave_ports" ]; then
+            eval "$1"
+        fi
+        shift
+    done
+    _g_migrate_ip=1
+    if [ -z "$slave_ports" ]; then
+        log_error "No valid slave_ports defined. Abort!" >&2
+        return 1
+    fi
+
+    # let's query "nicnetworks" table about its target "xcatnet" 
+    if [ -n "$ifname" -a -z "$xcatnet" -a -z "$_ipaddr" ]; then
+        xcatnet=`query_nicnetworks_net $ifname`
+        log_debug "Pickup xcatnet, \"$xcatnet\", from NICNETWORKS for interface \"$ifname\"." >&2
+    fi
+    # if target interface is a vlan interface, we need to setup raw interface
+    # first and then the vlan interface.
+    local vlan_wrapped=0
+    local vlanid=""
+    if echo "$ifname" | grep -sq ".*\.[0-9]\+"; then
+        vlanid=`echo "$ifname" | $cut -s -d. -f2-`
+        ifname=`echo "$ifname" | $cut -s -d. -f1`
+        vlan_wrapped=1
+    # NOTE: xCAT had problem to support nic interface with character dot
+    # in its name. This causes problem to chdef, mkdef commands and so on.
+    # in order to workaround this, here we support replacing dot as "vl(an)?"
+    # when planning and defining vlan interfaces. This script will recognize
+    # such vlan nic name and create correct nic and its ifcfg- definition files
+    # correctly.
+    elif echo "$ifname" | grep -sq ".*vla\?n\?[0-9]\+"; then
+        vlanid=`echo "$ifname" | $sed -e 's/^\(.*\)vla\?n\?\([0-9]\+\)$/\2/'`
+        ifname=`echo "$ifname" | $sed -e 's/^\(.*\)vla\?n\?\([0-9]\+\)$/\1/'`
+        vlan_wrapped=1
+    fi
+
+    local cnt
+    if [ $vlan_wrapped -eq 1 ]; then
+        # create bond interface first.
+        cnt=0
+        while [ $cnt -lt 2 ];
+        do
+            if [ -f /proc/net/bonding/$ifname -a /sys/devices/virtual/net/$ifname ]; then
+                break
+            fi
+        
+            [ $cnt -eq 0 ] && \
+            create_bond_interface \
+                ifname="$ifname" \
+                slave_ports="$slave_ports" \
+                _bonding_opts="$_bonding_opts" \
+                _mtu="$_mtu"
+
+            ((cnt+=1))
+        done
+
+        # create vlan interface on top of bond
+        [ $cnt -lt 2 ] && \
+        create_vlan_interface \
+            ifname="$ifname" \
+            vlanid="$vlanid" \
+            xcatnet="$xcatnet" \
+            _ipaddr="$_ipaddr" \
+            _netmask="$_netmask" \
+            _mtu="$_mtu" \
+            slave_ports="$slave_ports"
+
+        return $?
+    fi
+
+    # convert the delimitor of _bonding_opts from comma to blank
+    if [ -n "$_bonding_opts" ]; then
+        _bonding_opts=`echo "$_bonding_opts" | $sed -e 's/,/ /g'`
+    fi
+    
+    ##############################
+    # Create target bond interface
+    # if target bond device was already exists, assume succ.
+    # stage 0: create interface
+    # stage 1: setup bond options which need to bring down bond first
+    # stage 2: setup bond slaves, apply other options on the fly and bring interface up
+    # stage 3: check target interface up
+    cnt=0
+    while [ $cnt -lt 4 ];
+    do
+        # Stage 0:
+        # create raw bond device on the fly, if not created yet.
+        if [ $cnt -eq 0 -a ! -f /proc/net/bonding/$ifname ]; then
+            # load the bonding module if not loaded.
+            load_kmod module=bonding retry=10 interval=0.5
+    
+            # create required bond device
+            ((i=0))
+            while ! grep -sq "^$ifname$" /sys/class/net/bonding_masters;
+            do
+                [ $i -eq 0 ] && echo "+$ifname" >/sys/class/net/bonding_masters
+                $sleep 0.5
+                ((i+=1))
+                [ $i -ge 10 ] && break
+            done
+            if [ $i -ge 10 -o ! -f /proc/net/bonding/$ifname ]; then
+                log_error "stage 0: Fail to create bond device \"$ifname\"" >&2
+                break
+            fi
+
+        # Stage 1:
+        # setup bond options
+        elif [ $cnt -eq 1 -a -n "$_bonding_opts" ]; then
+            # 1.1) bring down bond interface before setup its attributes
+            $ip link set $ifname down
+            $ip link show $ifname | $sed -e 's/^/[bond.down] >> /g' | log_lines debug >&2
+
+            # 1.2) remove current slaves first
+            local saved_slaves=$(</sys/class/net/$ifname/bonding/slaves)
+            for ifslave in $saved_slaves
+            do
+                echo "-$ifslave" >/sys/class/net/$ifname/bonding/slaves
+            done
+            lines=`$(</sys/class/net/$ifname/bonding/slaves)`
+            if [ -n "$lines" ]; then
+                log_warn "stage 1: Cannot clean up bond slaves before setting up its attributes." >&2
+                echo "$lines" \
+                | $sed -e 's/[failed.slaves] >> /g' \
+                | log_lines debug >&2
+            fi
+    
+            # 1.3) apply bond options
+            local option
+            for option in $_bonding_opts
+            do
+                key=`echo "$option" | $cut -s -d= -f1`
+                val=`echo "$option" | $cut -s -d= -f2-`
+                echo "$val" >/sys/class/net/$ifname/bonding/$key
+                rc=$?
+                if [ $rc -ne 0 ]; then
+                    log_warn "stage 1: Fail to set bonding option \"$key\" to \"$val\" in device \"$ifname\"" >&2
+                    cat /sys/class/net/$ifname/bonding/$key \
+                    | $sed -e 's/^/[bond.'$key'] >>/g' \
+                    | log_lines debug >&2
+                fi
+            done
+    
+            # 1.4) restore saved bond slaves
+            for ifslave in $saved_slaves
+            do
+                echo "+$ifslave" >/sys/class/net/$ifname/bonding/slaves
+            done
+            log_debug "[bond.slavesAft] >> $(</sys/class/net/$ifname/bonding/slaves)" >&2
+
+        # Stage 2:
+        # add slave ports
+        elif [ $cnt -eq 2 ]; then
+            # 2.1) add new slaves
+            for ifslave in `echo "$slave_ports" | $sed -e 's/,/ /g'`
+            do
+                # if the interface was not bonded as slave of master, do it now.
+                if ! grep -sq "Slave Interface: *$ifslave *$" /proc/net/bonding/$ifname; then
+                    # bring it down before adding it to the master, or the operation will fail
+                    # the slave interface will be brought up implicitely after bonded to master.
+                    $ip link set $ifslave down
+
+                    # log for debug
+                    $ip link show $ifslave | $sed -e 's/^/[slave]: >> /g' \
+                    | log_lines debug >&2
+        
+                    echo "+$ifslave" >/sys/class/net/$ifname/bonding/slaves
+                fi
+
+                # define and bring up slave interfaces.
+                cfg=""
+                cfg="${cfg}${cfg:+,}ONBOOT=yes"
+                
+                if grep -q -i "release 6" /etc/redhat-release ; then
+                    cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+                fi
+
+                cfg="${cfg}${cfg:+,}USERCTL=no"
+                cfg="${cfg}${cfg:+,}TYPE=Ethernet"
+                cfg="${cfg}${cfg:+,}SLAVE=yes"
+                cfg="${cfg}${cfg:+,}MASTER=$ifname"
+                cfg="${cfg}${cfg:+,}BOOTPROTO=none"
+                [ -n "$_mtu" ] && \
+                cfg="${cfg}${cfg:+,}MTU=$_mtu"
+                create_persistent_ifcfg \
+                    ifname=$ifslave \
+                    inattrs="$cfg"
+            done
+            # log for debug
+            log_debug "[bond.slavesNew] >> $(</sys/class/net/$ifname/bonding/slaves)" >&2
+
+            # 2.2) apply other bond interface options on the fly
+            [ -n "$_mtu" ] && $ip link set $ifname mtu $_mtu
+
+            # 2.3) bring interface up
+            $ip link set $ifname up
+
+        elif [ $cnt -eq 3 ]; then
+            # 3.1) Check bond interface status
+            wait_for_ifstate $ifname UP 200 1
+            rc=$?
+
+            # log for debug
+            $ip link show $ifname | $sed -e 's/^/[ip.link] >> /g' | log_lines debug >&2
+
+            if [ $rc -ne 0 ]; then
+                log_warn "stage 3: Fail to bring up bond interface \"$ifname\"" >&2
+                break
+            fi
+        fi
+
+        ((cnt+=1))
+    done
+    test $cnt -eq 4
+    rc=$?
+    
+    # migrate slave ports ip and route to bond master
+    #[ $_g_migrate_ip -eq 1 ] && \
+    #migrate_ip ifname=$ifname sports="$slave_ports"
+
+    # define and bring up raw bond interface
+    # DHCLIENTARGS is optional, but default to have.
+    cfg=""
+    cfg="${cfg}${cfg:+,}ONBOOT=yes"
+                
+    if grep -q -i "release 6" /etc/redhat-release ; then
+        cfg="${cfg}${cfg:+,}NM_CONTROLLED=no"
+    fi
+    
+    cfg="${cfg}${cfg:+,}USERCTL=no"
+    cfg="${cfg}${cfg:+,}TYPE=Bond"
+    cfg="${cfg}${cfg:+,}BONDING_MASTER=yes"
+    cfg="${cfg}${cfg:+,}BONDING_OPTS='$_bonding_opts'"
+    cfg="${cfg}${cfg:+,}BOOTPROTO=none"
+    cfg="${cfg}${cfg:+,}DHCLIENTARGS='-timeout 200'"
+    [ -n "$_mtu" ] && \
+    cfg="${cfg}${cfg:+,}MTU=$_mtu"
+    create_persistent_ifcfg \
+        ifname=$ifname \
+        xcatnet=$xcatnet \
+        inattrs="$cfg"
+    lines=`$ifdown $ifname; $ifup $ifname 2>&1`
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        log_warn "ifup $ifname failed with return code equals to $rc"
+        echo "$lines" \
+        | $sed -e 's/^/'$ifname' ifup out >> /g' \
+        | log_lines debug
+    fi
+
+    wait_for_ifstate $ifname UP 200 1
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        log_error "Error! Interface \"$ifname\" was NOT in \"UP\" state eventually."
+        $ip link show $ifname \
+        | $sed -e 's/^/['$ifname' ip out >> /g' \
+        | log_lines debug
+    fi
+
+    return $rc
+}
+
+#############################################################################
+#
+# base64 encoded, decode first
+#
+# input : string 
+#
+############################################################################
+function decode_arguments {
+    local rc=1
+    local line=`echo "$1" | $base64 -d 2>/dev/null`
+    if echo "$line" | grep -sq "^{BASE64}:"; then
+        line=`echo "$line" | $sed -e 's/^{BASE64}:[ 	]*//'`
+        rc=0
+    fi
+    echo "$line"
+    return $rc
+}
+    # meta cme to configure the interface
+
+#############################################################################
+#
+# Main program to configure bond/bond->vlan
+#
+# input : ifname=<bond_name|vlan_name> mports=<ports> ifcmd=bond
+#
+############################################################################
+function config_bond_vlan {
+
+    local ifcmd=""
+    # target port
+    local ifname=""
+    # member ports
+    local mports=""
+    local other_args=""
+
+    # if arguments are base64 encoded, decode first
+    if [ $# -eq 1 ]; then
+        line=`decode_arguments "$1"`
+        if [ $? -eq 0 ]; then
+            eval set -- "$line"
+        fi
+    fi
+
+    # since comma was used as delimiter in xCAT DB and their scripts.
+    ARGS=`echo "$@" | $sed -e 's/%2c/,/g'`
+    eval set -- "$ARGS"
+    log_debug "input args is \"$@\"" >&2
+
+    # parser input arguments
+    while [ -n "$1" ];
+    do
+        key=`echo "$1" | cut -s -d= -f1`
+        if [ "$key" = "ifname" ] || \
+           [ "$key" = "ifcmd" ]; then
+            eval "$1"
+
+        elif [ "$key" = "mports" ]; then
+            val=`echo "$1" | cut -s -d= -f2-`
+            if [ -n "$val" ]; then
+                mports="${mports}${mports:+,}$val"
+            fi
+        else
+            other_args="${other_args}${other_args:+ }$1"
+        fi
+        shift
+    done
+
+    if [ -z "$ifcmd" ]; then
+        log_error "Empty input command, abort!" >&2
+        return 1
+    fi
+
+    local line=""
+    if [ -n "$mports" ]; then
+        line=`expand_ports $mports`
+        if [ -z "$line" ]; then
+            log_error "No member port found for \"mports=$mports\"!"
+            return 1
+        fi
+        mports="$line"
+    fi
+    if [ x$ifcmd == "xbond" ]; then 
+        create_bond_interface ifname=$ifname slave_ports="$mports" $other_args
+        rc=$?
+    else
+        log_error "Unknown input command, \"$cmd\". Abort!" >&2
+        return 1
+    fi 
+    log_status "$PROGNAME return code on host `hostname -s` equals to $rc" >&2
+    (exit $rc)
+}
+