useful_scripts/fix_cloud.sh

#!/bin/bash

# Used for debugging
# set -ax

# This script is required after a reboot of the cloud after the cloud has been
# shut down

check_unit_status()
{

    app_name=$1
    status_check="$2"

    unit_status=$(juju status --format json | jq -rc ".applications.${app_name}.units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}")

    app_units=$(echo ${unit_status} | jq .sub | sed s/\"//g)

    num=0
    for unit in ${app_units} ; do
      this_unit_status=$(echo $unit_status | jq -rc . | grep ${unit} | jq .status | sed s/\"//g)
      if [[ "${this_unit_status}" == "${status_check}" ]] ; then
        (( num++ ))
      fi
    done

    if [[ $num -ge 3 ]] ; then echo 1
    else echo 0
    fi
}

get_lead()
{
    app_name=$1

    cat ${juju_status} | jq -rc ".applications.${app_name}.units | to_entries[] | select(.value.leader == "true") | .key"
}

do_vault()
{
    vault_vip=$(juju config vault vip)
    echo export VAULT_ADDR="http://${vault_vip}:8200"
    export VAULT_ADDR="http://${vault_vip}:8200"

    echo " "

    IPS=$(cat ${juju_status} | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g)

    for ip in $IPS;do
      echo export VAULT_ADDR=http://${ip}:8200;
      export VAULT_ADDR=http://${ip}:8200;
      for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do
        echo vault operator unseal -tls-skip-verify $vault_key
        vault operator unseal -tls-skip-verify $vault_key
      done;
    done;

    juju run -a vault "hooks/update-status"
}

juju-wait -v

juju_status=$(mktemp)

juju status --format json > ${juju_status}

mysql_status=$(cat ${juju_status} | jq -rc ".applications.mysql.units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}")

#{"sub":"mysql/0","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 1"}
#{"sub":"mysql/1","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 0"}
#{"sub":"mysql/2","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 0"}

mysql_units=$(echo ${mysql_status} | jq .sub | sed s/\"//g)
bootstrap_unit=""

mysql_lead=$(get_lead mysql)

safe_to_bootstrap=$(echo $mysql_status | jq -rc . | grep "Safe To Bootstrap: 1" | jq .sub | sed s/\"//g)

if [[ -n "$safe_to_bootstrap" ]]
then 

  bootstrap_unit=$safe_to_bootstrap

else

  seq_number=$(echo $mysql_status | jq -rc . | grep "Sequence Number" )

  if [[ -n "${seq_number}" ]]
  then

     seqs=$(echo $seq_number | jq -rc ". | {sub:.sub,seq:(.status|split(\".\")[1]|split(\": \")[1])}")

     uniq_seqs=$(echo $seqs| jq .seq | sed s/\"//g | sort -n | uniq)
     seq_count=$(echo $uniq_seqs | xargs | wc -w)

     highest_seq=$(echo "${seqs}"| jq .seq | sed s/\"//g | sort -n | uniq | tail -n 1)
     lowest_seq=$(echo "${seqs}"| jq .seq | sed s/\"//g | sort -n | uniq | head -n 1)

     if [[ ${seq_count} -eq 1 ]]
     then # same seq numbers all round
       if [[ ${highest_seq} -eq -1 ]]
       then # if all seq numbers are -1
         echo "The sequence number is -1 ... exiting"
         exit 1
       fi
       bootstrap_unit=${mysql_lead}
     else # we have different seq numbers

       unit_high_seq=$(echo $seqs | jq -rc . | grep ${highest_seq} | jq .sub | sed s/\"//g | tail -n 1)

       bootstrap_unit=${unit_high_seq}
     fi
   fi
fi

if [[ -n ${bootstrap_unit} ]]
then
    juju run-action --wait ${bootstrap_unit} bootstrap-pxc
    juju run --application mysql "hooks/update-status"
    until [[ $(check_unit_status mysql "Unit waiting for cluster bootstrap") -eq 1 ]]
    do
      sleep 10
    done
    if [[ "${bootstrap_unit}" == "${mysql_lead}" ]] ; then
      for unit in ${mysql_units}; do
        if [[ "${unit}" != "${mysql_lead}" ]] ; then
            juju run-action --wait ${unit} notify-bootstrapped
            ran_bootstrap="true"
            break
        fi
      done
    else
      juju run-action --wait ${mysql_lead} notify-bootstrapped
      ran_bootstrap="true"
    fi
    juju run -a mysql "hooks/update-status"
    until [[ $(check_unit_status mysql "Unit is ready") -eq 1 ]]
    do
      sleep 10
    done
    # This is so that nagios doesn't report that the mysql daemon is down
    # although the process is running. juju will then automatically start
    # the mysqld process
    juju ssh ${bootstrap_unit} -- sudo reboot
fi

juju run -a nova-cloud-controller -- sudo systemctl restart nova-api-os-compute nova-conductor nova-consoleauth &
juju run -a heat -- sudo systemctl restart heat-engine &
juju run -a vault -- sudo systemctl restart vault &

wait

for app in nova-cloud-controller heat vault ; do
  juju run -a $app "hooks/update-status" &
done

wait

# cleanup all crm resources
cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g | xargs -i juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"'

do_vault

# Wait 10 seconds, and ensure that vault is unsealed
echo "Sleeping 10 seconds to wait for vault to finalise unseal"
sleep 10

ceph_osd_apps=$(cat ${juju_status} | jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key")

ceph_osds=""
for apps in ${ceph_osd_apps}
do
  ceph_osds="${ceph_osds} $(cat ${juju_status} | jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key")"
done


for ceph_osd in ${ceph_osds}
do
  juju ssh ${ceph_osd} -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ;
    sudo systemctl start --all --type=service vaultlocker-decrypt@* ;
    sudo systemctl start --all --type=service ceph-volume@*' &
done

wait

juju run -a ceph-osd "hooks/update-status"

lds_servers=$(cat ${juju_status} | jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key")

cat > /tmp/restart-landscape.sh << EOF
#!/bin/bash

sudo systemctl restart landscape-*
EOF

for lds_server in ${lds_servers}
do
  juju scp /tmp/restart-landscape.sh ${lds_server}:.
  juju ssh ${lds_server} chmod +x restart-landscape.sh
  juju ssh ${lds_server} sudo ./restart-landscape.sh &
done

wait

juju run --all -- sudo systemctl restart systemd-resolved