133 lines
4.0 KiB
Bash
Executable File
133 lines
4.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Used for debugging
|
|
# set -ax
|
|
|
|
# This script is required after a reboot of the cloud after the cloud has been
|
|
# shut down
|
|
|
|
get_lead()
|
|
{
|
|
app_name=$1
|
|
|
|
jq -rc ".applications.\"${app_name}\".units | to_entries[] | select(.value.leader == "true") | .key" "${juju_status}"
|
|
}
|
|
|
|
do_vault()
|
|
{
|
|
vault_vip=$(juju config vault vip)
|
|
echo export VAULT_ADDR="http://${vault_vip}:8200"
|
|
export VAULT_ADDR="http://${vault_vip}:8200"
|
|
|
|
echo " "
|
|
|
|
IPS=$(jq -r '.applications.vault.units | to_entries[].value."public-address"' "${juju_status}")
|
|
|
|
for ip in $IPS;do
|
|
echo export VAULT_ADDR=http://"${ip}":8200;
|
|
export VAULT_ADDR=http://"${ip}":8200;
|
|
for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do
|
|
echo vault operator unseal -tls-skip-verify "$vault_key"
|
|
vault operator unseal -tls-skip-verify "$vault_key"
|
|
done;
|
|
done;
|
|
|
|
juju run -a vault "hooks/update-status"
|
|
}
|
|
|
|
juju-wait -v
|
|
|
|
juju_status=$(mktemp)
|
|
|
|
juju status --format json > "${juju_status}"
|
|
|
|
bootstrap_unit=""
|
|
|
|
mysql_status=$(jq -rc ".applications.\"mysql-innodb-cluster\".units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}" "${juju_status}")
|
|
|
|
is_ready=$(echo "$mysql_status" | jq -rc . | grep "Mode: R/W, Cluster is ONLINE" | jq -r .sub)
|
|
|
|
if [[ -z "${is_ready}" ]] ; then
|
|
reboot_status=$(juju run-action mysql-innodb-cluster/leader reboot-cluster-from-complete-outage --wait --format json)
|
|
|
|
outcome=$(echo "$reboot_status" | jq .[].results.outcome)
|
|
|
|
if [[ ${outcome} == null ]] ; then
|
|
|
|
output=$(echo "$reboot_status" | jq .[].results.output)
|
|
|
|
mysql_ip=$(echo "$output" | sed -e 's/\\n/\n/g' 2>&1| grep Please | sed -e "s|.*Please use the most up to date instance: '\(.*\):.*|\1|")
|
|
|
|
bootstrap_unit=$(jq -r ".applications.\"mysql-innodb-cluster\".units | to_entries[] | select(.value.\"public-address\" == \"${mysql_ip}\") | .key" "${juju_status}")
|
|
|
|
juju run-action "${bootstrap_unit}" reboot-cluster-from-complete-outage --wait
|
|
|
|
fi
|
|
|
|
juju run --application mysql-innodb-cluster "hooks/update-status"
|
|
|
|
# Run update-status on all mysql-router units, to ensure that theyave connected to the mysql-innodb-cluster
|
|
jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-router\") | .key" "${juju_status}" \
|
|
| xargs -I{} juju run -a "{}" -- 'hooks/update-status'
|
|
|
|
fi
|
|
|
|
juju run -a heat -- sudo systemctl restart heat-engine &
|
|
juju run -a vault -- sudo systemctl restart vault &
|
|
|
|
wait
|
|
|
|
for app in heat vault ; do
|
|
juju run -a $app "hooks/update-status" &
|
|
done
|
|
|
|
wait
|
|
|
|
# cleanup all crm resources
|
|
jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" "${juju_status}" \
|
|
| xargs -I{} juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"'
|
|
|
|
do_vault
|
|
|
|
# Wait 10 seconds, and ensure that vault is unsealed
|
|
echo "Sleeping 10 seconds to wait for vault to finalise unseal"
|
|
sleep 10
|
|
|
|
ceph_osd_apps=$(jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key" "${juju_status}")
|
|
|
|
ceph_osds=""
|
|
for apps in ${ceph_osd_apps}
|
|
do
|
|
ceph_osds="${ceph_osds} $(jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key" "${juju_status}")"
|
|
done
|
|
|
|
juju run --all -- sudo systemctl restart systemd-resolved
|
|
|
|
for ceph_osd in ${ceph_osds}
|
|
do
|
|
juju ssh "${ceph_osd}" -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ;
|
|
sudo systemctl start --all --type=service vaultlocker-decrypt@* ;
|
|
sudo systemctl start --all --type=service ceph-volume@*' &
|
|
done
|
|
|
|
wait
|
|
|
|
juju run -a ceph-osd "hooks/update-status"
|
|
|
|
lds_servers=$(jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key" "${juju_status}")
|
|
|
|
cat > /tmp/restart-landscape.sh << EOF
|
|
#!/bin/bash
|
|
|
|
sudo systemctl restart landscape-*
|
|
EOF
|
|
|
|
for lds_server in ${lds_servers}
|
|
do
|
|
juju scp /tmp/restart-landscape.sh "${lds_server}":.
|
|
juju ssh "${lds_server}" chmod +x restart-landscape.sh
|
|
juju ssh "${lds_server}" sudo ./restart-landscape.sh &
|
|
done
|
|
|
|
wait
|