#!/bin/bash # Used for debugging # set -ax # This script is required after a reboot of the cloud after the cloud has been # shut down get_lead() { app_name=$1 cat ${juju_status} | jq -rc ".applications.\"${app_name}\".units | to_entries[] | select(.value.leader == "true") | .key" } do_vault() { vault_vip=$(juju config vault vip) echo export VAULT_ADDR="http://${vault_vip}:8200" export VAULT_ADDR="http://${vault_vip}:8200" echo " " IPS=$(cat ${juju_status} | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g) for ip in $IPS;do echo export VAULT_ADDR=http://${ip}:8200; export VAULT_ADDR=http://${ip}:8200; for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do echo vault operator unseal -tls-skip-verify $vault_key vault operator unseal -tls-skip-verify $vault_key done; done; juju run -a vault "hooks/update-status" } juju-wait -v juju_status=$(mktemp) juju status --format json > ${juju_status} mysql_status=$(cat ${juju_status} | jq -rc ".applications.\"mysql-innodb-cluster\".units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}") mysql_units=$(echo ${mysql_status} | jq .sub | sed s/\"//g) bootstrap_unit="" mysql_lead=$(get_lead mysql-innodb-cluster) reboot_status=$(juju run-action mysql-innodb-cluster/leader reboot-cluster-from-complete-outage --wait --format json) outcome=$(echo $reboot_status| jq .[].results.outcome) if [[ ${outcome} == null ]] ; then output=$(echo $reboot_status| jq .[].results.output) mysql_ip=$(echo $output | sed -e 's/\\n/\n/g' 2>&1| grep Please | sed -e "s|.*Please use the most up to date instance: '\(.*\):.*|\1|") bootstrap_unit=$(cat ${juju_status} | jq ".applications.\"mysql-innodb-cluster\".units | to_entries[] | select(.value.\"public-address\" == \"${mysql_ip}\") | .key" | sed s/\"//g) juju run-action ${bootstrap_unit} reboot-cluster-from-complete-outage --wait fi juju run --application mysql-innodb-cluster "hooks/update-status" # Run update-status on all mysql-router units, to ensure that theyave connected to the mysql-innodb-cluster cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-router\") | .key" | sed s/\"//g | xargs -i juju run -a "{}" -- 'hooks/update-status' juju run -a heat -- sudo systemctl restart heat-engine & juju run -a vault -- sudo systemctl restart vault & wait for app in heat vault ; do juju run -a $app "hooks/update-status" & done wait # cleanup all crm resources cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g | xargs -i juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"' do_vault # Wait 10 seconds, and ensure that vault is unsealed echo "Sleeping 10 seconds to wait for vault to finalise unseal" sleep 10 ceph_osd_apps=$(cat ${juju_status} | jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key") ceph_osds="" for apps in ${ceph_osd_apps} do ceph_osds="${ceph_osds} $(cat ${juju_status} | jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key")" done for ceph_osd in ${ceph_osds} do juju ssh ${ceph_osd} -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ; sudo systemctl start --all --type=service vaultlocker-decrypt@* ; sudo systemctl start --all --type=service ceph-volume@*' & done wait juju run -a ceph-osd "hooks/update-status" lds_servers=$(cat ${juju_status} | jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key") cat > /tmp/restart-landscape.sh << EOF #!/bin/bash sudo systemctl restart landscape-* EOF for lds_server in ${lds_servers} do juju scp /tmp/restart-landscape.sh ${lds_server}:. juju ssh ${lds_server} chmod +x restart-landscape.sh juju ssh ${lds_server} sudo ./restart-landscape.sh & done wait juju run --all -- sudo systemctl restart systemd-resolved