From 4ee03a988755189eca9c4ed5654643e2d5dbce8f Mon Sep 17 00:00:00 2001
From: Arif Ali <arif.ali@canonical.com>
Date: Fri, 1 Jul 2022 20:28:36 +0100
Subject: [PATCH] Many Updates

* Combine bionic and focal fix_cloud script
* Add latest vault keys for my lab
* Add new shudown script for my lab envs
---
 .gitignore         |   2 +
 fix_cloud.sh       | 193 +++++++++++++++++++++++++--------------------
 fix_cloud_focal.sh | 133 +------------------------------
 shutdown_nodes.sh  |  12 +++
 vault-secrets.txt  |  12 +--
 5 files changed, 128 insertions(+), 224 deletions(-)
 mode change 100755 => 120000 fix_cloud_focal.sh
 create mode 100755 shutdown_nodes.sh

diff --git a/.gitignore b/.gitignore
index e964dda..f4ae247 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
 vault-token.txt
+controller_cert.crt
+license.txt
diff --git a/fix_cloud.sh b/fix_cloud.sh
index fad26a4..c45ec27 100755
--- a/fix_cloud.sh
+++ b/fix_cloud.sh
@@ -14,11 +14,11 @@ check_unit_status()
 
     unit_status=$(juju status --format json | jq -rc ".applications.${app_name}.units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}")
 
-    app_units=$(echo ${unit_status} | jq .sub | sed s/\"//g)
+    app_units=$(echo "${unit_status}" | jq -r .sub)
 
     num=0
     for unit in ${app_units} ; do
-      this_unit_status=$(echo $unit_status | jq -rc . | grep ${unit} | jq .status | sed s/\"//g)
+      this_unit_status=$(echo "$unit_status" | jq -rc . | grep "${unit}" | jq -r .status)
       if [[ "${this_unit_status}" == "${status_check}" ]] ; then
         (( num++ ))
       fi
@@ -33,7 +33,7 @@ get_lead()
 {
     app_name=$1
 
-    cat ${juju_status} | jq -rc ".applications.${app_name}.units | to_entries[] | select(.value.leader == "true") | .key"
+    jq -rc ".applications.${app_name}.units | to_entries[] | select(.value.leader == "true") | .key" "${juju_status}"
 }
 
 do_vault()
@@ -44,14 +44,14 @@ do_vault()
 
     echo " "
 
-    IPS=$(cat ${juju_status} | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g)
+    IPS=$(jq -r '.applications.vault.units | to_entries[].value."public-address"' "${juju_status}")
 
     for ip in $IPS;do
-      echo export VAULT_ADDR=http://${ip}:8200;
-      export VAULT_ADDR=http://${ip}:8200;
+      echo export VAULT_ADDR=http://"${ip}":8200;
+      export VAULT_ADDR=http://"${ip}":8200;
       for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do
-        echo vault operator unseal -tls-skip-verify $vault_key
-        vault operator unseal -tls-skip-verify $vault_key
+        echo vault operator unseal -tls-skip-verify "$vault_key"
+        vault operator unseal -tls-skip-verify "$vault_key"
       done;
     done;
 
@@ -62,61 +62,66 @@ juju-wait -v
 
 juju_status=$(mktemp)
 
-juju status --format json > ${juju_status}
+juju status --format json > "${juju_status}"
 
-mysql_status=$(cat ${juju_status} | jq -rc ".applications.mysql.units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}")
+# Check if we're using percona-cluster or mysql-innodb-cluster
+percona_cluster=$(jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"percona-cluster\") | .key" "${juju_status}")
+mysql_innodb_cluster=$(jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-innodb-cluster\") | .key" "${juju_status}")
 
-#{"sub":"mysql/0","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 1"}
-#{"sub":"mysql/1","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 0"}
-#{"sub":"mysql/2","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 0"}
+if [[ -n "${percona_cluster}" ]] ; then
 
-mysql_units=$(echo ${mysql_status} | jq .sub | sed s/\"//g)
-bootstrap_unit=""
+  mysql_status=$(jq -rc ".applications.mysql.units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}" "${juju_status}")
 
-mysql_lead=$(get_lead mysql)
+  #{"sub":"mysql/0","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 1"}
+  #{"sub":"mysql/1","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 0"}
+  #{"sub":"mysql/2","status":"MySQL is down. Sequence Number: 102921. Safe To Bootstrap: 0"}
 
-safe_to_bootstrap=$(echo $mysql_status | jq -rc . | grep "Safe To Bootstrap: 1" | jq .sub | sed s/\"//g)
+  mysql_units=$(echo "${mysql_status}" | jq -r .sub)
+  bootstrap_unit=""
 
-if [[ -n "$safe_to_bootstrap" ]]
-then 
+  mysql_lead=$(get_lead mysql)
 
-  bootstrap_unit=$safe_to_bootstrap
+  safe_to_bootstrap=$(echo "$mysql_status" | jq -rc . | grep "Safe To Bootstrap: 1" | jq -r .sub)
 
-else
+  if [[ -n "$safe_to_bootstrap" ]]
+  then 
 
-  seq_number=$(echo $mysql_status | jq -rc . | grep "Sequence Number" )
+    bootstrap_unit=$safe_to_bootstrap
 
-  if [[ -n "${seq_number}" ]]
+  else
+
+    seq_number=$(echo "$mysql_status" | jq -rc . | grep "Sequence Number" )
+
+    if [[ -n "${seq_number}" ]]
+    then
+
+      seqs=$(echo "$seq_number" | jq -rc ". | {sub:.sub,seq:(.status|split(\".\")[1]|split(\": \")[1])}")
+
+      uniq_seqs=$(echo "$seqs" | jq -r .seq | sort -n | uniq)
+      seq_count=$(echo "$uniq_seqs" | xargs | wc -w)
+
+      highest_seq=$(echo "${seqs}" | jq -r .seq | sort -n | uniq | tail -n 1)
+
+      if [[ ${seq_count} -eq 1 ]]
+      then # same seq numbers all round
+        if [[ ${highest_seq} -eq -1 ]]
+        then # if all seq numbers are -1
+          echo "The sequence number is -1 ... exiting"
+          exit 1
+        fi
+        bootstrap_unit=${mysql_lead}
+      else # we have different seq numbers
+
+        unit_high_seq=$(echo "$seqs" | jq -rc . | grep "${highest_seq}" | jq -r .sub | tail -n 1)
+
+        bootstrap_unit=${unit_high_seq}
+      fi
+    fi
+  fi
+
+  if [[ -n ${bootstrap_unit} ]]
   then
-
-     seqs=$(echo $seq_number | jq -rc ". | {sub:.sub,seq:(.status|split(\".\")[1]|split(\": \")[1])}")
-
-     uniq_seqs=$(echo $seqs| jq .seq | sed s/\"//g | sort -n | uniq)
-     seq_count=$(echo $uniq_seqs | xargs | wc -w)
-
-     highest_seq=$(echo "${seqs}"| jq .seq | sed s/\"//g | sort -n | uniq | tail -n 1)
-     lowest_seq=$(echo "${seqs}"| jq .seq | sed s/\"//g | sort -n | uniq | head -n 1)
-
-     if [[ ${seq_count} -eq 1 ]]
-     then # same seq numbers all round
-       if [[ ${highest_seq} -eq -1 ]]
-       then # if all seq numbers are -1
-         echo "The sequence number is -1 ... exiting"
-         exit 1
-       fi
-       bootstrap_unit=${mysql_lead}
-     else # we have different seq numbers
-
-       unit_high_seq=$(echo $seqs | jq -rc . | grep ${highest_seq} | jq .sub | sed s/\"//g | tail -n 1)
-
-       bootstrap_unit=${unit_high_seq}
-     fi
-   fi
-fi
-
-if [[ -n ${bootstrap_unit} ]]
-then
-    juju run-action --wait ${bootstrap_unit} bootstrap-pxc
+    juju run-action --wait "${bootstrap_unit}" bootstrap-pxc
     juju run --application mysql "hooks/update-status"
     until [[ $(check_unit_status mysql "Unit waiting for cluster bootstrap") -eq 1 ]]
     do
@@ -125,14 +130,12 @@ then
     if [[ "${bootstrap_unit}" == "${mysql_lead}" ]] ; then
       for unit in ${mysql_units}; do
         if [[ "${unit}" != "${mysql_lead}" ]] ; then
-            juju run-action --wait ${unit} notify-bootstrapped
-            ran_bootstrap="true"
+            juju run-action --wait "${unit}" notify-bootstrapped
             break
         fi
       done
     else
-      juju run-action --wait ${mysql_lead} notify-bootstrapped
-      ran_bootstrap="true"
+      juju run-action --wait "${mysql_lead}" notify-bootstrapped
     fi
     juju run -a mysql "hooks/update-status"
     until [[ $(check_unit_status mysql "Unit is ready") -eq 1 ]]
@@ -142,51 +145,71 @@ then
     # This is so that nagios doesn't report that the mysql daemon is down
     # although the process is running. juju will then automatically start
     # the mysqld process
-    juju ssh ${bootstrap_unit} -- sudo reboot
+    juju ssh "${bootstrap_unit}" -- sudo reboot
+  fi
+
+  juju run -a nova-cloud-controller -- sudo systemctl restart nova-api-os-compute nova-conductor nova-consoleauth &
+fi
+
+if [[ -n "${mysql_innodb_cluster}" ]] ; then
+
+  mysql_status=$(jq -rc ".applications.\"mysql-innodb-cluster\".units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}" "${juju_status}")
+
+  is_ready=$(echo "$mysql_status" | jq -rc . | grep "Mode: R/W, Cluster is ONLINE" | jq -r .sub)
+
+  if [[ -z "${is_ready}" ]] ; then
+    reboot_status=$(juju run-action mysql-innodb-cluster/leader reboot-cluster-from-complete-outage --wait --format json)
+ 
+    outcome=$(echo "$reboot_status" | jq .[].results.outcome)
+
+    if [[ ${outcome} == null ]] ; then
+
+      output=$(echo "$reboot_status" | jq .[].results.output)
+
+      mysql_ip=$(echo "$output" | sed -e 's/\\n/\n/g' 2>&1| grep Please | sed -e "s|.*Please use the most up to date instance: '\(.*\):.*|\1|")
+
+      bootstrap_unit=$(jq -r ".applications.\"mysql-innodb-cluster\".units | to_entries[] | select(.value.\"public-address\" == \"${mysql_ip}\") | .key" "${juju_status}")
+
+      juju run-action "${bootstrap_unit}" reboot-cluster-from-complete-outage --wait
+
+    fi
+
+    juju run --application mysql-innodb-cluster "hooks/update-status"
+
+    # Run update-status on all mysql-router units, to ensure that theyave connected to the mysql-innodb-cluster
+    jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-router\") | .key" "${juju_status}" \
+        | xargs -I{} juju run -a "{}" -- 'hooks/update-status'
+
+  fi
+
 fi
 
-juju run -a nova-cloud-controller -- sudo systemctl restart nova-api-os-compute nova-conductor nova-consoleauth &
 juju run -a heat -- sudo systemctl restart heat-engine &
 juju run -a vault -- sudo systemctl restart vault &
 
 wait
 
-for app in nova-cloud-controller heat vault ; do
-  juju run -a $app "hooks/update-status" &
-done
-
-wait
-
 # cleanup all crm resources
-cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g | xargs -i juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"'
+jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" "${juju_status}" \
+    | xargs -I{} juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"'
 
 do_vault
 
-# Wait 10 seconds, and ensure that vault is unsealed
-echo "Sleeping 10 seconds to wait for vault to finalise unseal"
-sleep 10
+juju run --all -- sudo systemctl restart systemd-resolved
 
-ceph_osd_apps=$(cat ${juju_status} | jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key")
+ceph_osd_apps=$(jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key" "${juju_status}")
 
 ceph_osds=""
 for apps in ${ceph_osd_apps}
 do
-  ceph_osds="${ceph_osds} $(cat ${juju_status} | jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key")"
-done
-
-
-for ceph_osd in ${ceph_osds}
-do
-  juju ssh ${ceph_osd} -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ;
-    sudo systemctl start --all --type=service vaultlocker-decrypt@* ;
-    sudo systemctl start --all --type=service ceph-volume@*' &
+  juju run -a "${apps}" --timeout 30s -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@*'
+  juju run -a "${apps}" --timeout 30s -- 'sudo systemctl start --all --type=service vaultlocker-decrypt@*'
+  juju run -a "${apps}" --timeout 30s -- 'sudo systemctl start --all --type=service ceph-volume@*' &
 done
 
 wait
 
-juju run -a ceph-osd "hooks/update-status"
-
-lds_servers=$(cat ${juju_status} | jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key")
+lds_servers=$(jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key" "${juju_status}")
 
 cat > /tmp/restart-landscape.sh << EOF
 #!/bin/bash
@@ -196,11 +219,9 @@ EOF
 
 for lds_server in ${lds_servers}
 do
-  juju scp /tmp/restart-landscape.sh ${lds_server}:.
-  juju ssh ${lds_server} chmod +x restart-landscape.sh
-  juju ssh ${lds_server} sudo ./restart-landscape.sh &
+  juju scp /tmp/restart-landscape.sh "${lds_server}":.
+  juju ssh "${lds_server}" chmod +x restart-landscape.sh
+  juju ssh "${lds_server}" sudo ./restart-landscape.sh &
 done
 
 wait
-
-juju run --all -- sudo systemctl restart systemd-resolved
diff --git a/fix_cloud_focal.sh b/fix_cloud_focal.sh
deleted file mode 100755
index 3e2ad05..0000000
--- a/fix_cloud_focal.sh
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/bin/bash
-
-# Used for debugging
-# set -ax
-
-# This script is required after a reboot of the cloud after the cloud has been
-# shut down
-
-get_lead()
-{
-    app_name=$1
-
-    jq -rc ".applications.\"${app_name}\".units | to_entries[] | select(.value.leader == "true") | .key" "${juju_status}"
-}
-
-do_vault()
-{
-    vault_vip=$(juju config vault vip)
-    echo export VAULT_ADDR="http://${vault_vip}:8200"
-    export VAULT_ADDR="http://${vault_vip}:8200"
-
-    echo " "
-
-    IPS=$(jq -r '.applications.vault.units | to_entries[].value."public-address"' "${juju_status}")
-
-    for ip in $IPS;do
-      echo export VAULT_ADDR=http://"${ip}":8200;
-      export VAULT_ADDR=http://"${ip}":8200;
-      for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do
-        echo vault operator unseal -tls-skip-verify "$vault_key"
-        vault operator unseal -tls-skip-verify "$vault_key"
-      done;
-    done;
-
-    juju run -a vault "hooks/update-status"
-}
-
-juju-wait -v
-
-juju_status=$(mktemp)
-
-juju status --format json > "${juju_status}"
-
-bootstrap_unit=""
-
-mysql_status=$(jq -rc ".applications.\"mysql-innodb-cluster\".units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}" "${juju_status}")
-
-is_ready=$(echo "$mysql_status" | jq -rc . | grep "Mode: R/W, Cluster is ONLINE" | jq -r .sub)
-
-if [[ -z "${is_ready}" ]] ; then
-  reboot_status=$(juju run-action mysql-innodb-cluster/leader reboot-cluster-from-complete-outage --wait --format json)
- 
-  outcome=$(echo "$reboot_status" | jq .[].results.outcome)
-
-  if [[ ${outcome} == null ]] ; then
-
-    output=$(echo "$reboot_status" | jq .[].results.output)
-
-    mysql_ip=$(echo "$output" | sed -e 's/\\n/\n/g' 2>&1| grep Please | sed -e "s|.*Please use the most up to date instance: '\(.*\):.*|\1|")
-
-    bootstrap_unit=$(jq -r ".applications.\"mysql-innodb-cluster\".units | to_entries[] | select(.value.\"public-address\" == \"${mysql_ip}\") | .key" "${juju_status}")
-
-    juju run-action "${bootstrap_unit}" reboot-cluster-from-complete-outage --wait
-
-  fi
-
-  juju run --application mysql-innodb-cluster "hooks/update-status"
-
-  # Run update-status on all mysql-router units, to ensure that theyave connected to the mysql-innodb-cluster
-  jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-router\") | .key" "${juju_status}" \
-      | xargs -I{} juju run -a "{}" -- 'hooks/update-status'
-
-fi
-
-juju run -a heat -- sudo systemctl restart heat-engine &
-juju run -a vault -- sudo systemctl restart vault &
-
-wait
-
-for app in heat vault ; do
-  juju run -a $app "hooks/update-status" &
-done
-
-wait
-
-# cleanup all crm resources
-jq -r ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" "${juju_status}" \
-    | xargs -I{} juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"'
-
-do_vault
-
-# Wait 10 seconds, and ensure that vault is unsealed
-echo "Sleeping 10 seconds to wait for vault to finalise unseal"
-sleep 10
-
-ceph_osd_apps=$(jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key" "${juju_status}")
-
-ceph_osds=""
-for apps in ${ceph_osd_apps}
-do
-  ceph_osds="${ceph_osds} $(jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key" "${juju_status}")"
-done
-
-juju run --all -- sudo systemctl restart systemd-resolved
-
-for ceph_osd in ${ceph_osds}
-do
-  juju ssh "${ceph_osd}" -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ;
-    sudo systemctl start --all --type=service vaultlocker-decrypt@* ;
-    sudo systemctl start --all --type=service ceph-volume@*' &
-done
-
-wait
-
-juju run -a ceph-osd "hooks/update-status"
-
-lds_servers=$(jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key" "${juju_status}")
-
-cat > /tmp/restart-landscape.sh << EOF
-#!/bin/bash
-
-sudo systemctl restart landscape-*
-EOF
-
-for lds_server in ${lds_servers}
-do
-  juju scp /tmp/restart-landscape.sh "${lds_server}":.
-  juju ssh "${lds_server}" chmod +x restart-landscape.sh
-  juju ssh "${lds_server}" sudo ./restart-landscape.sh &
-done
-
-wait
diff --git a/fix_cloud_focal.sh b/fix_cloud_focal.sh
new file mode 120000
index 0000000..666bf1c
--- /dev/null
+++ b/fix_cloud_focal.sh
@@ -0,0 +1 @@
+fix_cloud.sh
\ No newline at end of file
diff --git a/shutdown_nodes.sh b/shutdown_nodes.sh
new file mode 100755
index 0000000..950fd6f
--- /dev/null
+++ b/shutdown_nodes.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+rcmd="ssh"
+cmd="sudo halt -p"
+
+if [[ "$1" == "asrock" ]] ; then
+  hosts="192.168.1.21[1-3]"
+elif [[ "$1" == "pi-k8s" ]] ; then
+  hosts="192.168.1.8[1-6]"
+fi
+
+pdsh -R $rcmd -l ubuntu -w $hosts -- $cmd
diff --git a/vault-secrets.txt b/vault-secrets.txt
index 1f00e61..df8617a 100644
--- a/vault-secrets.txt
+++ b/vault-secrets.txt
@@ -1,10 +1,10 @@
-Unseal Key 1: Tbel8zdD7vS8g1bSUaW5Q+8KS3No+sNVrqGKodiPGJ57
-Unseal Key 2: vioPM3fJJ8hitlBq7NuzsrlKCiHRo7dKkJXuskNJlERP
-Unseal Key 3: T0JJT7ATamUa+Quk9VDWb2WvstQ3W7rGUC+CgJbZHo2v
-Unseal Key 4: fC8m3accpSLsVhK3u3OA1kpsoUuXTp11xZcJolvu8uvo
-Unseal Key 5: YfFun6w9cihqYDGak7c5Q1ZQxafI1vJ0B/d/XJqaUygC
+Unseal Key 1: xb11HFT+6KBH0y8Jglvmr2ljnVIWZk9YruHLJpR4oVot
+Unseal Key 2: aG8MUmlY0NbzHyIp5MSj98q7eCalUhrYmYXXMGozocSn
+Unseal Key 3: 1BMKM86m6JMZL9DCypY1rbsfvuiO22Tm0i9E2T6a/UUU
+Unseal Key 4: VZ36NLxKn/xC32TGDhThIPlndqSaXhmm3FFEewzoeAXk
+Unseal Key 5: bAZ77ThRY6RfHM00q7kkSmknY0tqXla092PDPkuP5NfW
 
-Initial Root Token: s.90scV8c059y8IelnA33YAs6C
+Initial Root Token: s.EQOap6R44K8Hpx3o7lB5ORqs
 
 Vault initialized with 5 key shares and a key threshold of 3. Please securely
 distribute the key shares printed above. When the Vault is re-sealed,