From 85a5cadd4786cfb396d5389b646a34c8de4dc854 Mon Sep 17 00:00:00 2001
From: Arif Ali <arif.ali@canonical.com>
Date: Mon, 19 Jun 2023 14:45:13 +0100
Subject: [PATCH] Update and add moe scripts

---
 check_avail_pci.sh      | 21 +++++++++++++
 check_innodb.sh         |  6 ++--
 do_ceph.sh              |  6 ++++
 do_group_replication.sh |  4 ++-
 do_innodb_resolution.sh | 69 +++++++++++++++++++++++++++++++++++++++++
 fix_cloud.sh            |  8 ++---
 shutdown_nodes.sh       |  2 +-
 vault-secrets.txt       | 12 +++----
 vault.sh                |  2 ++
 9 files changed, 116 insertions(+), 14 deletions(-)
 create mode 100644 check_avail_pci.sh
 create mode 100755 do_ceph.sh
 create mode 100755 do_innodb_resolution.sh

diff --git a/check_avail_pci.sh b/check_avail_pci.sh
new file mode 100644
index 0000000..05a3c6c
--- /dev/null
+++ b/check_avail_pci.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+NODE=$1
+
+usage() { 
+    echo "Usage: ${0} [node-name]" 1>&2
+    exit 0
+}
+
+get_pci(){
+    RO_MYSQL_UNIT=$(juju status --format json | jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]|  select(.value."workload-status".message | contains("R/O")) | .key] | .[0]')
+    PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd)
+
+    juju ssh $RO_MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"select pci_stats from nova.compute_nodes where hypervisor_hostname like '%$NODE%' and deleted_at is NULL\G;\""
+}
+
+if [ $# -ne 1 ]; then
+    usage
+else
+    get_pci
+fi
diff --git a/check_innodb.sh b/check_innodb.sh
index daf95ce..2fc46b0 100755
--- a/check_innodb.sh
+++ b/check_innodb.sh
@@ -1,11 +1,13 @@
 #!/bin/bash
 
+
 check_controller()
 {
     controller=$1
-    model="cpe-focal"
+    model="${controller}:cpe-focal"
 
-    juju run-action -m "${controller}":${model} mysql-innodb-cluster/leader --wait cluster-status --format json | jq -rc '.[].results."cluster-status"'  | jq
+    RW_MYSQL_UNIT=$(juju status -m ${model} --format json | jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]|  select(.value."workload-status".message | contains("R/W")) | .key] | .[0]')
+    juju run-action -m ${model} ${RW_MYSQL_UNIT} --wait cluster-status --format json | jq -rc '.[].results."cluster-status"'  | jq
 
 }
 
diff --git a/do_ceph.sh b/do_ceph.sh
new file mode 100755
index 0000000..8f65197
--- /dev/null
+++ b/do_ceph.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+juju run --timeout=30s -a ceph-osd -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@*'
+juju run --timeout=30s -a ceph-osd -- 'sudo systemctl start --all --type=service vaultlocker-decrypt@*'
+juju run --timeout=30s -a ceph-osd -- 'sudo systemctl start --all --type=service ceph-volume@*'
+
diff --git a/do_group_replication.sh b/do_group_replication.sh
index 7296ab8..7c87190 100755
--- a/do_group_replication.sh
+++ b/do_group_replication.sh
@@ -11,7 +11,9 @@ do_gr(){
     MYSQL_UNIT=${NODE}
     PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd)
 
-    juju ssh $MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"stop group_replication; start group_replication;\""
+    juju ssh $MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"stop group_replication;\""
+    sleep 5
+    juju ssh $MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"start group_replication;\""
 }
 
 if [ $# -ne 1 ]; then
diff --git a/do_innodb_resolution.sh b/do_innodb_resolution.sh
new file mode 100755
index 0000000..ec4ce50
--- /dev/null
+++ b/do_innodb_resolution.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+NODE=$1
+
+set -ax
+
+usage() { 
+    echo "Usage: ${0} [node-name]" 1>&2
+    exit 0
+}
+
+do_recovery() {
+    juju_status=$(mktemp)
+    juju status --format json > ${juju_status}
+
+    MYSQL_UNIT=${NODE}
+    RW_MYSQL_UNIT=$(jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]|  select(.value."workload-status".message | contains("R/W")) | .key] | .[0]' ${juju_status})
+    PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd)
+
+    MYSQL_UNIT_IP=$(jq -rc '.applications."mysql-innodb-cluster".units."mysql-innodb-cluster/0"."public-address"' ${juju_status})
+
+    juju ssh ${MYSQL_UNIT} -- sudo systemctl stop mysql
+    juju run-action --wait ${RW_MYSQL_UNIT} remove-instance address=${MYSQL_UNIT_IP} force=true
+
+    cat > init_mysql.sh << EOF
+set -ax
+cd /var/lib
+mv mysql mysql.old.\$(date +%s)
+mkdir mysql
+chown mysql:mysql mysql
+chmod 700 mysql
+mysqld --initialize
+systemctl start mysql
+EOF
+
+    juju scp init_mysql.sh ${MYSQL_UNIT}:init_mysql.sh
+    juju ssh ${MYSQL_UNIT} -- "chmod +x init_mysql.sh && sudo ./init_mysql.sh"
+
+    #exit
+    cat > set_password.sh << EOF
+#!/bin/bash
+set -ax
+old_password_line=\$(grep 'temporary password' /var/log/mysql/error.log | tail -n 1)
+OLD_PASSWORD="\$(echo \${old_password_line} | awk '{print \$13}' | sed -e 's/\r//g')"
+
+cat > /tmp/alter_command << EOF2
+ALTER USER 'root'@'localhost' IDENTIFIED BY '${PASSWORD}';
+EOF2
+
+mysql -u root -p\${OLD_PASSWORD}  -e \"source /tmp/alter_command\"
+EOF
+
+    juju scp set_password.sh ${MYSQL_UNIT}:set_password.sh
+    juju ssh ${MYSQL_UNIT} -- "chmod +x set_password.sh && sudo ./set_password.sh"
+
+    juju run -u ${MYSQL_UNIT} -- charms.reactive clear_flag local.cluster.user-created
+    juju run -u ${MYSQL_UNIT} -- charms.reactive clear_flag local.cluster.all-users-created
+    juju run -u ${MYSQL_UNIT} -- ./hooks/update-status
+
+    sleep 10
+
+    juju run-action --wait ${RW_MYSQL_UNIT} add-instance address=${MYSQL_UNIT_IP}
+}
+
+if [ $# -ne 1 ]; then
+    usage
+else
+    do_recovery
+fi
diff --git a/fix_cloud.sh b/fix_cloud.sh
index 14b8f2e..b1dd02d 100755
--- a/fix_cloud.sh
+++ b/fix_cloud.sh
@@ -12,14 +12,14 @@ check_juju_version()
 {
     juju_version=$(juju version | cut -d'-' -f1 | cut -d'.' -f1)
 
-    juju_timeout="--timeout 30s"
+    juju_timeout="30s"
 
-    juju_run="juju run ${juju_timeout}"
+    juju_run="juju run --timeout ${juju_timeout}"
     juju_run_action="juju run-action --wait"
 
     if [[ ${juju_version} -ge 3 ]] ; then
-        juju_run="juju exec ${juju_timeout}"
-        juju_run_action="juju run --wait"
+        juju_run="juju exec --wait=${juju_timeout}"
+        juju_run_action="juju run"
     fi
 }
 
diff --git a/shutdown_nodes.sh b/shutdown_nodes.sh
index 950fd6f..e25dac5 100755
--- a/shutdown_nodes.sh
+++ b/shutdown_nodes.sh
@@ -4,7 +4,7 @@ rcmd="ssh"
 cmd="sudo halt -p"
 
 if [[ "$1" == "asrock" ]] ; then
-  hosts="192.168.1.21[1-3]"
+  hosts="192.168.1.21[1-4]"
 elif [[ "$1" == "pi-k8s" ]] ; then
   hosts="192.168.1.8[1-6]"
 fi
diff --git a/vault-secrets.txt b/vault-secrets.txt
index 3db4dfb..5417d37 100644
--- a/vault-secrets.txt
+++ b/vault-secrets.txt
@@ -1,10 +1,10 @@
-Unseal Key 1: rDjUVcpsiwv3SDmAAavfkMBJ2O2H9XfsmMXOXaI46zT0
-Unseal Key 2: ww7AgnhYINM7BpD1v2LHyBsPn10CWT2AnPm5KfCN48Ve
-Unseal Key 3: jH5WxSmY0I+InM0W4n4L9H9jKXW1BLPGFLCMfVTF8tFb
-Unseal Key 4: qfxj3ejTGD1tawonOjXRG+qdIFNZHSsRJIUqp6x2PLHw
-Unseal Key 5: HGT+smfQdbGKQsHa+Aci1x3u2RPjNQZHJ8ouUPNXPpvC
+Unseal Key 1: BXyPvDeMDzfzarrN2Gmtl0YBuCuhrSllhYindRR2Sdle
+Unseal Key 2: EhN9f3J1+FdawXsApoUNT3MlSMTk7zCBBWteuYj6azgf
+Unseal Key 3: XGTCoW5eMSykc+Gn1A01gTtM2wxpIi9i5hTydFni+AQJ
+Unseal Key 4: NcfyD1ORFugYg8wt7AXvEFpNlMUiPrxjDoRR359UbD8Y
+Unseal Key 5: UJMEzTKKVpNMtlhTDzOzwesBfI/Yfo5pfAViHnDvtZ3D
 
-Initial Root Token: hvs.iLynNrOHmG74IT29P6zMnShH
+Initial Root Token: hvs.tyA2bCvimuDETIFyeDw2rqU5
 
 Vault initialized with 5 key shares and a key threshold of 3. Please securely
 distribute the key shares printed above. When the Vault is re-sealed,
diff --git a/vault.sh b/vault.sh
index ee6896a..9711aad 100755
--- a/vault.sh
+++ b/vault.sh
@@ -2,6 +2,8 @@
 
 vault_file="vault-secrets.txt"
 
+juju run -a vault -- sudo systemctl restart vault
+
 IPS=$(juju status vault --format json  | jq -r '.applications.vault.units | to_entries[].value."public-address"')
 
 for ip in $IPS