Update to latest scripts

* do_innodb_resolution.sh: mostly now works, the timing between
  update-status and the add-instance may need to be tweaked still
* vault.sh: use juju status json to grab the IPs for the vault units
* check_juju.sh: add debug to the default status
* check_avail_pci.sh: add hypervisor_name to the output to make it
  easier to grab multiple nodes
* get_passwords.sh: add extra password for collection

check_avail_pci.sh

@@ -11,7 +11,7 @@ get_pci(){
     RO_MYSQL_UNIT=$(juju status --format json | jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]|  select(.value."workload-status".message | contains("R/O")) | .key] | .[0]')
     PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd)
 
-    juju ssh $RO_MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"select pci_stats from nova.compute_nodes where hypervisor_hostname like '%$NODE%' and deleted_at is NULL\G;\""
+    juju ssh $RO_MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"select pci_stats,hypervisor_hostname from nova.compute_nodes where hypervisor_hostname like '%$NODE%' and deleted_at is NULL\G;\""
 }
 
 if [ $# -ne 1 ]; then

check_juju.sh

@@ -5,7 +5,7 @@ check_controller()
     controller=$1
     model="cpe-focal"
 
-    juju status -m "${controller}":${model} --color | grep ^Unit -A 999999 | grep -E -v "started.*focal|started.*bionic|active.*idle"
+    juju status --debug -m "${controller}":${model} --color | grep ^Unit -A 999999 | grep -E -v "started.*focal|started.*bionic|active.*idle"
 
 }
 

do_innodb_resolution.sh

@@ -15,22 +15,25 @@ do_recovery() {
 
     MYSQL_UNIT=${NODE}
     RW_MYSQL_UNIT=$(jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]|  select(.value."workload-status".message | contains("R/W")) | .key] | .[0]' ${juju_status})
+    MYSQL_LEAD=$(jq -r '.applications."mysql-innodb-cluster".units | to_entries[] | select(.value.leader == true) | .key' ${juju_status})
     PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd)
 
-    MYSQL_UNIT_IP=$(jq -rc '.applications."mysql-innodb-cluster".units."mysql-innodb-cluster/0"."public-address"' ${juju_status})
+    MYSQL_UNIT_IP=$(jq -rc '.applications."mysql-innodb-cluster".units."'${NODE}'"."public-address"' ${juju_status})
+    MYSQL_UNIT_IP_HYPHENS=$(echo ${MYSQL_UNIT_IP} | tr '.' '-')
 
     juju ssh ${MYSQL_UNIT} -- sudo systemctl stop mysql
-    juju run-action --wait ${RW_MYSQL_UNIT} remove-instance address=${MYSQL_UNIT_IP} force=true
+    juju run-action --wait ${MYSQL_LEAD} remove-instance address=${MYSQL_UNIT_IP} force=true
 
     cat > init_mysql.sh << EOF
 set -ax
+systemctl stop mysql
 cd /var/lib
 mv mysql mysql.old.\$(date +%s)
 mkdir mysql
 chown mysql:mysql mysql
 chmod 700 mysql
-mysqld --initialize
-systemctl start mysql
+mysqld --initialize-insecure --user=mysql
+systemctl restart mysql
 EOF
 
     juju scp init_mysql.sh ${MYSQL_UNIT}:init_mysql.sh
@@ -40,14 +43,12 @@ EOF
     cat > set_password.sh << EOF
 #!/bin/bash
 set -ax
-old_password_line=\$(grep 'temporary password' /var/log/mysql/error.log | tail -n 1)
-OLD_PASSWORD="\$(echo \${old_password_line} | awk '{print \$13}' | sed -e 's/\r//g')"
 
 cat > /tmp/alter_command << EOF2
 ALTER USER 'root'@'localhost' IDENTIFIED BY '${PASSWORD}';
 EOF2
 
-mysql -u root -p\${OLD_PASSWORD}  -e \"source /tmp/alter_command\"
+mysql -u root -e "source /tmp/alter_command"
 EOF
 
     juju scp set_password.sh ${MYSQL_UNIT}:set_password.sh
@@ -57,9 +58,13 @@ EOF
     juju run -u ${MYSQL_UNIT} -- charms.reactive clear_flag local.cluster.all-users-created
     juju run -u ${MYSQL_UNIT} -- ./hooks/update-status
 
-    sleep 10
 
-    juju run-action --wait ${RW_MYSQL_UNIT} add-instance address=${MYSQL_UNIT_IP}
+    # just in case the remove-instance didn't unset these values
+    juju run -u ${MYSQL_LEAD} leader-set cluster-instance-clustered-${MYSQL_UNIT_IP_HYPHENS}="" cluster-instance-configured-${MYSQL_UNIT_IP_HYPHENS}=""
+
+    sleep 120
+
+    juju run-action --wait ${MYSQL_LEAD} add-instance address=${MYSQL_UNIT_IP}
 }
 
 if [ $# -ne 1 ]; then

get_passwords.sh

@@ -5,7 +5,8 @@ nagios_passwd=$(juju run --unit nagios/leader 'sudo cat /var/lib/juju/nagios.pas
 grafana_passwd=$(juju run-action --wait grafana/leader get-admin-password | grep password | awk '{print $2}')
 graylog_passwd=$(juju run-action --wait graylog/leader show-admin-password | grep admin-password | awk '{print $2}')
 mysql_passwd=$(juju run --unit mysql/leader 'leader-get root-password')
-innodb_passwd=$(juju run --unit mysql-innodb-cluster/leader 'leader-get mysql.passwd')
+innodb_admin_passwd=$(juju run --unit mysql-innodb-cluster/leader 'leader-get mysql.passwd')
+innodb_cluster_passwd=$(juju run --unit mysql-innodb-cluster/leader 'leader-get cluster-password')
 
 echo "Keystone admin password: ... ${keystone_passwd}"
 echo "nagios password: ... ${nagios_passwd}"

vault.sh

@@ -4,7 +4,7 @@ vault_file="vault-secrets.txt"
 
 juju run -a vault -- sudo systemctl restart vault
 
-IPS=$(juju status vault --format json  | jq -r '.applications.vault.units | to_entries[].value."public-address"')
+IPS=$(juju status --format json  | jq -r '.applications.vault.units | to_entries[].value."public-address"')
 
 for ip in $IPS
 do