Update + add scripts

* Added a simple check juju script
* fix cloud script for focal, similar to the bionic one
* initialise vault scipt
* Update get_password script to add innodb
* Added a new script for check_vip_issue_db.sh

check_juju.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+check_controller()
+{
+    controller=$1
+    model="cpe-focal"
+
+    juju status -m ${controller}:${model} --color | grep ^Unit -A 999999 | egrep -v "started.*focal|started.*bionic|active.*idle"
+
+}
+
+if [[ -z "$1" ]] ; then
+    check_controller $(juju controllers --format json | jq .\"current-controller\" | sed s/\"//g)
+else
+    check_controller $1
+fi
+

check_vip_issue_db.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+juju_status=$(mktemp)
+juju_db=$(mktemp)
+
+juju status --format json > ${juju_status}
+JUJU_DEV_FEATURE_FLAGS=developer-mode juju dump-db --format json > ${juju_db}
+
+#juju_status=/tmp/tmp.IG9jLOKBET
+#juju_db=/tmp/tmp.1kg5IV5I3k
+
+hacluster_apps=$(cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g)
+
+for app in ${hacluster_apps}
+do
+
+  super_app=$(cat ${juju_status} | jq .applications.\"${app}\".relations.ha[] | sed s/\"//g)
+  units=$(cat ${juju_status} | jq ".applications.\"${super_app}\".units | keys[]" | sed s/\"//g)
+
+  #vips=$(juju config ${super_app} vip)
+  vips=$(cat ${juju_db} | jq ".settings[] | select(._id | contains(\"a#${super_app}#cs\")) | .settings.vip")
+
+  bindings=$(cat ${juju_status} | jq ".applications.\"${super_app}\".\"endpoint-bindings\" | keys[]" | sed s/\"//g)
+
+  for r in ${bindings}
+  do
+     [[ ${r} != "cluster" ]] && [[ ${r} != "ha" ]] && continue
+     for unit in ${units}
+     do
+        #relation_ids=$(juju run --unit ${unit} "relation-ids ${r}" | awk -F\: '{print $2}' | sort)
+        relation_ids=$(cat ${juju_db} | jq ".relations[] | select(.endpoints[].relation.name == \"${r}\") | select(.endpoints[].applicationname == \"${super_app}\")| .id")
+        for i in ${relation_ids}
+        do
+                #relation_output=$(juju run --unit ${unit} "relation-get -r ${r}:${i} - ${unit}" | grep $vip)
+                if [[ ${r} == "cluster" ]] ; then
+                    relation_output=$(cat ${juju_db} | jq ".settings[] | select(._id | contains(\"r#${i}#peer#${unit}\"))")
+                else
+                    relation_output=$(cat ${juju_db} | jq ".settings[] | select(._id | contains(\"r#${i}#${unit}#req\"))")
+                fi
+
+                for vip in ${vips} ; do
+
+                  #include_vip=$(echo $relation_output | grep ${vip})
+                  include_vip=$(echo $relation_output | jq -rc ".settings | [.\"egress-subnets\", .\"private-address\", .\"ingress-address\"]" | grep ${vip})
+
+                  if [[ -n "$relation_output" ]] && [[ -n "${include_vip}" ]] ; then
+
+                      #echo $relation_output | jq -rc "._id"
+                      echo ${unit}: {vip: ${vip}, binding: ${r}:${i}}
+                      echo
+                      echo $relation_output | jq -rc ".settings | [.\"egress-subnets\", .\"private-address\", .\"ingress-address\"]"
+                      echo
+
+                  fi
+                done
+        done
+     done
+  done
+done
+

fix_cloud_focal.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+# Used for debugging
+# set -ax
+
+# This script is required after a reboot of the cloud after the cloud has been
+# shut down
+
+get_lead()
+{
+    app_name=$1
+
+    cat ${juju_status} | jq -rc ".applications.\"${app_name}\".units | to_entries[] | select(.value.leader == "true") | .key"
+}
+
+do_vault()
+{
+    vault_vip=$(juju config vault vip)
+    echo export VAULT_ADDR="http://${vault_vip}:8200"
+    export VAULT_ADDR="http://${vault_vip}:8200"
+
+    echo " "
+
+    IPS=$(cat ${juju_status} | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g)
+
+    for ip in $IPS;do
+      echo export VAULT_ADDR=http://${ip}:8200;
+      export VAULT_ADDR=http://${ip}:8200;
+      for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do
+        echo vault operator unseal -tls-skip-verify $vault_key
+        vault operator unseal -tls-skip-verify $vault_key
+      done;
+    done;
+
+    juju run -a vault "hooks/update-status"
+}
+
+juju-wait -v
+
+juju_status=$(mktemp)
+
+juju status --format json > ${juju_status}
+
+mysql_status=$(cat ${juju_status} | jq -rc ".applications.\"mysql-innodb-cluster\".units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}")
+
+mysql_units=$(echo ${mysql_status} | jq .sub | sed s/\"//g)
+bootstrap_unit=""
+
+mysql_lead=$(get_lead mysql-innodb-cluster)
+
+reboot_status=$(juju run-action mysql-innodb-cluster/leader reboot-cluster-from-complete-outage --wait --format json)
+ 
+outcome=$(echo $reboot_status| jq .[].results.outcome)
+
+if [[ ${outcome} == null ]] ; then
+
+  output=$(echo $reboot_status| jq .[].results.output)
+
+  mysql_ip=$(echo $output | sed -e 's/\\n/\n/g' 2>&1| grep Please | sed -e "s|.*Please use the most up to date instance: '\(.*\):.*|\1|")
+
+  bootstrap_unit=$(cat ${juju_status} | jq ".applications.\"mysql-innodb-cluster\".units | to_entries[] | select(.value.\"public-address\" == \"${mysql_ip}\") | .key" | sed s/\"//g)
+
+  juju run-action ${bootstrap_unit} reboot-cluster-from-complete-outage --wait
+
+fi
+
+juju run --application mysql-innodb-cluster "hooks/update-status"
+
+# Run update-status on all mysql-router units, to ensure that theyave connected to the mysql-innodb-cluster
+cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-router\") | .key" | sed s/\"//g | xargs -i juju run -a "{}" -- 'hooks/update-status'
+
+juju run -a heat -- sudo systemctl restart heat-engine &
+juju run -a vault -- sudo systemctl restart vault &
+
+wait
+
+for app in heat vault ; do
+  juju run -a $app "hooks/update-status" &
+done
+
+wait
+
+# cleanup all crm resources
+cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g | xargs -i juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"'
+
+do_vault
+
+# Wait 10 seconds, and ensure that vault is unsealed
+echo "Sleeping 10 seconds to wait for vault to finalise unseal"
+sleep 10
+
+ceph_osd_apps=$(cat ${juju_status} | jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key")
+
+ceph_osds=""
+for apps in ${ceph_osd_apps}
+do
+  ceph_osds="${ceph_osds} $(cat ${juju_status} | jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key")"
+done
+
+
+for ceph_osd in ${ceph_osds}
+do
+  juju ssh ${ceph_osd} -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ;
+    sudo systemctl start --all --type=service vaultlocker-decrypt@* ;
+    sudo systemctl start --all --type=service ceph-volume@*' &
+done
+
+wait
+
+juju run -a ceph-osd "hooks/update-status"
+
+lds_servers=$(cat ${juju_status} | jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key")
+
+cat > /tmp/restart-landscape.sh << EOF
+#!/bin/bash
+
+sudo systemctl restart landscape-*
+EOF
+
+for lds_server in ${lds_servers}
+do
+  juju scp /tmp/restart-landscape.sh ${lds_server}:.
+  juju ssh ${lds_server} chmod +x restart-landscape.sh
+  juju ssh ${lds_server} sudo ./restart-landscape.sh &
+done
+
+wait
+
+juju run --all -- sudo systemctl restart systemd-resolved

get_all_ips.sh

@@ -4,6 +4,10 @@ juju_status=$(mktemp)
 
 juju status --format json > $juju_status
 
-cat ${juju_status} | jq -rc '.machines | to_entries[] |[.key,.value.hostname,.value."ip-addresses"]'
-cat ${juju_status} | jq -rc '.machines | to_entries[] | select(.value.containers != null ) | .value.containers | to_entries[] | [.key,.value.hostname,.value."ip-addresses"]'
+# old method
+#cat ${juju_status} | jq -rc '.machines | to_entries[] |[.key,.value.hostname,.value."ip-addresses"]'
+#cat ${juju_status} | jq -rc '.machines | to_entries[] | select(.value.containers != null ) | .value.containers | to_entries[] | [.key,.value.hostname,.value."ip-addresses"]'
+
+# new method
+cat ${juju_status} | jq -rc '.machines | to_entries[] |[.key,.value.hostname,.value."ip-addresses", [(select(.value.containers != null ) | .value.containers | to_entries[] | [.key,.value.hostname,.value."ip-addresses"])]]'
 

get_passwords.sh

@@ -5,6 +5,7 @@ nagios_passwd=$(juju run --unit nagios/leader 'sudo cat /var/lib/juju/nagios.pas
 grafana_passwd=$(juju run-action --wait grafana/leader get-admin-password | grep password | awk '{print $2}')
 graylog_passwd=$(juju run-action --wait graylog/leader show-admin-password | grep admin-password | awk '{print $2}')
 mysql_passwd=$(juju run --unit mysql/leader 'leader-get root-password')
+innodb_passwd=$(juju run --unit mysql-innodb-cluster/leader 'leader-get mysql.passwd')
 
 echo "Keystone admin password: ... ${keystone_passwd}"
 echo "nagios password: ... ${nagios_passwd}"

vault-secrets.txt

@@ -1,10 +1,10 @@
-Unseal Key 1: Uqi9GhxPReMkPf0b1JKApVyDmXlAdhKbCEonyVuZeDT9
-Unseal Key 2: nJeypTRP2cJSKqENcup7S7CR9O1yxp3pwG/zLagj7ttP
-Unseal Key 3: 1FEGUTx45p4pmmGsZO4UGCae9IhsPfrPO2GcYaobyozb
-Unseal Key 4: 0XVF+/FUC+GpFep8YgFOOLkYPMeRjvWk5YRGt37uU+Ie
-Unseal Key 5: 6nsTnzm1xl6OlJ7i/TlOjJ4VRusJJJuJ3SYz7X4D2tqG
+Unseal Key 1: Tbel8zdD7vS8g1bSUaW5Q+8KS3No+sNVrqGKodiPGJ57
+Unseal Key 2: vioPM3fJJ8hitlBq7NuzsrlKCiHRo7dKkJXuskNJlERP
+Unseal Key 3: T0JJT7ATamUa+Quk9VDWb2WvstQ3W7rGUC+CgJbZHo2v
+Unseal Key 4: fC8m3accpSLsVhK3u3OA1kpsoUuXTp11xZcJolvu8uvo
+Unseal Key 5: YfFun6w9cihqYDGak7c5Q1ZQxafI1vJ0B/d/XJqaUygC
 
-Initial Root Token: s.EjAsxRkg3ai8FpuWWl44ceNr
+Initial Root Token: s.90scV8c059y8IelnA33YAs6C
 
 Vault initialized with 5 key shares and a key threshold of 3. Please securely
 distribute the key shares printed above. When the Vault is re-sealed,

vault.sh

@@ -1,10 +1,6 @@
 #!/bin/bash
 
-vault_vip=$(juju config vault vip)
-echo export VAULT_ADDR="http://${vault_vip}:8200"
-export VAULT_ADDR="http://${vault_vip}:8200"
-
-echo " "
+vault_file="vault-secrets.txt"
 
 IPS=$(juju status vault --format json  | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g)
 
@@ -12,7 +8,7 @@ for ip in $IPS
 do
   echo export VAULT_ADDR=http://${ip}:8200;
   export VAULT_ADDR=http://${ip}:8200;
-  for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do
+  for vault_key in $(head -n3 ${vault_file} | awk '{print $4}');do
     echo vault operator unseal -tls-skip-verify $vault_key
     vault operator unseal -tls-skip-verify $vault_key
   done

vault_init.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+vault_file="vault-secrets.txt"
+vault_token_file="vault-token.txt"
+
+vault_vip=$(juju config vault vip)
+
+export VAULT_ADDR="http://${vault_vip}:8200"
+
+vault operator init -key-shares=5 -key-threshold=3 > ${vault_file}
+
+IPS=$(juju status vault --format json  | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g)
+
+for ip in $IPS;do
+  echo export VAULT_ADDR=http://${ip}:8200;
+  export VAULT_ADDR=http://${ip}:8200;
+  for vault_key in $(head -n3 ${vault_file} | awk '{print $4}');do
+    echo vault operator unseal -tls-skip-verify $vault_key
+    vault operator unseal -tls-skip-verify $vault_key
+  done;
+done;
+
+initial_token=$(grep Initial ${vault_file} | awk '{print $4}')
+
+export VAULT_ADDR="http://${vault_vip}:8200"
+export VAULT_TOKEN=${initial_token}
+
+vault token create -ttl=10m > ${vault_token_file}
+
+token=$(cat ${vault_token_file} | grep token | head -n 1 | awk '{print $2}')
+
+juju run-action --wait vault/leader authorize-charm token=${token}