diff --git a/check_avail_pci.sh b/check_avail_pci.sh new file mode 100644 index 0000000..05a3c6c --- /dev/null +++ b/check_avail_pci.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +NODE=$1 + +usage() { + echo "Usage: ${0} [node-name]" 1>&2 + exit 0 +} + +get_pci(){ + RO_MYSQL_UNIT=$(juju status --format json | jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]| select(.value."workload-status".message | contains("R/O")) | .key] | .[0]') + PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd) + + juju ssh $RO_MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"select pci_stats from nova.compute_nodes where hypervisor_hostname like '%$NODE%' and deleted_at is NULL\G;\"" +} + +if [ $# -ne 1 ]; then + usage +else + get_pci +fi diff --git a/check_innodb.sh b/check_innodb.sh index daf95ce..2fc46b0 100755 --- a/check_innodb.sh +++ b/check_innodb.sh @@ -1,11 +1,13 @@ #!/bin/bash + check_controller() { controller=$1 - model="cpe-focal" + model="${controller}:cpe-focal" - juju run-action -m "${controller}":${model} mysql-innodb-cluster/leader --wait cluster-status --format json | jq -rc '.[].results."cluster-status"' | jq + RW_MYSQL_UNIT=$(juju status -m ${model} --format json | jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]| select(.value."workload-status".message | contains("R/W")) | .key] | .[0]') + juju run-action -m ${model} ${RW_MYSQL_UNIT} --wait cluster-status --format json | jq -rc '.[].results."cluster-status"' | jq } diff --git a/do_ceph.sh b/do_ceph.sh new file mode 100755 index 0000000..8f65197 --- /dev/null +++ b/do_ceph.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +juju run --timeout=30s -a ceph-osd -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@*' +juju run --timeout=30s -a ceph-osd -- 'sudo systemctl start --all --type=service vaultlocker-decrypt@*' +juju run --timeout=30s -a ceph-osd -- 'sudo systemctl start --all --type=service ceph-volume@*' + diff --git a/do_group_replication.sh b/do_group_replication.sh index 7296ab8..7c87190 100755 --- a/do_group_replication.sh +++ b/do_group_replication.sh @@ -11,7 +11,9 @@ do_gr(){ MYSQL_UNIT=${NODE} PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd) - juju ssh $MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"stop group_replication; start group_replication;\"" + juju ssh $MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"stop group_replication;\"" + sleep 5 + juju ssh $MYSQL_UNIT "sudo mysql -u root -p$PASSWORD -e \"start group_replication;\"" } if [ $# -ne 1 ]; then diff --git a/do_innodb_resolution.sh b/do_innodb_resolution.sh new file mode 100755 index 0000000..ec4ce50 --- /dev/null +++ b/do_innodb_resolution.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +NODE=$1 + +set -ax + +usage() { + echo "Usage: ${0} [node-name]" 1>&2 + exit 0 +} + +do_recovery() { + juju_status=$(mktemp) + juju status --format json > ${juju_status} + + MYSQL_UNIT=${NODE} + RW_MYSQL_UNIT=$(jq -r '[.applications."mysql-innodb-cluster".units | to_entries[]| select(.value."workload-status".message | contains("R/W")) | .key] | .[0]' ${juju_status}) + PASSWORD=$(juju run --unit mysql-innodb-cluster/leader leader-get mysql.passwd) + + MYSQL_UNIT_IP=$(jq -rc '.applications."mysql-innodb-cluster".units."mysql-innodb-cluster/0"."public-address"' ${juju_status}) + + juju ssh ${MYSQL_UNIT} -- sudo systemctl stop mysql + juju run-action --wait ${RW_MYSQL_UNIT} remove-instance address=${MYSQL_UNIT_IP} force=true + + cat > init_mysql.sh << EOF +set -ax +cd /var/lib +mv mysql mysql.old.\$(date +%s) +mkdir mysql +chown mysql:mysql mysql +chmod 700 mysql +mysqld --initialize +systemctl start mysql +EOF + + juju scp init_mysql.sh ${MYSQL_UNIT}:init_mysql.sh + juju ssh ${MYSQL_UNIT} -- "chmod +x init_mysql.sh && sudo ./init_mysql.sh" + + #exit + cat > set_password.sh << EOF +#!/bin/bash +set -ax +old_password_line=\$(grep 'temporary password' /var/log/mysql/error.log | tail -n 1) +OLD_PASSWORD="\$(echo \${old_password_line} | awk '{print \$13}' | sed -e 's/\r//g')" + +cat > /tmp/alter_command << EOF2 +ALTER USER 'root'@'localhost' IDENTIFIED BY '${PASSWORD}'; +EOF2 + +mysql -u root -p\${OLD_PASSWORD} -e \"source /tmp/alter_command\" +EOF + + juju scp set_password.sh ${MYSQL_UNIT}:set_password.sh + juju ssh ${MYSQL_UNIT} -- "chmod +x set_password.sh && sudo ./set_password.sh" + + juju run -u ${MYSQL_UNIT} -- charms.reactive clear_flag local.cluster.user-created + juju run -u ${MYSQL_UNIT} -- charms.reactive clear_flag local.cluster.all-users-created + juju run -u ${MYSQL_UNIT} -- ./hooks/update-status + + sleep 10 + + juju run-action --wait ${RW_MYSQL_UNIT} add-instance address=${MYSQL_UNIT_IP} +} + +if [ $# -ne 1 ]; then + usage +else + do_recovery +fi diff --git a/fix_cloud.sh b/fix_cloud.sh index 14b8f2e..b1dd02d 100755 --- a/fix_cloud.sh +++ b/fix_cloud.sh @@ -12,14 +12,14 @@ check_juju_version() { juju_version=$(juju version | cut -d'-' -f1 | cut -d'.' -f1) - juju_timeout="--timeout 30s" + juju_timeout="30s" - juju_run="juju run ${juju_timeout}" + juju_run="juju run --timeout ${juju_timeout}" juju_run_action="juju run-action --wait" if [[ ${juju_version} -ge 3 ]] ; then - juju_run="juju exec ${juju_timeout}" - juju_run_action="juju run --wait" + juju_run="juju exec --wait=${juju_timeout}" + juju_run_action="juju run" fi } diff --git a/shutdown_nodes.sh b/shutdown_nodes.sh index 950fd6f..e25dac5 100755 --- a/shutdown_nodes.sh +++ b/shutdown_nodes.sh @@ -4,7 +4,7 @@ rcmd="ssh" cmd="sudo halt -p" if [[ "$1" == "asrock" ]] ; then - hosts="192.168.1.21[1-3]" + hosts="192.168.1.21[1-4]" elif [[ "$1" == "pi-k8s" ]] ; then hosts="192.168.1.8[1-6]" fi diff --git a/vault-secrets.txt b/vault-secrets.txt index 3db4dfb..5417d37 100644 --- a/vault-secrets.txt +++ b/vault-secrets.txt @@ -1,10 +1,10 @@ -Unseal Key 1: rDjUVcpsiwv3SDmAAavfkMBJ2O2H9XfsmMXOXaI46zT0 -Unseal Key 2: ww7AgnhYINM7BpD1v2LHyBsPn10CWT2AnPm5KfCN48Ve -Unseal Key 3: jH5WxSmY0I+InM0W4n4L9H9jKXW1BLPGFLCMfVTF8tFb -Unseal Key 4: qfxj3ejTGD1tawonOjXRG+qdIFNZHSsRJIUqp6x2PLHw -Unseal Key 5: HGT+smfQdbGKQsHa+Aci1x3u2RPjNQZHJ8ouUPNXPpvC +Unseal Key 1: BXyPvDeMDzfzarrN2Gmtl0YBuCuhrSllhYindRR2Sdle +Unseal Key 2: EhN9f3J1+FdawXsApoUNT3MlSMTk7zCBBWteuYj6azgf +Unseal Key 3: XGTCoW5eMSykc+Gn1A01gTtM2wxpIi9i5hTydFni+AQJ +Unseal Key 4: NcfyD1ORFugYg8wt7AXvEFpNlMUiPrxjDoRR359UbD8Y +Unseal Key 5: UJMEzTKKVpNMtlhTDzOzwesBfI/Yfo5pfAViHnDvtZ3D -Initial Root Token: hvs.iLynNrOHmG74IT29P6zMnShH +Initial Root Token: hvs.tyA2bCvimuDETIFyeDw2rqU5 Vault initialized with 5 key shares and a key threshold of 3. Please securely distribute the key shares printed above. When the Vault is re-sealed, diff --git a/vault.sh b/vault.sh index ee6896a..9711aad 100755 --- a/vault.sh +++ b/vault.sh @@ -2,6 +2,8 @@ vault_file="vault-secrets.txt" +juju run -a vault -- sudo systemctl restart vault + IPS=$(juju status vault --format json | jq -r '.applications.vault.units | to_entries[].value."public-address"') for ip in $IPS