diff --git a/check_juju.sh b/check_juju.sh new file mode 100755 index 0000000..5fdcdad --- /dev/null +++ b/check_juju.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +check_controller() +{ + controller=$1 + model="cpe-focal" + + juju status -m ${controller}:${model} --color | grep ^Unit -A 999999 | egrep -v "started.*focal|started.*bionic|active.*idle" + +} + +if [[ -z "$1" ]] ; then + check_controller $(juju controllers --format json | jq .\"current-controller\" | sed s/\"//g) +else + check_controller $1 +fi + diff --git a/check_vip_issue_db.sh b/check_vip_issue_db.sh new file mode 100755 index 0000000..734f62d --- /dev/null +++ b/check_vip_issue_db.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +juju_status=$(mktemp) +juju_db=$(mktemp) + +juju status --format json > ${juju_status} +JUJU_DEV_FEATURE_FLAGS=developer-mode juju dump-db --format json > ${juju_db} + +#juju_status=/tmp/tmp.IG9jLOKBET +#juju_db=/tmp/tmp.1kg5IV5I3k + +hacluster_apps=$(cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g) + +for app in ${hacluster_apps} +do + + super_app=$(cat ${juju_status} | jq .applications.\"${app}\".relations.ha[] | sed s/\"//g) + units=$(cat ${juju_status} | jq ".applications.\"${super_app}\".units | keys[]" | sed s/\"//g) + + #vips=$(juju config ${super_app} vip) + vips=$(cat ${juju_db} | jq ".settings[] | select(._id | contains(\"a#${super_app}#cs\")) | .settings.vip") + + bindings=$(cat ${juju_status} | jq ".applications.\"${super_app}\".\"endpoint-bindings\" | keys[]" | sed s/\"//g) + + for r in ${bindings} + do + [[ ${r} != "cluster" ]] && [[ ${r} != "ha" ]] && continue + for unit in ${units} + do + #relation_ids=$(juju run --unit ${unit} "relation-ids ${r}" | awk -F\: '{print $2}' | sort) + relation_ids=$(cat ${juju_db} | jq ".relations[] | select(.endpoints[].relation.name == \"${r}\") | select(.endpoints[].applicationname == \"${super_app}\")| .id") + for i in ${relation_ids} + do + #relation_output=$(juju run --unit ${unit} "relation-get -r ${r}:${i} - ${unit}" | grep $vip) + if [[ ${r} == "cluster" ]] ; then + relation_output=$(cat ${juju_db} | jq ".settings[] | select(._id | contains(\"r#${i}#peer#${unit}\"))") + else + relation_output=$(cat ${juju_db} | jq ".settings[] | select(._id | contains(\"r#${i}#${unit}#req\"))") + fi + + for vip in ${vips} ; do + + #include_vip=$(echo $relation_output | grep ${vip}) + include_vip=$(echo $relation_output | jq -rc ".settings | [.\"egress-subnets\", .\"private-address\", .\"ingress-address\"]" | grep ${vip}) + + if [[ -n "$relation_output" ]] && [[ -n "${include_vip}" ]] ; then + + #echo $relation_output | jq -rc "._id" + echo ${unit}: {vip: ${vip}, binding: ${r}:${i}} + echo + echo $relation_output | jq -rc ".settings | [.\"egress-subnets\", .\"private-address\", .\"ingress-address\"]" + echo + + fi + done + done + done + done +done + diff --git a/fix_cloud_focal.sh b/fix_cloud_focal.sh new file mode 100755 index 0000000..6ecf799 --- /dev/null +++ b/fix_cloud_focal.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +# Used for debugging +# set -ax + +# This script is required after a reboot of the cloud after the cloud has been +# shut down + +get_lead() +{ + app_name=$1 + + cat ${juju_status} | jq -rc ".applications.\"${app_name}\".units | to_entries[] | select(.value.leader == "true") | .key" +} + +do_vault() +{ + vault_vip=$(juju config vault vip) + echo export VAULT_ADDR="http://${vault_vip}:8200" + export VAULT_ADDR="http://${vault_vip}:8200" + + echo " " + + IPS=$(cat ${juju_status} | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g) + + for ip in $IPS;do + echo export VAULT_ADDR=http://${ip}:8200; + export VAULT_ADDR=http://${ip}:8200; + for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do + echo vault operator unseal -tls-skip-verify $vault_key + vault operator unseal -tls-skip-verify $vault_key + done; + done; + + juju run -a vault "hooks/update-status" +} + +juju-wait -v + +juju_status=$(mktemp) + +juju status --format json > ${juju_status} + +mysql_status=$(cat ${juju_status} | jq -rc ".applications.\"mysql-innodb-cluster\".units | to_entries[] | {sub:.key,status:.value[\"workload-status\"].message}") + +mysql_units=$(echo ${mysql_status} | jq .sub | sed s/\"//g) +bootstrap_unit="" + +mysql_lead=$(get_lead mysql-innodb-cluster) + +reboot_status=$(juju run-action mysql-innodb-cluster/leader reboot-cluster-from-complete-outage --wait --format json) + +outcome=$(echo $reboot_status| jq .[].results.outcome) + +if [[ ${outcome} == null ]] ; then + + output=$(echo $reboot_status| jq .[].results.output) + + mysql_ip=$(echo $output | sed -e 's/\\n/\n/g' 2>&1| grep Please | sed -e "s|.*Please use the most up to date instance: '\(.*\):.*|\1|") + + bootstrap_unit=$(cat ${juju_status} | jq ".applications.\"mysql-innodb-cluster\".units | to_entries[] | select(.value.\"public-address\" == \"${mysql_ip}\") | .key" | sed s/\"//g) + + juju run-action ${bootstrap_unit} reboot-cluster-from-complete-outage --wait + +fi + +juju run --application mysql-innodb-cluster "hooks/update-status" + +# Run update-status on all mysql-router units, to ensure that theyave connected to the mysql-innodb-cluster +cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"mysql-router\") | .key" | sed s/\"//g | xargs -i juju run -a "{}" -- 'hooks/update-status' + +juju run -a heat -- sudo systemctl restart heat-engine & +juju run -a vault -- sudo systemctl restart vault & + +wait + +for app in heat vault ; do + juju run -a $app "hooks/update-status" & +done + +wait + +# cleanup all crm resources +cat ${juju_status} | jq ".applications | to_entries[] | select(.value[\"charm-name\"] == \"hacluster\") | .key" | sed s/\"//g | xargs -i juju run --unit "{}"/leader -- 'sudo crm_resource -l | sed s/:.*//g | uniq | xargs -i sudo crm resource cleanup \"\{\}\"' + +do_vault + +# Wait 10 seconds, and ensure that vault is unsealed +echo "Sleeping 10 seconds to wait for vault to finalise unseal" +sleep 10 + +ceph_osd_apps=$(cat ${juju_status} | jq -rc ".applications | to_entries[] | select(.value[\"charm-name\"] == \"ceph-osd\") | .key") + +ceph_osds="" +for apps in ${ceph_osd_apps} +do + ceph_osds="${ceph_osds} $(cat ${juju_status} | jq -rc ". | .applications[\"${apps}\"].units | to_entries[] | .key")" +done + + +for ceph_osd in ${ceph_osds} +do + juju ssh ${ceph_osd} -- 'sudo systemctl kill --all --type=service vaultlocker-decrypt@* ; + sudo systemctl start --all --type=service vaultlocker-decrypt@* ; + sudo systemctl start --all --type=service ceph-volume@*' & +done + +wait + +juju run -a ceph-osd "hooks/update-status" + +lds_servers=$(cat ${juju_status} | jq -rc ". | .applications[\"landscape-server\"].units | to_entries[] | .key") + +cat > /tmp/restart-landscape.sh << EOF +#!/bin/bash + +sudo systemctl restart landscape-* +EOF + +for lds_server in ${lds_servers} +do + juju scp /tmp/restart-landscape.sh ${lds_server}:. + juju ssh ${lds_server} chmod +x restart-landscape.sh + juju ssh ${lds_server} sudo ./restart-landscape.sh & +done + +wait + +juju run --all -- sudo systemctl restart systemd-resolved diff --git a/get_all_ips.sh b/get_all_ips.sh index 5ae7f05..014edc4 100755 --- a/get_all_ips.sh +++ b/get_all_ips.sh @@ -4,6 +4,10 @@ juju_status=$(mktemp) juju status --format json > $juju_status -cat ${juju_status} | jq -rc '.machines | to_entries[] |[.key,.value.hostname,.value."ip-addresses"]' -cat ${juju_status} | jq -rc '.machines | to_entries[] | select(.value.containers != null ) | .value.containers | to_entries[] | [.key,.value.hostname,.value."ip-addresses"]' +# old method +#cat ${juju_status} | jq -rc '.machines | to_entries[] |[.key,.value.hostname,.value."ip-addresses"]' +#cat ${juju_status} | jq -rc '.machines | to_entries[] | select(.value.containers != null ) | .value.containers | to_entries[] | [.key,.value.hostname,.value."ip-addresses"]' + +# new method +cat ${juju_status} | jq -rc '.machines | to_entries[] |[.key,.value.hostname,.value."ip-addresses", [(select(.value.containers != null ) | .value.containers | to_entries[] | [.key,.value.hostname,.value."ip-addresses"])]]' diff --git a/get_passwords.sh b/get_passwords.sh index 310ce16..bfede97 100755 --- a/get_passwords.sh +++ b/get_passwords.sh @@ -5,6 +5,7 @@ nagios_passwd=$(juju run --unit nagios/leader 'sudo cat /var/lib/juju/nagios.pas grafana_passwd=$(juju run-action --wait grafana/leader get-admin-password | grep password | awk '{print $2}') graylog_passwd=$(juju run-action --wait graylog/leader show-admin-password | grep admin-password | awk '{print $2}') mysql_passwd=$(juju run --unit mysql/leader 'leader-get root-password') +innodb_passwd=$(juju run --unit mysql-innodb-cluster/leader 'leader-get mysql.passwd') echo "Keystone admin password: ... ${keystone_passwd}" echo "nagios password: ... ${nagios_passwd}" diff --git a/vault-secrets.txt b/vault-secrets.txt index 4e6ce50..1f00e61 100644 --- a/vault-secrets.txt +++ b/vault-secrets.txt @@ -1,10 +1,10 @@ -Unseal Key 1: Uqi9GhxPReMkPf0b1JKApVyDmXlAdhKbCEonyVuZeDT9 -Unseal Key 2: nJeypTRP2cJSKqENcup7S7CR9O1yxp3pwG/zLagj7ttP -Unseal Key 3: 1FEGUTx45p4pmmGsZO4UGCae9IhsPfrPO2GcYaobyozb -Unseal Key 4: 0XVF+/FUC+GpFep8YgFOOLkYPMeRjvWk5YRGt37uU+Ie -Unseal Key 5: 6nsTnzm1xl6OlJ7i/TlOjJ4VRusJJJuJ3SYz7X4D2tqG +Unseal Key 1: Tbel8zdD7vS8g1bSUaW5Q+8KS3No+sNVrqGKodiPGJ57 +Unseal Key 2: vioPM3fJJ8hitlBq7NuzsrlKCiHRo7dKkJXuskNJlERP +Unseal Key 3: T0JJT7ATamUa+Quk9VDWb2WvstQ3W7rGUC+CgJbZHo2v +Unseal Key 4: fC8m3accpSLsVhK3u3OA1kpsoUuXTp11xZcJolvu8uvo +Unseal Key 5: YfFun6w9cihqYDGak7c5Q1ZQxafI1vJ0B/d/XJqaUygC -Initial Root Token: s.EjAsxRkg3ai8FpuWWl44ceNr +Initial Root Token: s.90scV8c059y8IelnA33YAs6C Vault initialized with 5 key shares and a key threshold of 3. Please securely distribute the key shares printed above. When the Vault is re-sealed, diff --git a/vault.sh b/vault.sh index 1b94fbd..85c87c4 100755 --- a/vault.sh +++ b/vault.sh @@ -1,10 +1,6 @@ #!/bin/bash -vault_vip=$(juju config vault vip) -echo export VAULT_ADDR="http://${vault_vip}:8200" -export VAULT_ADDR="http://${vault_vip}:8200" - -echo " " +vault_file="vault-secrets.txt" IPS=$(juju status vault --format json | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g) @@ -12,7 +8,7 @@ for ip in $IPS do echo export VAULT_ADDR=http://${ip}:8200; export VAULT_ADDR=http://${ip}:8200; - for vault_key in $(head -n3 vault-secrets.txt | awk '{print $4}');do + for vault_key in $(head -n3 ${vault_file} | awk '{print $4}');do echo vault operator unseal -tls-skip-verify $vault_key vault operator unseal -tls-skip-verify $vault_key done diff --git a/vault_init.sh b/vault_init.sh new file mode 100755 index 0000000..6f55539 --- /dev/null +++ b/vault_init.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +vault_file="vault-secrets.txt" +vault_token_file="vault-token.txt" + +vault_vip=$(juju config vault vip) + +export VAULT_ADDR="http://${vault_vip}:8200" + +vault operator init -key-shares=5 -key-threshold=3 > ${vault_file} + +IPS=$(juju status vault --format json | jq '.applications.vault.units | to_entries[] | .value."public-address"' | sed s/\"//g) + +for ip in $IPS;do + echo export VAULT_ADDR=http://${ip}:8200; + export VAULT_ADDR=http://${ip}:8200; + for vault_key in $(head -n3 ${vault_file} | awk '{print $4}');do + echo vault operator unseal -tls-skip-verify $vault_key + vault operator unseal -tls-skip-verify $vault_key + done; +done; + +initial_token=$(grep Initial ${vault_file} | awk '{print $4}') + +export VAULT_ADDR="http://${vault_vip}:8200" +export VAULT_TOKEN=${initial_token} + +vault token create -ttl=10m > ${vault_token_file} + +token=$(cat ${vault_token_file} | grep token | head -n 1 | awk '{print $2}') + +juju run-action --wait vault/leader authorize-charm token=${token}