confluent/confluent_server/bin/confluent_selfcheck

#!/usr/bin/python3

import os
import socket
import glob
import ssl
import sys
path = os.path.dirname(os.path.realpath(__file__))
path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
if path.startswith('/opt'):
    # if installed into system path, do not muck with things
    sys.path.append(path)
import confluent.sshutil as sshutil
import confluent.certutil as certutil
import confluent.config.configmanager as configmanager
import subprocess
import tempfile
import shutil

def fprint(txt):
    sys.stdout.write(txt)
    sys.stdout.flush()


def tftp_works():
    try:
        subprocess.check_call(['curl', '--connect-timeout', '2', '-sf', 'tftp://localhost/confluent/x86_64/ipxe.efi', '-o', '/dev/null'])
        return True
    except Exception:
        return False

def emprint(txt):
    if sys.stdout.isatty():
        print('\x1b[1m\x1b[4m' + txt + '\x1b[0m')
    else:
        print(txt)

def deployment_configured():
    return os.path.exists('/var/lib/confluent/public/site/confluent_uuid')

def webserver_listening():
    try:
        conn = socket.create_connection(('localhost', 443))
        return conn
    except Exception:
        return False


def certificates_missing_ips(conn):
    # check if the tls can verify by the right CAs, then further
    # check if all ip addresses are in the certificate offered
    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    ctx.check_hostname = False
    for cacert in glob.glob('/var/lib/confluent/public/site/tls/*.pem'):
        ctx.load_verify_locations(cacert)
    sock = ctx.wrap_socket(conn)
    crt = sock.getpeercert()
    sans = crt.get('subjectAltName', [])
    ips = certutil.get_ip_addresses()
    missing_ips = []
    for ip in ips:
        for san in sans:
            field, val = san
            if val[-1] == '\n':
                val = val[:-1]
            if ':' in val:
                # must normalize ipv6 to a sane value
                val = socket.getaddrinfo(val, 443, type=socket.SOCK_STREAM)[0][-1][0]
            if ip == val:
                break
        else:
            missing_ips.append(ip)
    return missing_ips


def web_download_works():
    try:
        subprocess.check_call(['curl', '-skf', 'https://localhost/confluent-public/site/confluent_uuid', '-o', '/dev/null'])
    except Exception:
        return False
    return True
            

def nics_missing_ipv6():
    # check for ability to create AF_INET6, for kernel disabled ipv6
    a = socket.socket(socket.AF_INET6)
    ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
    missingnics = []
    for line in ipaddrs:
        comps = line.split()
        if not comps:
            continue
        iname, state = comps[:2]
        if iname == b'lo':
            continue
        addrs = comps[2:]
        hasv6 = False
        hasv4 = False
        for addr in addrs:
            if b'.' in addr:
                hasv4 = True
            if addr.startswith(b'fe80::'):
                hasv6 = True
        if hasv4 and not hasv6:
            missingnics.append(iname.decode('utf8'))
    return missingnics

def insecure_boot_attempts():
    insecurenodes = set([])
    with open('/var/log/confluent/events') as eventin:
        line = True
        while line:
            line = eventin.readline()
            if 'insecure mode is disabled' in line:
                line = line.split()
                insecurenodes.add(line[7])
    for node in insecurenodes:
        currattr = subprocess.check_output(['nodeattrib', node, 'deployment.useinsecureprotocols'])
        currattr = currattr.split()
        if len(currattr) > 2 and currattr[2] == b'firmware':
            continue
        else:
            return True
    return False


def uuid_matches():
    with open('/var/lib/confluent/public/site/confluent_uuid', 'r') as uuidf:
        fsuuid = uuidf.read().strip()
    dbuuid = configmanager.get_global('confluent_uuid')
    return dbuuid == fsuuid

if __name__ == '__main__':
    sys.stdout.write('OS Deployment: ')
    sys.stdout.flush()
    if deployment_configured():
        print("Initialized")
        sys.stdout.write('Confluent UUID: ')
        sys.stdout.flush()
        if uuid_matches():
            print('Consistent')
        else:
            emprint('Inconsistent between confluent database and /var/lib/confluent (Example resolution: confetty set /uuid resync=1)')
        fprint('Web Server: ')
        conn = webserver_listening()
        if conn:
            print('Running')
            fprint('Web Certificate: ')
            cert = certificates_missing_ips(conn)
            if cert:
                cert = ', '.join(cert)
                emprint('Addresses missing from certificate: {0} (Example resolution: osdeploy initialize -t)'.format(cert))
            else:
                print('OK')
                fprint('Checking web download: ')
                if web_download_works():
                    print('OK')
                else:
                    emprint('Failed to download /confluent-public/site/confluent_uuid')
        else:
            emprint('Not Running (Example resolution: systemctl enable httpd --now)')
        fprint('TFTP Status: ')
        if tftp_works():
            print('OK')
        else:
            emprint('TFTP failure, PXE will not work, though media and HTTP boot can still work. (Example resolution: osdeploy initialize -p)')
        fprint('SSH root user public key: ')
        if glob.glob('/var/lib/confluent/public/site/ssh/*.rootpubkey'):
            print('OK')
        else:
            emprint('No trusted ssh keys for root user, passwordless SSH from managers to nodes may not work (Example resolution: osdeploy initialize -u)')
        if sshutil.sshver() > 7.6:
            fprint('Checking SSH Certificate authority: ')
            try:
                sshutil.prep_ssh_key('/etc/confluent/ssh/ca')
                print('OK')
            except Exception:
                emprint('Failed to load SSH authority key, deployed servers will not have host certificates for known_hosts and users may be unable to ssh between nodes without a password (Example resolution: osdeploy initialize -s)')            
            fprint('Checking confluent SSH automation key: ')
            try:
                sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
                print('OK')
            except subprocess.CalledProcessError:
                emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)')
        fprint('Checking for blocked insecure boot: ')
        if insecure_boot_attempts():
            emprint('Some nodes are attempting network boot using PXE or HTTP boot, but the node is not configured to allow this (Example resolution: nodegroupattrib everything deployment.useinsecureprotocols=firmware)')
        else:
            print('OK')
        fprint('Checking IPv6 enablement: ')
        nics = nics_missing_ipv6()
        if nics:
            snics = ','.join(nics)
            emprint('Some interfaces ({0}) have ipv6 disabled, and may be unable to fully perform discovery or deployment (Example resolution: nmcli c m {1} ipv6.method link-local )'.format(snics, nics[0]))
        else:
            print('OK')
    else:
        print("Uninitialized, further OS deployment checks skipped, see `osdeploy initialize` to set up OS deployment feature")