From 792e6472e418e4be06f415f93e22c588543e7f75 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 23 Jan 2023 11:24:25 -0500 Subject: [PATCH 1/5] Fix IPv6 addresses_match fe80:: could be submitted during collective startup, handle that problem appropriately. --- confluent_server/confluent/netutil.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index e38d2a00..b98228e5 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -661,6 +661,8 @@ def addresses_match(addr1, addr2): :param addr2: :return: True if the given addresses refer to the same thing """ + if '%' in addr1 or '%' in addr2: + return False for addrinfo in socket.getaddrinfo(addr1, 0, 0, socket.SOCK_STREAM): rootaddr1 = socket.inet_pton(addrinfo[0], addrinfo[4][0]) if addrinfo[0] == socket.AF_INET6 and rootaddr1[:12] == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff': From 2e059b5887ef5d05b920b6085502c8b1eabc7209 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 23 Jan 2023 11:47:33 -0500 Subject: [PATCH 2/5] Make an API for getting full discovery data in one fetch This makes for faster nodediscover being possible, also makes web management of the data easier --- confluent_server/confluent/discovery/core.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index ddbec819..d2f73555 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -359,6 +359,14 @@ def show_info(mac): for i in send_discovery_datum(known_info[mac]): yield i +def dump_discovery(): + infobymac = {} + for mac in known_info: + infobymac[mac] = {} + for i in send_discovery_datum(known_info[mac]): + for kn in i.kvpairs: + infobymac[mac][kn] = i.kvpairs[kn] + yield msg.KeyValueData(infobymac) list_info = { 'by-node': list_matching_nodes, @@ -602,11 +610,14 @@ def handle_read_api_request(pathcomponents): # starting at 2 are parameters to previous index if pathcomponents == ['discovery', 'rescan']: return (msg.KeyValueData({'scanning': bool(scanner)}),) + if pathcomponents == ['discovery', 'alldata']: + return dump_discovery() subcats, queryparms, indexof, coll = _parameterize_path(pathcomponents[1:]) if len(pathcomponents) == 1: dirlist = [msg.ChildCollection(x + '/') for x in sorted(list(subcats))] dirlist.append(msg.ChildCollection('rescan')) dirlist.append(msg.ChildCollection('autosense')) + dirlist.append(msg.ChildCollection('alldata')) dirlist.append(msg.ChildCollection('subscriptions/')) return dirlist if not coll: From 000899868043dd7e779ed76e9e5c23a098f5e060 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 23 Jan 2023 13:37:29 -0500 Subject: [PATCH 3/5] Add api method to request all mac data This will provide easy way for client to get FDB data, potentially for use in conjunction with discovery data. For now, leave LLDP out, as that isn't currently cached at the confluent layer. --- confluent_server/confluent/networking/macmap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/networking/macmap.py b/confluent_server/confluent/networking/macmap.py index f44e8c93..6069fa6f 100644 --- a/confluent_server/confluent/networking/macmap.py +++ b/confluent_server/confluent/networking/macmap.py @@ -601,7 +601,7 @@ def handle_read_api_request(pathcomponents, configmanager): elif len(pathcomponents) == 2: if pathcomponents[-1] == 'macs': return [msg.ChildCollection(x) for x in (# 'by-node/', - 'by-mac/', 'by-switch/', + 'alldata', 'by-mac/', 'by-switch/', 'rescan')] elif pathcomponents[-1] == 'neighbors': return [msg.ChildCollection('by-switch/')] @@ -616,6 +616,8 @@ def handle_read_api_request(pathcomponents, configmanager): elif len(pathcomponents) == 4: macaddr = pathcomponents[-1].replace('-', ':') return dump_macinfo(macaddr) + elif pathcomponents[2] == 'alldata': + return [msg.KeyValueData(_apimacmap)] elif pathcomponents[2] == 'by-mac': if len(pathcomponents) == 3: return [msg.ChildCollection(x.replace(':', '-')) From d14d28caf83a7d5ad8655915cef3f0af6d30bbe3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 24 Jan 2023 08:22:00 -0500 Subject: [PATCH 4/5] Confirm TLS connectivity when scanning hosts In certain environments, Confluent may have an IP address that is fake, but then there is elsewhere with that same IP for real. To mitigate this, follow up basic connectivity with proof of having an associated certificate. --- .../common/initramfs/opt/confluent/bin/apiclient | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient b/confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient index d5bdbf6b..fc1aad87 100644 --- a/confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient +++ b/confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient @@ -304,6 +304,10 @@ class HTTPSClient(client.HTTPConnection, object): def check_connections(self): foundsrv = None hosts = self.hosts + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.load_verify_locations('/etc/confluent/ca.pem') + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = True for timeo in (0.1, 5): for host in hosts: try: @@ -311,11 +315,14 @@ class HTTPSClient(client.HTTPConnection, object): psock = socket.socket(addrinf[0]) psock.settimeout(timeo) psock.connect(addrinf[4]) + ctx.wrap_socket(psock, server_hostname=host) foundsrv = host psock.close() break except OSError: continue + except ssl.SSLError: + continue else: continue break From ce324e90f773a90e392efb057d91b454c4ddf4f5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 25 Jan 2023 12:54:03 -0500 Subject: [PATCH 5/5] Draft spec to generate addons-aarch64 files --- .../confluent_osdeploy-aarch64.spec.tmpl | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl diff --git a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl new file mode 100644 index 00000000..d282acad --- /dev/null +++ b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl @@ -0,0 +1,94 @@ +Name: confluent_osdeploy-aarch64 +Version: #VERSION# +Release: 1 +Summary: OS Deployment support for confluent + +License: Apache2 +URL: https://hpc.lenovo.com/ +Source0: confluent_osdeploy.tar.xz +Source1: confluent_el9bin.tar.xz +BuildArch: noarch +Requires: confluent_ipxe mtools tar +BuildRoot: /tmp + +%description +This contains support utilities for enabling deployment of aarch64 architecture systems + + +%define debug_package %{nil} + +%prep +%setup -n confluent_osdeploy -a 1 + +%build +mkdir -p opt/confluent/bin +mkdir -p stateless-bin +cd utils +make all +cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin +cp start_root urlmount ../stateless-bin/ +cd .. +ln -s el8 el9 +for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do + mkdir ${os}out + cd ${os}out + if [ -d ../${os}bin ]; then + cp -a ../${os}bin/opt . + else + cp -a ../opt . + fi + cp -a ../${os}/initramfs/* . + cp -a ../common/initramfs/* . + find . | cpio -H newc -o > ../addons-aarch64.cpio + mv ../addons-aarch64.cpio . + cd .. +done +for os in el7 el8 suse15 el9 ubuntu20.04; do + mkdir ${os}disklessout + cd ${os}disklessout + if [ -d ../${os}bin ]; then + cp -a ../${os}bin/opt . + else + cp -a ../opt . + fi + cp -a ../${os}-diskless/initramfs/* . + cp -a ../common/initramfs/* . + if [ -d ../${os}bin ]; then + cp -a ../${os}bin/stateless-bin/* opt/confluent/bin + else + cp -a ../stateless-bin/* opt/confluent/bin + fi + find . | cpio -H newc -o > ../addons-aarch64.cpio + mv ../addons-aarch64.cpio . + cd .. +done +mkdir esxi7out +cd esxi7out +cp -a ../opt . +cp -a ../esxi7/initramfs/* . +cp -a ../common/initramfs/* . +chmod +x bin/* opt/confluent/bin/* +tar zcvf ../addons-aarch64.tgz * +mv ../addons-aarch64.tgz . +cd .. +cp -a esxi7out esxi6out +cp -a esxi7 esxi6 +cp -a esxi7out esxi8out +cp -a esxi7 esxi8 + +%install +mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ +cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ +for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do + mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs + cp ${os}out/addons-aarch64.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs + if [ -d ${os}disklessout ]; then + mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs + cp ${os}disklessout/addons-aarch64.* %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs + fi +done +find %{buildroot}/opt/confluent/lib/osdeploy/ -name .gitignore -exec rm -f {} + + +%files +/opt/confluent/lib/osdeploy +%license /opt/confluent/share/licenses/confluent_osdeploy/LICENSE