diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 9159b74..91e77f5 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -33,6 +33,7 @@ import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest import zaza.openstack.utilities.openstack as openstack_utils +import zaza.utilities.machine_os class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest): @@ -835,6 +836,10 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): # the external provider network cls.attach_to_external_network = False + # Override this if you want your test to launch instances with a + # specific flavor + cls.instance_flavor = None + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, @@ -1112,13 +1117,177 @@ class NeutronNetworkingTest(NeutronNetworkingBase): instance_1, instance_2 = self.retrieve_guests() if not all([instance_1, instance_2]): self.launch_guests( - attach_to_external_network=self.attach_to_external_network) + attach_to_external_network=self.attach_to_external_network, + flavor_name=self.instance_flavor) instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) self.run_resource_cleanup = self.get_my_tests_options( 'run_resource_cleanup', True) +class DPDKNeutronNetworkingTest(NeutronNetworkingTest): + """Ensure that openstack instances have valid networking with DPDK.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Neutron API Networking tests.""" + super(DPDKNeutronNetworkingTest, cls).setUpClass() + + # At this point in time the charms do not support configuring overlay + # networks with DPDK. To perform end to end validation we need to + # attach instances directly to the provider network and subsequently + # DHCP needs to be enabled on that network. + # + # Note that for instances wired with DPDK the DHCP request/response is + # handled as private communication between the ovn-controller and the + # instance, and as such there is no risk of rogue DHCP replies escaping + # to the surrounding network. + cls.attach_to_external_network = True + cls.instance_flavor = 'hugepages' + cls.external_subnet = cls.neutron_client.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name']) + if ('dhcp_enabled' not in cls.external_subnet or + not cls.external_subnet['dhcp_enabled']): + logging.info('Enabling DHCP on subnet {}' + .format(cls.external_subnet['name'])) + openstack_utils.update_subnet_dhcp( + cls.neutron_client, cls.external_subnet, True) + + cls.nr_1g_hugepages = 4 + # Prepare hypervisor units for the test + for unit in zaza.model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=cls.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=cls.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Checking CPU topology on {}'.format(unit.name)) + cls._assert_unit_cpu_topology(cls, unit, model_name=cls.model_name) + logging.info('Enabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.enable_hugepages( + unit, cls.nr_1g_hugepages, model_name=cls.model_name) + logging.info('Enabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode( + unit, model_name=cls.model_name) + + def _assert_unit_cpu_topology(self, unit, model_name=None): + r"""Assert unit under test CPU topology. + + When using OpenStack as CI substrate: + + By default, when instance NUMA placement is not specified, + a topology of N sockets, each with one core and one thread, + is used for an instance, where N corresponds to the number of + instance vCPUs requested. + + In this context a socket is a physical socket on the motherboard + where a CPU is connected. + + The DPDK Environment Abstraction Layer (EAL) allocates memory per + CPU socket, so we want the CPU topology inside the instance to + mimic something we would be likely to find in the real world and + at the same time not make the test too heavy. + + The charm default is to have Open vSwitch allocate 1GB RAM per + CPU socket. + + The following command would set the apropriate CPU topology for a + 4 VCPU flavor: + + openstack flavor set m1.large \ + --property hw:cpu_sockets=2 \ + --property hw:cpu_cores=2 \ + --property hw:cpu_threads=2 + """ + # Get number of sockets + cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l' + sockets = int(zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=model_name, fatal=True).rstrip()) + + # Get total memory + cmd = 'cat /proc/meminfo |grep ^MemTotal' + _, meminfo_value, _ = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=model_name, + fatal=True).rstrip().split() + mbtotal = int(meminfo_value) * 1024 / 1000 / 1000 + mbtotalhugepages = self.nr_1g_hugepages * 1024 + + # headroom for operating system and daemons in instance + mbsystemheadroom = 2048 + # memory to be consumed by the nested instance + mbinstance = 1024 + + # the amount of hugepage memory OVS / DPDK EAL will allocate + mbovshugepages = sockets * 1024 + # the amount of hugepage memory available for nested instance + mbfreehugepages = mbtotalhugepages - mbovshugepages + + assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and + mbfreehugepages >= mbinstance), ( + 'Unit {} is not suitable for test, please adjust instance ' + 'type CPU topology or provide suitable physical machine. ' + 'CPU Sockets: {} ' + 'Available memory: {} MB ' + 'Details:\n{}' + .format(unit.name, + sockets, + mbtotal, + self._assert_unit_cpu_topology.__doc__)) + + def test_instances_have_networking(self): + """Enable DPDK then Validate North/South and East/West networking.""" + with self.config_change( + { + 'enable-dpdk': False, + 'dpdk-driver': '', + }, + { + 'enable-dpdk': True, + 'dpdk-driver': 'vfio-pci', + }, + application_name='ovn-chassis'): + super().test_instances_have_networking() + self.run_resource_cleanup = self.get_my_tests_options( + 'run_resource_cleanup', True) + + def resource_cleanup(self): + """Extend to also revert VFIO NOIOMMU mode on units under test.""" + super().resource_cleanup() + if not self.run_resource_cleanup: + return + + if ('dhcp_enabled' not in self.external_subnet or + not self.external_subnet['dhcp_enabled']): + logging.info('Disabling DHCP on subnet {}' + .format(self.external_subnet['name'])) + openstack_utils.update_subnet_dhcp( + self.neutron_client, self.external_subnet, False) + + for unit in zaza.model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=self.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=self.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Disabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.disable_hugepages( + unit, model_name=self.model_name) + logging.info('Disabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode( + unit, model_name=self.model_name) + + class NeutronNetworkingVRRPTests(NeutronNetworkingBase): """Check networking when gateways are restarted.""" diff --git a/zaza/openstack/charm_tests/nova/utils.py b/zaza/openstack/charm_tests/nova/utils.py index f2c8af9..87af7d5 100644 --- a/zaza/openstack/charm_tests/nova/utils.py +++ b/zaza/openstack/charm_tests/nova/utils.py @@ -65,5 +65,14 @@ FLAVORS = { 'hw:tpm_model': 'tpm-crb', }, }, + 'hugepages': { + 'flavorid': 'auto', + 'ram': 1024, + 'disk': 20, + 'vcpus': 1, + 'extra-specs': { + 'hw:mem_page_size': 'large', + }, + }, } KEYPAIR_NAME = 'zaza' diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 6a1b567..7abd331 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -686,7 +686,8 @@ class OpenStackBaseTest(BaseCharmTest): flavor_name=flavor_name, attach_to_external_network=attach_to_external_network) - def launch_guests(self, userdata=None, attach_to_external_network=False): + def launch_guests(self, userdata=None, attach_to_external_network=False, + flavor_name=None): """Launch two guests to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -706,7 +707,8 @@ class OpenStackBaseTest(BaseCharmTest): self.launch_guest( guest_name='ins-{}'.format(guest_number), userdata=userdata, - attach_to_external_network=attach_to_external_network)) + attach_to_external_network=attach_to_external_network, + flavor_name=flavor_name)) return launched_instances def retrieve_guest(self, guest_name):