mirror of
https://github.com/xcat2/xNBA.git
synced 2024-11-23 01:51:58 +00:00
[infiniband] Add the concept of an Infiniband upper-layer driver
Replace the explicit calls from the Infiniband core to the IPoIB layer with the general concept of an Infiniband upper-layer driver (analogous to a PCI driver) which can create arbitrary devices on top of Infiniband devices. Signed-off-by: Michael Brown <mcb30@ipxe.org>
This commit is contained in:
Michael Brown
parent
ca4df90a63
commit
35b19d8848
@ -597,6 +597,42 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle link status change
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
|
||||
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
||||
struct ipoib_device *ipoib = netdev->priv;
|
||||
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
|
||||
int rc;
|
||||
|
||||
/* Leave existing broadcast group */
|
||||
ipoib_leave_broadcast_group ( ipoib );
|
||||
|
||||
/* Update MAC address based on potentially-new GID prefix */
|
||||
memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
|
||||
sizeof ( mac->gid.u.half[0] ) );
|
||||
|
||||
/* Update broadcast GID based on potentially-new partition key */
|
||||
ipoib->broadcast.gid.u.words[2] =
|
||||
htons ( ibdev->pkey | IB_PKEY_FULL );
|
||||
|
||||
/* Set net device link state to reflect Infiniband link state */
|
||||
rc = ib_link_rc ( ibdev );
|
||||
netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
|
||||
|
||||
/* Join new broadcast group */
|
||||
if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
|
||||
( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
|
||||
DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
|
||||
"%s\n", ipoib, strerror ( rc ) );
|
||||
netdev_link_err ( netdev, rc );
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Open IPoIB network device
|
||||
*
|
||||
@ -690,49 +726,13 @@ static struct net_device_operations ipoib_operations = {
|
||||
.irq = ipoib_irq,
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle link status change
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ipoib_link_state_changed ( struct ib_device *ibdev ) {
|
||||
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
||||
struct ipoib_device *ipoib = netdev->priv;
|
||||
struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
|
||||
int rc;
|
||||
|
||||
/* Leave existing broadcast group */
|
||||
ipoib_leave_broadcast_group ( ipoib );
|
||||
|
||||
/* Update MAC address based on potentially-new GID prefix */
|
||||
memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
|
||||
sizeof ( mac->gid.u.half[0] ) );
|
||||
|
||||
/* Update broadcast GID based on potentially-new partition key */
|
||||
ipoib->broadcast.gid.u.words[2] =
|
||||
htons ( ibdev->pkey | IB_PKEY_FULL );
|
||||
|
||||
/* Set net device link state to reflect Infiniband link state */
|
||||
rc = ib_link_rc ( ibdev );
|
||||
netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
|
||||
|
||||
/* Join new broadcast group */
|
||||
if ( ib_link_ok ( ibdev ) &&
|
||||
( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
|
||||
DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
|
||||
"%s\n", ipoib, strerror ( rc ) );
|
||||
netdev_link_err ( netdev, rc );
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe IPoIB device
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
int ipoib_probe ( struct ib_device *ibdev ) {
|
||||
static int ipoib_probe ( struct ib_device *ibdev ) {
|
||||
struct net_device *netdev;
|
||||
struct ipoib_device *ipoib;
|
||||
int rc;
|
||||
@ -775,10 +775,18 @@ int ipoib_probe ( struct ib_device *ibdev ) {
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ipoib_remove ( struct ib_device *ibdev ) {
|
||||
static void ipoib_remove ( struct ib_device *ibdev ) {
|
||||
struct net_device *netdev = ib_get_ownerdata ( ibdev );
|
||||
|
||||
unregister_netdev ( netdev );
|
||||
netdev_nullify ( netdev );
|
||||
netdev_put ( netdev );
|
||||
}
|
||||
|
||||
/** IPoIB driver */
|
||||
struct ib_driver ipoib_driver __ib_driver = {
|
||||
.name = "IPoIB",
|
||||
.probe = ipoib_probe,
|
||||
.notify = ipoib_link_state_changed,
|
||||
.remove = ipoib_remove,
|
||||
};
|
||||
|
||||
@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||
#include <stdint.h>
|
||||
#include <ipxe/refcnt.h>
|
||||
#include <ipxe/device.h>
|
||||
#include <ipxe/tables.h>
|
||||
#include <ipxe/ib_packet.h>
|
||||
#include <ipxe/ib_mad.h>
|
||||
|
||||
@ -432,6 +433,34 @@ struct ib_device {
|
||||
void *owner_priv;
|
||||
};
|
||||
|
||||
/** An Infiniband upper-layer driver */
|
||||
struct ib_driver {
|
||||
/** Name */
|
||||
const char *name;
|
||||
/** Probe device
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
int ( * probe ) ( struct ib_device *ibdev );
|
||||
/** Notify of device or link state change
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ( * notify ) ( struct ib_device *ibdev );
|
||||
/** Remove device
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ( * remove ) ( struct ib_device *ibdev );
|
||||
};
|
||||
|
||||
/** Infiniband driver table */
|
||||
#define IB_DRIVERS __table ( struct ib_driver, "ib_drivers" )
|
||||
|
||||
/** Declare an Infiniband driver */
|
||||
#define __ib_driver __table_entry ( IB_DRIVERS, 01 )
|
||||
|
||||
extern struct ib_completion_queue *
|
||||
ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
|
||||
struct ib_completion_queue_operations *op );
|
||||
@ -492,7 +521,7 @@ extern struct list_head ib_devices;
|
||||
list_for_each_entry ( (ibdev), &ib_devices, list )
|
||||
|
||||
/**
|
||||
* Check link state
|
||||
* Check link state of Infiniband device
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret link_up Link is up
|
||||
@ -502,6 +531,17 @@ ib_link_ok ( struct ib_device *ibdev ) {
|
||||
return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether or not Infiniband device is open
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v is_open Infiniband device is open
|
||||
*/
|
||||
static inline __attribute__ (( always_inline )) int
|
||||
ib_is_open ( struct ib_device *ibdev ) {
|
||||
return ( ibdev->open_count > 0 );
|
||||
}
|
||||
|
||||
/**
|
||||
* Get reference to Infiniband device
|
||||
*
|
||||
|
||||
@ -53,9 +53,6 @@ struct ipoib_hdr {
|
||||
} __attribute__ (( packed ));
|
||||
|
||||
extern const char * ipoib_ntoa ( const void *ll_addr );
|
||||
extern void ipoib_link_state_changed ( struct ib_device *ibdev );
|
||||
extern int ipoib_probe ( struct ib_device *ibdev );
|
||||
extern void ipoib_remove ( struct ib_device *ibdev );
|
||||
extern struct net_device * alloc_ipoibdev ( size_t priv_size );
|
||||
|
||||
#endif /* _IPXE_IPOIB_H */
|
||||
|
||||
@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||
#include <ipxe/if_arp.h>
|
||||
#include <ipxe/netdevice.h>
|
||||
#include <ipxe/iobuf.h>
|
||||
#include <ipxe/ipoib.h>
|
||||
#include <ipxe/process.h>
|
||||
#include <ipxe/infiniband.h>
|
||||
#include <ipxe/ib_mi.h>
|
||||
@ -538,6 +537,64 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get link state
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Link status code
|
||||
*/
|
||||
int ib_link_rc ( struct ib_device *ibdev ) {
|
||||
switch ( ibdev->port_state ) {
|
||||
case IB_PORT_STATE_DOWN: return -ENOTCONN;
|
||||
case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT;
|
||||
case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED;
|
||||
case IB_PORT_STATE_ACTIVE: return 0;
|
||||
default: return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Textual representation of Infiniband link state
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret link_text Link state text
|
||||
*/
|
||||
static const char * ib_link_state_text ( struct ib_device *ibdev ) {
|
||||
switch ( ibdev->port_state ) {
|
||||
case IB_PORT_STATE_DOWN: return "DOWN";
|
||||
case IB_PORT_STATE_INIT: return "INIT";
|
||||
case IB_PORT_STATE_ARMED: return "ARMED";
|
||||
case IB_PORT_STATE_ACTIVE: return "ACTIVE";
|
||||
default: return "UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify drivers of Infiniband device or link state change
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
static void ib_notify ( struct ib_device *ibdev ) {
|
||||
struct ib_driver *driver;
|
||||
|
||||
for_each_table_entry ( driver, IB_DRIVERS )
|
||||
driver->notify ( ibdev );
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify of Infiniband link state change
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ib_link_state_changed ( struct ib_device *ibdev ) {
|
||||
|
||||
DBGC ( ibdev, "IBDEV %p link state is %s\n",
|
||||
ibdev, ib_link_state_text ( ibdev ) );
|
||||
|
||||
/* Notify drivers of link state change */
|
||||
ib_notify ( ibdev );
|
||||
}
|
||||
|
||||
/**
|
||||
* Open port
|
||||
*
|
||||
@ -586,6 +643,9 @@ int ib_open ( struct ib_device *ibdev ) {
|
||||
/* Add to head of open devices list */
|
||||
list_add ( &ibdev->open_list, &open_ib_devices );
|
||||
|
||||
/* Notify drivers of device state change */
|
||||
ib_notify ( ibdev );
|
||||
|
||||
assert ( ibdev->open_count == 1 );
|
||||
return 0;
|
||||
|
||||
@ -614,6 +674,7 @@ void ib_close ( struct ib_device *ibdev ) {
|
||||
|
||||
/* Close device if this was the last remaining requested opening */
|
||||
if ( ibdev->open_count == 0 ) {
|
||||
ib_notify ( ibdev );
|
||||
list_del ( &ibdev->open_list );
|
||||
ib_destroy_mi ( ibdev, ibdev->gsi );
|
||||
ib_destroy_sma ( ibdev, ibdev->smi );
|
||||
@ -622,22 +683,6 @@ void ib_close ( struct ib_device *ibdev ) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get link state
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Link status code
|
||||
*/
|
||||
int ib_link_rc ( struct ib_device *ibdev ) {
|
||||
switch ( ibdev->port_state ) {
|
||||
case IB_PORT_STATE_DOWN: return -ENOTCONN;
|
||||
case IB_PORT_STATE_INIT: return -EINPROGRESS_INIT;
|
||||
case IB_PORT_STATE_ARMED: return -EINPROGRESS_ARMED;
|
||||
case IB_PORT_STATE_ACTIVE: return 0;
|
||||
default: return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Multicast
|
||||
@ -799,17 +844,6 @@ int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad ) {
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Handle Infiniband link state change
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ib_link_state_changed ( struct ib_device *ibdev ) {
|
||||
|
||||
/* Notify IPoIB of link state change */
|
||||
ipoib_link_state_changed ( ibdev );
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll event queue
|
||||
*
|
||||
@ -883,24 +917,29 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
int register_ibdev ( struct ib_device *ibdev ) {
|
||||
struct ib_driver *driver;
|
||||
int rc;
|
||||
|
||||
/* Add to device list */
|
||||
ibdev_get ( ibdev );
|
||||
list_add_tail ( &ibdev->list, &ib_devices );
|
||||
|
||||
/* Add IPoIB device */
|
||||
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
|
||||
DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
|
||||
ibdev, strerror ( rc ) );
|
||||
goto err_ipoib_probe;
|
||||
}
|
||||
|
||||
DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
|
||||
ibdev->dev->name );
|
||||
|
||||
/* Probe device */
|
||||
for_each_table_entry ( driver, IB_DRIVERS ) {
|
||||
if ( ( rc = driver->probe ( ibdev ) ) != 0 ) {
|
||||
DBGC ( ibdev, "IBDEV %p could not add %s device: %s\n",
|
||||
ibdev, driver->name, strerror ( rc ) );
|
||||
goto err_probe;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_ipoib_probe:
|
||||
err_probe:
|
||||
for_each_table_entry_continue_reverse ( driver, IB_DRIVERS )
|
||||
driver->remove ( ibdev );
|
||||
list_del ( &ibdev->list );
|
||||
ibdev_put ( ibdev );
|
||||
return rc;
|
||||
@ -912,9 +951,11 @@ int register_ibdev ( struct ib_device *ibdev ) {
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void unregister_ibdev ( struct ib_device *ibdev ) {
|
||||
struct ib_driver *driver;
|
||||
|
||||
/* Close device */
|
||||
ipoib_remove ( ibdev );
|
||||
/* Remove device */
|
||||
for_each_table_entry_reverse ( driver, IB_DRIVERS )
|
||||
driver->remove ( ibdev );
|
||||
|
||||
/* Remove from device list */
|
||||
list_del ( &ibdev->list );
|
||||
@ -953,3 +994,6 @@ struct ib_device * last_opened_ibdev ( void ) {
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Drag in IPoIB */
|
||||
REQUIRE_OBJECT ( ipoib );
|
||||
|
||||
Loading…
Reference in New Issue
Block a user