mirror of
https://github.com/xcat2/xNBA.git
synced 2024-11-29 04:39:42 +00:00
[Infiniband] Centralise MAD operations
Pull out common code for handling management datagrams from arbel.c and hermon.c into infiniband.c. Add port number to struct ib_device. Add open(), close() and mad() methods to struct ib_device_operations.
This commit is contained in:
Michael Brown
parent
03d1fc1da3
commit
6570203571
@ -34,7 +34,6 @@
|
||||
#include <gpxe/iobuf.h>
|
||||
#include <gpxe/netdevice.h>
|
||||
#include <gpxe/infiniband.h>
|
||||
#include <gpxe/ipoib.h>
|
||||
#include "arbel.h"
|
||||
|
||||
/**
|
||||
@ -349,12 +348,13 @@ arbel_cmd_2rst_qpee ( struct arbel *arbel, unsigned long qpn ) {
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_mad_ifc ( struct arbel *arbel, union arbelprm_mad *mad ) {
|
||||
arbel_cmd_mad_ifc ( struct arbel *arbel, unsigned int port,
|
||||
union arbelprm_mad *mad ) {
|
||||
return arbel_cmd ( arbel,
|
||||
ARBEL_HCR_INOUT_CMD ( ARBEL_HCR_MAD_IFC,
|
||||
1, sizeof ( *mad ),
|
||||
1, sizeof ( *mad ) ),
|
||||
0x03, mad, PXE_IB_PORT, mad );
|
||||
0x03, mad, port, mad );
|
||||
}
|
||||
|
||||
static inline int
|
||||
@ -776,7 +776,7 @@ static int arbel_create_qp ( struct ib_device *ibdev,
|
||||
MLX_FILL_1 ( &qpctx, 5,
|
||||
qpc_eec_data.usr_page, arbel->limits.reserved_uars );
|
||||
MLX_FILL_1 ( &qpctx, 10, qpc_eec_data.primary_address_path.port_number,
|
||||
PXE_IB_PORT );
|
||||
ibdev->port );
|
||||
MLX_FILL_1 ( &qpctx, 27, qpc_eec_data.pd, ARBEL_GLOBAL_PD );
|
||||
MLX_FILL_1 ( &qpctx, 29, qpc_eec_data.wqe_lkey, arbel->reserved_lkey );
|
||||
MLX_FILL_1 ( &qpctx, 30, qpc_eec_data.ssc, 1 );
|
||||
@ -955,7 +955,7 @@ static int arbel_post_send ( struct ib_device *ibdev,
|
||||
memset ( &wqe->ud, 0, sizeof ( wqe->ud ) );
|
||||
MLX_FILL_2 ( &wqe->ud, 0,
|
||||
ud_address_vector.pd, ARBEL_GLOBAL_PD,
|
||||
ud_address_vector.port_number, PXE_IB_PORT );
|
||||
ud_address_vector.port_number, ibdev->port );
|
||||
MLX_FILL_2 ( &wqe->ud, 1,
|
||||
ud_address_vector.rlid, av->dlid,
|
||||
ud_address_vector.g, av->gid_present );
|
||||
@ -1206,6 +1206,57 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Infiniband link-layer operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Initialise Infiniband link
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int arbel_open ( struct ib_device *ibdev ) {
|
||||
struct arbel *arbel = ibdev->dev_priv;
|
||||
struct arbelprm_init_ib init_ib;
|
||||
int rc;
|
||||
|
||||
memset ( &init_ib, 0, sizeof ( init_ib ) );
|
||||
MLX_FILL_3 ( &init_ib, 0,
|
||||
mtu_cap, ARBEL_MTU_2048,
|
||||
port_width_cap, 3,
|
||||
vl_cap, 1 );
|
||||
MLX_FILL_1 ( &init_ib, 1, max_gid, 1 );
|
||||
MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
|
||||
if ( ( rc = arbel_cmd_init_ib ( arbel, ibdev->port,
|
||||
&init_ib ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not intialise IB: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close Infiniband link
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
static void arbel_close ( struct ib_device *ibdev ) {
|
||||
struct arbel *arbel = ibdev->dev_priv;
|
||||
int rc;
|
||||
|
||||
if ( ( rc = arbel_cmd_close_ib ( arbel, ibdev->port ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not close IB: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
/* Nothing we can do about this */
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Multicast group operations
|
||||
@ -1302,6 +1353,51 @@ static void arbel_mcast_detach ( struct ib_device *ibdev,
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* MAD operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Issue management datagram
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v mad Management datagram
|
||||
* @v len Length of management datagram
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int arbel_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
|
||||
size_t len ) {
|
||||
struct arbel *arbel = ibdev->dev_priv;
|
||||
union arbelprm_mad mad_ifc;
|
||||
int rc;
|
||||
|
||||
/* Copy in request packet */
|
||||
memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
|
||||
assert ( len <= sizeof ( mad_ifc.mad ) );
|
||||
memcpy ( &mad_ifc.mad, mad, len );
|
||||
|
||||
/* Issue MAD */
|
||||
if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
|
||||
&mad_ifc ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Copy out reply packet */
|
||||
memcpy ( mad, &mad_ifc.mad, len );
|
||||
|
||||
if ( mad->status != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
|
||||
arbel, ntohs ( mad->status ) );
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Arbel Infiniband operations */
|
||||
static struct ib_device_operations arbel_ib_operations = {
|
||||
.create_cq = arbel_create_cq,
|
||||
@ -1311,206 +1407,13 @@ static struct ib_device_operations arbel_ib_operations = {
|
||||
.post_send = arbel_post_send,
|
||||
.post_recv = arbel_post_recv,
|
||||
.poll_cq = arbel_poll_cq,
|
||||
.open = arbel_open,
|
||||
.close = arbel_close,
|
||||
.mcast_attach = arbel_mcast_attach,
|
||||
.mcast_detach = arbel_mcast_detach,
|
||||
.mad = arbel_mad,
|
||||
};
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* MAD IFC operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
static int arbel_mad_ifc ( struct arbel *arbel,
|
||||
union arbelprm_mad *mad ) {
|
||||
struct ib_mad_hdr *hdr = &mad->mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
hdr->base_version = IB_MGMT_BASE_VERSION;
|
||||
if ( ( rc = arbel_cmd_mad_ifc ( arbel, mad ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
if ( hdr->status != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
|
||||
arbel, ntohs ( hdr->status ) );
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arbel_get_port_info ( struct arbel *arbel,
|
||||
struct ib_mad_port_info *port_info ) {
|
||||
union arbelprm_mad mad;
|
||||
struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
memset ( &mad, 0, sizeof ( mad ) );
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
|
||||
hdr->attr_mod = htonl ( PXE_IB_PORT );
|
||||
if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not get port info: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
memcpy ( port_info, &mad.mad.port_info, sizeof ( *port_info ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arbel_get_guid_info ( struct arbel *arbel,
|
||||
struct ib_mad_guid_info *guid_info ) {
|
||||
union arbelprm_mad mad;
|
||||
struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
memset ( &mad, 0, sizeof ( mad ) );
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
|
||||
if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not get GUID info: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
memcpy ( guid_info, &mad.mad.guid_info, sizeof ( *guid_info ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arbel_get_pkey_table ( struct arbel *arbel,
|
||||
struct ib_mad_pkey_table *pkey_table ) {
|
||||
union arbelprm_mad mad;
|
||||
struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
memset ( &mad, 0, sizeof ( mad ) );
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
|
||||
if ( ( rc = arbel_mad_ifc ( arbel, &mad ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not get pkey table: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
memcpy ( pkey_table, &mad.mad.pkey_table, sizeof ( *pkey_table ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arbel_get_port_gid ( struct arbel *arbel,
|
||||
struct ib_gid *port_gid ) {
|
||||
union {
|
||||
/* This union exists just to save stack space */
|
||||
struct ib_mad_port_info port_info;
|
||||
struct ib_mad_guid_info guid_info;
|
||||
} u;
|
||||
int rc;
|
||||
|
||||
/* Port info gives us the first half of the port GID */
|
||||
if ( ( rc = arbel_get_port_info ( arbel, &u.port_info ) ) != 0 )
|
||||
return rc;
|
||||
memcpy ( &port_gid->u.bytes[0], u.port_info.gid_prefix, 8 );
|
||||
|
||||
/* GUID info gives us the second half of the port GID */
|
||||
if ( ( rc = arbel_get_guid_info ( arbel, &u.guid_info ) ) != 0 )
|
||||
return rc;
|
||||
memcpy ( &port_gid->u.bytes[8], u.guid_info.gid_local, 8 );
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arbel_get_sm_lid ( struct arbel *arbel,
|
||||
unsigned long *sm_lid ) {
|
||||
struct ib_mad_port_info port_info;
|
||||
int rc;
|
||||
|
||||
if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
|
||||
return rc;
|
||||
*sm_lid = ntohs ( port_info.mastersm_lid );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arbel_get_pkey ( struct arbel *arbel, unsigned int *pkey ) {
|
||||
struct ib_mad_pkey_table pkey_table;
|
||||
int rc;
|
||||
|
||||
if ( ( rc = arbel_get_pkey_table ( arbel, &pkey_table ) ) != 0 )
|
||||
return rc;
|
||||
*pkey = ntohs ( pkey_table.pkey[0][0] );
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for link up
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
* @ret rc Return status code
|
||||
*
|
||||
* This function shouldn't really exist. Unfortunately, IB links take
|
||||
* a long time to come up, and we can't get various key parameters
|
||||
* e.g. our own IPoIB MAC address without information from the subnet
|
||||
* manager). We should eventually make link-up an asynchronous event.
|
||||
*/
|
||||
static int arbel_wait_for_link ( struct arbel *arbel ) {
|
||||
struct ib_mad_port_info port_info;
|
||||
unsigned int retries;
|
||||
int rc;
|
||||
|
||||
printf ( "Waiting for Infiniband link-up..." );
|
||||
for ( retries = 20 ; retries ; retries-- ) {
|
||||
if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
|
||||
continue;
|
||||
if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
|
||||
== 4 ) {
|
||||
printf ( "ok\n" );
|
||||
return 0;
|
||||
}
|
||||
printf ( "." );
|
||||
sleep ( 1 );
|
||||
}
|
||||
printf ( "failed\n" );
|
||||
return -ENODEV;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get MAD parameters
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int arbel_get_mad_params ( struct ib_device *ibdev ) {
|
||||
struct arbel *arbel = ibdev->dev_priv;
|
||||
int rc;
|
||||
|
||||
/* Get subnet manager LID */
|
||||
if ( ( rc = arbel_get_sm_lid ( arbel, &ibdev->sm_lid ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not determine subnet manager "
|
||||
"LID: %s\n", arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Get port GID */
|
||||
if ( ( rc = arbel_get_port_gid ( arbel, &ibdev->port_gid ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not determine port GID: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Get partition key */
|
||||
if ( ( rc = arbel_get_pkey ( arbel, &ibdev->pkey ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not determine partition key: "
|
||||
"%s\n", arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Firmware control
|
||||
@ -1882,55 +1785,6 @@ static void arbel_free_icm ( struct arbel *arbel ) {
|
||||
arbel->icm = UNULL;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Infiniband link-layer operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Initialise Infiniband link
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int arbel_init_ib ( struct arbel *arbel ) {
|
||||
struct arbelprm_init_ib init_ib;
|
||||
int rc;
|
||||
|
||||
memset ( &init_ib, 0, sizeof ( init_ib ) );
|
||||
MLX_FILL_3 ( &init_ib, 0,
|
||||
mtu_cap, ARBEL_MTU_2048,
|
||||
port_width_cap, 3,
|
||||
vl_cap, 1 );
|
||||
MLX_FILL_1 ( &init_ib, 1, max_gid, 1 );
|
||||
MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
|
||||
if ( ( rc = arbel_cmd_init_ib ( arbel, PXE_IB_PORT,
|
||||
&init_ib ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not intialise IB: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close Infiniband link
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
*/
|
||||
static void arbel_close_ib ( struct arbel *arbel ) {
|
||||
int rc;
|
||||
|
||||
if ( ( rc = arbel_cmd_close_ib ( arbel, PXE_IB_PORT ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not close IB: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
/* Nothing we can do about this */
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* PCI interface
|
||||
@ -1997,6 +1851,7 @@ static int arbel_probe ( struct pci_device *pci,
|
||||
ibdev->op = &arbel_ib_operations;
|
||||
pci_set_drvdata ( pci, ibdev );
|
||||
ibdev->dev = &pci->dev;
|
||||
ibdev->port = PXE_IB_PORT;
|
||||
arbel = ibdev->dev_priv;
|
||||
memset ( arbel, 0, sizeof ( *arbel ) );
|
||||
|
||||
@ -2047,38 +1902,16 @@ static int arbel_probe ( struct pci_device *pci,
|
||||
if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
|
||||
goto err_setup_mpt;
|
||||
|
||||
/* Bring up IB layer */
|
||||
if ( ( rc = arbel_init_ib ( arbel ) ) != 0 )
|
||||
goto err_init_ib;
|
||||
|
||||
/* Wait for link */
|
||||
if ( ( rc = arbel_wait_for_link ( arbel ) ) != 0 )
|
||||
goto err_wait_for_link;
|
||||
|
||||
/* Get MAD parameters */
|
||||
if ( ( rc = arbel_get_mad_params ( ibdev ) ) != 0 )
|
||||
goto err_get_mad_params;
|
||||
|
||||
DBGC ( arbel, "Arbel %p port GID is %08lx:%08lx:%08lx:%08lx\n", arbel,
|
||||
htonl ( ibdev->port_gid.u.dwords[0] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[1] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[2] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[3] ) );
|
||||
|
||||
/* Add IPoIB device */
|
||||
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not add IPoIB device: %s\n",
|
||||
/* Register Infiniband device */
|
||||
if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not register IB device: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
goto err_ipoib_probe;
|
||||
goto err_register_ibdev;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_ipoib_probe:
|
||||
err_get_mad_params:
|
||||
err_wait_for_link:
|
||||
arbel_close_ib ( arbel );
|
||||
err_init_ib:
|
||||
err_register_ibdev:
|
||||
err_setup_mpt:
|
||||
arbel_cmd_close_hca ( arbel );
|
||||
err_init_hca:
|
||||
@ -2105,8 +1938,7 @@ static void arbel_remove ( struct pci_device *pci ) {
|
||||
struct ib_device *ibdev = pci_get_drvdata ( pci );
|
||||
struct arbel *arbel = ibdev->dev_priv;
|
||||
|
||||
ipoib_remove ( ibdev );
|
||||
arbel_close_ib ( arbel );
|
||||
unregister_ibdev ( ibdev );
|
||||
arbel_cmd_close_hca ( arbel );
|
||||
arbel_free_icm ( arbel );
|
||||
arbel_stop_firmware ( arbel );
|
||||
|
||||
@ -32,7 +32,6 @@
|
||||
#include <gpxe/iobuf.h>
|
||||
#include <gpxe/netdevice.h>
|
||||
#include <gpxe/infiniband.h>
|
||||
#include <gpxe/ipoib.h>
|
||||
#include "hermon.h"
|
||||
|
||||
/**
|
||||
@ -390,12 +389,13 @@ hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) {
|
||||
}
|
||||
|
||||
static inline int
|
||||
hermon_cmd_mad_ifc ( struct hermon *hermon, union hermonprm_mad *mad ) {
|
||||
hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port,
|
||||
union hermonprm_mad *mad ) {
|
||||
return hermon_cmd ( hermon,
|
||||
HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC,
|
||||
1, sizeof ( *mad ),
|
||||
1, sizeof ( *mad ) ),
|
||||
0x03, mad, PXE_IB_PORT, mad );
|
||||
0x03, mad, port, mad );
|
||||
}
|
||||
|
||||
static inline int
|
||||
@ -785,7 +785,7 @@ static int hermon_create_qp ( struct ib_device *ibdev,
|
||||
goto err_alloc_mtt;
|
||||
}
|
||||
|
||||
/* Hand queue over to hardware */
|
||||
/* Transition queue to INIT state */
|
||||
memset ( &qpctx, 0, sizeof ( qpctx ) );
|
||||
MLX_FILL_2 ( &qpctx, 2,
|
||||
qpc_eec_data.pm_state, 0x03 /* Always 0x03 for UD */,
|
||||
@ -817,6 +817,7 @@ static int hermon_create_qp ( struct ib_device *ibdev,
|
||||
goto err_rst2init_qp;
|
||||
}
|
||||
|
||||
/* Transition queue to RTR state */
|
||||
memset ( &qpctx, 0, sizeof ( qpctx ) );
|
||||
MLX_FILL_2 ( &qpctx, 4,
|
||||
qpc_eec_data.mtu, HERMON_MTU_2048,
|
||||
@ -824,7 +825,7 @@ static int hermon_create_qp ( struct ib_device *ibdev,
|
||||
MLX_FILL_1 ( &qpctx, 16,
|
||||
qpc_eec_data.primary_address_path.sched_queue,
|
||||
( 0x83 /* default policy */ |
|
||||
( ( PXE_IB_PORT - 1 ) << 6 ) ) );
|
||||
( ( ibdev->port - 1 ) << 6 ) ) );
|
||||
if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn,
|
||||
&qpctx ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p INIT2RTR_QP failed: %s\n",
|
||||
@ -949,7 +950,7 @@ static int hermon_post_send ( struct ib_device *ibdev,
|
||||
MLX_FILL_1 ( &wqe->ctrl, 2, c, 0x03 /* generate completion */ );
|
||||
MLX_FILL_2 ( &wqe->ud, 0,
|
||||
ud_address_vector.pd, HERMON_GLOBAL_PD,
|
||||
ud_address_vector.port_number, PXE_IB_PORT );
|
||||
ud_address_vector.port_number, ibdev->port );
|
||||
MLX_FILL_2 ( &wqe->ud, 1,
|
||||
ud_address_vector.rlid, av->dlid,
|
||||
ud_address_vector.g, av->gid_present );
|
||||
@ -1162,6 +1163,58 @@ static void hermon_poll_cq ( struct ib_device *ibdev,
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Infiniband link-layer operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Initialise Infiniband link
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int hermon_open ( struct ib_device *ibdev ) {
|
||||
struct hermon *hermon = ibdev->dev_priv;
|
||||
struct hermonprm_init_port init_port;
|
||||
int rc;
|
||||
|
||||
memset ( &init_port, 0, sizeof ( init_port ) );
|
||||
MLX_FILL_2 ( &init_port, 0,
|
||||
port_width_cap, 3,
|
||||
vl_cap, 1 );
|
||||
MLX_FILL_2 ( &init_port, 1,
|
||||
mtu, HERMON_MTU_2048,
|
||||
max_gid, 1 );
|
||||
MLX_FILL_1 ( &init_port, 2, max_pkey, 64 );
|
||||
if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port,
|
||||
&init_port ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not intialise port: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close Infiniband link
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
static void hermon_close ( struct ib_device *ibdev ) {
|
||||
struct hermon *hermon = ibdev->dev_priv;
|
||||
int rc;
|
||||
|
||||
if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not close port: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
/* Nothing we can do about this */
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Multicast group operations
|
||||
@ -1257,6 +1310,51 @@ static void hermon_mcast_detach ( struct ib_device *ibdev,
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* MAD operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Issue management datagram
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v mad Management datagram
|
||||
* @v len Length of management datagram
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
|
||||
size_t len ) {
|
||||
struct hermon *hermon = ibdev->dev_priv;
|
||||
union hermonprm_mad mad_ifc;
|
||||
int rc;
|
||||
|
||||
/* Copy in request packet */
|
||||
memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
|
||||
assert ( len <= sizeof ( mad_ifc.mad ) );
|
||||
memcpy ( &mad_ifc.mad, mad, len );
|
||||
|
||||
/* Issue MAD */
|
||||
if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
|
||||
&mad_ifc ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Copy out reply packet */
|
||||
memcpy ( mad, &mad_ifc.mad, len );
|
||||
|
||||
if ( mad->status != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
|
||||
hermon, ntohs ( mad->status ) );
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Hermon Infiniband operations */
|
||||
static struct ib_device_operations hermon_ib_operations = {
|
||||
.create_cq = hermon_create_cq,
|
||||
@ -1266,207 +1364,13 @@ static struct ib_device_operations hermon_ib_operations = {
|
||||
.post_send = hermon_post_send,
|
||||
.post_recv = hermon_post_recv,
|
||||
.poll_cq = hermon_poll_cq,
|
||||
.open = hermon_open,
|
||||
.close = hermon_close,
|
||||
.mcast_attach = hermon_mcast_attach,
|
||||
.mcast_detach = hermon_mcast_detach,
|
||||
.mad = hermon_mad,
|
||||
};
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* MAD IFC operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
static int hermon_mad_ifc ( struct hermon *hermon,
|
||||
union hermonprm_mad *mad ) {
|
||||
struct ib_mad_hdr *hdr = &mad->mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
hdr->base_version = IB_MGMT_BASE_VERSION;
|
||||
if ( ( rc = hermon_cmd_mad_ifc ( hermon, mad ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
if ( hdr->status != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
|
||||
hermon, ntohs ( hdr->status ) );
|
||||
return -EIO;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hermon_get_port_info ( struct hermon *hermon,
|
||||
struct ib_mad_port_info *port_info ) {
|
||||
union hermonprm_mad mad;
|
||||
struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
memset ( &mad, 0, sizeof ( mad ) );
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
|
||||
hdr->attr_mod = htonl ( PXE_IB_PORT );
|
||||
if ( ( rc = hermon_mad_ifc ( hermon, &mad ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not get port info: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
memcpy ( port_info, &mad.mad.port_info, sizeof ( *port_info ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hermon_get_guid_info ( struct hermon *hermon,
|
||||
struct ib_mad_guid_info *guid_info ) {
|
||||
union hermonprm_mad mad;
|
||||
struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
memset ( &mad, 0, sizeof ( mad ) );
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
|
||||
if ( ( rc = hermon_mad_ifc ( hermon, &mad ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not get GUID info: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
memcpy ( guid_info, &mad.mad.guid_info, sizeof ( *guid_info ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hermon_get_pkey_table ( struct hermon *hermon,
|
||||
struct ib_mad_pkey_table *pkey_table ) {
|
||||
union hermonprm_mad mad;
|
||||
struct ib_mad_hdr *hdr = &mad.mad.mad_hdr;
|
||||
int rc;
|
||||
|
||||
memset ( &mad, 0, sizeof ( mad ) );
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
|
||||
if ( ( rc = hermon_mad_ifc ( hermon, &mad ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not get pkey table: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
memcpy ( pkey_table, &mad.mad.pkey_table, sizeof ( *pkey_table ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hermon_get_port_gid ( struct hermon *hermon,
|
||||
struct ib_gid *port_gid ) {
|
||||
union {
|
||||
/* This union exists just to save stack space */
|
||||
struct ib_mad_port_info port_info;
|
||||
struct ib_mad_guid_info guid_info;
|
||||
} u;
|
||||
int rc;
|
||||
|
||||
/* Port info gives us the first half of the port GID */
|
||||
if ( ( rc = hermon_get_port_info ( hermon, &u.port_info ) ) != 0 )
|
||||
return rc;
|
||||
memcpy ( &port_gid->u.bytes[0], u.port_info.gid_prefix, 8 );
|
||||
|
||||
/* GUID info gives us the second half of the port GID */
|
||||
if ( ( rc = hermon_get_guid_info ( hermon, &u.guid_info ) ) != 0 )
|
||||
return rc;
|
||||
memcpy ( &port_gid->u.bytes[8], u.guid_info.gid_local, 8 );
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hermon_get_sm_lid ( struct hermon *hermon,
|
||||
unsigned long *sm_lid ) {
|
||||
struct ib_mad_port_info port_info;
|
||||
int rc;
|
||||
|
||||
if ( ( rc = hermon_get_port_info ( hermon, &port_info ) ) != 0 )
|
||||
return rc;
|
||||
*sm_lid = ntohs ( port_info.mastersm_lid );
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hermon_get_pkey ( struct hermon *hermon, unsigned int *pkey ) {
|
||||
struct ib_mad_pkey_table pkey_table;
|
||||
int rc;
|
||||
|
||||
if ( ( rc = hermon_get_pkey_table ( hermon, &pkey_table ) ) != 0 )
|
||||
return rc;
|
||||
*pkey = ntohs ( pkey_table.pkey[0][0] );
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for link up
|
||||
*
|
||||
* @v hermon Hermon device
|
||||
* @ret rc Return status code
|
||||
*
|
||||
* This function shouldn't really exist. Unfortunately, IB links take
|
||||
* a long time to come up, and we can't get various key parameters
|
||||
* e.g. our own IPoIB MAC address without information from the subnet
|
||||
* manager). We should eventually make link-up an asynchronous event.
|
||||
*/
|
||||
static int hermon_wait_for_link ( struct hermon *hermon ) {
|
||||
struct ib_mad_port_info port_info;
|
||||
unsigned int retries;
|
||||
int rc;
|
||||
|
||||
printf ( "Waiting for Infiniband link-up..." );
|
||||
for ( retries = 20 ; retries ; retries-- ) {
|
||||
if ( ( rc = hermon_get_port_info ( hermon,
|
||||
&port_info ) ) != 0 )
|
||||
continue;
|
||||
if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
|
||||
== 4 ) {
|
||||
printf ( "ok\n" );
|
||||
return 0;
|
||||
}
|
||||
printf ( "." );
|
||||
sleep ( 1 );
|
||||
}
|
||||
printf ( "failed\n" );
|
||||
return -ENODEV;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get MAD parameters
|
||||
*
|
||||
* @v hermon Hermon device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int hermon_get_mad_params ( struct ib_device *ibdev ) {
|
||||
struct hermon *hermon = ibdev->dev_priv;
|
||||
int rc;
|
||||
|
||||
/* Get subnet manager LID */
|
||||
if ( ( rc = hermon_get_sm_lid ( hermon, &ibdev->sm_lid ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not determine subnet manager "
|
||||
"LID: %s\n", hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Get port GID */
|
||||
if ( ( rc = hermon_get_port_gid ( hermon, &ibdev->port_gid ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not determine port GID: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Get partition key */
|
||||
if ( ( rc = hermon_get_pkey ( hermon, &ibdev->pkey ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not determine partition key: "
|
||||
"%s\n", hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Firmware control
|
||||
@ -1914,56 +1818,6 @@ static void hermon_free_icm ( struct hermon *hermon ) {
|
||||
hermon->icm = UNULL;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Infiniband link-layer operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Initialise Infiniband link
|
||||
*
|
||||
* @v hermon Hermon device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int hermon_init_port ( struct hermon *hermon ) {
|
||||
struct hermonprm_init_port init_port;
|
||||
int rc;
|
||||
|
||||
memset ( &init_port, 0, sizeof ( init_port ) );
|
||||
MLX_FILL_2 ( &init_port, 0,
|
||||
port_width_cap, 3,
|
||||
vl_cap, 1 );
|
||||
MLX_FILL_2 ( &init_port, 1,
|
||||
mtu, HERMON_MTU_2048,
|
||||
max_gid, 1 );
|
||||
MLX_FILL_1 ( &init_port, 2, max_pkey, 64 );
|
||||
if ( ( rc = hermon_cmd_init_port ( hermon, PXE_IB_PORT,
|
||||
&init_port ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not intialise port: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close Infiniband link
|
||||
*
|
||||
* @v hermon Hermon device
|
||||
*/
|
||||
static void hermon_close_port ( struct hermon *hermon ) {
|
||||
int rc;
|
||||
|
||||
if ( ( rc = hermon_cmd_close_port ( hermon, PXE_IB_PORT ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not close port: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
/* Nothing we can do about this */
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* PCI interface
|
||||
@ -2030,6 +1884,7 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
ibdev->op = &hermon_ib_operations;
|
||||
pci_set_drvdata ( pci, ibdev );
|
||||
ibdev->dev = &pci->dev;
|
||||
ibdev->port = PXE_IB_PORT;
|
||||
hermon = ibdev->dev_priv;
|
||||
memset ( hermon, 0, sizeof ( *hermon ) );
|
||||
|
||||
@ -2084,38 +1939,16 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
|
||||
goto err_setup_mpt;
|
||||
|
||||
/* Bring up IB layer */
|
||||
if ( ( rc = hermon_init_port ( hermon ) ) != 0 )
|
||||
goto err_init_port;
|
||||
|
||||
/* Wait for link */
|
||||
if ( ( rc = hermon_wait_for_link ( hermon ) ) != 0 )
|
||||
goto err_wait_for_link;
|
||||
|
||||
/* Get MAD parameters */
|
||||
if ( ( rc = hermon_get_mad_params ( ibdev ) ) != 0 )
|
||||
goto err_get_mad_params;
|
||||
|
||||
DBGC ( hermon, "Hermon %p port GID is %08lx:%08lx:%08lx:%08lx\n",
|
||||
hermon, htonl ( ibdev->port_gid.u.dwords[0] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[1] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[2] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[3] ) );
|
||||
|
||||
/* Add IPoIB device */
|
||||
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not add IPoIB device: %s\n",
|
||||
/* Register Infiniband device */
|
||||
if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not register IB device: %s\n",
|
||||
hermon, strerror ( rc ) );
|
||||
goto err_ipoib_probe;
|
||||
goto err_register_ibdev;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_ipoib_probe:
|
||||
err_get_mad_params:
|
||||
err_wait_for_link:
|
||||
hermon_close_port ( hermon );
|
||||
err_init_port:
|
||||
err_register_ibdev:
|
||||
err_setup_mpt:
|
||||
hermon_cmd_close_hca ( hermon );
|
||||
err_init_hca:
|
||||
@ -2142,8 +1975,7 @@ static void hermon_remove ( struct pci_device *pci ) {
|
||||
struct ib_device *ibdev = pci_get_drvdata ( pci );
|
||||
struct hermon *hermon = ibdev->dev_priv;
|
||||
|
||||
ipoib_remove ( ibdev );
|
||||
hermon_close_port ( hermon );
|
||||
unregister_ibdev ( ibdev );
|
||||
hermon_cmd_close_hca ( hermon );
|
||||
hermon_free_icm ( hermon );
|
||||
hermon_stop_firmware ( hermon );
|
||||
|
||||
@ -156,6 +156,8 @@ struct ib_address_vector {
|
||||
struct ib_gid gid;
|
||||
};
|
||||
|
||||
struct ib_mad_hdr;
|
||||
|
||||
/**
|
||||
* Infiniband device operations
|
||||
*
|
||||
@ -237,6 +239,19 @@ struct ib_device_operations {
|
||||
struct ib_completion_queue *cq,
|
||||
ib_completer_t complete_send,
|
||||
ib_completer_t complete_recv );
|
||||
/**
|
||||
* Open port
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
int ( * open ) ( struct ib_device *ibdev );
|
||||
/**
|
||||
* Close port
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void ( * close ) ( struct ib_device *ibdev );
|
||||
/** Attach to multicast group
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
@ -256,20 +271,32 @@ struct ib_device_operations {
|
||||
void ( * mcast_detach ) ( struct ib_device *ibdev,
|
||||
struct ib_queue_pair *qp,
|
||||
struct ib_gid *gid );
|
||||
/**
|
||||
* Issue management datagram
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v mad Management datagram
|
||||
* @v len Length of management datagram
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
int ( * mad ) ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
|
||||
size_t len );
|
||||
};
|
||||
|
||||
/** An Infiniband device */
|
||||
struct ib_device {
|
||||
/** Underlying device */
|
||||
struct device *dev;
|
||||
/** Infiniband operations */
|
||||
struct ib_device_operations *op;
|
||||
/** Port number */
|
||||
unsigned int port;
|
||||
/** Port GID */
|
||||
struct ib_gid port_gid;
|
||||
/** Subnet manager LID */
|
||||
unsigned long sm_lid;
|
||||
/** Partition key */
|
||||
unsigned int pkey;
|
||||
/** Underlying device */
|
||||
struct device *dev;
|
||||
/** Infiniband operations */
|
||||
struct ib_device_operations *op;
|
||||
/** Device private data */
|
||||
void *dev_priv;
|
||||
/** Owner private data */
|
||||
@ -289,6 +316,8 @@ extern void ib_destroy_qp ( struct ib_device *ibdev,
|
||||
extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
|
||||
unsigned long qpn, int is_send );
|
||||
extern struct ib_device * alloc_ibdev ( size_t priv_size );
|
||||
extern int register_ibdev ( struct ib_device *ibdev );
|
||||
extern void unregister_ibdev ( struct ib_device *ibdev );
|
||||
extern void free_ibdev ( struct ib_device *ibdev );
|
||||
|
||||
/**
|
||||
@ -334,6 +363,26 @@ ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq,
|
||||
ibdev->op->poll_cq ( ibdev, cq, complete_send, complete_recv );
|
||||
}
|
||||
|
||||
/**
|
||||
* Open port
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static inline __attribute__ (( always_inline )) int
|
||||
ib_open ( struct ib_device *ibdev ) {
|
||||
return ibdev->op->open ( ibdev );
|
||||
}
|
||||
|
||||
/**
|
||||
* Close port
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
static inline __attribute__ (( always_inline )) void
|
||||
ib_close ( struct ib_device *ibdev ) {
|
||||
ibdev->op->close ( ibdev );
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach to multicast group
|
||||
@ -362,14 +411,27 @@ ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
||||
ibdev->op->mcast_detach ( ibdev, qp, gid );
|
||||
}
|
||||
|
||||
/**
|
||||
* Issue management datagram
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v mad Management datagram
|
||||
* @v len Length of management datagram
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static inline __attribute__ (( always_inline )) int
|
||||
ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) {
|
||||
return ibdev->op->mad ( ibdev, mad, len );
|
||||
}
|
||||
|
||||
/**
|
||||
* Set Infiniband owner-private data
|
||||
*
|
||||
* @v pci Infiniband device
|
||||
* @v priv Private data
|
||||
*/
|
||||
static inline void ib_set_ownerdata ( struct ib_device *ibdev,
|
||||
void *owner_priv ) {
|
||||
static inline __attribute__ (( always_inline )) void
|
||||
ib_set_ownerdata ( struct ib_device *ibdev, void *owner_priv ) {
|
||||
ibdev->owner_priv = owner_priv;
|
||||
}
|
||||
|
||||
@ -379,7 +441,8 @@ static inline void ib_set_ownerdata ( struct ib_device *ibdev,
|
||||
* @v pci Infiniband device
|
||||
* @ret priv Private data
|
||||
*/
|
||||
static inline void * ib_get_ownerdata ( struct ib_device *ibdev ) {
|
||||
static inline __attribute__ (( always_inline )) void *
|
||||
ib_get_ownerdata ( struct ib_device *ibdev ) {
|
||||
return ibdev->owner_priv;
|
||||
}
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <byteswap.h>
|
||||
#include <errno.h>
|
||||
#include <assert.h>
|
||||
@ -27,6 +28,7 @@
|
||||
#include <gpxe/if_arp.h>
|
||||
#include <gpxe/netdevice.h>
|
||||
#include <gpxe/iobuf.h>
|
||||
#include <gpxe/ipoib.h>
|
||||
#include <gpxe/infiniband.h>
|
||||
|
||||
/** @file
|
||||
@ -184,6 +186,177 @@ struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Management datagram operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get port information
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v port_info Port information datagram to fill in
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int ib_get_port_info ( struct ib_device *ibdev,
|
||||
struct ib_mad_port_info *port_info ) {
|
||||
struct ib_mad_hdr *hdr = &port_info->mad_hdr;
|
||||
int rc;
|
||||
|
||||
/* Construct MAD */
|
||||
memset ( port_info, 0, sizeof ( *port_info ) );
|
||||
hdr->base_version = IB_MGMT_BASE_VERSION;
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
|
||||
hdr->attr_mod = htonl ( ibdev->port );
|
||||
|
||||
if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *port_info ) ) ) != 0 ) {
|
||||
DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
|
||||
ibdev, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get GUID information
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v guid_info GUID information datagram to fill in
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int ib_get_guid_info ( struct ib_device *ibdev,
|
||||
struct ib_mad_guid_info *guid_info ) {
|
||||
struct ib_mad_hdr *hdr = &guid_info->mad_hdr;
|
||||
int rc;
|
||||
|
||||
/* Construct MAD */
|
||||
memset ( guid_info, 0, sizeof ( *guid_info ) );
|
||||
hdr->base_version = IB_MGMT_BASE_VERSION;
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
|
||||
|
||||
if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *guid_info ) ) ) != 0 ) {
|
||||
DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
|
||||
ibdev, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get partition key table
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @v guid_info Partition key table datagram to fill in
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int ib_get_pkey_table ( struct ib_device *ibdev,
|
||||
struct ib_mad_pkey_table *pkey_table ) {
|
||||
struct ib_mad_hdr *hdr = &pkey_table->mad_hdr;
|
||||
int rc;
|
||||
|
||||
/* Construct MAD */
|
||||
memset ( pkey_table, 0, sizeof ( *pkey_table ) );
|
||||
hdr->base_version = IB_MGMT_BASE_VERSION;
|
||||
hdr->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
|
||||
hdr->class_version = 1;
|
||||
hdr->method = IB_MGMT_METHOD_GET;
|
||||
hdr->attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
|
||||
|
||||
if ( ( rc = ib_mad ( ibdev, hdr, sizeof ( *pkey_table ) ) ) != 0 ) {
|
||||
DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
|
||||
ibdev, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for link up
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*
|
||||
* This function shouldn't really exist. Unfortunately, IB links take
|
||||
* a long time to come up, and we can't get various key parameters
|
||||
* e.g. our own IPoIB MAC address without information from the subnet
|
||||
* manager). We should eventually make link-up an asynchronous event.
|
||||
*/
|
||||
static int ib_wait_for_link ( struct ib_device *ibdev ) {
|
||||
struct ib_mad_port_info port_info;
|
||||
unsigned int retries;
|
||||
int rc;
|
||||
|
||||
printf ( "Waiting for Infiniband link-up..." );
|
||||
for ( retries = 20 ; retries ; retries-- ) {
|
||||
if ( ( rc = ib_get_port_info ( ibdev, &port_info ) ) != 0 )
|
||||
continue;
|
||||
if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
|
||||
== 4 ) {
|
||||
printf ( "ok\n" );
|
||||
return 0;
|
||||
}
|
||||
printf ( "." );
|
||||
sleep ( 1 );
|
||||
}
|
||||
printf ( "failed\n" );
|
||||
return -ENODEV;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get MAD parameters
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int ib_get_mad_params ( struct ib_device *ibdev ) {
|
||||
union {
|
||||
/* This union exists just to save stack space */
|
||||
struct ib_mad_port_info port_info;
|
||||
struct ib_mad_guid_info guid_info;
|
||||
struct ib_mad_pkey_table pkey_table;
|
||||
} u;
|
||||
int rc;
|
||||
|
||||
/* Port info gives us the first half of the port GID and the SM LID */
|
||||
if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 )
|
||||
return rc;
|
||||
memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 );
|
||||
ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid );
|
||||
|
||||
/* GUID info gives us the second half of the port GID */
|
||||
if ( ( rc = ib_get_guid_info ( ibdev, &u.guid_info ) ) != 0 )
|
||||
return rc;
|
||||
memcpy ( &ibdev->port_gid.u.bytes[8], u.guid_info.gid_local, 8 );
|
||||
|
||||
/* Get partition key */
|
||||
if ( ( rc = ib_get_pkey_table ( ibdev, &u.pkey_table ) ) != 0 )
|
||||
return rc;
|
||||
ibdev->pkey = ntohs ( u.pkey_table.pkey[0][0] );
|
||||
|
||||
DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n",
|
||||
ibdev, htonl ( ibdev->port_gid.u.dwords[0] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[1] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[2] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[3] ) );
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Infiniband device creation/destruction
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Allocate Infiniband device
|
||||
*
|
||||
@ -202,6 +375,54 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
|
||||
return ibdev;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register Infiniband device
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
int register_ibdev ( struct ib_device *ibdev ) {
|
||||
int rc;
|
||||
|
||||
/* Open link */
|
||||
if ( ( rc = ib_open ( ibdev ) ) != 0 )
|
||||
goto err_open;
|
||||
|
||||
/* Wait for link */
|
||||
if ( ( rc = ib_wait_for_link ( ibdev ) ) != 0 )
|
||||
goto err_wait_for_link;
|
||||
|
||||
/* Get MAD parameters */
|
||||
if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 )
|
||||
goto err_get_mad_params;
|
||||
|
||||
/* Add IPoIB device */
|
||||
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
|
||||
DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
|
||||
ibdev, strerror ( rc ) );
|
||||
goto err_ipoib_probe;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_ipoib_probe:
|
||||
err_get_mad_params:
|
||||
err_wait_for_link:
|
||||
ib_close ( ibdev );
|
||||
err_open:
|
||||
return rc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister Infiniband device
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
*/
|
||||
void unregister_ibdev ( struct ib_device *ibdev ) {
|
||||
ipoib_remove ( ibdev );
|
||||
ib_close ( ibdev );
|
||||
}
|
||||
|
||||
/**
|
||||
* Free Infiniband device
|
||||
*
|
||||
@ -210,3 +431,4 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
|
||||
void free_ibdev ( struct ib_device *ibdev ) {
|
||||
free ( ibdev );
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user