xCAT/xNBA
2
0
Fork 0
mirror of https://github.com/xcat2/xNBA.git synced 2025-04-23 13:25:54 +00:00
Code Issues Projects Releases Wiki Activity

[infiniband] Update all other MAD users to use a management interface

Browse Source
This commit is contained in:
Michael Brown 2009-08-06 01:18:38 +01:00
parent 44251ebb9a
commit 34bfc04e4c
12 changed files with 752 additions and 997 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/drivers/net/ipoib.c
View File

@ -58,12 +58,14 @@ struct ipoib_device {
struct ib_queue_pair *qp;
/** Broadcast MAC */
struct ipoib_mac broadcast;
/** Joined to multicast group
/** Joined to IPv4 broadcast multicast group
*
* This flag indicates whether or not we have initiated the
* join to the IPv4 multicast group.
* join to the IPv4 broadcast multicast group.
*/
int broadcast_joined;
/** IPv4 broadcast multicast group membership */
struct ib_mc_membership broadcast_membership;
};
/** Broadcast IPoIB address */
@ -456,6 +458,26 @@ static void ipoib_irq ( struct net_device *netdev __unused,
/* No implementation */
}
/**
* Handle IPv4 broadcast multicast group join completion
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v membership Multicast group membership
* @v rc Status code
* @v mad Response MAD (or NULL on error)
*/
void ipoib_join_complete ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp __unused,
struct ib_mc_membership *membership, int rc,
union ib_mad *mad __unused ) {
struct ipoib_device *ipoib = container_of ( membership,
struct ipoib_device, broadcast_membership );
/* Record join status as link status */
netdev_link_err ( ipoib->netdev, rc );
}
/**
* Join IPv4 broadcast multicast group
*
@ -466,7 +488,9 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
int rc;
if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp,
&ipoib->broadcast.gid ) ) != 0 ) {
&ipoib->broadcast_membership,
&ipoib->broadcast.gid,
ipoib_join_complete ) ) != 0 ) {
DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
ipoib, strerror ( rc ) );
return rc;
@ -485,7 +509,7 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
if ( ipoib->broadcast_joined ) {
ib_mcast_leave ( ipoib->ibdev, ipoib->qp,
&ipoib->broadcast.gid );
&ipoib->broadcast_membership );
ipoib->broadcast_joined = 0;
}
}

60
src/include/gpxe/ib_cm.h
View File

@ -10,12 +10,60 @@
FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/infiniband.h>
#include <gpxe/retry.h>
extern int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid,
struct ib_gid_half *service_id,
void *private_data, size_t private_data_len,
void ( * notify ) ( struct ib_queue_pair *qp,
int rc, void *private_data,
size_t private_data_len ) );
struct ib_mad_transaction;
struct ib_connection;
/** Infiniband connection operations */
struct ib_connection_operations {
/** Handle change of connection status
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v conn Connection
* @v rc Connection status code
* @v private_data Private data, if available
* @v private_data_len Length of private data
*/
void ( * changed ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_connection *conn, int rc,
void *private_data, size_t private_data_len );
};
/** An Infiniband connection */
struct ib_connection {
/** Infiniband device */
struct ib_device *ibdev;
/** Queue pair */
struct ib_queue_pair *qp;
/** Local communication ID */
uint32_t local_id;
/** Remote communication ID */
uint32_t remote_id;
/** Target service ID */
struct ib_gid_half service_id;
/** Connection operations */
struct ib_connection_operations *op;
/** Path to target */
struct ib_path *path;
/** Connection request management transaction */
struct ib_mad_transaction *madx;
/** Length of connection request private data */
size_t private_data_len;
/** Connection request private data */
uint8_t private_data[0];
};
extern struct ib_connection *
ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *dgid, struct ib_gid_half *service_id,
void *req_private_data, size_t req_private_data_len,
struct ib_connection_operations *op );
extern void ib_destroy_conn ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_connection *conn );
#endif /* _GPXE_IB_CM_H */

65
src/include/gpxe/ib_gma.h
View File

@ -1,65 +0,0 @@
#ifndef _GPXE_IB_GMA_H
#define _GPXE_IB_GMA_H
/** @file
*
* Infiniband General Management Agent
*
*/
FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/list.h>
#include <gpxe/retry.h>
#include <gpxe/tables.h>
#include <gpxe/infiniband.h>
struct ib_gma;
/** A GMA attribute handler */
struct ib_gma_handler {
/** Management class */
uint8_t mgmt_class;
/** Management class don't-care bits */
uint8_t mgmt_class_ignore;
/** Class version */
uint8_t class_version;
/** Method */
uint8_t method;
/** Attribute (in network byte order) */
uint16_t attr_id;
/** Handle attribute
*
* @v gma General management agent
* @v mad MAD
* @ret response MAD response, or NULL to send no response
*/
union ib_mad * ( * handle ) ( struct ib_gma *gma, union ib_mad *mad );
};
/** GMA attribute handlers */
#define IB_GMA_HANDLERS __table ( struct ib_gma_handler, "ib_gma_handlers" )
/** Declare a GMA attribute handler */
#define __ib_gma_handler __table_entry ( IB_GMA_HANDLERS, 01 )
/** An Infiniband General Management Agent */
struct ib_gma {
/** Infiniband device */
struct ib_device *ibdev;
/** Completion queue */
struct ib_completion_queue *cq;
/** Queue pair */
struct ib_queue_pair *qp;
/** List of outstanding MAD requests */
struct list_head requests;
};
extern int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad,
struct ib_address_vector *av, int retry );
extern struct ib_gma * ib_create_gma ( struct ib_device *ibdev,
enum ib_queue_pair_type type );
extern void ib_destroy_gma ( struct ib_gma *gma );
#endif /* _GPXE_IB_GMA_H */

11
src/include/gpxe/ib_mad.h
View File

@ -303,6 +303,16 @@ union ib_sa_data {
#define IB_CM_ATTR_LOAD_ALTERNATE_PATH 0x0019
#define IB_CM_ATTR_ALTERNATE_PATH_RESPONSE 0x001a
/** Communication management common fields */
struct ib_cm_common {
/** Local communication ID */
uint32_t local_id;
/** Remote communication ID */
uint32_t remote_id;
/** Reserved */
uint8_t reserved[224];
} __attribute__ (( packed ));
/** A communication management path */
struct ib_cm_path {
/** Local port LID */
@ -438,6 +448,7 @@ struct ib_cm_ready_to_use {
/** A communication management attribute */
union ib_cm_data {
struct ib_cm_common common;
struct ib_cm_connect_request connect_request;
struct ib_cm_connect_reject connect_reject;
struct ib_cm_connect_reply connect_reply;

33
src/include/gpxe/ib_mcast.h
View File

@ -11,9 +11,38 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/infiniband.h>
struct ib_mad_transaction;
/** An Infiniband multicast group membership */
struct ib_mc_membership {
/** Queue pair */
struct ib_queue_pair *qp;
/** Multicast GID */
struct ib_gid gid;
/** Multicast group join transaction */
struct ib_mad_transaction *madx;
/** Handle join success/failure
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v membership Multicast group membership
* @v rc Status code
* @v mad Response MAD (or NULL on error)
*/
void ( * complete ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_mc_membership *membership, int rc,
union ib_mad *mad );
};
extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid );
struct ib_mc_membership *membership,
struct ib_gid *gid,
void ( * joined ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_mc_membership *memb,
int rc, union ib_mad *mad ) );
extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid );
struct ib_mc_membership *membership );
#endif /* _GPXE_IB_MCAST_H */

59
src/include/gpxe/ib_pathrec.h
View File

@ -11,6 +11,65 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/infiniband.h>
struct ib_mad_transaction;
struct ib_path;
/** Infiniband path operations */
struct ib_path_operations {
/** Handle path transaction completion
*
* @v ibdev Infiniband device
* @v path Path
* @v rc Status code
* @v av Address vector, or NULL on error
*/
void ( * complete ) ( struct ib_device *ibdev,
struct ib_path *path, int rc,
struct ib_address_vector *av );
};
/** An Infiniband path */
struct ib_path {
/** Infiniband device */
struct ib_device *ibdev;
/** Address vector */
struct ib_address_vector av;
/** Management transaction */
struct ib_mad_transaction *madx;
/** Path operations */
struct ib_path_operations *op;
/** Owner private data */
void *owner_priv;
};
/**
* Set Infiniband path owner-private data
*
* @v path Path
* @v priv Private data
*/
static inline __always_inline void
ib_path_set_ownerdata ( struct ib_path *path, void *priv ) {
path->owner_priv = priv;
}
/**
* Get Infiniband path owner-private data
*
* @v path Path
* @ret priv Private data
*/
static inline __always_inline void *
ib_path_get_ownerdata ( struct ib_path *path ) {
return path->owner_priv;
}
extern struct ib_path *
ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
struct ib_path_operations *op );
extern void ib_destroy_path ( struct ib_device *ibdev,
struct ib_path *path );
extern int ib_resolve_path ( struct ib_device *ibdev,
struct ib_address_vector *av );

5
src/include/gpxe/infiniband.h
View File

@ -46,7 +46,6 @@ struct ib_queue_pair;
struct ib_address_vector;
struct ib_completion_queue;
struct ib_mad_interface;
struct ib_gma;
/** Infiniband transmission rates */
enum ib_rate {
@ -416,8 +415,8 @@ struct ib_device {
/** Subnet management interface */
struct ib_mad_interface *smi;
/** General management agent */
struct ib_gma *gma;
/** General services interface */
struct ib_mad_interface *gsi;
/** Driver private data */
void *drv_priv;

17
src/net/infiniband.c
View File

@ -35,7 +35,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <gpxe/infiniband.h>
#include <gpxe/ib_mi.h>
#include <gpxe/ib_sma.h>
#include <gpxe/ib_gma.h>
/** @file
*
@ -551,12 +550,12 @@ int ib_open ( struct ib_device *ibdev ) {
goto err_create_sma;
}
/* Create general management agent */
ibdev->gma = ib_create_gma ( ibdev, IB_QPT_GSI );
if ( ! ibdev->gma ) {
DBGC ( ibdev, "IBDEV %p could not create GMA\n", ibdev );
/* Create general services interface */
ibdev->gsi = ib_create_mi ( ibdev, IB_QPT_GSI );
if ( ! ibdev->gsi ) {
DBGC ( ibdev, "IBDEV %p could not create GSI\n", ibdev );
rc = -ENOMEM;
goto err_create_gma;
goto err_create_gsi;
}
/* Open device */
@ -571,8 +570,8 @@ int ib_open ( struct ib_device *ibdev ) {
ibdev->op->close ( ibdev );
err_open:
ib_destroy_gma ( ibdev->gma );
err_create_gma:
ib_destroy_mi ( ibdev, ibdev->gsi );
err_create_gsi:
ib_destroy_sma ( ibdev, ibdev->smi );
err_create_sma:
ib_destroy_mi ( ibdev, ibdev->smi );
@ -594,7 +593,7 @@ void ib_close ( struct ib_device *ibdev ) {
/* Close device if this was the last remaining requested opening */
if ( ibdev->open_count == 0 ) {
ib_destroy_gma ( ibdev->gma );
ib_destroy_mi ( ibdev, ibdev->gsi );
ib_destroy_sma ( ibdev, ibdev->smi );
ib_destroy_mi ( ibdev, ibdev->smi );
ibdev->op->close ( ibdev );

477
src/net/infiniband/ib_cm.c
View File

@ -24,10 +24,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <byteswap.h>
#include <errno.h>
#include <assert.h>
#include <gpxe/list.h>
#include <gpxe/process.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_gma.h>
#include <gpxe/ib_mi.h>
#include <gpxe/ib_pathrec.h>
#include <gpxe/ib_cm.h>
@ -38,64 +36,170 @@ FILE_LICENCE ( GPL2_OR_LATER );
*
*/
/** An outstanding connection request */
struct ib_cm_request {
/** List of all outstanding requests */
struct list_head list;
/** Local communication ID */
uint32_t local_id;
/** Remote communication ID */
uint32_t remote_id;
/** Queue pair */
struct ib_queue_pair *qp;
/** Target service ID */
struct ib_gid_half service_id;
/** Connection process */
struct process process;
/** Notification handler
*
* @v qp Queue pair
* @v rc Connection status code
* @v private_data Private data
* @v private_data_len Length of private data
*/
void ( * notify ) ( struct ib_queue_pair *qp, int rc,
void *private_data, size_t private_data_len );
/** Private data length */
size_t private_data_len;
/** Private data */
uint8_t private_data[0];
};
/** List of all outstanding connection requests */
static LIST_HEAD ( ib_cm_requests );
/**
* Send connection request
* Send "ready to use" response
*
* @v request Connection request
* @v ibdev Infiniband device
* @v mi Management interface
* @v conn Connection
* @v av Address vector
* @ret rc Return status code
*/
static int ib_cm_send_request ( struct ib_cm_request *request ) {
struct ib_queue_pair *qp = request->qp;
struct ib_device *ibdev = qp->ibdev;
struct ib_gma *gma = ibdev->gma;
static int ib_cm_send_rtu ( struct ib_device *ibdev,
struct ib_mad_interface *mi,
struct ib_connection *conn,
struct ib_address_vector *av ) {
union ib_mad mad;
struct ib_mad_cm *cm = &mad.cm;
struct ib_cm_connect_request *connect_req =
&cm->cm_data.connect_request;
size_t private_data_len;
struct ib_cm_ready_to_use *ready =
&mad.cm.cm_data.ready_to_use;
int rc;
/* Construct "ready to use" response */
memset ( &mad, 0, sizeof ( mad ) );
mad.hdr.mgmt_class = IB_MGMT_CLASS_CM;
mad.hdr.class_version = IB_CM_CLASS_VERSION;
mad.hdr.method = IB_MGMT_METHOD_SEND;
mad.hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE );
ready->local_id = htonl ( conn->local_id );
ready->remote_id = htonl ( conn->remote_id );
if ( ( rc = ib_mi_send ( ibdev, mi, &mad, av ) ) != 0 ){
DBGC ( conn, "CM %p could not send RTU: %s\n",
conn, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Handle connection request transaction completion
*
* @v ibdev Infiniband device
* @v mi Management interface
* @v madx Management transaction
* @v rc Status code
* @v mad Received MAD (or NULL on error)
* @v av Source address vector (or NULL on error)
*/
static void ib_cm_req_complete ( struct ib_device *ibdev,
struct ib_mad_interface *mi,
struct ib_mad_transaction *madx,
int rc, union ib_mad *mad,
struct ib_address_vector *av ) {
struct ib_connection *conn = ib_madx_get_ownerdata ( madx );
struct ib_queue_pair *qp = conn->qp;
struct ib_cm_common *common = &mad->cm.cm_data.common;
struct ib_cm_connect_reply *connect_rep =
&mad->cm.cm_data.connect_reply;
struct ib_cm_connect_reject *connect_rej =
&mad->cm.cm_data.connect_reject;
void *private_data = NULL;
size_t private_data_len = 0;
/* Report failures */
if ( rc != 0 ) {
DBGC ( conn, "CM %p connection request failed: %s\n",
conn, strerror ( rc ) );
goto out;
}
/* Record remote communication ID */
conn->remote_id = ntohl ( common->local_id );
/* Handle response */
switch ( mad->hdr.attr_id ) {
case htons ( IB_CM_ATTR_CONNECT_REPLY ) :
/* Extract fields */
qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 );
qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 );
private_data = &connect_rep->private_data;
private_data_len = sizeof ( connect_rep->private_data );
DBGC ( conn, "CM %p connected to QPN %lx PSN %x\n",
conn, qp->av.qpn, qp->send.psn );
/* Modify queue pair */
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
DBGC ( conn, "CM %p could not modify queue pair: %s\n",
conn, strerror ( rc ) );
goto out;
}
/* Send "ready to use" reply */
if ( ( rc = ib_cm_send_rtu ( ibdev, mi, conn, av ) ) != 0 ) {
/* Treat as non-fatal */
rc = 0;
}
break;
case htons ( IB_CM_ATTR_CONNECT_REJECT ) :
/* Extract fields */
DBGC ( conn, "CM %p connection rejected (reason %d)\n",
conn, ntohs ( connect_rej->reason ) );
private_data = &connect_rej->private_data;
private_data_len = sizeof ( connect_rej->private_data );
rc = -ENOTCONN;
break;
default:
DBGC ( conn, "CM %p unexpected response (attribute %04x)\n",
conn, ntohs ( mad->hdr.attr_id ) );
rc = -EIO;
break;
}
out:
/* Destroy the completed transaction */
ib_destroy_madx ( ibdev, ibdev->gsi, madx );
conn->madx = NULL;
/* Hand off to the upper completion handler */
conn->op->changed ( ibdev, qp, conn, rc, private_data,
private_data_len );
}
/** Connection request operations */
static struct ib_mad_transaction_operations ib_cm_req_op = {
.complete = ib_cm_req_complete,
};
/**
* Handle connection path transaction completion
*
* @v ibdev Infiniband device
* @v path Path
* @v rc Status code
* @v av Address vector, or NULL on error
*/
static void ib_cm_path_complete ( struct ib_device *ibdev,
struct ib_path *path, int rc,
struct ib_address_vector *av ) {
struct ib_connection *conn = ib_path_get_ownerdata ( path );
struct ib_queue_pair *qp = conn->qp;
union ib_mad mad;
struct ib_cm_connect_request *connect_req =
&mad.cm.cm_data.connect_request;
size_t private_data_len;
/* Report failures */
if ( rc != 0 ) {
DBGC ( conn, "CM %p path lookup failed: %s\n",
conn, strerror ( rc ) );
conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 );
goto out;
}
/* Update queue pair peer path */
memcpy ( &qp->av, av, sizeof ( qp->av ) );
/* Construct connection request */
memset ( cm, 0, sizeof ( *cm ) );
cm->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
cm->mad_hdr.mgmt_class = IB_MGMT_CLASS_CM;
cm->mad_hdr.class_version = IB_CM_CLASS_VERSION;
cm->mad_hdr.method = IB_MGMT_METHOD_SEND;
cm->mad_hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST );
connect_req->local_id = htonl ( request->local_id );
memcpy ( &connect_req->service_id, &request->service_id,
memset ( &mad, 0, sizeof ( mad ) );
mad.hdr.mgmt_class = IB_MGMT_CLASS_CM;
mad.hdr.class_version = IB_CM_CLASS_VERSION;
mad.hdr.method = IB_MGMT_METHOD_SEND;
mad.hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST );
connect_req->local_id = htonl ( conn->local_id );
memcpy ( &connect_req->service_id, &conn->service_id,
sizeof ( connect_req->service_id ) );
ib_get_hca_info ( ibdev, &connect_req->local_ca );
connect_req->local_qpn__responder_resources =
@ -113,217 +217,116 @@ static int ib_cm_send_request ( struct ib_cm_request *request ) {
connect_req->max_cm_retries__srq =
( ( 0x0f << 4 ) | ( 0 << 3 ) );
connect_req->primary.local_lid = htons ( ibdev->lid );
connect_req->primary.remote_lid = htons ( request->qp->av.lid );
connect_req->primary.remote_lid = htons ( conn->qp->av.lid );
memcpy ( &connect_req->primary.local_gid, &ibdev->gid,
sizeof ( connect_req->primary.local_gid ) );
memcpy ( &connect_req->primary.remote_gid, &request->qp->av.gid,
memcpy ( &connect_req->primary.remote_gid, &conn->qp->av.gid,
sizeof ( connect_req->primary.remote_gid ) );
connect_req->primary.flow_label__rate =
htonl ( ( 0 << 12 ) | ( request->qp->av.rate << 0 ) );
htonl ( ( 0 << 12 ) | ( conn->qp->av.rate << 0 ) );
connect_req->primary.hop_limit = 0;
connect_req->primary.sl__subnet_local =
( ( request->qp->av.sl << 4 ) | ( 1 << 3 ) );
( ( conn->qp->av.sl << 4 ) | ( 1 << 3 ) );
connect_req->primary.local_ack_timeout = ( 0x13 << 3 );
private_data_len = request->private_data_len;
private_data_len = conn->private_data_len;
if ( private_data_len > sizeof ( connect_req->private_data ) )
private_data_len = sizeof ( connect_req->private_data );
memcpy ( &connect_req->private_data, &request->private_data,
memcpy ( &connect_req->private_data, &conn->private_data,
private_data_len );
/* Send request */
if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) {
DBGC ( gma, "GMA %p could not send connection request: %s\n",
gma, strerror ( rc ) );
return rc;
/* Create connection request */
conn->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
&ib_cm_req_op );
if ( ! conn->madx ) {
DBGC ( conn, "CM %p could not create connection request\n",
conn );
conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 );
goto out;
}
ib_madx_set_ownerdata ( conn->madx, conn );
return 0;
out:
/* Destroy the completed transaction */
ib_destroy_path ( ibdev, path );
conn->path = NULL;
}
/**
* Connection request process step
*
* @v process Connection request process
*/
static void ib_cm_step ( struct process *process ) {
struct ib_cm_request *request =
container_of ( process, struct ib_cm_request, process );
struct ib_queue_pair *qp = request->qp;
struct ib_device *ibdev = qp->ibdev;
int rc;
/* Wait until path can be resolved */
if ( ( rc = ib_resolve_path ( ibdev, &request->qp->av ) ) != 0 )
return;
/* Wait until request can be sent */
if ( ( rc = ib_cm_send_request ( request ) ) != 0 )
return;
/* Stop process */
process_del ( process );
}
/**
* Identify connection request by communication ID
*
* @v local_id Local communication ID
* @v remote_id Remote communication ID
* @ret request Connection request, or NULL
*/
static struct ib_cm_request * ib_cm_find_request ( uint32_t local_id,
uint32_t remote_id ) {
struct ib_cm_request *request;
list_for_each_entry ( request, &ib_cm_requests, list ) {
if ( request->local_id == local_id ) {
request->remote_id = remote_id;
return request;
}
}
return NULL;
}
/**
* Handle connection reply
*
* @v gma General management agent
* @v mad MAD
* @ret response MAD response
*/
static union ib_mad * ib_cm_connect_reply ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_cm_connect_reply *connect_rep =
&mad->cm.cm_data.connect_reply;
struct ib_cm_ready_to_use *ready =
&mad->cm.cm_data.ready_to_use;
struct ib_cm_request *request;
int rc;
/* Identify request */
request = ib_cm_find_request ( ntohl ( connect_rep->remote_id ),
ntohl ( connect_rep->local_id ) );
if ( ! request ) {
DBGC ( gma, "GMA %p received connection reply with unknown "
"ID %08x\n", gma, ntohl ( connect_rep->remote_id ) );
return NULL;
}
/* Extract fields */
request->qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 );
request->qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 );
DBGC ( gma, "GMA %p QPN %lx connected to QPN %lx PSN %x\n", gma,
request->qp->qpn, request->qp->av.qpn, request->qp->send.psn );
/* Modify queue pair */
if ( ( rc = ib_modify_qp ( request->qp->ibdev, request->qp ) ) != 0 ) {
DBGC ( gma, "GMA %p QPN %lx could not modify queue pair: %s\n",
gma, request->qp->qpn, strerror ( rc ) );
return NULL;
}
/* Inform recipient that we are now connected */
request->notify ( request->qp, 0, &connect_rep->private_data,
sizeof ( connect_rep->private_data ) );
/* Construct ready to use reply */
mad->hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE );
memset ( ready, 0, sizeof ( *ready ) );
ready->local_id = htonl ( request->local_id );
ready->remote_id = htonl ( request->remote_id );
return mad;
}
/**
* Handle connection rejection
*
* @v gma General management agent
* @v mad MAD
* @ret response MAD response
*/
static union ib_mad * ib_cm_connect_reject ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_cm_connect_reject *connect_rej =
&mad->cm.cm_data.connect_reject;
struct ib_cm_request *request;
uint16_t reason;
/* Identify request */
request = ib_cm_find_request ( ntohl ( connect_rej->remote_id ),
ntohl ( connect_rej->local_id ) );
if ( ! request ) {
DBGC ( gma, "GMA %p received connection rejection with "
"unknown ID %08x\n", gma,
ntohl ( connect_rej->remote_id ) );
return NULL;
}
/* Extract fields */
reason = ntohs ( connect_rej->reason );
DBGC ( gma, "GMA %p QPN %lx connection rejected (reason %d)\n",
gma, request->qp->qpn, reason );
/* Inform recipient that we are now disconnected */
request->notify ( request->qp, -ENOTCONN, &connect_rej->private_data,
sizeof ( connect_rej->private_data ) );
return NULL;
}
/** Communication management MAD handlers */
struct ib_gma_handler ib_cm_handlers[] __ib_gma_handler = {
{
.mgmt_class = IB_MGMT_CLASS_CM,
.class_version = IB_CM_CLASS_VERSION,
.method = IB_MGMT_METHOD_SEND,
.attr_id = htons ( IB_CM_ATTR_CONNECT_REPLY ),
.handle = ib_cm_connect_reply,
},
{
.mgmt_class = IB_MGMT_CLASS_CM,
.class_version = IB_CM_CLASS_VERSION,
.method = IB_MGMT_METHOD_SEND,
.attr_id = htons ( IB_CM_ATTR_CONNECT_REJECT ),
.handle = ib_cm_connect_reject,
},
/** Connection path operations */
static struct ib_path_operations ib_cm_path_op = {
.complete = ib_cm_path_complete,
};
/**
* Connect to remote QP
* Create connection to remote QP
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dgid Target GID
* @v service_id Target service ID
* @v private_data Private data
* @v private_data_len Length of private data
* @ret rc Return status code
* @v private_data Connection request private data
* @v private_data_len Length of connection request private data
* @v op Connection operations
* @ret conn Connection
*/
int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid,
struct ib_gid_half *service_id,
void *private_data, size_t private_data_len,
void ( * notify ) ( struct ib_queue_pair *qp, int rc,
void *private_data,
size_t private_data_len ) ) {
struct ib_cm_request *request;
struct ib_connection *
ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *dgid, struct ib_gid_half *service_id,
void *private_data, size_t private_data_len,
struct ib_connection_operations *op ) {
struct ib_connection *conn;
/* Allocate and initialise request */
request = zalloc ( sizeof ( *request ) + private_data_len );
if ( ! request )
return -ENOMEM;
list_add ( &request->list, &ib_cm_requests );
request->local_id = random();
request->qp = qp;
conn = zalloc ( sizeof ( *conn ) + private_data_len );
if ( ! conn )
goto err_alloc_conn;
conn->ibdev = ibdev;
conn->qp = qp;
memset ( &qp->av, 0, sizeof ( qp->av ) );
qp->av.gid_present = 1;
memcpy ( &qp->av.gid, dgid, sizeof ( qp->av.gid ) );
memcpy ( &request->service_id, service_id,
sizeof ( request->service_id ) );
request->notify = notify;
request->private_data_len = private_data_len;
memcpy ( &request->private_data, private_data, private_data_len );
process_init ( &request->process, ib_cm_step, NULL );
conn->local_id = random();
memcpy ( &conn->service_id, service_id, sizeof ( conn->service_id ) );
conn->op = op;
conn->private_data_len = private_data_len;
memcpy ( &conn->private_data, private_data, private_data_len );
return 0;
/* Create path */
conn->path = ib_create_path ( ibdev, &qp->av, &ib_cm_path_op );
if ( ! conn->path )
goto err_create_path;
ib_path_set_ownerdata ( conn->path, conn );
DBGC ( conn, "CM %p created for IBDEV %p QPN %lx\n",
conn, ibdev, qp->qpn );
DBGC ( conn, "CM %p connecting to %08x:%08x:%08x:%08x %08x:%08x\n",
conn, ntohl ( dgid->u.dwords[0] ), ntohl ( dgid->u.dwords[1] ),
ntohl ( dgid->u.dwords[2] ), ntohl ( dgid->u.dwords[3] ),
ntohl ( service_id->u.dwords[0] ),
ntohl ( service_id->u.dwords[1] ) );
return conn;
ib_destroy_path ( ibdev, conn->path );
err_create_path:
free ( conn );
err_alloc_conn:
return NULL;
}
/**
* Destroy connection to remote QP
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v conn Connection
*/
void ib_destroy_conn ( struct ib_device *ibdev,
struct ib_queue_pair *qp __unused,
struct ib_connection *conn ) {
if ( conn->madx )
ib_destroy_madx ( ibdev, ibdev->gsi, conn->madx );
if ( conn->path )
ib_destroy_path ( ibdev, conn->path );
free ( conn );
}

403
src/net/infiniband/ib_gma.c
View File

@ -1,403 +0,0 @@
/*
* Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
FILE_LICENCE ( GPL2_OR_LATER );
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <byteswap.h>
#include <gpxe/infiniband.h>
#include <gpxe/iobuf.h>
#include <gpxe/ib_gma.h>
/**
* @file
*
* Infiniband General Management Agent
*
*/
/** A MAD request */
struct ib_mad_request {
/** Associated GMA */
struct ib_gma *gma;
/** List of outstanding MAD requests */
struct list_head list;
/** Retry timer */
struct retry_timer timer;
/** Destination address */
struct ib_address_vector av;
/** MAD request */
union ib_mad mad;
};
/** GMA number of send WQEs
*
* This is a policy decision.
*/
#define IB_GMA_NUM_SEND_WQES 4
/** GMA number of receive WQEs
*
* This is a policy decision.
*/
#define IB_GMA_NUM_RECV_WQES 2
/** GMA number of completion queue entries
*
* This is a policy decision
*/
#define IB_GMA_NUM_CQES 8
/** TID magic signature */
#define IB_GMA_TID_MAGIC ( ( 'g' << 24 ) | ( 'P' << 16 ) | ( 'X' << 8 ) | 'E' )
/** TID to use for next MAD request */
static unsigned int next_request_tid;
/*****************************************************************************
*
* General management agent
*
*****************************************************************************
*/
/**
* Call attribute handler
*
* @v gma General management agent
* @v mad MAD
* @ret mad MAD response
*/
static union ib_mad * ib_handle_mad ( struct ib_gma *gma, union ib_mad *mad ) {
struct ib_mad_hdr *hdr = &mad->hdr;
struct ib_gma_handler *handler;
for_each_table_entry ( handler, IB_GMA_HANDLERS ) {
if ( ( ( handler->mgmt_class & ~handler->mgmt_class_ignore ) ==
( hdr->mgmt_class & ~handler->mgmt_class_ignore ) ) &&
( handler->class_version == hdr->class_version ) &&
( handler->method == hdr->method ) &&
( handler->attr_id == hdr->attr_id ) ) {
return handler->handle ( gma, mad );
}
}
hdr->method = IB_MGMT_METHOD_TRAP;
hdr->status = htons ( IB_MGMT_STATUS_UNSUPPORTED_METHOD_ATTR );
return mad;
}
/**
* Complete GMA receive
*
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v av Address vector
* @v iobuf I/O buffer
* @v rc Completion status code
*/
static void ib_gma_complete_recv ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_address_vector *av,
struct io_buffer *iobuf, int rc ) {
struct ib_gma *gma = ib_qp_get_ownerdata ( qp );
struct ib_mad_request *request;
union ib_mad *mad;
struct ib_mad_hdr *hdr;
union ib_mad *response;
/* Ignore errors */
if ( rc != 0 ) {
DBGC ( gma, "GMA %p RX error: %s\n", gma, strerror ( rc ) );
goto out;
}
/* Sanity checks */
if ( iob_len ( iobuf ) != sizeof ( *mad ) ) {
DBGC ( gma, "GMA %p RX bad size (%zd bytes)\n",
gma, iob_len ( iobuf ) );
DBGC_HDA ( gma, 0, iobuf->data, iob_len ( iobuf ) );
goto out;
}
mad = iobuf->data;
hdr = &mad->hdr;
if ( hdr->base_version != IB_MGMT_BASE_VERSION ) {
DBGC ( gma, "GMA %p unsupported base version %x\n",
gma, hdr->base_version );
DBGC_HDA ( gma, 0, mad, sizeof ( *mad ) );
goto out;
}
DBGC ( gma, "GMA %p RX TID %08x%08x (%02x,%02x,%02x,%04x) status "
"%04x\n", gma, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
hdr->mgmt_class, hdr->class_version, hdr->method,
ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
DBGC2_HDA ( gma, 0, mad, sizeof ( *mad ) );
/* Dequeue request if applicable */
list_for_each_entry ( request, &gma->requests, list ) {
if ( memcmp ( &request->mad.hdr.tid, &hdr->tid,
sizeof ( request->mad.hdr.tid ) ) == 0 ) {
stop_timer ( &request->timer );
list_del ( &request->list );
free ( request );
break;
}
}
/* Handle MAD */
if ( ( response = ib_handle_mad ( gma, mad ) ) == NULL )
goto out;
/* Re-use I/O buffer for response */
memcpy ( mad, response, sizeof ( *mad ) );
DBGC ( gma, "GMA %p TX TID %08x%08x (%02x,%02x,%02x,%04x) status "
"%04x\n", gma, ntohl ( hdr->tid[0] ), ntohl ( hdr->tid[1] ),
hdr->mgmt_class, hdr->class_version, hdr->method,
ntohs ( hdr->attr_id ), ntohs ( hdr->status ) );
DBGC2_HDA ( gma, 0, mad, sizeof ( *mad ) );
/* Send MAD response, if applicable */
if ( ( rc = ib_post_send ( ibdev, qp, av,
iob_disown ( iobuf ) ) ) != 0 ) {
DBGC ( gma, "GMA %p could not send MAD response: %s\n",
gma, strerror ( rc ) );
goto out;
}
out:
free_iob ( iobuf );
}
/** GMA completion operations */
static struct ib_completion_queue_operations ib_gma_completion_ops = {
.complete_recv = ib_gma_complete_recv,
};
/**
* Transmit MAD request
*
* @v gma General management agent
* @v request MAD request
* @ret rc Return status code
*/
static int ib_gma_send ( struct ib_gma *gma, struct ib_mad_request *request ) {
struct io_buffer *iobuf;
int rc;
DBGC ( gma, "GMA %p TX TID %08x%08x (%02x,%02x,%02x,%04x)\n",
gma, ntohl ( request->mad.hdr.tid[0] ),
ntohl ( request->mad.hdr.tid[1] ), request->mad.hdr.mgmt_class,
request->mad.hdr.class_version, request->mad.hdr.method,
ntohs ( request->mad.hdr.attr_id ) );
DBGC2_HDA ( gma, 0, &request->mad, sizeof ( request->mad ) );
/* Construct I/O buffer */
iobuf = alloc_iob ( sizeof ( request->mad ) );
if ( ! iobuf ) {
DBGC ( gma, "GMA %p could not allocate buffer for TID "
"%08x%08x\n", gma, ntohl ( request->mad.hdr.tid[0] ),
ntohl ( request->mad.hdr.tid[1] ) );
return -ENOMEM;
}
memcpy ( iob_put ( iobuf, sizeof ( request->mad ) ), &request->mad,
sizeof ( request->mad ) );
/* Send I/O buffer */
if ( ( rc = ib_post_send ( gma->ibdev, gma->qp, &request->av,
iobuf ) ) != 0 ) {
DBGC ( gma, "GMA %p could not send TID %08x%08x: %s\n",
gma, ntohl ( request->mad.hdr.tid[0] ),
ntohl ( request->mad.hdr.tid[1] ), strerror ( rc ) );
free_iob ( iobuf );
return rc;
}
return 0;
}
/**
* Handle MAD request timer expiry
*
* @v timer Retry timer
* @v expired Failure indicator
*/
static void ib_gma_timer_expired ( struct retry_timer *timer, int expired ) {
struct ib_mad_request *request =
container_of ( timer, struct ib_mad_request, timer );
struct ib_gma *gma = request->gma;
/* Abandon TID if we have tried too many times */
if ( expired ) {
DBGC ( gma, "GMA %p abandoning TID %08x%08x\n",
gma, ntohl ( request->mad.hdr.tid[0] ),
ntohl ( request->mad.hdr.tid[1] ) );
list_del ( &request->list );
free ( request );
return;
}
/* Restart retransmission timer */
start_timer ( timer );
/* Resend request */
ib_gma_send ( gma, request );
}
/**
* Issue MAD request
*
* @v gma General management agent
* @v mad MAD request
* @v av Destination address, or NULL for SM
* @v retry Request should be retried until a response arrives
* @ret rc Return status code
*/
int ib_gma_request ( struct ib_gma *gma, union ib_mad *mad,
struct ib_address_vector *av, int retry ) {
struct ib_device *ibdev = gma->ibdev;
struct ib_mad_request *request;
/* Allocate and initialise structure */
request = zalloc ( sizeof ( *request ) );
if ( ! request ) {
DBGC ( gma, "GMA %p could not allocate MAD request\n", gma );
return -ENOMEM;
}
request->gma = gma;
request->timer.expired = ib_gma_timer_expired;
/* Determine address vector */
if ( av ) {
memcpy ( &request->av, av, sizeof ( request->av ) );
} else {
request->av.lid = ibdev->sm_lid;
request->av.sl = ibdev->sm_sl;
request->av.qpn = IB_QPN_GSI;
request->av.qkey = IB_QKEY_GSI;
}
/* Copy MAD body */
memcpy ( &request->mad, mad, sizeof ( request->mad ) );
/* Allocate TID */
request->mad.hdr.tid[0] = htonl ( IB_GMA_TID_MAGIC );
request->mad.hdr.tid[1] = htonl ( ++next_request_tid );
/* Send initial request. Ignore errors; the retry timer will
* take care of those we care about.
*/
ib_gma_send ( gma, request );
/* Add to list and start timer if applicable */
if ( retry ) {
list_add ( &request->list, &gma->requests );
start_timer ( &request->timer );
} else {
free ( request );
}
return 0;
}
/**
* Create GMA
*
* @v ibdev Infiniband device
* @v type Queue pair type
* @ret gma General management agent, or NULL
*/
struct ib_gma * ib_create_gma ( struct ib_device *ibdev,
enum ib_queue_pair_type type ) {
struct ib_gma *gma;
int rc;
/* Allocate and initialise fields */
gma = zalloc ( sizeof ( *gma ) );
if ( ! gma )
goto err_alloc;
gma->ibdev = ibdev;
INIT_LIST_HEAD ( &gma->requests );
/* Create completion queue */
gma->cq = ib_create_cq ( ibdev, IB_GMA_NUM_CQES,
&ib_gma_completion_ops );
if ( ! gma->cq ) {
DBGC ( gma, "GMA %p could not allocate completion queue\n",
gma );
goto err_create_cq;
}
/* Create queue pair */
gma->qp = ib_create_qp ( ibdev, type, IB_GMA_NUM_SEND_WQES, gma->cq,
IB_GMA_NUM_RECV_WQES, gma->cq );
if ( ! gma->qp ) {
DBGC ( gma, "GMA %p could not allocate queue pair\n", gma );
goto err_create_qp;
}
ib_qp_set_ownerdata ( gma->qp, gma );
DBGC ( gma, "GMA %p running on QPN %#lx\n", gma, gma->qp->qpn );
/* Set queue key */
gma->qp->qkey = ( ( type == IB_QPT_SMI ) ? IB_QKEY_SMI : IB_QKEY_GSI );
if ( ( rc = ib_modify_qp ( ibdev, gma->qp ) ) != 0 ) {
DBGC ( gma, "GMA %p could not set queue key: %s\n",
gma, strerror ( rc ) );
goto err_modify_qp;
}
/* Fill receive ring */
ib_refill_recv ( ibdev, gma->qp );
return gma;
err_modify_qp:
ib_destroy_qp ( ibdev, gma->qp );
err_create_qp:
ib_destroy_cq ( ibdev, gma->cq );
err_create_cq:
free ( gma );
err_alloc:
return NULL;
}
/**
* Destroy GMA
*
* @v gma General management agent
*/
void ib_destroy_gma ( struct ib_gma *gma ) {
struct ib_device *ibdev = gma->ibdev;
struct ib_mad_request *request;
struct ib_mad_request *tmp;
/* Flush any outstanding requests */
list_for_each_entry_safe ( request, tmp, &gma->requests, list ) {
stop_timer ( &request->timer );
list_del ( &request->list );
free ( request );
}
ib_destroy_qp ( ibdev, gma->qp );
ib_destroy_cq ( ibdev, gma->cq );
free ( gma );
}

254
src/net/infiniband/ib_mcast.c
View File

@ -24,7 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <errno.h>
#include <gpxe/list.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_gma.h>
#include <gpxe/ib_mi.h>
#include <gpxe/ib_mcast.h>
/** @file
@ -34,22 +34,19 @@ FILE_LICENCE ( GPL2_OR_LATER );
*/
/**
* Transmit multicast group membership request
* Generate multicast membership MAD
*
* @v gma General management agent
* @v ibdev Infiniband device
* @v gid Multicast GID
* @v join Join (rather than leave) group
* @ret rc Return status code
* @v mad MAD to fill in
*/
static int ib_mc_member_request ( struct ib_gma *gma, struct ib_gid *gid,
int join ) {
union ib_mad mad;
struct ib_mad_sa *sa = &mad.sa;
int rc;
static void ib_mcast_mad ( struct ib_device *ibdev, struct ib_gid *gid,
int join, union ib_mad *mad ) {
struct ib_mad_sa *sa = &mad->sa;
/* Construct multicast membership record request */
memset ( sa, 0, sizeof ( *sa ) );
sa->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
sa->mad_hdr.method =
@ -61,51 +58,123 @@ static int ib_mc_member_request ( struct ib_gma *gma, struct ib_gid *gid,
sa->sa_data.mc_member_record.scope__join_state = 1;
memcpy ( &sa->sa_data.mc_member_record.mgid, gid,
sizeof ( sa->sa_data.mc_member_record.mgid ) );
memcpy ( &sa->sa_data.mc_member_record.port_gid, &gma->ibdev->gid,
memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
sizeof ( sa->sa_data.mc_member_record.port_gid ) );
}
/* Issue multicast membership record request */
if ( ( rc = ib_gma_request ( gma, &mad, NULL, join ) ) != 0 ) {
DBGC ( gma, "GMA %p could not join group: %s\n",
gma, strerror ( rc ) );
return rc;
/**
* Handle multicast membership record join response
*
* @v ibdev Infiniband device
* @v mi Management interface
* @v madx Management transaction
* @v rc Status code
* @v mad Received MAD (or NULL on error)
* @v av Source address vector (or NULL on error)
*/
static void ib_mcast_complete ( struct ib_device *ibdev,
struct ib_mad_interface *mi __unused,
struct ib_mad_transaction *madx,
int rc, union ib_mad *mad,
struct ib_address_vector *av __unused ) {
struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
struct ib_queue_pair *qp = membership->qp;
struct ib_gid *gid = &membership->gid;
struct ib_mc_member_record *mc_member_record =
&mad->sa.sa_data.mc_member_record;
int joined;
unsigned long qkey;
/* Report failures */
if ( rc != 0 ) {
DBGC ( ibdev, "IBDEV %p QPN %lx join failed: %s\n",
ibdev, qp->qpn, strerror ( rc ) );
goto out;
}
return 0;
/* Extract values from MAD */
joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
qkey = ntohl ( mc_member_record->qkey );
DBGC ( ibdev, "IBDEV %p QPN %lx %s %08x:%08x:%08x:%08x qkey %lx\n",
ibdev, qp->qpn, ( joined ? "joined" : "left" ),
ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ),
ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ),
qkey );
/* Set queue key */
qp->qkey = qkey;
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p QPN %lx could not modify qkey: %s\n",
ibdev, qp->qpn, strerror ( rc ) );
goto out;
}
out:
/* Destroy the completed transaction */
ib_destroy_madx ( ibdev, mi, madx );
membership->madx = NULL;
/* Hand off to upper completion handler */
membership->complete ( ibdev, qp, membership, rc, mad );
}
/** Multicast membership management transaction completion operations */
static struct ib_mad_transaction_operations ib_mcast_op = {
.complete = ib_mcast_complete,
};
/**
* Join multicast group
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v gid Multicast GID
* @v membership Multicast group membership
* @v gid Multicast GID to join
* @v joined Join completion handler
* @ret rc Return status code
*/
int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid ) {
struct ib_gma *gma = ibdev->gma;
struct ib_mc_membership *membership, struct ib_gid *gid,
void ( * complete ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_mc_membership *membership,
int rc, union ib_mad *mad ) ) {
union ib_mad mad;
int rc;
DBGC ( gma, "GMA %p QPN %lx joining %08x:%08x:%08x:%08x\n",
gma, qp->qpn, ntohl ( gid->u.dwords[0] ),
DBGC ( ibdev, "IBDEV %p QPN %lx joining %08x:%08x:%08x:%08x\n",
ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ),
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
ntohl ( gid->u.dwords[3] ) );
/* Initialise structure */
membership->qp = qp;
memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
membership->complete = complete;
/* Attach queue pair to multicast GID */
if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) {
DBGC ( gma, "GMA %p could not attach: %s\n",
gma, strerror ( rc ) );
DBGC ( ibdev, "IBDEV %p QPN %lx could not attach: %s\n",
ibdev, qp->qpn, strerror ( rc ) );
goto err_mcast_attach;
}
/* Initiate multicast membership join */
if ( ( rc = ib_mc_member_request ( gma, gid, 1 ) ) != 0 )
goto err_mc_member_record;
ib_mcast_mad ( ibdev, gid, 1, &mad );
membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
&ib_mcast_op );
if ( ! membership->madx ) {
DBGC ( ibdev, "IBDEV %p QPN %lx could not create join "
"transaction\n", ibdev, qp->qpn );
rc = -ENOMEM;
goto err_create_madx;
}
ib_madx_set_ownerdata ( membership->madx, membership );
return 0;
err_mc_member_record:
ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
err_create_madx:
ib_mcast_detach ( ibdev, qp, gid );
err_mcast_attach:
return rc;
@ -116,121 +185,32 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v gid Multicast GID
* @v membership Multicast group membership
*/
void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *gid ) {
struct ib_gma *gma = ibdev->gma;
struct ib_mc_membership *membership ) {
struct ib_gid *gid = &membership->gid;
union ib_mad mad;
int rc;
DBGC ( gma, "GMA %p QPN %lx leaving %08x:%08x:%08x:%08x\n",
gma, qp->qpn, ntohl ( gid->u.dwords[0] ),
DBGC ( ibdev, "IBDEV %p QPN %lx leaving %08x:%08x:%08x:%08x\n",
ibdev, qp->qpn, ntohl ( gid->u.dwords[0] ),
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
ntohl ( gid->u.dwords[3] ) );
/* Detach queue pair from multicast GID */
ib_mcast_detach ( ibdev, qp, gid );
/* Detach from multicast GID */
ib_mcast_detach ( ibdev, qp, &membership->gid );
/* Initiate multicast membership leave */
ib_mc_member_request ( gma, gid, 0 );
/* Cancel multicast membership join, if applicable */
if ( membership->madx ) {
ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
membership->madx = NULL;
}
/* Send a single group leave MAD */
ib_mcast_mad ( ibdev, &membership->gid, 0, &mad );
if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
DBGC ( ibdev, "IBDEV %p QPN %lx could not send leave request: "
"%s\n", ibdev, qp->qpn, strerror ( rc ) );
}
}
/**
* Handle multicast membership record join response
*
* @v gma General management agent
* @v mad MAD
* @ret mad MAD response
*/
static union ib_mad * ib_handle_mc_member_join ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_device *ibdev = gma->ibdev;
struct ib_mc_member_record *mc_member_record =
&mad->sa.sa_data.mc_member_record;
struct ib_queue_pair *qp;
struct ib_gid *gid;
unsigned long qkey;
int rc;
/* Ignore if not a success */
if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) {
DBGC ( gma, "GMA %p join failed with status %04x\n",
gma, ntohs ( mad->hdr.status ) );
return NULL;
}
/* Extract MAD parameters */
gid = &mc_member_record->mgid;
qkey = ntohl ( mc_member_record->qkey );
/* Locate matching queue pair */
qp = ib_find_qp_mgid ( ibdev, gid );
if ( ! qp ) {
DBGC ( gma, "GMA %p has no QP to join %08x:%08x:%08x:%08x\n",
gma, ntohl ( gid->u.dwords[0] ),
ntohl ( gid->u.dwords[1] ),
ntohl ( gid->u.dwords[2] ),
ntohl ( gid->u.dwords[3] ) );
return NULL;
}
DBGC ( gma, "GMA %p QPN %lx joined %08x:%08x:%08x:%08x qkey %lx\n",
gma, qp->qpn, ntohl ( gid->u.dwords[0] ),
ntohl ( gid->u.dwords[1] ), ntohl ( gid->u.dwords[2] ),
ntohl ( gid->u.dwords[3] ), qkey );
/* Set queue key */
qp->qkey = qkey;
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
DBGC ( gma, "GMA %p QPN %lx could not modify qkey: %s\n",
gma, qp->qpn, strerror ( rc ) );
return NULL;
}
return NULL;
}
/**
* Handle multicast membership record leave response
*
* @v gma General management agent
* @v mad MAD
* @v response MAD response
*/
static union ib_mad * ib_handle_mc_member_leave ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_mc_member_record *mc_member_record =
&mad->sa.sa_data.mc_member_record;
struct ib_gid *gid;
/* Ignore if not a success */
if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) {
DBGC ( gma, "GMA %p leave failed with status %04x\n",
gma, ntohs ( mad->hdr.status ) );
return NULL;
}
/* Extract MAD parameters */
gid = &mc_member_record->mgid;
DBGC ( gma, "GMA %p left %08x:%08x:%08x:%08x\n", gma,
ntohl ( gid->u.dwords[0] ), ntohl ( gid->u.dwords[1] ),
ntohl ( gid->u.dwords[2] ), ntohl ( gid->u.dwords[3] ) );
return NULL;
}
/** Multicast membership record response handler */
struct ib_gma_handler ib_mc_member_record_handlers[] __ib_gma_handler = {
{
.mgmt_class = IB_MGMT_CLASS_SUBN_ADM,
.class_version = IB_SA_CLASS_VERSION,
.method = IB_MGMT_METHOD_GET_RESP,
.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ),
.handle = ib_handle_mc_member_join,
},
{
.mgmt_class = IB_MGMT_CLASS_SUBN_ADM,
.class_version = IB_SA_CLASS_VERSION,
.method = IB_SA_METHOD_DELETE_RESP,
.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC ),
.handle = ib_handle_mc_member_leave,
},
};

333
src/net/infiniband/ib_pathrec.c
View File

@ -19,11 +19,12 @@
FILE_LICENCE ( GPL2_OR_LATER );
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <byteswap.h>
#include <errno.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_gma.h>
#include <gpxe/ib_mi.h>
#include <gpxe/ib_pathrec.h>
/** @file
@ -32,56 +33,162 @@ FILE_LICENCE ( GPL2_OR_LATER );
*
*/
/** Number of path record cache entries
/**
* Handle path transaction completion
*
* @v ibdev Infiniband device
* @v mi Management interface
* @v madx Management transaction
* @v rc Status code
* @v mad Received MAD (or NULL on error)
* @v av Source address vector (or NULL on error)
*/
static void ib_path_complete ( struct ib_device *ibdev,
struct ib_mad_interface *mi,
struct ib_mad_transaction *madx,
int rc, union ib_mad *mad,
struct ib_address_vector *av __unused ) {
struct ib_path *path = ib_madx_get_ownerdata ( madx );
struct ib_gid *dgid = &path->av.gid;
struct ib_path_record *pathrec = &mad->sa.sa_data.path_record;
/* Report failures */
if ( rc != 0 ) {
DBGC ( ibdev, "IBDEV %p path lookup for %08x:%08x:%08x:%08x "
"failed: %s\n", ibdev, htonl ( dgid->u.dwords[0] ),
htonl ( dgid->u.dwords[1] ),
htonl ( dgid->u.dwords[2] ),
htonl ( dgid->u.dwords[3] ), strerror ( rc ) );
goto out;
}
/* Extract values from MAD */
path->av.lid = ntohs ( pathrec->dlid );
path->av.sl = ( pathrec->reserved__sl & 0x0f );
path->av.rate = ( pathrec->rate_selector__rate & 0x3f );
DBGC ( ibdev, "IBDEV %p path to %08x:%08x:%08x:%08x is %04x sl %d "
"rate %d\n", ibdev, htonl ( dgid->u.dwords[0] ),
htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ),
htonl ( dgid->u.dwords[3] ), path->av.lid, path->av.sl,
path->av.rate );
out:
/* Destroy the completed transaction */
ib_destroy_madx ( ibdev, mi, madx );
path->madx = NULL;
/* Hand off to upper completion handler */
path->op->complete ( ibdev, path, rc, &path->av );
}
/** Path transaction completion operations */
static struct ib_mad_transaction_operations ib_path_op = {
.complete = ib_path_complete,
};
/**
* Create path
*
* @v ibdev Infiniband device
* @v av Address vector to complete
* @v op Path operations
* @ret path Path
*/
struct ib_path *
ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
struct ib_path_operations *op ) {
struct ib_path *path;
union ib_mad mad;
struct ib_mad_sa *sa = &mad.sa;
/* Allocate and initialise structure */
path = zalloc ( sizeof ( *path ) );
if ( ! path )
goto err_alloc_path;
path->ibdev = ibdev;
memcpy ( &path->av, av, sizeof ( path->av ) );
path->op = op;
/* Construct path request */
memset ( sa, 0, sizeof ( *sa ) );
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
sa->mad_hdr.method = IB_MGMT_METHOD_GET;
sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
sa->sa_hdr.comp_mask[1] =
htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
memcpy ( &sa->sa_data.path_record.dgid, &path->av.gid,
sizeof ( sa->sa_data.path_record.dgid ) );
memcpy ( &sa->sa_data.path_record.sgid, &ibdev->gid,
sizeof ( sa->sa_data.path_record.sgid ) );
/* Create management transaction */
path->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
&ib_path_op );
if ( ! path->madx )
goto err_create_madx;
ib_madx_set_ownerdata ( path->madx, path );
return path;
ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
err_create_madx:
free ( path );
err_alloc_path:
return NULL;
}
/**
* Destroy path
*
* @v ibdev Infiniband device
* @v path Path
*/
void ib_destroy_path ( struct ib_device *ibdev, struct ib_path *path ) {
if ( path->madx )
ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
free ( path );
}
/** Number of path cache entries
*
* Must be a power of two.
*/
#define IB_NUM_CACHED_PATHS 4
/** A path record cache entry */
struct ib_cached_path_record {
/** Infiniband device's port GID
*
* Used to disambiguate cache entries when we have multiple
* Infiniband devices, without having to maintain a pointer to
* the Infiniband device.
*/
struct ib_gid sgid;
/** Destination GID */
struct ib_gid dgid;
/** Destination LID */
unsigned int dlid;
/** Rate */
unsigned int rate;
/** Service level */
unsigned int sl;
/** A cached path */
struct ib_cached_path {
/** Path */
struct ib_path *path;
};
/** Path record cache */
static struct ib_cached_path_record ib_path_cache[IB_NUM_CACHED_PATHS];
/** Path cache */
static struct ib_cached_path ib_path_cache[IB_NUM_CACHED_PATHS];
/** Oldest path record cache entry index */
/** Oldest path cache entry index */
static unsigned int ib_path_cache_idx;
/**
* Find path record cache entry
* Find path cache entry
*
* @v ibdev Infiniband device
* @v dgid Destination GID
* @ret cached Path record cache entry, or NULL
* @ret path Path cache entry, or NULL
*/
static struct ib_cached_path_record *
static struct ib_cached_path *
ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
struct ib_cached_path_record *cached;
struct ib_cached_path *cached;
unsigned int i;
for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) {
cached = &ib_path_cache[i];
if ( memcmp ( &cached->sgid, &ibdev->gid,
sizeof ( cached->sgid ) ) != 0 )
if ( ! cached->path )
continue;
if ( memcmp ( &cached->dgid, dgid,
sizeof ( cached->dgid ) ) != 0 )
if ( cached->path->ibdev != ibdev )
continue;
if ( memcmp ( &cached->path->av.gid, dgid,
sizeof ( cached->path->av.gid ) ) != 0 )
continue;
return cached;
}
@ -90,134 +197,98 @@ ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
}
/**
* Resolve path record
* Handle cached path transaction completion
*
* @v ibdev Infiniband device
* @v path Path
* @v rc Status code
* @v av Address vector, or NULL on error
*/
static void ib_cached_path_complete ( struct ib_device *ibdev,
struct ib_path *path, int rc,
struct ib_address_vector *av __unused ) {
struct ib_cached_path *cached = ib_path_get_ownerdata ( path );
/* If the transaction failed, erase the cache entry */
if ( rc != 0 ) {
/* Destroy the old cache entry */
ib_destroy_path ( ibdev, path );
memset ( cached, 0, sizeof ( *cached ) );
return;
}
/* Do not destroy the completed transaction; we still need to
* refer to the resolved path.
*/
}
/** Cached path transaction completion operations */
static struct ib_path_operations ib_cached_path_op = {
.complete = ib_cached_path_complete,
};
/**
* Resolve path
*
* @v ibdev Infiniband device
* @v av Address vector to complete
* @ret rc Return status code
*
* This provides a non-transactional way to resolve a path, via a
* cache similar to ARP.
*/
int ib_resolve_path ( struct ib_device *ibdev,
struct ib_address_vector *av ) {
struct ib_gma *gma = ibdev->gma;
int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ) {
struct ib_gid *gid = &av->gid;
struct ib_cached_path_record *cached;
union ib_mad mad;
struct ib_mad_sa *sa = &mad.sa;
struct ib_cached_path *cached;
unsigned int cache_idx;
int rc;
/* Sanity check */
if ( ! av->gid_present ) {
DBGC ( gma, "GMA %p attempt to look up path record "
"without GID\n", gma );
DBGC ( ibdev, "IBDEV %p attempt to look up path "
"without GID\n", ibdev );
return -EINVAL;
}
/* Look in cache for a matching entry */
cached = ib_find_path_cache_entry ( ibdev, gid );
if ( cached && cached->dlid ) {
if ( cached && cached->path->av.lid ) {
/* Populated entry found */
av->lid = cached->dlid;
av->rate = cached->rate;
av->sl = cached->sl;
DBGC2 ( gma, "GMA %p cache hit for %08x:%08x:%08x:%08x\n",
gma, htonl ( gid->u.dwords[0] ),
av->lid = cached->path->av.lid;
av->rate = cached->path->av.rate;
av->sl = cached->path->av.sl;
DBGC2 ( ibdev, "IBDEV %p cache hit for %08x:%08x:%08x:%08x\n",
ibdev, htonl ( gid->u.dwords[0] ),
htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ),
htonl ( gid->u.dwords[3] ) );
return 0;
}
DBGC ( gma, "GMA %p cache miss for %08x:%08x:%08x:%08x%s\n", gma,
htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ),
DBGC ( ibdev, "IBDEV %p cache miss for %08x:%08x:%08x:%08x%s\n",
ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ),
htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ),
( cached ? " (in progress)" : "" ) );
/* If no unresolved entry was found, then create a new one */
if ( ! cached ) {
cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS );
cached = &ib_path_cache[cache_idx];
memset ( cached, 0, sizeof ( *cached ) );
memcpy ( &cached->sgid, &ibdev->gid, sizeof ( cached->sgid ) );
memcpy ( &cached->dgid, gid, sizeof ( cached->dgid ) );
}
/* If lookup is already in progress, do nothing */
if ( cached )
return -ENOENT;
/* Construct path record request */
memset ( sa, 0, sizeof ( *sa ) );
sa->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
sa->mad_hdr.method = IB_MGMT_METHOD_GET;
sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
sa->sa_hdr.comp_mask[1] =
htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
memcpy ( &sa->sa_data.path_record.dgid, &cached->dgid,
sizeof ( sa->sa_data.path_record.dgid ) );
memcpy ( &sa->sa_data.path_record.sgid, &cached->sgid,
sizeof ( sa->sa_data.path_record.sgid ) );
/* Locate a new cache entry to use */
cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS );
cached = &ib_path_cache[cache_idx];
/* Issue path record request */
if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) {
DBGC ( gma, "GMA %p could not get path record: %s\n",
gma, strerror ( rc ) );
return rc;
/* Destroy the old cache entry */
if ( cached->path )
ib_destroy_path ( ibdev, cached->path );
memset ( cached, 0, sizeof ( *cached ) );
/* Create new path */
cached->path = ib_create_path ( ibdev, av, &ib_cached_path_op );
if ( ! cached->path ) {
DBGC ( ibdev, "IBDEV %p could not create path\n",
ibdev );
return -ENOMEM;
}
ib_path_set_ownerdata ( cached->path, cached );
/* Not found yet */
return -ENOENT;
}
/**
* Handle path record response
*
* @v gma General management agent
* @v mad MAD
* @ret response MAD response
*/
static union ib_mad * ib_handle_path_record ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_device *ibdev = gma->ibdev;
struct ib_path_record *path_record = &mad->sa.sa_data.path_record;
struct ib_gid *dgid = &path_record->dgid;
struct ib_cached_path_record *cached;
unsigned int dlid;
unsigned int sl;
unsigned int rate;
/* Ignore if not a success */
if ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) {
DBGC ( gma, "GMA %p path record lookup failed with status "
"%04x\n", gma, ntohs ( mad->hdr.status ) );
return NULL;
}
/* Extract values from MAD */
dlid = ntohs ( path_record->dlid );
sl = ( path_record->reserved__sl & 0x0f );
rate = ( path_record->rate_selector__rate & 0x3f );
DBGC ( gma, "GMA %p path to %08x:%08x:%08x:%08x is %04x sl %d "
"rate %d\n", gma, htonl ( dgid->u.dwords[0] ),
htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ),
htonl ( dgid->u.dwords[3] ), dlid, sl, rate );
/* Look for a matching cache entry to fill in */
if ( ( cached = ib_find_path_cache_entry ( ibdev, dgid ) ) != NULL ) {
DBGC ( gma, "GMA %p cache add for %08x:%08x:%08x:%08x\n",
gma, htonl ( dgid->u.dwords[0] ),
htonl ( dgid->u.dwords[1] ),
htonl ( dgid->u.dwords[2] ),
htonl ( dgid->u.dwords[3] ) );
cached->dlid = dlid;
cached->rate = rate;
cached->sl = sl;
}
return NULL;
}
/** Path record response handler */
struct ib_gma_handler ib_path_record_handler __ib_gma_handler = {
.mgmt_class = IB_MGMT_CLASS_SUBN_ADM,
.class_version = IB_SA_CLASS_VERSION,
.method = IB_MGMT_METHOD_GET_RESP,
.attr_id = htons ( IB_SA_ATTR_PATH_REC ),
.handle = ib_handle_path_record,
};