From 53a7dd26cd0aff8409e16c6cc1c7423d4a551f4b Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Fri, 2 Jan 2009 21:04:31 +0000 Subject: [PATCH] [infiniband] Call ib_open() only when opening the IPoIB net device Defer the call to ib_open() until we want to actually open the device, rather than when the device is registered. --- src/drivers/infiniband/arbel.c | 4 +++ src/drivers/infiniband/hermon.c | 4 +++ src/drivers/net/ipoib.c | 12 +++++++++ src/include/gpxe/infiniband.h | 25 +++---------------- src/net/infiniband.c | 43 +++++++++++++++++++++++++++------ 5 files changed, 60 insertions(+), 28 deletions(-) diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c index e9bc260e..1756a6e2 100644 --- a/src/drivers/infiniband/arbel.c +++ b/src/drivers/infiniband/arbel.c @@ -2175,6 +2175,10 @@ static int arbel_probe ( struct pci_device *pci, if ( ( rc = arbel_create_eq ( arbel ) ) != 0 ) goto err_create_eq; + /* Update MAD parameters */ + for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) + ib_smc_update ( arbel->ibdev[i], arbel_mad ); + /* Register Infiniband devices */ for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) { if ( ( rc = register_ibdev ( arbel->ibdev[i] ) ) != 0 ) { diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c index f598a31f..40add28a 100644 --- a/src/drivers/infiniband/hermon.c +++ b/src/drivers/infiniband/hermon.c @@ -2244,6 +2244,10 @@ static int hermon_probe ( struct pci_device *pci, if ( ( rc = hermon_create_eq ( hermon ) ) != 0 ) goto err_create_eq; + /* Update MAD parameters */ + for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) + ib_smc_update ( hermon->ibdev[i], hermon_mad ); + /* Register Infiniband devices */ for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) { if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) { diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c index 3a78dcef..8ad2c29f 100644 --- a/src/drivers/net/ipoib.c +++ b/src/drivers/net/ipoib.c @@ -926,6 +926,13 @@ static int ipoib_open ( struct net_device *netdev ) { struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr ); int rc; + /* Open IB device */ + if ( ( rc = ib_open ( ipoib->ibdev ) ) != 0 ) { + DBGC ( ipoib, "IPoIB %p could not open device: %s\n", + ipoib, strerror ( rc ) ); + goto err_ib_open; + } + /* Allocate metadata queue set */ if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta, IPOIB_META_NUM_CQES, @@ -971,6 +978,8 @@ static int ipoib_open ( struct net_device *netdev ) { err_create_data_qset: ipoib_destroy_qset ( ipoib, &ipoib->meta ); err_create_meta_qset: + ib_close ( ipoib->ibdev ); + err_ib_open: return rc; } @@ -992,6 +1001,9 @@ static void ipoib_close ( struct net_device *netdev ) { /* Tear down the queues */ ipoib_destroy_qset ( ipoib, &ipoib->data ); ipoib_destroy_qset ( ipoib, &ipoib->meta ); + + /* Close IB device */ + ib_close ( ipoib->ibdev ); } /** IPoIB network device operations */ diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 1bb82401..196b5950 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -306,6 +306,8 @@ struct ib_device { struct ib_device_operations *op; /** Port number */ unsigned int port; + /** Port open request counter */ + unsigned int open_count; /** Port state */ uint8_t port_state; @@ -364,6 +366,8 @@ extern void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *av, struct io_buffer *iobuf, int rc ); +extern int ib_open ( struct ib_device *ibdev ); +extern void ib_close ( struct ib_device *ibdev ); extern int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_gid *gid ); extern void ib_mcast_detach ( struct ib_device *ibdev, @@ -389,27 +393,6 @@ ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { ibdev->op->poll_cq ( ibdev, cq ); } -/** - * Open port - * - * @v ibdev Infiniband device - * @ret rc Return status code - */ -static inline __always_inline int -ib_open ( struct ib_device *ibdev ) { - return ibdev->op->open ( ibdev ); -} - -/** - * Close port - * - * @v ibdev Infiniband device - */ -static inline __always_inline void -ib_close ( struct ib_device *ibdev ) { - ibdev->op->close ( ibdev ); -} - /** * Check link state * diff --git a/src/net/infiniband.c b/src/net/infiniband.c index fb6fca7d..d79bdc2c 100644 --- a/src/net/infiniband.c +++ b/src/net/infiniband.c @@ -391,6 +391,42 @@ void ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, qp->recv.fill--; } +/** + * Open port + * + * @v ibdev Infiniband device + * @ret rc Return status code + */ +int ib_open ( struct ib_device *ibdev ) { + int rc; + + /* Open device if this is the first requested opening */ + if ( ibdev->open_count == 0 ) { + if ( ( rc = ibdev->op->open ( ibdev ) ) != 0 ) + return rc; + } + + /* Increment device open request counter */ + ibdev->open_count++; + + return 0; +} + +/** + * Close port + * + * @v ibdev Infiniband device + */ +void ib_close ( struct ib_device *ibdev ) { + + /* Decrement device open request counter */ + ibdev->open_count--; + + /* Close device if this was the last remaining requested opening */ + if ( ibdev->open_count == 0 ) + ibdev->op->close ( ibdev ); +} + /** * Attach to multicast group * @@ -530,10 +566,6 @@ int register_ibdev ( struct ib_device *ibdev ) { ibdev_get ( ibdev ); list_add_tail ( &ibdev->list, &ib_devices ); - /* Open link */ - if ( ( rc = ib_open ( ibdev ) ) != 0 ) - goto err_open; - /* Add IPoIB device */ if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) { DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n", @@ -546,8 +578,6 @@ int register_ibdev ( struct ib_device *ibdev ) { return 0; err_ipoib_probe: - ib_close ( ibdev ); - err_open: list_del ( &ibdev->list ); ibdev_put ( ibdev ); return rc; @@ -562,7 +592,6 @@ void unregister_ibdev ( struct ib_device *ibdev ) { /* Close device */ ipoib_remove ( ibdev ); - ib_close ( ibdev ); /* Remove from device list */ list_del ( &ibdev->list );