mirror of
https://github.com/xcat2/xNBA.git
synced 2025-04-18 11:09:26 +00:00
[hermon] Add support for multiple ports and detecting non-IB ports
Originally-fixed-by: Itay Gazit <itaygazit@gmail.com>
This commit is contained in:
Michael Brown
parent
b0c563824b
commit
ad66465b3c
@ -2071,7 +2071,11 @@ struct hermonprm_query_dev_cap_st { /* Little Endian */
|
||||
pseudo_bit_t pkv[0x00001]; /* PKey Violation Counter Supported */
|
||||
pseudo_bit_t qkv[0x00001]; /* QKey Violation Coutner Supported */
|
||||
pseudo_bit_t vmm[0x00001]; /* Hermon New */
|
||||
pseudo_bit_t reserved27[0x00005];
|
||||
pseudo_bit_t fcoe[0x00001];
|
||||
pseudo_bit_t dpdp[0x00001]; /* Dual Port Different Protocols */
|
||||
pseudo_bit_t raw_ethertype[0x00001];
|
||||
pseudo_bit_t raw_ipv6[0x00001];
|
||||
pseudo_bit_t blh[0x00001];
|
||||
pseudo_bit_t mw[0x00001]; /* Memory windows supported */
|
||||
pseudo_bit_t apm[0x00001]; /* Automatic Path Migration Supported */
|
||||
pseudo_bit_t atm[0x00001]; /* Atomic operations supported (atomicity is guaranteed between QPs on this HCA) */
|
||||
|
||||
@ -541,6 +541,16 @@ hermon_cmd_map_fa ( struct hermon *hermon,
|
||||
0, map, 1, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port,
|
||||
struct hermonprm_sense_port *port_type ) {
|
||||
return hermon_cmd ( hermon,
|
||||
HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT,
|
||||
1, sizeof ( *port_type ) ),
|
||||
0, NULL, port, port_type );
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Memory translation table operations
|
||||
@ -1664,7 +1674,7 @@ static void hermon_event_port_state_change ( struct hermon *hermon,
|
||||
( link_up ? "up" : "down" ) );
|
||||
|
||||
/* Sanity check */
|
||||
if ( port >= HERMON_NUM_PORTS ) {
|
||||
if ( port >= hermon->cap.num_ports ) {
|
||||
DBGC ( hermon, "Hermon %p port %d does not exist!\n",
|
||||
hermon, ( port + 1 ) );
|
||||
return;
|
||||
@ -1735,6 +1745,36 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) {
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Sense port type
|
||||
*
|
||||
* @v ibdev Infiniband device
|
||||
* @ret port_type Port type, or negative error
|
||||
*/
|
||||
static int hermon_sense_port_type ( struct ib_device *ibdev ) {
|
||||
struct hermon *hermon = ib_get_drvdata ( ibdev );
|
||||
struct hermonprm_sense_port sense_port;
|
||||
int port_type;
|
||||
int rc;
|
||||
|
||||
/* If DPDP is not supported, always assume Infiniband */
|
||||
if ( ! hermon->cap.dpdp )
|
||||
return HERMON_PORT_TYPE_IB;
|
||||
|
||||
/* Sense the port type */
|
||||
if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port,
|
||||
&sense_port ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p port %d sense failed: %s\n",
|
||||
hermon, ibdev->port, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
port_type = MLX_GET ( &sense_port, port_type );
|
||||
|
||||
DBGC ( hermon, "Hermon %p port %d type %d\n",
|
||||
hermon, ibdev->port, port_type );
|
||||
return port_type;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialise Infiniband link
|
||||
*
|
||||
@ -1744,8 +1784,19 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) {
|
||||
static int hermon_open ( struct ib_device *ibdev ) {
|
||||
struct hermon *hermon = ib_get_drvdata ( ibdev );
|
||||
struct hermonprm_init_port init_port;
|
||||
int port_type;
|
||||
int rc;
|
||||
|
||||
/* Check we are connected to an Infiniband network */
|
||||
if ( ( rc = port_type = hermon_sense_port_type ( ibdev ) ) < 0 )
|
||||
return rc;
|
||||
if ( port_type != HERMON_PORT_TYPE_IB ) {
|
||||
DBGC ( hermon, "Hermon %p port %d not connected to an "
|
||||
"Infiniband network", hermon, ibdev->port );
|
||||
return -ENOTCONN;
|
||||
}
|
||||
|
||||
/* Init Port */
|
||||
memset ( &init_port, 0, sizeof ( init_port ) );
|
||||
MLX_FILL_2 ( &init_port, 0,
|
||||
port_width_cap, 3,
|
||||
@ -2099,6 +2150,15 @@ static int hermon_get_cap ( struct hermon *hermon ) {
|
||||
( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) );
|
||||
hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz );
|
||||
hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars );
|
||||
hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports );
|
||||
hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp );
|
||||
|
||||
/* Sanity check */
|
||||
if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) {
|
||||
DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n",
|
||||
hermon, hermon->cap.num_ports, HERMON_MAX_PORTS );
|
||||
hermon->cap.num_ports = HERMON_MAX_PORTS;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -2478,7 +2538,7 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
struct hermon *hermon;
|
||||
struct ib_device *ibdev;
|
||||
struct hermonprm_init_hca init_hca;
|
||||
int i;
|
||||
unsigned int i;
|
||||
int rc;
|
||||
|
||||
/* Allocate Hermon device */
|
||||
@ -2489,20 +2549,6 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
}
|
||||
pci_set_drvdata ( pci, hermon );
|
||||
|
||||
/* Allocate Infiniband devices */
|
||||
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
|
||||
ibdev = alloc_ibdev ( 0 );
|
||||
if ( ! ibdev ) {
|
||||
rc = -ENOMEM;
|
||||
goto err_alloc_ibdev;
|
||||
}
|
||||
hermon->ibdev[i] = ibdev;
|
||||
ibdev->op = &hermon_ib_operations;
|
||||
ibdev->dev = &pci->dev;
|
||||
ibdev->port = ( HERMON_PORT_BASE + i );
|
||||
ib_set_drvdata ( ibdev, hermon );
|
||||
}
|
||||
|
||||
/* Fix up PCI device */
|
||||
adjust_pci_device ( pci );
|
||||
|
||||
@ -2534,6 +2580,20 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
if ( ( rc = hermon_get_cap ( hermon ) ) != 0 )
|
||||
goto err_get_cap;
|
||||
|
||||
/* Allocate Infiniband devices */
|
||||
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
|
||||
ibdev = alloc_ibdev ( 0 );
|
||||
if ( ! ibdev ) {
|
||||
rc = -ENOMEM;
|
||||
goto err_alloc_ibdev;
|
||||
}
|
||||
hermon->ibdev[i] = ibdev;
|
||||
ibdev->op = &hermon_ib_operations;
|
||||
ibdev->dev = &pci->dev;
|
||||
ibdev->port = ( HERMON_PORT_BASE + i );
|
||||
ib_set_drvdata ( ibdev, hermon );
|
||||
}
|
||||
|
||||
/* Allocate ICM */
|
||||
memset ( &init_hca, 0, sizeof ( init_hca ) );
|
||||
if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 )
|
||||
@ -2552,7 +2612,7 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
/* Set up memory protection */
|
||||
if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
|
||||
goto err_setup_mpt;
|
||||
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ )
|
||||
for ( i = 0 ; i < hermon->cap.num_ports ; i++ )
|
||||
hermon->ibdev[i]->rdma_key = hermon->lkey;
|
||||
|
||||
/* Set up event queue */
|
||||
@ -2563,12 +2623,13 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 )
|
||||
goto err_conf_special_qps;
|
||||
|
||||
/* Update MAD parameters */
|
||||
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ )
|
||||
/* Update IPoIB MAC address */
|
||||
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
|
||||
ib_smc_update ( hermon->ibdev[i], hermon_mad );
|
||||
}
|
||||
|
||||
/* Register Infiniband devices */
|
||||
for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
|
||||
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
|
||||
if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) {
|
||||
DBGC ( hermon, "Hermon %p could not register IB "
|
||||
"device: %s\n", hermon, strerror ( rc ) );
|
||||
@ -2578,9 +2639,9 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
|
||||
return 0;
|
||||
|
||||
i = HERMON_NUM_PORTS;
|
||||
i = hermon->cap.num_ports;
|
||||
err_register_ibdev:
|
||||
for ( i-- ; i >= 0 ; i-- )
|
||||
for ( i-- ; ( signed int ) i >= 0 ; i-- )
|
||||
unregister_ibdev ( hermon->ibdev[i] );
|
||||
err_conf_special_qps:
|
||||
hermon_destroy_eq ( hermon );
|
||||
@ -2590,6 +2651,10 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
err_init_hca:
|
||||
hermon_free_icm ( hermon );
|
||||
err_alloc_icm:
|
||||
i = hermon->cap.num_ports;
|
||||
err_alloc_ibdev:
|
||||
for ( i-- ; ( signed int ) i >= 0 ; i-- )
|
||||
ibdev_put ( hermon->ibdev[i] );
|
||||
err_get_cap:
|
||||
hermon_stop_firmware ( hermon );
|
||||
err_start_firmware:
|
||||
@ -2597,10 +2662,6 @@ static int hermon_probe ( struct pci_device *pci,
|
||||
err_mailbox_out:
|
||||
free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
|
||||
err_mailbox_in:
|
||||
i = HERMON_NUM_PORTS;
|
||||
err_alloc_ibdev:
|
||||
for ( i-- ; i >= 0 ; i-- )
|
||||
ibdev_put ( hermon->ibdev[i] );
|
||||
free ( hermon );
|
||||
err_alloc_hermon:
|
||||
return rc;
|
||||
@ -2615,7 +2676,7 @@ static void hermon_remove ( struct pci_device *pci ) {
|
||||
struct hermon *hermon = pci_get_drvdata ( pci );
|
||||
int i;
|
||||
|
||||
for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
|
||||
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
|
||||
unregister_ibdev ( hermon->ibdev[i] );
|
||||
hermon_destroy_eq ( hermon );
|
||||
hermon_cmd_close_hca ( hermon );
|
||||
@ -2624,7 +2685,7 @@ static void hermon_remove ( struct pci_device *pci ) {
|
||||
hermon_stop_firmware ( hermon );
|
||||
free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
|
||||
free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
|
||||
for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
|
||||
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
|
||||
ibdev_put ( hermon->ibdev[i] );
|
||||
free ( hermon );
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||
*/
|
||||
|
||||
/* Ports in existence */
|
||||
#define HERMON_NUM_PORTS 2
|
||||
#define HERMON_MAX_PORTS 2
|
||||
#define HERMON_PORT_BASE 1
|
||||
|
||||
/* PCI BARs */
|
||||
@ -61,6 +61,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
|
||||
#define HERMON_HCR_READ_MCG 0x0025
|
||||
#define HERMON_HCR_WRITE_MCG 0x0026
|
||||
#define HERMON_HCR_MGID_HASH 0x0027
|
||||
#define HERMON_HCR_SENSE_PORT 0x004d
|
||||
#define HERMON_HCR_RUN_FW 0x0ff6
|
||||
#define HERMON_HCR_DISABLE_LAM 0x0ff7
|
||||
#define HERMON_HCR_ENABLE_LAM 0x0ff8
|
||||
@ -164,6 +165,14 @@ struct hermonprm_port_state_change_event_st {
|
||||
struct hermonprm_port_state_change_st data;
|
||||
} __attribute__ (( packed ));
|
||||
|
||||
/** Hermon sense port */
|
||||
struct hermonprm_sense_port_st {
|
||||
pseudo_bit_t port_type[0x00020];
|
||||
/* -------------- */
|
||||
pseudo_bit_t reserved[0x00020];
|
||||
};
|
||||
#define HERMON_PORT_TYPE_IB 1
|
||||
|
||||
/*
|
||||
* Wrapper structures for hardware datatypes
|
||||
*
|
||||
@ -192,6 +201,7 @@ struct MLX_DECLARE_STRUCT ( hermonprm_query_dev_cap );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_query_fw );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_queue_pair_ee_context_entry );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_scalar_parameter );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_sense_port );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_send_db_register );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_ud_address_vector );
|
||||
struct MLX_DECLARE_STRUCT ( hermonprm_virtual_physical_mapping );
|
||||
@ -296,6 +306,10 @@ struct hermon_dev_cap {
|
||||
size_t dmpt_entry_size;
|
||||
/** Number of reserved UARs */
|
||||
unsigned int reserved_uars;
|
||||
/** Number of ports */
|
||||
unsigned int num_ports;
|
||||
/** Dual-port different protocol */
|
||||
int dpdp;
|
||||
};
|
||||
|
||||
/** Number of cMPT entries of each type */
|
||||
@ -523,7 +537,7 @@ struct hermon {
|
||||
unsigned long qpn_base;
|
||||
|
||||
/** Infiniband devices */
|
||||
struct ib_device *ibdev[HERMON_NUM_PORTS];
|
||||
struct ib_device *ibdev[HERMON_MAX_PORTS];
|
||||
};
|
||||
|
||||
/** Global protection domain */
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user