mirror of
https://github.com/xcat2/xNBA.git
synced 2024-12-13 23:01:31 +00:00
Gets most of the way through initialisation.
This commit is contained in:
Michael Brown
parent
def5ae9127
commit
8423b0b109
@ -40,7 +40,12 @@
|
||||
#define ARBEL_HCR_QUERY_FW 0x0004
|
||||
#define ARBEL_HCR_INIT_HCA 0x0007
|
||||
#define ARBEL_HCR_CLOSE_HCA 0x0008
|
||||
#define ARBEL_HCR_INIT_IB 0x0009
|
||||
#define ARBEL_HCR_CLOSE_IB 0x000a
|
||||
#define ARBEL_HCR_SW2HW_MPT 0x000d
|
||||
#define ARBEL_HCR_MAP_EQ 0x0012
|
||||
#define ARBEL_HCR_SW2HW_EQ 0x0013
|
||||
#define ARBEL_HCR_HW2SW_EQ 0x0014
|
||||
#define ARBEL_HCR_SW2HW_CQ 0x0016
|
||||
#define ARBEL_HCR_HW2SW_CQ 0x0017
|
||||
#define ARBEL_HCR_RST2INIT_QPEE 0x0019
|
||||
@ -68,6 +73,8 @@
|
||||
/* MTUs */
|
||||
#define ARBEL_MTU_2048 0x04
|
||||
|
||||
#define ARBEL_NO_EQ 64
|
||||
|
||||
#define ARBEL_INVALID_LKEY 0x00000100UL
|
||||
|
||||
/*
|
||||
@ -98,8 +105,10 @@ struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_eqc );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_init_hca );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_init_ib );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_mad_ifc );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry );
|
||||
struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash );
|
||||
@ -321,8 +330,6 @@ struct arbel {
|
||||
* Used to get unrestricted memory access.
|
||||
*/
|
||||
unsigned long reserved_lkey;
|
||||
/** Event queue number */
|
||||
unsigned long eqn;
|
||||
|
||||
/** Completion queue in-use bitmask */
|
||||
arbel_bitmask_t cq_inuse[ ARBEL_BITMASK_SIZE ( ARBEL_MAX_CQS ) ];
|
||||
|
||||
@ -811,7 +811,7 @@ static int setup_hca(__u8 port, void **eq_p)
|
||||
DBG ( "mpt_base_addr_l = %lx\n", icm_start );
|
||||
init_hca.mpt_base_addr_l = icm_start;
|
||||
init_hca.log_mpt_sz = log2_entries;
|
||||
DBG ( "log2_entries for mpt = %d\n", log2_entries );
|
||||
DBG ( "log2_entries for mpt = %ld\n", log2_entries );
|
||||
icm_start += (tmp << 12);
|
||||
icm_size += (tmp << 12);
|
||||
|
||||
|
||||
@ -30,9 +30,8 @@ Skeleton NIC driver for Etherboot
|
||||
#include "arbel.h"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/* Port to use */
|
||||
#define PXE_IB_PORT 1
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
@ -242,6 +241,22 @@ arbel_cmd_close_hca ( struct arbel *arbel ) {
|
||||
0, NULL, 0, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_init_ib ( struct arbel *arbel, unsigned int port,
|
||||
const struct arbelprm_init_ib *init_ib ) {
|
||||
return arbel_cmd ( arbel,
|
||||
ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT_IB,
|
||||
1, sizeof ( *init_ib ) ),
|
||||
0, init_ib, port, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_close_ib ( struct arbel *arbel, unsigned int port ) {
|
||||
return arbel_cmd ( arbel,
|
||||
ARBEL_HCR_VOID_CMD ( ARBEL_HCR_CLOSE_IB ),
|
||||
0, NULL, port, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index,
|
||||
const struct arbelprm_mpt *mpt ) {
|
||||
@ -251,6 +266,22 @@ arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index,
|
||||
0, mpt, index, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_sw2hw_eq ( struct arbel *arbel, unsigned int index,
|
||||
const struct arbelprm_eqc *eqc ) {
|
||||
return arbel_cmd ( arbel,
|
||||
ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_EQ,
|
||||
1, sizeof ( *eqc ) ),
|
||||
0, eqc, index, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_hw2sw_eq ( struct arbel *arbel, unsigned int index ) {
|
||||
return arbel_cmd ( arbel,
|
||||
ARBEL_HCR_VOID_CMD ( ARBEL_HCR_HW2SW_EQ ),
|
||||
1, NULL, index, NULL );
|
||||
}
|
||||
|
||||
static inline int
|
||||
arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
|
||||
const struct arbelprm_completion_queue_context *cqctx ) {
|
||||
@ -421,16 +452,6 @@ arbel_cmd_map_fa ( struct arbel *arbel,
|
||||
0, map, 1, NULL );
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Event queue operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
static int arbel_create_eq ( struct arbel *arbel ) {
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Completion queue operations
|
||||
@ -508,7 +529,7 @@ static int arbel_create_cq ( struct ib_device *ibdev,
|
||||
MLX_FILL_2 ( &cqctx, 3,
|
||||
usr_page, arbel->limits.reserved_uars,
|
||||
log_cq_size, fls ( cq->num_cqes - 1 ) );
|
||||
MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eqn );
|
||||
MLX_FILL_1 ( &cqctx, 5, c_eqn, ARBEL_NO_EQ );
|
||||
MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
|
||||
MLX_FILL_1 ( &cqctx, 7, l_key, arbel->reserved_lkey );
|
||||
MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn );
|
||||
@ -1423,6 +1444,38 @@ static int arbel_get_pkey ( struct arbel *arbel, unsigned int *pkey ) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for link up
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
* @ret rc Return status code
|
||||
*
|
||||
* This function shouldn't really exist. Unfortunately, IB links take
|
||||
* a long time to come up, and we can't get various key parameters
|
||||
* e.g. our own IPoIB MAC address without information from the subnet
|
||||
* manager). We should eventually make link-up an asynchronous event.
|
||||
*/
|
||||
static int arbel_wait_for_link ( struct arbel *arbel ) {
|
||||
struct ib_mad_port_info port_info;
|
||||
unsigned int retries;
|
||||
int rc;
|
||||
|
||||
printf ( "Waiting for Infiniband link-up..." );
|
||||
for ( retries = 20 ; retries ; retries-- ) {
|
||||
if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
|
||||
continue;
|
||||
if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
|
||||
== 4 ) {
|
||||
printf ( "ok\n" );
|
||||
return 0;
|
||||
}
|
||||
printf ( "." );
|
||||
sleep ( 1 );
|
||||
}
|
||||
printf ( "failed\n" );
|
||||
return -ENODEV;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get MAD parameters
|
||||
*
|
||||
@ -1819,6 +1872,54 @@ static void arbel_free_icm ( struct arbel *arbel ) {
|
||||
arbel->icm = UNULL;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* Infiniband link-layer operations
|
||||
*
|
||||
***************************************************************************
|
||||
*/
|
||||
|
||||
/**
|
||||
* Initialise Infiniband link
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
* @ret rc Return status code
|
||||
*/
|
||||
static int arbel_init_ib ( struct arbel *arbel ) {
|
||||
struct arbelprm_init_ib init_ib;
|
||||
int rc;
|
||||
|
||||
memset ( &init_ib, 0, sizeof ( init_ib ) );
|
||||
MLX_FILL_3 ( &init_ib, 0,
|
||||
mtu_cap, ARBEL_MTU_2048,
|
||||
port_width_cap, 3,
|
||||
vl_cap, 1 );
|
||||
MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
|
||||
if ( ( rc = arbel_cmd_init_ib ( arbel, PXE_IB_PORT,
|
||||
&init_ib ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not intialise IB: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close Infiniband link
|
||||
*
|
||||
* @v arbel Arbel device
|
||||
*/
|
||||
static void arbel_close_ib ( struct arbel *arbel ) {
|
||||
int rc;
|
||||
|
||||
if ( ( rc = arbel_cmd_close_ib ( arbel, PXE_IB_PORT ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not close IB: %s\n",
|
||||
arbel, strerror ( rc ) );
|
||||
/* Nothing we can do about this */
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
* PCI interface
|
||||
@ -1933,6 +2034,16 @@ static int arbel_probe ( struct pci_device *pci,
|
||||
if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
|
||||
goto err_alloc_icm;
|
||||
|
||||
|
||||
unsigned long uar_offset = ( arbel->limits.reserved_uars * 4096 );
|
||||
arbel->db_rec = phys_to_virt ( user_to_phys ( arbel->icm,
|
||||
uar_offset ) );
|
||||
memset ( arbel->db_rec, 0, 4096 );
|
||||
union arbelprm_doorbell_record *db_rec;
|
||||
db_rec = &arbel->db_rec[ARBEL_GROUP_SEPARATOR_DOORBELL];
|
||||
MLX_FILL_1 ( &db_rec->qp, 1, res, ARBEL_UAR_RES_GROUP_SEP );
|
||||
|
||||
|
||||
/* Initialise HCA */
|
||||
MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 1 );
|
||||
if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
|
||||
@ -1945,7 +2056,14 @@ static int arbel_probe ( struct pci_device *pci,
|
||||
if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
|
||||
goto err_setup_mpt;
|
||||
|
||||
|
||||
/* Bring up IB layer */
|
||||
if ( ( rc = arbel_init_ib ( arbel ) ) != 0 )
|
||||
goto err_init_ib;
|
||||
|
||||
/* Wait for link */
|
||||
if ( ( rc = arbel_wait_for_link ( arbel ) ) != 0 )
|
||||
goto err_wait_for_link;
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@ -1957,17 +2075,24 @@ static int arbel_probe ( struct pci_device *pci,
|
||||
arbel->mailbox_in = dev_buffers_p->inprm_buf;
|
||||
arbel->mailbox_out = dev_buffers_p->outprm_buf;
|
||||
#endif
|
||||
#if ! SELF_INIT
|
||||
#if SELF_INIT
|
||||
#else
|
||||
arbel->reserved_lkey = dev_ib_data.mkey;
|
||||
#endif
|
||||
arbel->db_rec = dev_ib_data.uar_context_base;
|
||||
arbel->eqn = dev_ib_data.eq.eqn;
|
||||
#endif
|
||||
// arbel->eqn = dev_ib_data.eq.eqn;
|
||||
|
||||
|
||||
/* Get MAD parameters */
|
||||
if ( ( rc = arbel_get_mad_params ( ibdev ) ) != 0 )
|
||||
goto err_get_mad_params;
|
||||
|
||||
DBGC ( arbel, "Arbel %p port GID is %08lx:%08lx:%08lx:%08lx\n", arbel,
|
||||
htonl ( ibdev->port_gid.u.dwords[0] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[1] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[2] ),
|
||||
htonl ( ibdev->port_gid.u.dwords[3] ) );
|
||||
|
||||
/* Add IPoIB device */
|
||||
if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
|
||||
DBGC ( arbel, "Arbel %p could not add IPoIB device: %s\n",
|
||||
@ -1982,6 +2107,9 @@ static int arbel_probe ( struct pci_device *pci,
|
||||
ib_driver_close ( 0 );
|
||||
err_ib_driver_init:
|
||||
|
||||
err_wait_for_link:
|
||||
arbel_close_ib ( arbel );
|
||||
err_init_ib:
|
||||
err_setup_mpt:
|
||||
arbel_cmd_close_hca ( arbel );
|
||||
err_init_hca:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user