xCAT/xNBA
2
0
Fork 0
mirror of https://github.com/xcat2/xNBA.git synced 2024-12-14 07:11:32 +00:00
Code Issues Projects Releases Wiki Activity

create_cq() implemented (but not tested).

Browse Source
This commit is contained in:
Michael Brown 2007-09-16 14:18:17 +01:00
parent 725a574042
commit 18edcf6634
2 changed files with 78 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/drivers/net/mlx_ipoib/arbel.h
View File

@ -12,10 +12,23 @@
*
*/
/* UAR context table (UCE) resource types */
#define ARBEL_UAR_RES_NONE 0x00
#define ARBEL_UAR_RES_CQ_CI 0x01
#define ARBEL_UAR_RES_CQ_ARM 0x02
#define ARBEL_UAR_RES_SQ 0x03
#define ARBEL_UAR_RES_RQ 0x04
#define ARBEL_UAR_RES_GROUP_SEP 0x07
/* Work queue entry and completion queue entry opcodes */
#define ARBEL_OPCODE_SEND 0x0a
#define ARBEL_OPCODE_RECV_ERROR 0xfe
#define ARBEL_OPCODE_SEND_ERROR 0xff
/* HCA command register opcodes */
#define ARBEL_HCR_QUERY_DEV_LIM 0x0003
#define ARBEL_HCR_SW2HW_CQ 0x0016
/*
* Wrapper structures for hardware datatypes
*
@ -24,6 +37,7 @@
struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_context );
struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry );
struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record );
struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record );
@ -71,6 +85,7 @@ union arbelprm_completion_entry {
} __attribute__ (( packed ));
union arbelprm_doorbell_record {
struct arbelprm_cq_arm_db_record cq_arm;
struct arbelprm_cq_ci_db_record cq_ci;
struct arbelprm_qp_db_record qp;
} __attribute__ (( packed ));
@ -87,6 +102,8 @@ union arbelprm_doorbell_register {
/** Arbel device limits */
struct arbel_dev_limits {
/** Number of reserver UARs */
unsigned long reserved_uars;
/** Number of reserved CQs */
unsigned long reserved_cqs;
};
@ -177,6 +194,8 @@ struct arbel {
* Used to get unrestricted memory access.
*/
unsigned long reserved_lkey;
/** Event queue number */
unsigned long eqn;
/** Completion queue in-use bitmask */
arbel_bitmask_t cq_inuse[ ARBEL_BITMASK_SIZE ( ARBEL_MAX_CQS ) ];
@ -185,14 +204,14 @@ struct arbel {
struct arbel_dev_limits limits;
};
/** Global protection domain */
#define ARBEL_GLOBAL_PD 0x123456
/*
* HCA commands
*
*/
#define ARBEL_HCR_QUERY_DEV_LIM 0x0003
#define ARBEL_HCR_SW2HW_CQ 0x0016
#define ARBEL_HCR_BASE 0x80680
#define ARBEL_HCR_REG(x) ( ARBEL_HCR_BASE + 4 * (x) )
#define ARBEL_HCR_MAX_WAIT_MS 2000
@ -251,7 +270,7 @@ struct arbel {
* @ret doorbell_idx Doorbell index
*/
static inline unsigned int
arbel_arm_cq_doorbell_idx ( unsigned int cqn_offset ) {
arbel_cq_arm_doorbell_idx ( unsigned int cqn_offset ) {
return cqn_offset;
}

81
src/drivers/net/mlx_ipoib/mt25218.c
View File

@ -455,26 +455,37 @@ arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
* Create completion queue
*
* @v ibdev Infiniband device
* @v
* @v log2_num_cqes Log2 of the number of completion queue entries
* @ret new_cq New completion queue
* @ret rc Return status code
*/
static int arbel_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
static int arbel_create_cq ( struct ib_device *ibdev,
unsigned int log2_num_cqes,
struct ib_completion_queue **new_cq ) {
struct arbel *arbel = ibdev->priv;
struct arbel_completion_queue *arbel_cq;
struct arbelprm_completion_queue_context cqctx;
struct arbelprm_cq_ci_db_record *ci_db_rec;
struct arbelprm_cq_arm_db_record *arm_db_rec;
int cqn_offset;
unsigned int cqn;
unsigned int num_cqes;
size_t cqe_size;
unsigned int ci_doorbell_idx;
unsigned int arm_doorbell_idx;
unsigned int i;
int rc;
/* Find a free completion queue number */
cqn_offset = arbel_alloc_qn_offset ( arbel->cq_inuse, ARBEL_MAX_CQS );
if ( cqn_offset < 0 ) {
DBGC ( arbel, "Arbel %p out of completion queues\n", arbel );
rc = cqn_offset;
goto err_cqn_offset;
}
cqn = ( arbel->limits.reserved_cqs + cqn_offset );
ci_doorbell_idx = arbel_cq_ci_doorbell_idx ( cqn_offset );
arm_doorbell_idx = arbel_cq_arm_doorbell_idx ( cqn_offset );
/* Allocate control structures */
arbel_cq = zalloc ( sizeof ( *arbel_cq ) );
@ -485,9 +496,10 @@ static int arbel_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
arbel_cq->cq.cqn = cqn;
arbel_cq->cq.num_cqes = num_cqes;
INIT_LIST_HEAD ( &arbel_cq->cq.work_queues );
arbel_cq->doorbell_idx = arbel_cq_ci_doorbell_idx ( cqn_offset );
arbel_cq->doorbell_idx = ci_doorbell_idx;
/* Allocate completion queue itself */
num_cqes = ( 1 << log2_num_cqes );
cqe_size = ( num_cqes * sizeof ( arbel_cq->cqe[0] ) );
arbel_cq->cqe = malloc_dma ( cqe_size, sizeof ( arbel_cq->cqe[0] ) );
if ( ! arbel_cq->cqe ) {
@ -501,28 +513,43 @@ static int arbel_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
barrier();
/* Initialise doorbell records */
// ...
ci_db_rec = &arbel->db_rec[ci_doorbell_idx].cq_ci;
MLX_FILL_1 ( ci_db_rec, 0, counter, 0 );
MLX_FILL_2 ( ci_db_rec, 1,
res, ARBEL_UAR_RES_CQ_CI,
cq_number, cqn );
arm_db_rec = &arbel->db_rec[arm_doorbell_idx].cq_arm;
MLX_FILL_1 ( arm_db_rec, 0, counter, 0 );
MLX_FILL_2 ( arm_db_rec, 1,
res, ARBEL_UAR_RES_CQ_ARM,
cq_number, cqn );
/* Hand queue over to hardware */
memset ( &cqctx, 0, sizeof ( cqctx ) );
MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
MLX_FILL_1 ( &cqctx, 2, start_address_l,
virt_to_bus ( arbel_cq->cqe ) );
/// ....
MLX_FILL_2 ( &cqctx, 3,
usr_page, arbel->limits.reserved_uars,
log_cq_size, log2_num_cqes );
MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eqn );
MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
MLX_FILL_1 ( &cqctx, 7, l_key, arbel->reserved_lkey );
MLX_FILL_1 ( &cqctx, 12, cqn, cqn );
MLX_FILL_1 ( &cqctx, 13, cq_ci_db_record, ci_doorbell_idx );
MLX_FILL_1 ( &cqctx, 14, cq_state_db_record, arm_doorbell_idx );
if ( ( rc = arbel_cmd_sw2hw_cq ( arbel, cqn, &cqctx ) ) != 0 ) {
// ...
DBGC ( arbel, "Arbel %p SW2HW_CQ failed: %s\n",
arbel, strerror ( rc ) );
goto err_sw2hw;
}
// completion queue number
// doorbell index
*new_cq = &arbel_cq->cq;
return 0;
err_sw2hw:
memset ( ci_db_rec, 0, sizeof ( *ci_db_rec ) );
memset ( arm_db_rec, 0, sizeof ( *arm_db_rec ) );
err_cqe:
free ( arbel_cq );
err_arbel_cq:
@ -580,7 +607,7 @@ static int arbel_post_send ( struct ib_device *ibdev,
struct arbel_send_work_queue *arbel_send_wq = &arbel_qp->send;
struct arbelprm_ud_send_wqe *prev_wqe;
struct arbelprm_ud_send_wqe *wqe;
union arbelprm_doorbell_record *db_rec;
struct arbelprm_qp_db_record *qp_db_rec;
union arbelprm_doorbell_register db_reg;
const struct ib_gid *gid;
unsigned int wqe_idx_mask;
@ -602,7 +629,7 @@ static int arbel_post_send ( struct ib_device *ibdev,
MLX_FILL_1 ( &wqe->ctrl, 0, always1, 1 );
memset ( &wqe->ud, 0, sizeof ( wqe->ud ) );
MLX_FILL_2 ( &wqe->ud, 0,
ud_address_vector.pd, GLOBAL_PD,
ud_address_vector.pd, ARBEL_GLOBAL_PD,
ud_address_vector.port_number, PXE_IB_PORT );
MLX_FILL_2 ( &wqe->ud, 1,
ud_address_vector.rlid, av->dlid,
@ -631,8 +658,8 @@ static int arbel_post_send ( struct ib_device *ibdev,
/* Update doorbell record */
barrier();
db_rec = &arbel->db_rec[arbel_send_wq->doorbell_idx];
MLX_FILL_1 ( &db_rec->qp, 0,
qp_db_rec = &arbel->db_rec[arbel_send_wq->doorbell_idx].qp;
MLX_FILL_1 ( qp_db_rec, 0,
counter, ( ( wq->next_idx + 1 ) & 0xffff ) );
/* Ring doorbell register */
@ -800,7 +827,7 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
struct arbel *arbel = ibdev->priv;
struct arbel_completion_queue *arbel_cq
= container_of ( cq, struct arbel_completion_queue, cq );
union arbelprm_doorbell_record *db_rec;
struct arbelprm_cq_ci_db_record *ci_db_rec;
union arbelprm_completion_entry *cqe;
unsigned int cqe_idx_mask;
int rc;
@ -828,8 +855,8 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
/* Update completion queue's index */
cq->next_idx++;
/* Update doorbell record */
db_rec = &arbel->db_rec[arbel_cq->doorbell_idx];
MLX_FILL_1 ( &db_rec->cq_ci, 0,
ci_db_rec = &arbel->db_rec[arbel_cq->doorbell_idx].cq_ci;
MLX_FILL_1 ( ci_db_rec, 0,
counter, ( cq->next_idx & 0xffffffffUL ) );
}
}
@ -897,12 +924,13 @@ static int arbel_probe ( struct pci_device *pci,
memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) );
/* Hack up IB structures */
static_arbel.config = memfree_pci_dev.cr_space;
static_arbel.mailbox_in = dev_buffers_p->inprm_buf;
static_arbel.mailbox_out = dev_buffers_p->outprm_buf;
static_arbel.uar = memfree_pci_dev.uar;
static_arbel.db_rec = dev_ib_data.uar_context_base;
static_arbel.reserved_lkey = dev_ib_data.mkey;
arbel->config = memfree_pci_dev.cr_space;
arbel->mailbox_in = dev_buffers_p->inprm_buf;
arbel->mailbox_out = dev_buffers_p->outprm_buf;
arbel->uar = memfree_pci_dev.uar;
arbel->db_rec = dev_ib_data.uar_context_base;
arbel->reserved_lkey = dev_ib_data.mkey;
arbel->eqn = dev_ib_data.eq.eqn;
static_ipoib_qp.send.wqe =
( ( struct udqp_st * ) qph )->snd_wq;
static_ipoib_qp.recv.wqe =
@ -924,6 +952,7 @@ static int arbel_probe ( struct pci_device *pci,
arbel, strerror ( rc ) );
goto err_query_dev_lim;
}
arbel->limits.reserved_uars = MLX_GET ( &dev_lim, num_rsvd_uars );
arbel->limits.reserved_cqs =
( 1 << MLX_GET ( &dev_lim, log2_rsvd_cqs ) );
DBG ( "Device limits:\n ");