2
0
mirror of https://github.com/xcat2/xNBA.git synced 2024-12-14 23:31:39 +00:00

First (working) draft of command interface.

This commit is contained in:
Michael Brown 2007-09-16 01:44:57 +01:00
parent baa885ee8e
commit bf9bd93856
2 changed files with 159 additions and 4 deletions

View File

@ -16,14 +16,34 @@
#define ARBEL_OPCODE_RECV_ERROR 0xfe
#define ARBEL_OPCODE_SEND_ERROR 0xff
/*
* HCA commands
*
*/
#define ARBEL_HCR_BASE 0x80680
#define ARBEL_HCR_REG(x) ( ARBEL_HCR_BASE + 4 * (x) )
#define ARBEL_HCR_MAX_WAIT_MS 2000
#define ARBEL_HCR_OPCODE_MASK 0x0000ffffUL
#define ARBEL_HCR_IN_IMMEDIATE 0x00010000UL
#define ARBEL_HCR_IN_MAILBOX 0x00020000UL
#define ARBEL_HCR_OUT_IMMEDIATE 0x00040000UL
#define ARBEL_HCR_OUT_MAILBOX 0x00080000UL
#define ARBEL_HCR_OP_SW2HW_CQ ( 0x0016 | ARBEL_HCR_IN_MAILBOX )
#define ARBEL_HCR_OP_NOP ( 0x0031 )
/*
* Wrapper structures for hardware datatypes
*
*/
struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_context );
struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry );
struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record );
struct MLX_DECLARE_STRUCT ( arbelprm_recv_wqe_segment_next );
struct MLX_DECLARE_STRUCT ( arbelprm_send_doorbell );
@ -126,6 +146,13 @@ struct arbel_completion_queue {
/** An Arbel device */
struct arbel {
/** Configuration registers */
void *config;
/** Command input mailbox */
void *mailbox_in;
/** Command output mailbox */
void *mailbox_out;
/** User Access Region */
void *uar;
/** Doorbell records */

View File

@ -26,6 +26,11 @@ Skeleton NIC driver for Etherboot
#include "arbel.h"
static const struct ib_gid arbel_no_gid = {
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }
};
#define MLX_RX_MAX_FILL NUM_IPOIB_RCV_WQES
struct mlx_nic {
@ -263,11 +268,121 @@ static struct net_device_operations mlx_operations = {
.irq = mlx_irq,
};
/**
* Wait for Arbel command completion
*
* @v arbel Arbel device
* @ret rc Return status code
*/
static int arbel_command_wait ( struct arbel *arbel,
struct arbelprm_hca_command_register *hcr ) {
unsigned int wait;
for ( wait = ARBEL_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
hcr->u.dwords[6] =
readl ( arbel->config + ARBEL_HCR_REG ( 6 ) );
if ( MLX_GET ( hcr, go ) == 0 )
return 0;
mdelay ( 1 );
}
return -EBUSY;
}
/**
* Issue HCA command
*
* @v arbel Arbel device
* @v op_fl Opcode (plus implied flags)
* @v op_mod Opcode modifier (0 if no modifier applicable)
* @v in_param Input parameter
* @v in_param_len Input parameter length
* @v in_mod Input modifier (0 if no modifier applicable)
* @v out_param Output parameter
* @ret rc Return status code
*/
static int arbel_command ( struct arbel *arbel, unsigned int op_fl,
unsigned int op_mod, const void *in_param,
size_t in_param_len, unsigned int in_mod,
void *out_param, size_t out_param_len ) {
struct arbelprm_hca_command_register hcr;
unsigned int status;
unsigned int i;
int rc;
/* Check that HCR is free */
if ( ( rc = arbel_command_wait ( arbel, &hcr ) ) != 0 ) {
DBGC ( arbel, "Arbel %p command interface locked\n", arbel );
return rc;
}
/* Prepare HCR */
memset ( &hcr, 0, sizeof ( hcr ) );
if ( op_fl & ARBEL_HCR_IN_IMMEDIATE ) {
memcpy ( &hcr.u.dwords[0], in_param, 8 );
} else if ( op_fl & ARBEL_HCR_IN_MAILBOX ) {
memcpy ( arbel->mailbox_in, in_param, in_param_len );
MLX_FILL_1 ( &hcr, 1, in_param_l,
virt_to_bus ( arbel->mailbox_in ) );
}
MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
if ( op_fl & ARBEL_HCR_OUT_MAILBOX ) {
MLX_FILL_1 ( &hcr, 4, out_param_l,
virt_to_bus ( arbel->mailbox_out ) );
}
MLX_FILL_3 ( &hcr, 6,
opcode, ( op_fl & ARBEL_HCR_OPCODE_MASK ),
opcode_modifier, op_mod,
go, 1 );
/* Issue command */
for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
i++ ) {
writel ( hcr.u.dwords[i],
arbel->config + ARBEL_HCR_REG ( i ) );
barrier();
}
/* Wait for command completion */
if ( ( rc = arbel_command_wait ( arbel, &hcr ) ) != 0 ) {
DBGC ( arbel, "Arbel %p timed out waiting for command:\n",
arbel );
DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
return rc;
}
/* Check command status */
status = MLX_GET ( &hcr, status );
if ( status != 0 ) {
DBGC ( arbel, "Arbel %p command failed with status %02x:\n",
arbel, status );
DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
return -EIO;
}
/* Read output parameters, if any */
hcr.u.dwords[3] = readl ( arbel->config + ARBEL_HCR_REG ( 3 ) );
hcr.u.dwords[4] = readl ( arbel->config + ARBEL_HCR_REG ( 4 ) );
if ( op_fl & ARBEL_HCR_OUT_IMMEDIATE ) {
memcpy ( out_param, &hcr.u.dwords[3], 8 );
} else if ( op_fl & ARBEL_HCR_OUT_MAILBOX ) {
memcpy ( out_param, arbel->mailbox_out, out_param_len );
}
return 0;
}
/**
* Create completion queue
*
* @v ibdev Infiniband device
* @v
*/
static int arbel_create_cq ( struct ib_device *ibdev ) {
struct arbelprm_completion_queue_context *cqctx;
static struct ib_gid arbel_no_gid = {
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }
};
}
/**
* Ring doorbell register in UAR
@ -310,7 +425,7 @@ static int arbel_post_send ( struct ib_device *ibdev,
struct arbelprm_ud_send_wqe *wqe;
union arbelprm_doorbell_record *db_rec;
union arbelprm_doorbell_register db_reg;
struct ib_gid *gid;
const struct ib_gid *gid;
unsigned int wqe_idx_mask;
size_t nds;
@ -616,6 +731,9 @@ static int arbel_probe ( struct pci_device *pci,
memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) );
/* Hack up IB structures */
static_arbel.config = memfree_pci_dev.cr_space;
static_arbel.mailbox_in = dev_buffers_p->inprm_buf;
static_arbel.mailbox_out = dev_buffers_p->outprm_buf;
static_arbel.uar = memfree_pci_dev.uar;
static_arbel.db_rec = dev_ib_data.uar_context_base;
static_arbel.reserved_lkey = dev_ib_data.mkey;
@ -634,6 +752,16 @@ static int arbel_probe ( struct pci_device *pci,
list_add ( &static_ipoib_qp.recv.list,
&static_ipoib_recv_cq.work_queues );
uint8_t buf[512];
memset ( buf, 0xaa, sizeof ( buf ) );
if ( ( rc = arbel_command ( &static_arbel,
( 0x03 | ARBEL_HCR_OUT_MAILBOX ), 0,
NULL, 0, 0, buf, 256 ) ) != 0 ) {
DBG ( "QUERY_DEV_LIM failed: %s\n", strerror ( rc ) );
}
DBG ( "Device limits:\n ");
DBG_HD ( &buf[0], sizeof ( buf ) );
/* Register network device */
if ( ( rc = register_netdev ( netdev ) ) != 0 )
goto err_register_netdev;