Skip to content

Commit

Permalink
IB/rdmavt, hfi1, qib: Enhance rdmavt and hfi1 to use 32 bit lids
Browse files Browse the repository at this point in the history 
Increase lid used in hfi1 driver to 32 bits. qib continues
to use 16 bit lids.

Reviewed-by: Dennis Dalessandro <[email protected]>
Signed-off-by: Dasaratharaman Chandramouli <[email protected]>
Signed-off-by: Don Hiatt <[email protected]>
Signed-off-by: Dennis Dalessandro <[email protected]>
Signed-off-by: Doug Ledford <[email protected]>
  • Loading branch information
@dledford
Dasaratharaman Chandramouli authored and dledford committed Aug 22, 2017
1 parent 863cf89 commit 51e658f
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 43 deletions.
12 changes: 9 additions & 3 deletions drivers/infiniband/hw/hfi1/chip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10067,10 +10067,16 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
struct hfi1_devdata *dd = ppd->dd;
u32 mask = ~((1U << ppd->lmc) - 1);
u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
u32 lid;

/*
* Program 0 in CSR if port lid is extended. This prevents
* 9B packets being sent out for large lids.
*/
lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
| DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
<< DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
<< DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
Expand All @@ -10081,7 +10087,7 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
*/
sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
(((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
(((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
SEND_CTXT_CHECK_SLID_VALUE_SHIFT);

for (i = 0; i < dd->chip_send_contexts; i++) {
Expand All @@ -10091,7 +10097,7 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
}

/* Now we have to do the same thing for the sdma engines */
sdma_update_lmc(dd, mask, ppd->lid);
sdma_update_lmc(dd, mask, lid);
}

static const char *state_completed_string(u32 completed)
Expand Down
2 changes: 1 addition & 1 deletion drivers/infiniband/hw/hfi1/hfi.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,7 @@ struct hfi1_pportdata {
u32 ibmaxlen;
u32 current_egress_rate; /* units [10^6 bits/sec] */
/* LID programmed for this instance */
u16 lid;
u32 lid;
/* list of pkeys programmed; 0 if not set */
u16 pkeys[MAX_PKEY_VALUES];
u16 link_width_supported;
Expand Down
91 changes: 79 additions & 12 deletions drivers/infiniband/hw/hfi1/mad.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,61 @@ static void subn_handle_opa_trap_repress(struct hfi1_ibport *ibp,
spin_unlock_irqrestore(&ibp->rvp.lock, flags);
}

static void hfi1_update_sm_ah_attr(struct hfi1_ibport *ibp,
struct rdma_ah_attr *attr, u32 dlid)
{
rdma_ah_set_dlid(attr, dlid);
rdma_ah_set_port_num(attr, ppd_from_ibp(ibp)->port);
if (dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);

rdma_ah_set_ah_flags(attr, IB_AH_GRH);
grh->sgid_index = 0;
grh->hop_limit = 1;
grh->dgid.global.subnet_prefix =
ibp->rvp.gid_prefix;
grh->dgid.global.interface_id = OPA_MAKE_ID(dlid);
}
}

static int hfi1_modify_qp0_ah(struct hfi1_ibport *ibp,
struct rvt_ah *ah, u32 dlid)
{
struct rdma_ah_attr attr;
struct rvt_qp *qp0;
int ret = -EINVAL;

memset(&attr, 0, sizeof(attr));
attr.type = ah->ibah.type;
hfi1_update_sm_ah_attr(ibp, &attr, dlid);
rcu_read_lock();
qp0 = rcu_dereference(ibp->rvp.qp[0]);
if (qp0)
ret = rdma_modify_ah(&ah->ibah, &attr);
rcu_read_unlock();
return ret;
}

static struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u32 dlid)
{
struct rdma_ah_attr attr;
struct ib_ah *ah = ERR_PTR(-EINVAL);
struct rvt_qp *qp0;
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
struct hfi1_devdata *dd = dd_from_ppd(ppd);
u8 port_num = ppd->port;

memset(&attr, 0, sizeof(attr));
attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
hfi1_update_sm_ah_attr(ibp, &attr, dlid);
rcu_read_lock();
qp0 = rcu_dereference(ibp->rvp.qp[0]);
if (qp0)
ah = rdma_create_ah(qp0->ibqp.pd, &attr);
rcu_read_unlock();
return ah;
}

static void send_trap(struct hfi1_ibport *ibp, struct trap_node *trap)
{
struct ib_mad_send_buf *send_buf;
Expand Down Expand Up @@ -1283,8 +1338,8 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
struct hfi1_ibport *ibp;
u8 clientrereg;
unsigned long flags;
u32 smlid, opa_lid; /* tmp vars to hold LID values */
u16 lid;
u32 smlid;
u32 lid;
u8 ls_old, ls_new, ps_new;
u8 vls;
u8 msl;
Expand All @@ -1301,22 +1356,20 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
return reply((struct ib_mad_hdr *)smp);
}

opa_lid = be32_to_cpu(pi->lid);
if (opa_lid & 0xFFFF0000) {
pr_warn("OPA_PortInfo lid out of range: %X\n", opa_lid);
lid = be32_to_cpu(pi->lid);
if (lid & 0xFF000000) {
pr_warn("OPA_PortInfo lid out of range: %X\n", lid);
smp->status |= IB_SMP_INVALID_FIELD;
goto get_only;
}

lid = (u16)(opa_lid & 0x0000FFFF);

smlid = be32_to_cpu(pi->sm_lid);
if (smlid & 0xFFFF0000) {
if (smlid & 0xFF000000) {
pr_warn("OPA_PortInfo SM lid out of range: %X\n", smlid);
smp->status |= IB_SMP_INVALID_FIELD;
goto get_only;
}
smlid &= 0x0000FFFF;

clientrereg = (pi->clientrereg_subnettimeout &
OPA_PI_MASK_CLIENT_REREGISTER);
Expand All @@ -1331,12 +1384,16 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
ls_old = driver_lstate(ppd);

ibp->rvp.mkey = pi->mkey;
ibp->rvp.gid_prefix = pi->subnet_prefix;
if (ibp->rvp.gid_prefix != pi->subnet_prefix) {
ibp->rvp.gid_prefix = pi->subnet_prefix;
event.event = IB_EVENT_GID_CHANGE;
ib_dispatch_event(&event);
}
ibp->rvp.mkey_lease_period = be16_to_cpu(pi->mkey_lease_period);

/* Must be a valid unicast LID address. */
if ((lid == 0 && ls_old > IB_PORT_INIT) ||
lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
(hfi1_is_16B_mcast(lid))) {
smp->status |= IB_SMP_INVALID_FIELD;
pr_warn("SubnSet(OPA_PortInfo) lid invalid 0x%x\n",
lid);
Expand All @@ -1349,6 +1406,16 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
hfi1_set_lid(ppd, lid, pi->mkeyprotect_lmc & OPA_PI_MASK_LMC);
event.event = IB_EVENT_LID_CHANGE;
ib_dispatch_event(&event);

if (HFI1_PORT_GUID_INDEX + 1 < HFI1_GUIDS_PER_PORT) {
/* Manufacture GID from LID to support extended
* addresses
*/
ppd->guids[HFI1_PORT_GUID_INDEX + 1] =
be64_to_cpu(OPA_MAKE_ID(lid));
event.event = IB_EVENT_GID_CHANGE;
ib_dispatch_event(&event);
}
}

msl = pi->smsl & OPA_PI_MASK_SMSL;
Expand All @@ -1359,15 +1426,15 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,

/* Must be a valid unicast LID address. */
if ((smlid == 0 && ls_old > IB_PORT_INIT) ||
smlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
(hfi1_is_16B_mcast(smlid))) {
smp->status |= IB_SMP_INVALID_FIELD;
pr_warn("SubnSet(OPA_PortInfo) smlid invalid 0x%x\n", smlid);
} else if (smlid != ibp->rvp.sm_lid || msl != ibp->rvp.sm_sl) {
pr_warn("SubnSet(OPA_PortInfo) smlid 0x%x\n", smlid);
spin_lock_irqsave(&ibp->rvp.lock, flags);
if (ibp->rvp.sm_ah) {
if (smlid != ibp->rvp.sm_lid)
rdma_ah_set_dlid(&ibp->rvp.sm_ah->attr, smlid);
hfi1_modify_qp0_ah(ibp, ibp->rvp.sm_ah, smlid);
if (msl != ibp->rvp.sm_sl)
rdma_ah_set_sl(&ibp->rvp.sm_ah->attr, msl);
}
Expand Down
23 changes: 1 addition & 22 deletions drivers/infiniband/hw/hfi1/verbs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1394,7 +1394,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,
struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
struct hfi1_pportdata *ppd = &dd->pport[port_num - 1];
u16 lid = ppd->lid;
u32 lid = ppd->lid;

/* props being zeroed by the caller, avoid zeroing it here */
props->lid = lid ? lid : 0;
Expand Down Expand Up @@ -1555,27 +1555,6 @@ static void hfi1_notify_new_ah(struct ib_device *ibdev,
ah->log_pmtu = ilog2(dd->vld[ah->vl].mtu);
}

struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid)
{
struct rdma_ah_attr attr;
struct ib_ah *ah = ERR_PTR(-EINVAL);
struct rvt_qp *qp0;
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
struct hfi1_devdata *dd = dd_from_ppd(ppd);
u8 port_num = ppd->port;

memset(&attr, 0, sizeof(attr));
attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
rdma_ah_set_dlid(&attr, dlid);
rdma_ah_set_port_num(&attr, ppd_from_ibp(ibp)->port);
rcu_read_lock();
qp0 = rcu_dereference(ibp->rvp.qp[0]);
if (qp0)
ah = rdma_create_ah(qp0->ibqp.pd, &attr);
rcu_read_unlock();
return ah;
}

/**
* hfi1_get_npkeys - return the size of the PKEY table for context 0
* @dd: the hfi1_ib device
Expand Down
2 changes: 0 additions & 2 deletions drivers/infiniband/hw/hfi1/verbs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,8 +334,6 @@ void hfi1_rc_hdrerr(

u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);

struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid);

void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);

void hfi1_ud_rcv(struct hfi1_packet *packet);
Expand Down
4 changes: 2 additions & 2 deletions drivers/infiniband/hw/qib/qib_mad.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ static void qib_send_trap(struct qib_ibport *ibp, void *data, unsigned len)
if (ibp->rvp.sm_lid != be16_to_cpu(IB_LID_PERMISSIVE)) {
struct ib_ah *ah;

ah = qib_create_qp0_ah(ibp, ibp->rvp.sm_lid);
ah = qib_create_qp0_ah(ibp, (u16)ibp->rvp.sm_lid);
if (IS_ERR(ah))
ret = PTR_ERR(ah);
else {
Expand Down Expand Up @@ -496,7 +496,7 @@ static int subn_get_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
pip->mkey = ibp->rvp.mkey;
pip->gid_prefix = ibp->rvp.gid_prefix;
pip->lid = cpu_to_be16(ppd->lid);
pip->sm_lid = cpu_to_be16(ibp->rvp.sm_lid);
pip->sm_lid = cpu_to_be16((u16)ibp->rvp.sm_lid);
pip->cap_mask = cpu_to_be32(ibp->rvp.port_cap_flags);
/* pip->diag_code; */
pip->mkey_lease_period = cpu_to_be16(ibp->rvp.mkey_lease_period);
Expand Down
2 changes: 1 addition & 1 deletion include/rdma/rdma_vt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ struct rvt_ibport {
__be16 pma_counter_select[5];
u16 pma_tag;
u16 mkey_lease_period;
u16 sm_lid;
u32 sm_lid;
u8 sm_sl;
u8 mkeyprot;
u8 subnet_timeout;
Expand Down

0 comments on commit 51e658f

Please sign in to comment.