Skip to content

Commit

Permalink
IB/core: Add gid_type to gid attribute
Browse files Browse the repository at this point in the history 
In order to support multiple GID types, we need to store the gid_type
with each GID. This is also aligned with the RoCE v2 annex "RoCEv2 PORT
GID table entries shall have a "GID type" attribute that denotes the L3
Address type". The currently supported GID is IB_GID_TYPE_IB which is
also RoCE v1 GID type.

This implies that gid_type should be added to roce_gid_table meta-data.

Signed-off-by: Matan Barak <[email protected]>
Signed-off-by: Doug Ledford <[email protected]>
  • Loading branch information
@matanb10 @dledford
matanb10 authored and dledford committed Dec 23, 2015
1 parent cee3c4d commit b39ffa1
Show file tree
Hide file tree
Showing 13 changed files with 185 additions and 62 deletions.
144 changes: 98 additions & 46 deletions drivers/infiniband/core/cache.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ enum gid_attr_find_mask {
GID_ATTR_FIND_MASK_GID = 1UL << 0,
GID_ATTR_FIND_MASK_NETDEV = 1UL << 1,
GID_ATTR_FIND_MASK_DEFAULT = 1UL << 2,
GID_ATTR_FIND_MASK_GID_TYPE = 1UL << 3,
};

enum gid_table_entry_props {
Expand Down Expand Up @@ -125,6 +126,19 @@ static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)
}
}

static const char * const gid_type_str[] = {
[IB_GID_TYPE_IB] = "IB/RoCE v1",
};

const char *ib_cache_gid_type_str(enum ib_gid_type gid_type)
{
if (gid_type < ARRAY_SIZE(gid_type_str) && gid_type_str[gid_type])
return gid_type_str[gid_type];

return "Invalid GID type";
}
EXPORT_SYMBOL(ib_cache_gid_type_str);

/* This function expects that rwlock will be write locked in all
* scenarios and that lock will be locked in sleep-able (RoCE)
* scenarios.
Expand Down Expand Up @@ -233,6 +247,10 @@ static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,
if (found >= 0)
continue;

if (mask & GID_ATTR_FIND_MASK_GID_TYPE &&
attr->gid_type != val->gid_type)
continue;

if (mask & GID_ATTR_FIND_MASK_GID &&
memcmp(gid, &data->gid, sizeof(*gid)))
continue;
Expand Down Expand Up @@ -296,6 +314,7 @@ int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
write_lock_irq(&table->rwlock);

ix = find_gid(table, gid, attr, false, GID_ATTR_FIND_MASK_GID |
GID_ATTR_FIND_MASK_GID_TYPE |
GID_ATTR_FIND_MASK_NETDEV, &empty);
if (ix >= 0)
goto out_unlock;
Expand Down Expand Up @@ -329,6 +348,7 @@ int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,

ix = find_gid(table, gid, attr, false,
GID_ATTR_FIND_MASK_GID |
GID_ATTR_FIND_MASK_GID_TYPE |
GID_ATTR_FIND_MASK_NETDEV |
GID_ATTR_FIND_MASK_DEFAULT,
NULL);
Expand Down Expand Up @@ -427,11 +447,13 @@ static int _ib_cache_gid_table_find(struct ib_device *ib_dev,

static int ib_cache_gid_find(struct ib_device *ib_dev,
const union ib_gid *gid,
enum ib_gid_type gid_type,
struct net_device *ndev, u8 *port,
u16 *index)
{
unsigned long mask = GID_ATTR_FIND_MASK_GID;
struct ib_gid_attr gid_attr_val = {.ndev = ndev};
unsigned long mask = GID_ATTR_FIND_MASK_GID |
GID_ATTR_FIND_MASK_GID_TYPE;
struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};

if (ndev)
mask |= GID_ATTR_FIND_MASK_NETDEV;
Expand All @@ -442,14 +464,16 @@ static int ib_cache_gid_find(struct ib_device *ib_dev,

int ib_find_cached_gid_by_port(struct ib_device *ib_dev,
const union ib_gid *gid,
enum ib_gid_type gid_type,
u8 port, struct net_device *ndev,
u16 *index)
{
int local_index;
struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
struct ib_gid_table *table;
unsigned long mask = GID_ATTR_FIND_MASK_GID;
struct ib_gid_attr val = {.ndev = ndev};
unsigned long mask = GID_ATTR_FIND_MASK_GID |
GID_ATTR_FIND_MASK_GID_TYPE;
struct ib_gid_attr val = {.ndev = ndev, .gid_type = gid_type};
unsigned long flags;

if (port < rdma_start_port(ib_dev) ||
Expand Down Expand Up @@ -607,71 +631,98 @@ static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,

void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
struct net_device *ndev,
unsigned long gid_type_mask,
enum ib_cache_gid_default_mode mode)
{
struct ib_gid_table **ports_table = ib_dev->cache.gid_cache;
union ib_gid gid;
struct ib_gid_attr gid_attr;
struct ib_gid_attr zattr_type = zattr;
struct ib_gid_table *table;
int ix;
union ib_gid current_gid;
struct ib_gid_attr current_gid_attr = {};
unsigned int gid_type;

table = ports_table[port - rdma_start_port(ib_dev)];

make_default_gid(ndev, &gid);
memset(&gid_attr, 0, sizeof(gid_attr));
gid_attr.ndev = ndev;

mutex_lock(&table->lock);
write_lock_irq(&table->rwlock);
ix = find_gid(table, NULL, NULL, true, GID_ATTR_FIND_MASK_DEFAULT, NULL);

/* Coudn't find default GID location */
WARN_ON(ix < 0);

if (!__ib_cache_gid_get(ib_dev, port, ix,
&current_gid, &current_gid_attr) &&
mode == IB_CACHE_GID_DEFAULT_MODE_SET &&
!memcmp(&gid, &current_gid, sizeof(gid)) &&
!memcmp(&gid_attr, &current_gid_attr, sizeof(gid_attr)))
goto unlock;

if (memcmp(&current_gid, &zgid, sizeof(current_gid)) ||
memcmp(&current_gid_attr, &zattr,
sizeof(current_gid_attr))) {
if (del_gid(ib_dev, port, table, ix, true)) {
pr_warn("ib_cache_gid: can't delete index %d for default gid %pI6\n",
ix, gid.raw);
goto unlock;
} else {
dispatch_gid_change_event(ib_dev, port);
for (gid_type = 0; gid_type < IB_GID_TYPE_SIZE; ++gid_type) {
int ix;
union ib_gid current_gid;
struct ib_gid_attr current_gid_attr = {};

if (1UL << gid_type & ~gid_type_mask)
continue;

gid_attr.gid_type = gid_type;

mutex_lock(&table->lock);
write_lock_irq(&table->rwlock);
ix = find_gid(table, NULL, &gid_attr, true,
GID_ATTR_FIND_MASK_GID_TYPE |
GID_ATTR_FIND_MASK_DEFAULT,
NULL);

/* Coudn't find default GID location */
WARN_ON(ix < 0);

zattr_type.gid_type = gid_type;

if (!__ib_cache_gid_get(ib_dev, port, ix,
&current_gid, &current_gid_attr) &&
mode == IB_CACHE_GID_DEFAULT_MODE_SET &&
!memcmp(&gid, &current_gid, sizeof(gid)) &&
!memcmp(&gid_attr, &current_gid_attr, sizeof(gid_attr)))
goto release;

if (memcmp(&current_gid, &zgid, sizeof(current_gid)) ||
memcmp(&current_gid_attr, &zattr_type,
sizeof(current_gid_attr))) {
if (del_gid(ib_dev, port, table, ix, true)) {
pr_warn("ib_cache_gid: can't delete index %d for default gid %pI6\n",
ix, gid.raw);
goto release;
} else {
dispatch_gid_change_event(ib_dev, port);
}
}
}

if (mode == IB_CACHE_GID_DEFAULT_MODE_SET) {
if (add_gid(ib_dev, port, table, ix, &gid, &gid_attr, true)) {
pr_warn("ib_cache_gid: unable to add default gid %pI6\n",
gid.raw);
} else {
dispatch_gid_change_event(ib_dev, port);
if (mode == IB_CACHE_GID_DEFAULT_MODE_SET) {
if (add_gid(ib_dev, port, table, ix, &gid, &gid_attr, true))
pr_warn("ib_cache_gid: unable to add default gid %pI6\n",
gid.raw);
else
dispatch_gid_change_event(ib_dev, port);
}
}

unlock:
if (current_gid_attr.ndev)
dev_put(current_gid_attr.ndev);
write_unlock_irq(&table->rwlock);
mutex_unlock(&table->lock);
release:
if (current_gid_attr.ndev)
dev_put(current_gid_attr.ndev);
write_unlock_irq(&table->rwlock);
mutex_unlock(&table->lock);
}
}

static int gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
struct ib_gid_table *table)
{
if (rdma_protocol_roce(ib_dev, port)) {
struct ib_gid_table_entry *entry = &table->data_vec[0];
unsigned int i;
unsigned long roce_gid_type_mask;
unsigned int num_default_gids;
unsigned int current_gid = 0;

roce_gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
num_default_gids = hweight_long(roce_gid_type_mask);
for (i = 0; i < num_default_gids && i < table->sz; i++) {
struct ib_gid_table_entry *entry =
&table->data_vec[i];

entry->props |= GID_TABLE_ENTRY_DEFAULT;
current_gid = find_next_bit(&roce_gid_type_mask,
BITS_PER_LONG,
current_gid);
entry->attr.gid_type = current_gid++;
}

return 0;
Expand Down Expand Up @@ -794,11 +845,12 @@ EXPORT_SYMBOL(ib_get_cached_gid);

int ib_find_cached_gid(struct ib_device *device,
const union ib_gid *gid,
enum ib_gid_type gid_type,
struct net_device *ndev,
u8 *port_num,
u16 *index)
{
return ib_cache_gid_find(device, gid, ndev, port_num, index);
return ib_cache_gid_find(device, gid, gid_type, ndev, port_num, index);
}
EXPORT_SYMBOL(ib_find_cached_gid);

Expand Down
2 changes: 1 addition & 1 deletion drivers/infiniband/core/cm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
read_lock_irqsave(&cm.device_lock, flags);
list_for_each_entry(cm_dev, &cm.device_list, list) {
if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
ndev, &p, NULL)) {
IB_GID_TYPE_IB, ndev, &p, NULL)) {
port = cm_dev->port[p-1];
break;
}
Expand Down
3 changes: 2 additions & 1 deletion drivers/infiniband/core/cma.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,8 @@ static inline int cma_validate_port(struct ib_device *device, u8 port,
if (dev_type == ARPHRD_ETHER)
ndev = dev_get_by_index(&init_net, bound_if_index);

ret = ib_find_cached_gid_by_port(device, gid, port, ndev, NULL);
ret = ib_find_cached_gid_by_port(device, gid, IB_GID_TYPE_IB, port,
ndev, NULL);

if (ndev)
dev_put(ndev);
Expand Down
4 changes: 4 additions & 0 deletions drivers/infiniband/core/core_priv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,11 @@ enum ib_cache_gid_default_mode {
IB_CACHE_GID_DEFAULT_MODE_DELETE
};

const char *ib_cache_gid_type_str(enum ib_gid_type gid_type);

void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
struct net_device *ndev,
unsigned long gid_type_mask,
enum ib_cache_gid_default_mode mode);

int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
Expand All @@ -87,6 +90,7 @@ int roce_gid_mgmt_init(void);
void roce_gid_mgmt_cleanup(void);

int roce_rescan_device(struct ib_device *ib_dev);
unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port);

int ib_cache_setup_one(struct ib_device *device);
void ib_cache_cleanup_one(struct ib_device *device);
Expand Down
9 changes: 7 additions & 2 deletions drivers/infiniband/core/device.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -815,26 +815,31 @@ EXPORT_SYMBOL(ib_modify_port);
* a specified GID value occurs.
* @device: The device to query.
* @gid: The GID value to search for.
* @gid_type: Type of GID.
* @ndev: The ndev related to the GID to search for.
* @port_num: The port number of the device where the GID value was found.
* @index: The index into the GID table where the GID was found. This
* parameter may be NULL.
*/
int ib_find_gid(struct ib_device *device, union ib_gid *gid,
struct net_device *ndev, u8 *port_num, u16 *index)
enum ib_gid_type gid_type, struct net_device *ndev,
u8 *port_num, u16 *index)
{
union ib_gid tmp_gid;
int ret, port, i;

for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) {
if (rdma_cap_roce_gid_table(device, port)) {
if (!ib_find_cached_gid_by_port(device, gid, port,
if (!ib_find_cached_gid_by_port(device, gid, gid_type, port,
ndev, index)) {
*port_num = port;
return 0;
}
}

if (gid_type != IB_GID_TYPE_IB)
continue;

for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) {
ret = ib_query_gid(device, port, i, &tmp_gid, NULL);
if (ret)
Expand Down
2 changes: 1 addition & 1 deletion drivers/infiniband/core/multicast.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -729,7 +729,7 @@ int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,
u16 gid_index;
u8 p;

ret = ib_find_cached_gid(device, &rec->port_gid,
ret = ib_find_cached_gid(device, &rec->port_gid, IB_GID_TYPE_IB,
NULL, &p, &gid_index);
if (ret)
return ret;
Expand Down
Loading

0 comments on commit b39ffa1

Please sign in to comment.