Skip to content

Commit

Permalink
iommu/iova: Make the rcache depot scale better
Browse files Browse the repository at this point in the history 
The algorithm in the original paper specifies the storage of full
magazines in the depot as an unbounded list rather than a fixed-size
array. It turns out to be pretty straightforward to do this in our
implementation with no significant loss of efficiency. This allows
the depot to scale up to the working set sizes of larger systems,
while also potentially saving some memory on smaller ones too.

Since this involves touching struct iova_magazine with the requisite
care, we may as well reinforce the comment with a proper assertion too.

Reviewed-by: John Garry <[email protected]>
Reviewed-by: Jerry Snitselaar <[email protected]>
Signed-off-by: Robin Murphy <[email protected]>
Link: https://lore.kernel.org/r/f597aa72fc3e1d315bc4574af0ce0ebe5c31cd22.1694535580.git.robin.murphy@arm.com
Signed-off-by: Joerg Roedel <[email protected]>
  • Loading branch information
@rmurphy-arm @joergroedel
rmurphy-arm authored and joergroedel committed Sep 25, 2023
1 parent afad94a commit 911aa12
Showing 1 changed file with 35 additions and 30 deletions.
65 changes: 35 additions & 30 deletions drivers/iommu/iova.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -622,15 +622,19 @@ EXPORT_SYMBOL_GPL(reserve_iova);
/*
* As kmalloc's buffer size is fixed to power of 2, 127 is chosen to
* assure size of 'iova_magazine' to be 1024 bytes, so that no memory
* will be wasted.
* will be wasted. Since only full magazines are inserted into the depot,
* we don't need to waste PFN capacity on a separate list head either.
*/
#define IOVA_MAG_SIZE 127
#define MAX_GLOBAL_MAGS 32 /* magazines per bin */

struct iova_magazine {
unsigned long size;
union {
unsigned long size;
struct iova_magazine *next;
};
unsigned long pfns[IOVA_MAG_SIZE];
};
static_assert(!(sizeof(struct iova_magazine) & (sizeof(struct iova_magazine) - 1)));

struct iova_cpu_rcache {
spinlock_t lock;
Expand All @@ -640,8 +644,7 @@ struct iova_cpu_rcache {

struct iova_rcache {
spinlock_t lock;
unsigned long depot_size;
struct iova_magazine *depot[MAX_GLOBAL_MAGS];
struct iova_magazine *depot;
struct iova_cpu_rcache __percpu *cpu_rcaches;
};

Expand Down Expand Up @@ -717,6 +720,21 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
mag->pfns[mag->size++] = pfn;
}

static struct iova_magazine *iova_depot_pop(struct iova_rcache *rcache)
{
struct iova_magazine *mag = rcache->depot;

rcache->depot = mag->next;
mag->size = IOVA_MAG_SIZE;
return mag;
}

static void iova_depot_push(struct iova_rcache *rcache, struct iova_magazine *mag)
{
mag->next = rcache->depot;
rcache->depot = mag;
}

int iova_domain_init_rcaches(struct iova_domain *iovad)
{
unsigned int cpu;
Expand All @@ -734,7 +752,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad)

rcache = &iovad->rcaches[i];
spin_lock_init(&rcache->lock);
rcache->depot_size = 0;
rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
cache_line_size());
if (!rcache->cpu_rcaches) {
Expand Down Expand Up @@ -776,7 +793,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
struct iova_rcache *rcache,
unsigned long iova_pfn)
{
struct iova_magazine *mag_to_free = NULL;
struct iova_cpu_rcache *cpu_rcache;
bool can_insert = false;
unsigned long flags;
Expand All @@ -794,12 +810,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,

if (new_mag) {
spin_lock(&rcache->lock);
if (rcache->depot_size < MAX_GLOBAL_MAGS) {
rcache->depot[rcache->depot_size++] =
cpu_rcache->loaded;
} else {
mag_to_free = cpu_rcache->loaded;
}
iova_depot_push(rcache, cpu_rcache->loaded);
spin_unlock(&rcache->lock);

cpu_rcache->loaded = new_mag;
Expand All @@ -812,11 +823,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,

spin_unlock_irqrestore(&cpu_rcache->lock, flags);

if (mag_to_free) {
iova_magazine_free_pfns(mag_to_free, iovad);
iova_magazine_free(mag_to_free);
}

return can_insert;
}

Expand Down Expand Up @@ -854,9 +860,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
has_pfn = true;
} else {
spin_lock(&rcache->lock);
if (rcache->depot_size > 0) {
if (rcache->depot) {
iova_magazine_free(cpu_rcache->loaded);
cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
cpu_rcache->loaded = iova_depot_pop(rcache);
has_pfn = true;
}
spin_unlock(&rcache->lock);
Expand Down Expand Up @@ -895,9 +901,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
struct iova_rcache *rcache;
struct iova_cpu_rcache *cpu_rcache;
unsigned int cpu;
int i, j;

for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
for (int i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
rcache = &iovad->rcaches[i];
if (!rcache->cpu_rcaches)
break;
Expand All @@ -907,8 +912,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
iova_magazine_free(cpu_rcache->prev);
}
free_percpu(rcache->cpu_rcaches);
for (j = 0; j < rcache->depot_size; ++j)
iova_magazine_free(rcache->depot[j]);
while (rcache->depot)
iova_magazine_free(iova_depot_pop(rcache));
}

kfree(iovad->rcaches);
Expand Down Expand Up @@ -942,16 +947,16 @@ static void free_global_cached_iovas(struct iova_domain *iovad)
{
struct iova_rcache *rcache;
unsigned long flags;
int i, j;

for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
for (int i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
rcache = &iovad->rcaches[i];
spin_lock_irqsave(&rcache->lock, flags);
for (j = 0; j < rcache->depot_size; ++j) {
iova_magazine_free_pfns(rcache->depot[j], iovad);
iova_magazine_free(rcache->depot[j]);
while (rcache->depot) {
struct iova_magazine *mag = iova_depot_pop(rcache);

iova_magazine_free_pfns(mag, iovad);
iova_magazine_free(mag);
}
rcache->depot_size = 0;
spin_unlock_irqrestore(&rcache->lock, flags);
}
}
Expand Down

0 comments on commit 911aa12

Please sign in to comment.