Skip to content

Commit

Permalink
lib/interval_tree: fast overlap detection
Browse files Browse the repository at this point in the history
Allow interval trees to quickly check for overlaps to avoid unnecesary
tree lookups in interval_tree_iter_first().

As of this patch, all interval tree flavors will require using a
'rb_root_cached' such that we can have the leftmost node easily
available.  While most users will make use of this feature, those with
special functions (in addition to the generic insert, delete, search
calls) will avoid using the cached option as they can do funky things
with insertions -- for example, vma_interval_tree_insert_after().

[[email protected]: fix deadlock from typo vm_lock_anon_vma()]
  Link: http://lkml.kernel.org/r/[email protected]
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Davidlohr Bueso <[email protected]>
Signed-off-by: Jérôme Glisse <[email protected]>
Acked-by: Christian König <[email protected]>
Acked-by: Peter Zijlstra (Intel) <[email protected]>
Acked-by: Doug Ledford <[email protected]>
Acked-by: Michael S. Tsirkin <[email protected]>
Cc: David Airlie <[email protected]>
Cc: Jason Wang <[email protected]>
Cc: Christian Benvenuti <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
Davidlohr Bueso authored and torvalds committed Sep 9, 2017
1 parent 09663c8 commit f808c13
Show file tree
Hide file tree
Showing 33 changed files with 145 additions and 105 deletions.
8 changes: 4 additions & 4 deletions drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ struct amdgpu_mn {

/* objects protected by lock */
struct mutex lock;
struct rb_root objects;
struct rb_root_cached objects;
};

struct amdgpu_mn_node {
Expand All @@ -76,8 +76,8 @@ static void amdgpu_mn_destroy(struct work_struct *work)
mutex_lock(&adev->mn_lock);
mutex_lock(&rmn->lock);
hash_del(&rmn->node);
rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,
it.rb) {
rbtree_postorder_for_each_entry_safe(node, next_node,
&rmn->objects.rb_root, it.rb) {
list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
bo->mn = NULL;
list_del_init(&bo->mn_list);
Expand Down Expand Up @@ -221,7 +221,7 @@ static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
rmn->mm = mm;
rmn->mn.ops = &amdgpu_mn_ops;
mutex_init(&rmn->lock);
rmn->objects = RB_ROOT;
rmn->objects = RB_ROOT_CACHED;

r = __mmu_notifier_register(&rmn->mn, mm);
if (r)
Expand Down
7 changes: 4 additions & 3 deletions drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
Original file line number Diff line number Diff line change
Expand Up @@ -2475,7 +2475,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
u64 flags;
uint64_t init_pde_value = 0;

vm->va = RB_ROOT;
vm->va = RB_ROOT_CACHED;
vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
vm->reserved_vmid[i] = NULL;
Expand Down Expand Up @@ -2596,10 +2596,11 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)

amd_sched_entity_fini(vm->entity.sched, &vm->entity);

if (!RB_EMPTY_ROOT(&vm->va)) {
if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
dev_err(adev->dev, "still active bo inside vm\n");
}
rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
rbtree_postorder_for_each_entry_safe(mapping, tmp,
&vm->va.rb_root, rb) {
list_del(&mapping->list);
amdgpu_vm_it_remove(mapping, &vm->va);
kfree(mapping);
Expand Down
2 changes: 1 addition & 1 deletion drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ struct amdgpu_vm_pt {

struct amdgpu_vm {
/* tree of virtual addresses mapped */
struct rb_root va;
struct rb_root_cached va;

/* protecting invalidated */
spinlock_t status_lock;
Expand Down
19 changes: 11 additions & 8 deletions drivers/gpu/drm/drm_mm.c
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
struct drm_mm_node *
__drm_mm_interval_first(const struct drm_mm *mm, u64 start, u64 last)
{
return drm_mm_interval_tree_iter_first((struct rb_root *)&mm->interval_tree,
return drm_mm_interval_tree_iter_first((struct rb_root_cached *)&mm->interval_tree,
start, last) ?: (struct drm_mm_node *)&mm->head_node;
}
EXPORT_SYMBOL(__drm_mm_interval_first);
Expand All @@ -180,6 +180,7 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
struct drm_mm *mm = hole_node->mm;
struct rb_node **link, *rb;
struct drm_mm_node *parent;
bool leftmost = true;

node->__subtree_last = LAST(node);

Expand All @@ -196,9 +197,10 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,

rb = &hole_node->rb;
link = &hole_node->rb.rb_right;
leftmost = false;
} else {
rb = NULL;
link = &mm->interval_tree.rb_node;
link = &mm->interval_tree.rb_root.rb_node;
}

while (*link) {
Expand All @@ -208,14 +210,15 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
parent->__subtree_last = node->__subtree_last;
if (node->start < parent->start)
link = &parent->rb.rb_left;
else
else {
link = &parent->rb.rb_right;
leftmost = true;
}
}

rb_link_node(&node->rb, rb, link);
rb_insert_augmented(&node->rb,
&mm->interval_tree,
&drm_mm_interval_tree_augment);
rb_insert_augmented_cached(&node->rb, &mm->interval_tree, leftmost,
&drm_mm_interval_tree_augment);
}

#define RB_INSERT(root, member, expr) do { \
Expand Down Expand Up @@ -577,7 +580,7 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
*new = *old;

list_replace(&old->node_list, &new->node_list);
rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree);
rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree.rb_root);

if (drm_mm_hole_follows(old)) {
list_replace(&old->hole_stack, &new->hole_stack);
Expand Down Expand Up @@ -863,7 +866,7 @@ void drm_mm_init(struct drm_mm *mm, u64 start, u64 size)
mm->color_adjust = NULL;

INIT_LIST_HEAD(&mm->hole_stack);
mm->interval_tree = RB_ROOT;
mm->interval_tree = RB_ROOT_CACHED;
mm->holes_size = RB_ROOT;
mm->holes_addr = RB_ROOT;

Expand Down
2 changes: 1 addition & 1 deletion drivers/gpu/drm/drm_vma_manager.c
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_m
struct rb_node *iter;
unsigned long offset;

iter = mgr->vm_addr_space_mm.interval_tree.rb_node;
iter = mgr->vm_addr_space_mm.interval_tree.rb_root.rb_node;
best = NULL;

while (likely(iter)) {
Expand Down
6 changes: 3 additions & 3 deletions drivers/gpu/drm/i915/i915_gem_userptr.c
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ struct i915_mmu_notifier {
spinlock_t lock;
struct hlist_node node;
struct mmu_notifier mn;
struct rb_root objects;
struct rb_root_cached objects;
struct workqueue_struct *wq;
};

Expand Down Expand Up @@ -123,7 +123,7 @@ static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
struct interval_tree_node *it;
LIST_HEAD(cancelled);

if (RB_EMPTY_ROOT(&mn->objects))
if (RB_EMPTY_ROOT(&mn->objects.rb_root))
return;

/* interval ranges are inclusive, but invalidate range is exclusive */
Expand Down Expand Up @@ -172,7 +172,7 @@ i915_mmu_notifier_create(struct mm_struct *mm)

spin_lock_init(&mn->lock);
mn->mn.ops = &i915_gem_userptr_notifier;
mn->objects = RB_ROOT;
mn->objects = RB_ROOT_CACHED;
mn->wq = alloc_workqueue("i915-userptr-release", WQ_UNBOUND, 0);
if (mn->wq == NULL) {
kfree(mn);
Expand Down
2 changes: 1 addition & 1 deletion drivers/gpu/drm/radeon/radeon.h
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,7 @@ struct radeon_vm_id {
struct radeon_vm {
struct mutex mutex;

struct rb_root va;
struct rb_root_cached va;

/* protecting invalidated and freed */
spinlock_t status_lock;
Expand Down
8 changes: 4 additions & 4 deletions drivers/gpu/drm/radeon/radeon_mn.c
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ struct radeon_mn {

/* objects protected by lock */
struct mutex lock;
struct rb_root objects;
struct rb_root_cached objects;
};

struct radeon_mn_node {
Expand All @@ -75,8 +75,8 @@ static void radeon_mn_destroy(struct work_struct *work)
mutex_lock(&rdev->mn_lock);
mutex_lock(&rmn->lock);
hash_del(&rmn->node);
rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,
it.rb) {
rbtree_postorder_for_each_entry_safe(node, next_node,
&rmn->objects.rb_root, it.rb) {

interval_tree_remove(&node->it, &rmn->objects);
list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
Expand Down Expand Up @@ -205,7 +205,7 @@ static struct radeon_mn *radeon_mn_get(struct radeon_device *rdev)
rmn->mm = mm;
rmn->mn.ops = &radeon_mn_ops;
mutex_init(&rmn->lock);
rmn->objects = RB_ROOT;
rmn->objects = RB_ROOT_CACHED;

r = __mmu_notifier_register(&rmn->mn, mm);
if (r)
Expand Down
7 changes: 4 additions & 3 deletions drivers/gpu/drm/radeon/radeon_vm.c
Original file line number Diff line number Diff line change
Expand Up @@ -1185,7 +1185,7 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
vm->ids[i].last_id_use = NULL;
}
mutex_init(&vm->mutex);
vm->va = RB_ROOT;
vm->va = RB_ROOT_CACHED;
spin_lock_init(&vm->status_lock);
INIT_LIST_HEAD(&vm->invalidated);
INIT_LIST_HEAD(&vm->freed);
Expand Down Expand Up @@ -1232,10 +1232,11 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
struct radeon_bo_va *bo_va, *tmp;
int i, r;

if (!RB_EMPTY_ROOT(&vm->va)) {
if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
dev_err(rdev->dev, "still active bo inside vm\n");
}
rbtree_postorder_for_each_entry_safe(bo_va, tmp, &vm->va, it.rb) {
rbtree_postorder_for_each_entry_safe(bo_va, tmp,
&vm->va.rb_root, it.rb) {
interval_tree_remove(&bo_va->it, &vm->va);
r = radeon_bo_reserve(bo_va->bo, false);
if (!r) {
Expand Down
4 changes: 2 additions & 2 deletions drivers/infiniband/core/umem_rbtree.c
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
/* @last is not a part of the interval. See comment for function
* node_last.
*/
int rbt_ib_umem_for_each_in_range(struct rb_root *root,
int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,
u64 start, u64 last,
umem_call_back cb,
void *cookie)
Expand All @@ -95,7 +95,7 @@ int rbt_ib_umem_for_each_in_range(struct rb_root *root,
}
EXPORT_SYMBOL(rbt_ib_umem_for_each_in_range);

struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root *root,
struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root_cached *root,
u64 addr, u64 length)
{
struct umem_odp_node *node;
Expand Down
2 changes: 1 addition & 1 deletion drivers/infiniband/core/uverbs_cmd.c
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
ucontext->closing = 0;

#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
ucontext->umem_tree = RB_ROOT;
ucontext->umem_tree = RB_ROOT_CACHED;
init_rwsem(&ucontext->umem_rwsem);
ucontext->odp_mrs_count = 0;
INIT_LIST_HEAD(&ucontext->no_private_counters);
Expand Down
10 changes: 5 additions & 5 deletions drivers/infiniband/hw/hfi1/mmu_rb.c
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@

struct mmu_rb_handler {
struct mmu_notifier mn;
struct rb_root root;
struct rb_root_cached root;
void *ops_arg;
spinlock_t lock; /* protect the RB tree */
struct mmu_rb_ops *ops;
Expand Down Expand Up @@ -108,7 +108,7 @@ int hfi1_mmu_rb_register(void *ops_arg, struct mm_struct *mm,
if (!handlr)
return -ENOMEM;

handlr->root = RB_ROOT;
handlr->root = RB_ROOT_CACHED;
handlr->ops = ops;
handlr->ops_arg = ops_arg;
INIT_HLIST_NODE(&handlr->mn.hlist);
Expand Down Expand Up @@ -149,9 +149,9 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
INIT_LIST_HEAD(&del_list);

spin_lock_irqsave(&handler->lock, flags);
while ((node = rb_first(&handler->root))) {
while ((node = rb_first_cached(&handler->root))) {
rbnode = rb_entry(node, struct mmu_rb_node, node);
rb_erase(node, &handler->root);
rb_erase_cached(node, &handler->root);
/* move from LRU list to delete list */
list_move(&rbnode->list, &del_list);
}
Expand Down Expand Up @@ -300,7 +300,7 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn,
{
struct mmu_rb_handler *handler =
container_of(mn, struct mmu_rb_handler, mn);
struct rb_root *root = &handler->root;
struct rb_root_cached *root = &handler->root;
struct mmu_rb_node *node, *ptr = NULL;
unsigned long flags;
bool added = false;
Expand Down
6 changes: 3 additions & 3 deletions drivers/infiniband/hw/usnic/usnic_uiom.c
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ static void __usnic_uiom_reg_release(struct usnic_uiom_pd *pd,
vpn_last = vpn_start + npages - 1;

spin_lock(&pd->lock);
usnic_uiom_remove_interval(&pd->rb_root, vpn_start,
usnic_uiom_remove_interval(&pd->root, vpn_start,
vpn_last, &rm_intervals);
usnic_uiom_unmap_sorted_intervals(&rm_intervals, pd);

Expand Down Expand Up @@ -379,7 +379,7 @@ struct usnic_uiom_reg *usnic_uiom_reg_get(struct usnic_uiom_pd *pd,
err = usnic_uiom_get_intervals_diff(vpn_start, vpn_last,
(writable) ? IOMMU_WRITE : 0,
IOMMU_WRITE,
&pd->rb_root,
&pd->root,
&sorted_diff_intervals);
if (err) {
usnic_err("Failed disjoint interval vpn [0x%lx,0x%lx] err %d\n",
Expand All @@ -395,7 +395,7 @@ struct usnic_uiom_reg *usnic_uiom_reg_get(struct usnic_uiom_pd *pd,

}

err = usnic_uiom_insert_interval(&pd->rb_root, vpn_start, vpn_last,
err = usnic_uiom_insert_interval(&pd->root, vpn_start, vpn_last,
(writable) ? IOMMU_WRITE : 0);
if (err) {
usnic_err("Failed insert interval vpn [0x%lx,0x%lx] err %d\n",
Expand Down
2 changes: 1 addition & 1 deletion drivers/infiniband/hw/usnic/usnic_uiom.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ struct usnic_uiom_dev {
struct usnic_uiom_pd {
struct iommu_domain *domain;
spinlock_t lock;
struct rb_root rb_root;
struct rb_root_cached root;
struct list_head devs;
int dev_cnt;
};
Expand Down
15 changes: 8 additions & 7 deletions drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ static int interval_cmp(void *priv, struct list_head *a, struct list_head *b)
}

static void
find_intervals_intersection_sorted(struct rb_root *root, unsigned long start,
unsigned long last,
struct list_head *list)
find_intervals_intersection_sorted(struct rb_root_cached *root,
unsigned long start, unsigned long last,
struct list_head *list)
{
struct usnic_uiom_interval_node *node;

Expand All @@ -118,7 +118,7 @@ find_intervals_intersection_sorted(struct rb_root *root, unsigned long start,

int usnic_uiom_get_intervals_diff(unsigned long start, unsigned long last,
int flags, int flag_mask,
struct rb_root *root,
struct rb_root_cached *root,
struct list_head *diff_set)
{
struct usnic_uiom_interval_node *interval, *tmp;
Expand Down Expand Up @@ -175,7 +175,7 @@ void usnic_uiom_put_interval_set(struct list_head *intervals)
kfree(interval);
}

int usnic_uiom_insert_interval(struct rb_root *root, unsigned long start,
int usnic_uiom_insert_interval(struct rb_root_cached *root, unsigned long start,
unsigned long last, int flags)
{
struct usnic_uiom_interval_node *interval, *tmp;
Expand Down Expand Up @@ -246,8 +246,9 @@ int usnic_uiom_insert_interval(struct rb_root *root, unsigned long start,
return err;
}

void usnic_uiom_remove_interval(struct rb_root *root, unsigned long start,
unsigned long last, struct list_head *removed)
void usnic_uiom_remove_interval(struct rb_root_cached *root,
unsigned long start, unsigned long last,
struct list_head *removed)
{
struct usnic_uiom_interval_node *interval;

Expand Down
Loading

0 comments on commit f808c13

Please sign in to comment.