Skip to content

Commit

Permalink
KVM: Move MMU notifier's mmu_lock acquisition into common helper
Browse files Browse the repository at this point in the history 
Acquire and release mmu_lock in the __kvm_handle_hva_range() helper
instead of requiring the caller to do the same.  This paves the way for
future patches to take mmu_lock if and only if an overlapping memslot is
found, without also having to introduce the on_lock() shenanigans used
to manipulate the notifier count and sequence.

No functional change intended.

Signed-off-by: Sean Christopherson <[email protected]>
Message-Id: <[email protected]>
Signed-off-by: Paolo Bonzini <[email protected]>
  • Loading branch information
@sean-jc @bonzini
sean-jc authored and bonzini committed Apr 17, 2021
1 parent b4c5936 commit f922bd9
Showing 1 changed file with 80 additions and 41 deletions.
121 changes: 80 additions & 41 deletions virt/kvm/kvm_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,28 +453,57 @@ static void kvm_mmu_notifier_invalidate_range(struct mmu_notifier *mn,

typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);

typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start,
unsigned long end);

struct kvm_hva_range {
unsigned long start;
unsigned long end;
pte_t pte;
hva_handler_t handler;
on_lock_fn_t on_lock;
bool flush_on_ret;
bool may_block;
};

/*
* Use a dedicated stub instead of NULL to indicate that there is no callback
* function/handler. The compiler technically can't guarantee that a real
* function will have a non-zero address, and so it will generate code to
* check for !NULL, whereas comparing against a stub will be elided at compile
* time (unless the compiler is getting long in the tooth, e.g. gcc 4.9).
*/
static void kvm_null_fn(void)
{

}
#define IS_KVM_NULL_FN(fn) ((fn) == (void *)kvm_null_fn)

static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
const struct kvm_hva_range *range)
{
struct kvm_gfn_range gfn_range;
struct kvm_memory_slot *slot;
struct kvm_memslots *slots;
struct kvm_gfn_range gfn_range;
bool ret = false;
int i, idx;

lockdep_assert_held_write(&kvm->mmu_lock);
/* A null handler is allowed if and only if on_lock() is provided. */
if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) &&
IS_KVM_NULL_FN(range->handler)))
return 0;

KVM_MMU_LOCK(kvm);

idx = srcu_read_lock(&kvm->srcu);

if (!IS_KVM_NULL_FN(range->on_lock)) {
range->on_lock(kvm, range->start, range->end);

if (IS_KVM_NULL_FN(range->handler))
goto out_unlock;
}

for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
slots = __kvm_memslots(kvm, i);
kvm_for_each_memslot(slot, slots) {
Expand Down Expand Up @@ -510,6 +539,9 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
if (range->flush_on_ret && (ret || kvm->tlbs_dirty))
kvm_flush_remote_tlbs(kvm);

out_unlock:
KVM_MMU_UNLOCK(kvm);

srcu_read_unlock(&kvm->srcu, idx);

/* The notifiers are averse to booleans. :-( */
Expand All @@ -528,16 +560,12 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
.end = end,
.pte = pte,
.handler = handler,
.on_lock = (void *)kvm_null_fn,
.flush_on_ret = true,
.may_block = false,
};
int ret;

KVM_MMU_LOCK(kvm);
ret = __kvm_handle_hva_range(kvm, &range);
KVM_MMU_UNLOCK(kvm);

return ret;
return __kvm_handle_hva_range(kvm, &range);
}

static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
Expand All @@ -551,16 +579,12 @@ static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn
.end = end,
.pte = __pte(0),
.handler = handler,
.on_lock = (void *)kvm_null_fn,
.flush_on_ret = false,
.may_block = false,
};
int ret;

KVM_MMU_LOCK(kvm);
ret = __kvm_handle_hva_range(kvm, &range);
KVM_MMU_UNLOCK(kvm);

return ret;
return __kvm_handle_hva_range(kvm, &range);
}
static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
struct mm_struct *mm,
Expand All @@ -581,31 +605,18 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
kvm_handle_hva_range(mn, address, address + 1, pte, kvm_set_spte_gfn);
}

static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
const struct mmu_notifier_range *range)
static void kvm_inc_notifier_count(struct kvm *kvm, unsigned long start,
unsigned long end)
{
struct kvm *kvm = mmu_notifier_to_kvm(mn);
const struct kvm_hva_range hva_range = {
.start = range->start,
.end = range->end,
.pte = __pte(0),
.handler = kvm_unmap_gfn_range,
.flush_on_ret = true,
.may_block = mmu_notifier_range_blockable(range),
};

trace_kvm_unmap_hva_range(range->start, range->end);

KVM_MMU_LOCK(kvm);
/*
* The count increase must become visible at unlock time as no
* spte can be established without taking the mmu_lock and
* count is also read inside the mmu_lock critical section.
*/
kvm->mmu_notifier_count++;
if (likely(kvm->mmu_notifier_count == 1)) {
kvm->mmu_notifier_range_start = range->start;
kvm->mmu_notifier_range_end = range->end;
kvm->mmu_notifier_range_start = start;
kvm->mmu_notifier_range_end = end;
} else {
/*
* Fully tracking multiple concurrent ranges has dimishing
Expand All @@ -617,24 +628,36 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
* complete.
*/
kvm->mmu_notifier_range_start =
min(kvm->mmu_notifier_range_start, range->start);
min(kvm->mmu_notifier_range_start, start);
kvm->mmu_notifier_range_end =
max(kvm->mmu_notifier_range_end, range->end);
max(kvm->mmu_notifier_range_end, end);
}
}

__kvm_handle_hva_range(kvm, &hva_range);
static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
const struct mmu_notifier_range *range)
{
struct kvm *kvm = mmu_notifier_to_kvm(mn);
const struct kvm_hva_range hva_range = {
.start = range->start,
.end = range->end,
.pte = __pte(0),
.handler = kvm_unmap_gfn_range,
.on_lock = kvm_inc_notifier_count,
.flush_on_ret = true,
.may_block = mmu_notifier_range_blockable(range),
};

KVM_MMU_UNLOCK(kvm);
trace_kvm_unmap_hva_range(range->start, range->end);

__kvm_handle_hva_range(kvm, &hva_range);

return 0;
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
const struct mmu_notifier_range *range)
static void kvm_dec_notifier_count(struct kvm *kvm, unsigned long start,
unsigned long end)
{
struct kvm *kvm = mmu_notifier_to_kvm(mn);

KVM_MMU_LOCK(kvm);
/*
* This sequence increase will notify the kvm page fault that
* the page that is going to be mapped in the spte could have
Expand All @@ -648,7 +671,23 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
* in conjunction with the smp_rmb in mmu_notifier_retry().
*/
kvm->mmu_notifier_count--;
KVM_MMU_UNLOCK(kvm);
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
const struct mmu_notifier_range *range)
{
struct kvm *kvm = mmu_notifier_to_kvm(mn);
const struct kvm_hva_range hva_range = {
.start = range->start,
.end = range->end,
.pte = __pte(0),
.handler = (void *)kvm_null_fn,
.on_lock = kvm_dec_notifier_count,
.flush_on_ret = false,
.may_block = mmu_notifier_range_blockable(range),
};

__kvm_handle_hva_range(kvm, &hva_range);

BUG_ON(kvm->mmu_notifier_count < 0);
}
Expand Down

0 comments on commit f922bd9

Please sign in to comment.