Skip to content

Commit

Permalink
Merge branch 'memory-observability' into x86/amd
Browse files Browse the repository at this point in the history
  • Loading branch information
@joergroedel
joergroedel committed Apr 26, 2024
2 parents a4eecd7 + 212c5c0 commit 5dc72c8
Show file tree
Hide file tree
Showing 24 changed files with 360 additions and 202 deletions.
2 changes: 1 addition & 1 deletion Documentation/admin-guide/cgroup-v2.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1432,7 +1432,7 @@ PAGE_SIZE multiple when read back.
sec_pagetables
Amount of memory allocated for secondary page tables,
this currently includes KVM mmu allocations on x86
and arm64.
and arm64 and IOMMU page tables.

percpu (npn)
Amount of memory used for storing per-cpu kernel
Expand Down
4 changes: 2 additions & 2 deletions Documentation/filesystems/proc.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1110,8 +1110,8 @@ KernelStack
PageTables
Memory consumed by userspace page tables
SecPageTables
Memory consumed by secondary page tables, this currently
currently includes KVM mmu allocations on x86 and arm64.
Memory consumed by secondary page tables, this currently includes
KVM mmu and IOMMU allocations on x86 and arm64.
NFS_Unstable
Always zero. Previous counted pages which had been written to
the server, but has not been committed to stable storage.
Expand Down
8 changes: 0 additions & 8 deletions drivers/iommu/amd/amd_iommu.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,14 +162,6 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev)
return PCI_SEG_DEVID_TO_SBDF(seg, devid);
}

static inline void *alloc_pgtable_page(int nid, gfp_t gfp)
{
struct page *page;

page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0);
return page ? page_address(page) : NULL;
}

/*
* This must be called after device probe completes. During probe
* use rlookup_amd_iommu() get the iommu.
Expand Down
86 changes: 40 additions & 46 deletions drivers/iommu/amd/init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

#include "amd_iommu.h"
#include "../irq_remapping.h"
#include "../iommu-pages.h"

/*
* definitions for the ACPI scanning code
Expand Down Expand Up @@ -649,8 +650,8 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_
/* Allocate per PCI segment device table */
static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
{
pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32,
get_order(pci_seg->dev_table_size));
pci_seg->dev_table = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
get_order(pci_seg->dev_table_size));
if (!pci_seg->dev_table)
return -ENOMEM;

Expand All @@ -659,17 +660,16 @@ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)

static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)
{
free_pages((unsigned long)pci_seg->dev_table,
get_order(pci_seg->dev_table_size));
iommu_free_pages(pci_seg->dev_table,
get_order(pci_seg->dev_table_size));
pci_seg->dev_table = NULL;
}

/* Allocate per PCI segment IOMMU rlookup table. */
static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
{
pci_seg->rlookup_table = (void *)__get_free_pages(
GFP_KERNEL | __GFP_ZERO,
get_order(pci_seg->rlookup_table_size));
pci_seg->rlookup_table = iommu_alloc_pages(GFP_KERNEL,
get_order(pci_seg->rlookup_table_size));
if (pci_seg->rlookup_table == NULL)
return -ENOMEM;

Expand All @@ -678,16 +678,15 @@ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)

static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
{
free_pages((unsigned long)pci_seg->rlookup_table,
get_order(pci_seg->rlookup_table_size));
iommu_free_pages(pci_seg->rlookup_table,
get_order(pci_seg->rlookup_table_size));
pci_seg->rlookup_table = NULL;
}

static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
{
pci_seg->irq_lookup_table = (void *)__get_free_pages(
GFP_KERNEL | __GFP_ZERO,
get_order(pci_seg->rlookup_table_size));
pci_seg->irq_lookup_table = iommu_alloc_pages(GFP_KERNEL,
get_order(pci_seg->rlookup_table_size));
kmemleak_alloc(pci_seg->irq_lookup_table,
pci_seg->rlookup_table_size, 1, GFP_KERNEL);
if (pci_seg->irq_lookup_table == NULL)
Expand All @@ -699,17 +698,17 @@ static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_se
static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
{
kmemleak_free(pci_seg->irq_lookup_table);
free_pages((unsigned long)pci_seg->irq_lookup_table,
get_order(pci_seg->rlookup_table_size));
iommu_free_pages(pci_seg->irq_lookup_table,
get_order(pci_seg->rlookup_table_size));
pci_seg->irq_lookup_table = NULL;
}

static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)
{
int i;

pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL,
get_order(pci_seg->alias_table_size));
pci_seg->alias_table = iommu_alloc_pages(GFP_KERNEL,
get_order(pci_seg->alias_table_size));
if (!pci_seg->alias_table)
return -ENOMEM;

Expand All @@ -724,8 +723,8 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)

static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
{
free_pages((unsigned long)pci_seg->alias_table,
get_order(pci_seg->alias_table_size));
iommu_free_pages(pci_seg->alias_table,
get_order(pci_seg->alias_table_size));
pci_seg->alias_table = NULL;
}

Expand All @@ -736,8 +735,8 @@ static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
*/
static int __init alloc_command_buffer(struct amd_iommu *iommu)
{
iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(CMD_BUFFER_SIZE));
iommu->cmd_buf = iommu_alloc_pages(GFP_KERNEL,
get_order(CMD_BUFFER_SIZE));

return iommu->cmd_buf ? 0 : -ENOMEM;
}
Expand Down Expand Up @@ -834,19 +833,19 @@ static void iommu_disable_command_buffer(struct amd_iommu *iommu)

static void __init free_command_buffer(struct amd_iommu *iommu)
{
free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
iommu_free_pages(iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
}

void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
size_t size)
{
int order = get_order(size);
void *buf = (void *)__get_free_pages(gfp, order);
void *buf = iommu_alloc_pages(gfp, order);

if (buf &&
check_feature(FEATURE_SNP) &&
set_memory_4k((unsigned long)buf, (1 << order))) {
free_pages((unsigned long)buf, order);
iommu_free_pages(buf, order);
buf = NULL;
}

Expand All @@ -856,7 +855,7 @@ void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
/* allocates the memory where the IOMMU will log its events to */
static int __init alloc_event_buffer(struct amd_iommu *iommu)
{
iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL,
EVT_BUFFER_SIZE);

return iommu->evt_buf ? 0 : -ENOMEM;
Expand Down Expand Up @@ -890,14 +889,14 @@ static void iommu_disable_event_buffer(struct amd_iommu *iommu)

static void __init free_event_buffer(struct amd_iommu *iommu)
{
free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
iommu_free_pages(iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
}

static void free_ga_log(struct amd_iommu *iommu)
{
#ifdef CONFIG_IRQ_REMAP
free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE));
free_pages((unsigned long)iommu->ga_log_tail, get_order(8));
iommu_free_pages(iommu->ga_log, get_order(GA_LOG_SIZE));
iommu_free_pages(iommu->ga_log_tail, get_order(8));
#endif
}

Expand Down Expand Up @@ -942,13 +941,11 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
return 0;

iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(GA_LOG_SIZE));
iommu->ga_log = iommu_alloc_pages(GFP_KERNEL, get_order(GA_LOG_SIZE));
if (!iommu->ga_log)
goto err_out;

iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(8));
iommu->ga_log_tail = iommu_alloc_pages(GFP_KERNEL, get_order(8));
if (!iommu->ga_log_tail)
goto err_out;

Expand All @@ -961,15 +958,15 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)

static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
{
iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1);
iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1);

return iommu->cmd_sem ? 0 : -ENOMEM;
}

static void __init free_cwwb_sem(struct amd_iommu *iommu)
{
if (iommu->cmd_sem)
free_page((unsigned long)iommu->cmd_sem);
iommu_free_page((void *)iommu->cmd_sem);
}

static void iommu_enable_xt(struct amd_iommu *iommu)
Expand Down Expand Up @@ -1034,7 +1031,6 @@ static bool __copy_device_table(struct amd_iommu *iommu)
u32 lo, hi, devid, old_devtb_size;
phys_addr_t old_devtb_phys;
u16 dom_id, dte_v, irq_v;
gfp_t gfp_flag;
u64 tmp;

/* Each IOMMU use separate device table with the same size */
Expand Down Expand Up @@ -1068,9 +1064,8 @@ static bool __copy_device_table(struct amd_iommu *iommu)
if (!old_devtb)
return false;

gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32;
pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,
get_order(pci_seg->dev_table_size));
pci_seg->old_dev_tbl_cpy = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
get_order(pci_seg->dev_table_size));
if (pci_seg->old_dev_tbl_cpy == NULL) {
pr_err("Failed to allocate memory for copying old device table!\n");
memunmap(old_devtb);
Expand Down Expand Up @@ -2769,8 +2764,8 @@ static void early_enable_iommus(void)

for_each_pci_segment(pci_seg) {
if (pci_seg->old_dev_tbl_cpy != NULL) {
free_pages((unsigned long)pci_seg->old_dev_tbl_cpy,
get_order(pci_seg->dev_table_size));
iommu_free_pages(pci_seg->old_dev_tbl_cpy,
get_order(pci_seg->dev_table_size));
pci_seg->old_dev_tbl_cpy = NULL;
}
}
Expand All @@ -2783,8 +2778,8 @@ static void early_enable_iommus(void)
pr_info("Copied DEV table from previous kernel.\n");

for_each_pci_segment(pci_seg) {
free_pages((unsigned long)pci_seg->dev_table,
get_order(pci_seg->dev_table_size));
iommu_free_pages(pci_seg->dev_table,
get_order(pci_seg->dev_table_size));
pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;
}

Expand Down Expand Up @@ -2989,8 +2984,8 @@ static bool __init check_ioapic_information(void)

static void __init free_dma_resources(void)
{
free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
get_order(MAX_DOMAIN_ID/8));
iommu_free_pages(amd_iommu_pd_alloc_bitmap,
get_order(MAX_DOMAIN_ID / 8));
amd_iommu_pd_alloc_bitmap = NULL;

free_unity_maps();
Expand Down Expand Up @@ -3062,9 +3057,8 @@ static int __init early_amd_iommu_init(void)
/* Device table - directly used by all IOMMUs */
ret = -ENOMEM;

amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
GFP_KERNEL | __GFP_ZERO,
get_order(MAX_DOMAIN_ID/8));
amd_iommu_pd_alloc_bitmap = iommu_alloc_pages(GFP_KERNEL,
get_order(MAX_DOMAIN_ID / 8));
if (amd_iommu_pd_alloc_bitmap == NULL)
goto out;

Expand Down
13 changes: 7 additions & 6 deletions drivers/iommu/amd/io_pgtable.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

#include "amd_iommu_types.h"
#include "amd_iommu.h"
#include "../iommu-pages.h"

static void v1_tlb_flush_all(void *cookie)
{
Expand Down Expand Up @@ -156,7 +157,7 @@ static bool increase_address_space(struct protection_domain *domain,
bool ret = true;
u64 *pte;

pte = alloc_pgtable_page(domain->nid, gfp);
pte = iommu_alloc_page_node(domain->nid, gfp);
if (!pte)
return false;

Expand Down Expand Up @@ -187,7 +188,7 @@ static bool increase_address_space(struct protection_domain *domain,

out:
spin_unlock_irqrestore(&domain->lock, flags);
free_page((unsigned long)pte);
iommu_free_page(pte);

return ret;
}
Expand Down Expand Up @@ -250,7 +251,7 @@ static u64 *alloc_pte(struct protection_domain *domain,

if (!IOMMU_PTE_PRESENT(__pte) ||
pte_level == PAGE_MODE_NONE) {
page = alloc_pgtable_page(domain->nid, gfp);
page = iommu_alloc_page_node(domain->nid, gfp);

if (!page)
return NULL;
Expand All @@ -259,7 +260,7 @@ static u64 *alloc_pte(struct protection_domain *domain,

/* pte could have been changed somewhere. */
if (!try_cmpxchg64(pte, &__pte, __npte))
free_page((unsigned long)page);
iommu_free_page(page);
else if (IOMMU_PTE_PRESENT(__pte))
*updated = true;

Expand Down Expand Up @@ -431,7 +432,7 @@ static int iommu_v1_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
}

/* Everything flushed out, free pages now */
put_pages_list(&freelist);
iommu_put_pages_list(&freelist);

return ret;
}
Expand Down Expand Up @@ -580,7 +581,7 @@ static void v1_free_pgtable(struct io_pgtable *iop)
/* Make changes visible to IOMMUs */
amd_iommu_domain_update(dom);

put_pages_list(&freelist);
iommu_put_pages_list(&freelist);
}

static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
Expand Down
Loading

0 comments on commit 5dc72c8

Please sign in to comment.