From: Joerg Roedel jroedel@suse.de
[ Upstream commit 19c6978fba68a2cdedee7d55fb8c3063d47982d9 ]
The Device Table needs to be updated before the new page-table root can be published in domain->pt_root. Otherwise a concurrent call to fetch_pte might fetch a PTE which is not reachable through the Device Table Entry.
Fixes: 92d420ec028d ("iommu/amd: Relax locking in dma_ops path") Reported-by: Qian Cai cai@lca.pw Signed-off-by: Joerg Roedel jroedel@suse.de Tested-by: Qian Cai cai@lca.pw Link: https://lore.kernel.org/r/20200504125413.16798-5-joro@8bytes.org Signed-off-by: Joerg Roedel jroedel@suse.de Signed-off-by: Sasha Levin sashal@kernel.org --- drivers/iommu/amd_iommu.c | 49 ++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 13 deletions(-)
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 28229a38af4d2..500d0a8c966fc 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -101,6 +101,8 @@ struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); static int protection_domain_init(struct protection_domain *domain); static void detach_device(struct device *dev); +static void update_and_flush_device_table(struct protection_domain *domain, + struct domain_pgtable *pgtable);
/**************************************************************************** * @@ -1461,8 +1463,16 @@ static bool increase_address_space(struct protection_domain *domain,
*pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
- root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode + 1); + pgtable.root = pte; + pgtable.mode += 1; + update_and_flush_device_table(domain, &pgtable); + domain_flush_complete(domain);
+ /* + * Device Table needs to be updated and flushed before the new root can + * be published. + */ + root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode); atomic64_set(&domain->pt_root, root);
ret = true; @@ -1882,19 +1892,17 @@ static bool dma_ops_domain(struct protection_domain *domain) }
static void set_dte_entry(u16 devid, struct protection_domain *domain, + struct domain_pgtable *pgtable, bool ats, bool ppr) { - struct domain_pgtable pgtable; u64 pte_root = 0; u64 flags = 0; u32 old_domid;
- amd_iommu_domain_get_pgtable(domain, &pgtable); + if (pgtable->mode != PAGE_MODE_NONE) + pte_root = iommu_virt_to_phys(pgtable->root);
- if (pgtable.mode != PAGE_MODE_NONE) - pte_root = iommu_virt_to_phys(pgtable.root); - - pte_root |= (pgtable.mode & DEV_ENTRY_MODE_MASK) + pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK) << DEV_ENTRY_MODE_SHIFT; pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV;
@@ -1967,6 +1975,7 @@ static void clear_dte_entry(u16 devid) static void do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { + struct domain_pgtable pgtable; struct amd_iommu *iommu; bool ats;
@@ -1982,7 +1991,9 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_cnt += 1;
/* Update device table */ - set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2); + amd_iommu_domain_get_pgtable(domain, &pgtable); + set_dte_entry(dev_data->devid, domain, &pgtable, + ats, dev_data->iommu_v2); clone_aliases(dev_data->pdev);
device_flush_dte(dev_data); @@ -2293,22 +2304,34 @@ static int amd_iommu_domain_get_attr(struct iommu_domain *domain, * *****************************************************************************/
-static void update_device_table(struct protection_domain *domain) +static void update_device_table(struct protection_domain *domain, + struct domain_pgtable *pgtable) { struct iommu_dev_data *dev_data;
list_for_each_entry(dev_data, &domain->dev_list, list) { - set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled, - dev_data->iommu_v2); + set_dte_entry(dev_data->devid, domain, pgtable, + dev_data->ats.enabled, dev_data->iommu_v2); clone_aliases(dev_data->pdev); } }
+static void update_and_flush_device_table(struct protection_domain *domain, + struct domain_pgtable *pgtable) +{ + update_device_table(domain, pgtable); + domain_flush_devices(domain); +} + static void update_domain(struct protection_domain *domain) { - update_device_table(domain); + struct domain_pgtable pgtable;
- domain_flush_devices(domain); + /* Update device table */ + amd_iommu_domain_get_pgtable(domain, &pgtable); + update_and_flush_device_table(domain, &pgtable); + + /* Flush domain TLB(s) and wait for completion */ domain_flush_tlb_pde(domain); }