RE: [PATCH v7 3/6] vfio/type1: Update iova list on detach

From: Shameerali Kolothum Thodi
Date: Thu Jul 04 2019 - 08:53:54 EST




> -----Original Message-----
> From: kvm-owner@xxxxxxxxxxxxxxx [mailto:kvm-owner@xxxxxxxxxxxxxxx] On
> Behalf Of Alex Williamson
> Sent: 03 July 2019 21:35
> To: Shameerali Kolothum Thodi <shameerali.kolothum.thodi@xxxxxxxxxx>
> Cc: eric.auger@xxxxxxxxxx; pmorel@xxxxxxxxxxxxxxxxxx;
> kvm@xxxxxxxxxxxxxxx; linux-kernel@xxxxxxxxxxxxxxx;
> iommu@xxxxxxxxxxxxxxxxxxxxxxxxxx; Linuxarm <linuxarm@xxxxxxxxxx>; John
> Garry <john.garry@xxxxxxxxxx>; xuwei (O) <xuwei5@xxxxxxxxxx>;
> kevin.tian@xxxxxxxxx
> Subject: Re: [PATCH v7 3/6] vfio/type1: Update iova list on detach
>
> On Wed, 26 Jun 2019 16:12:45 +0100
> Shameer Kolothum <shameerali.kolothum.thodi@xxxxxxxxxx> wrote:
>
> > Get a copy of iova list on _group_detach and try to update the list.
> > On success replace the current one with the copy. Leave the list as
> > it is if update fails.
> >
> > Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@xxxxxxxxxx>
> > ---
> > drivers/vfio/vfio_iommu_type1.c | 91
> +++++++++++++++++++++++++++++++++
> > 1 file changed, 91 insertions(+)
> >
> > diff --git a/drivers/vfio/vfio_iommu_type1.c
> b/drivers/vfio/vfio_iommu_type1.c
> > index b6bfdfa16c33..e872fb3a0f39 100644
> > --- a/drivers/vfio/vfio_iommu_type1.c
> > +++ b/drivers/vfio/vfio_iommu_type1.c
> > @@ -1873,12 +1873,88 @@ static void vfio_sanity_check_pfn_list(struct
> vfio_iommu *iommu)
> > WARN_ON(iommu->notifier.head);
> > }
> >
> > +/*
> > + * Called when a domain is removed in detach. It is possible that
> > + * the removed domain decided the iova aperture window. Modify the
> > + * iova aperture with the smallest window among existing domains.
> > + */
> > +static void vfio_iommu_aper_expand(struct vfio_iommu *iommu,
> > + struct list_head *iova_copy)
> > +{
> > + struct vfio_domain *domain;
> > + struct iommu_domain_geometry geo;
> > + struct vfio_iova *node;
> > + dma_addr_t start = 0;
> > + dma_addr_t end = (dma_addr_t)~0;
> > +
> > + list_for_each_entry(domain, &iommu->domain_list, next) {
> > + iommu_domain_get_attr(domain->domain,
> DOMAIN_ATTR_GEOMETRY,
> > + &geo);
> > + if (geo.aperture_start > start)
> > + start = geo.aperture_start;
> > + if (geo.aperture_end < end)
> > + end = geo.aperture_end;
> > + }
> > +
> > + /* Modify aperture limits. The new aper is either same or bigger */
> > + node = list_first_entry(iova_copy, struct vfio_iova, list);
> > + node->start = start;
> > + node = list_last_entry(iova_copy, struct vfio_iova, list);
> > + node->end = end;
> > +}
> > +
> > +/*
> > + * Called when a group is detached. The reserved regions for that
> > + * group can be part of valid iova now. But since reserved regions
> > + * may be duplicated among groups, populate the iova valid regions
> > + * list again.
> > + */
> > +static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu,
> > + struct list_head *iova_copy)
> > +{
> > + struct vfio_domain *d;
> > + struct vfio_group *g;
> > + struct vfio_iova *node;
> > + dma_addr_t start, end;
> > + LIST_HEAD(resv_regions);
> > + int ret;
> > +
> > + list_for_each_entry(d, &iommu->domain_list, next) {
> > + list_for_each_entry(g, &d->group_list, next)
> > + iommu_get_group_resv_regions(g->iommu_group,
> > + &resv_regions);
>
> Need to account for failure case here too.

Ok.

> > + }
> > +
> > + if (list_empty(&resv_regions))
> > + return 0;
> > +
> > + node = list_first_entry(iova_copy, struct vfio_iova, list);
> > + start = node->start;
> > + node = list_last_entry(iova_copy, struct vfio_iova, list);
> > + end = node->end;
> > +
> > + /* purge the iova list and create new one */
> > + vfio_iommu_iova_free(iova_copy);
> > +
> > + ret = vfio_iommu_aper_resize(iova_copy, start, end);
> > + if (ret)
> > + goto done;
> > +
> > + /* Exclude current reserved regions from iova ranges */
> > + ret = vfio_iommu_resv_exclude(iova_copy, &resv_regions);
> > +done:
> > + vfio_iommu_resv_free(&resv_regions);
> > + return ret;
> > +}
> > +
> > static void vfio_iommu_type1_detach_group(void *iommu_data,
> > struct iommu_group *iommu_group)
> > {
> > struct vfio_iommu *iommu = iommu_data;
> > struct vfio_domain *domain;
> > struct vfio_group *group;
> > + bool iova_copy_fail;
> > + LIST_HEAD(iova_copy);
> >
> > mutex_lock(&iommu->lock);
> >
> > @@ -1901,6 +1977,12 @@ static void vfio_iommu_type1_detach_group(void
> *iommu_data,
> > }
> > }
> >
> > + /*
> > + * Get a copy of iova list. If success, use copy to update the
> > + * list and to replace the current one.
> > + */
> > + iova_copy_fail = !!vfio_iommu_iova_get_copy(iommu, &iova_copy);
> > +
> > list_for_each_entry(domain, &iommu->domain_list, next) {
> > group = find_iommu_group(domain, iommu_group);
> > if (!group)
> > @@ -1926,10 +2008,19 @@ static void
> vfio_iommu_type1_detach_group(void *iommu_data,
> > iommu_domain_free(domain->domain);
> > list_del(&domain->next);
> > kfree(domain);
> > + if (!iova_copy_fail && !list_empty(&iommu->domain_list))
> > + vfio_iommu_aper_expand(iommu, &iova_copy);
> > }
> > break;
> > }
> >
> > + if (!iova_copy_fail && !list_empty(&iommu->domain_list)) {
> > + if (!vfio_iommu_resv_refresh(iommu, &iova_copy))
> > + vfio_iommu_iova_insert_copy(iommu, &iova_copy);
> > + else
> > + vfio_iommu_iova_free(&iova_copy);
> > + }
>
> The iova_copy_fail and list_empty tests are rather ugly, could we avoid
> them by pushing the tests to the expand and refresh functions? ie. it
> looks like vfio_iommu_aper_expand() could test list_empty(iova_copy),
> the list_for_each on domain_list doesn't need special handling. Same
> for vfio_iommu_resv_refresh(). This would also fix the bug above that
> I think we don't free iova_copy if domain_list becomes empty during
> this operation. Thanks,

Agree. I will change that in next revision.

Thanks,
Shameer