Re: [PATCH v3 4/8] vfio/pci: Add support for setting driver data inside core layer

[Abhishek Sahu]

On 5/4/2022 5:50 AM, Jason Gunthorpe wrote:
> On Tue, May 03, 2022 at 11:11:24AM -0600, Alex Williamson wrote:
>>> @@ -1843,6 +1845,17 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
>>>  		return -EBUSY;
>>>  	}
>>>  
>>> +	/*
>>> +	 * The 'struct vfio_pci_core_device' should be the first member of the
>>> +	 * of the structure referenced by 'driver_data' so that it can be
>>> +	 * retrieved with dev_get_drvdata() inside vfio-pci core layer.
>>> +	 */
>>> +	if ((struct vfio_pci_core_device *)driver_data != vdev) {
>>> +		pci_warn(pdev, "Invalid driver data\n");
>>> +		return -EINVAL;
>>> +	}
>>
>> It seems a bit odd to me to add a driver_data arg to the function,
>> which is actually required to point to the same thing as the existing
>> function arg.  Is this just to codify the requirement?  Maybe others
>> can suggest alternatives.

 Yes. It was mainly for enforcing this requirement, otherwise in future
 if someone tries to add new driver (or done changes in the existing
 structure) and does not follow this convention then the pointer will
 be wrong.

>>
>> We also need to collaborate with Jason's patch:
>>
>> https://lore.kernel.org/all/0-v2-0f36bcf6ec1e+64d-vfio_get_from_dev_jgg@xxxxxxxxxx/
>>
>> (and maybe others)
>>
>> If we implement a change like proposed here that vfio-pci-core sets
>> drvdata then we don't need for each variant driver to implement their
>> own wrapper around err_handler or err_detected as Jason proposes in the
>> linked patch.  Thanks,
> 
> Oh, I forgot about this series completely.
> 
> Yes, we need to pick a method, either drvdata always points at the
> core struct, or we wrapper the core functions.
> 
> I have an independent version of the above patch that uses the
> drvdata, but I chucked it because it was unnecessary for just a couple
> of AER functions. 
> 
> We should probably go back to it though if we are adding more
> functions, as the wrapping is a bit repetitive. I'll go and respin
> that series then. Abhishek can base on top of it.
> 

 Sure. I will rebase on top of Jason patch series.

> My approach was more type-sane though:
> 
 This is also fine.

 Initially I wanted to do the same but it requires to have a new
 wrapper function for each driver so I implemented in the core layer.
 
 Thanks,
 Abhishek

> commit 12ba94a72d7aa134af8752d6ff78193acdac93ae
> Author: Jason Gunthorpe <jgg@xxxxxxxx>
> Date:   Tue Mar 29 16:32:32 2022 -0300
> 
>     vfio/pci: Have all VFIO PCI drivers store the vfio_pci_core_device in drvdata
>     
>     Having a consistent pointer in the drvdata will allow the next patch to
>     make use of the drvdata from some of the core code helpers.
>     
>     Use a WARN_ON inside vfio_pci_core_unregister_device() to detect drivers
>     that miss this.
>     
>     Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxx>
> 
> diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> index 767b5d47631a49..665691967a030c 100644
> --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
> @@ -337,6 +337,14 @@ static int vf_qm_cache_wb(struct hisi_qm *qm)
>  	return 0;
>  }
>  
> +static struct hisi_acc_vf_core_device *hssi_acc_drvdata(struct pci_dev *pdev)
> +{
> +	struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
> +
> +	return container_of(core_device, struct hisi_acc_vf_core_device,
> +			    core_device);
> +}
> +
>  static void vf_qm_fun_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev,
>  			    struct hisi_qm *qm)
>  {
> @@ -962,7 +970,7 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
>  
>  static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
>  {
> -	struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(&pdev->dev);
> +	struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev);
>  
>  	if (hisi_acc_vdev->core_device.vdev.migration_flags !=
>  				VFIO_MIGRATION_STOP_COPY)
> @@ -1278,7 +1286,7 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device
>  	if (ret)
>  		goto out_free;
>  
> -	dev_set_drvdata(&pdev->dev, hisi_acc_vdev);
> +	dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device);
>  	return 0;
>  
>  out_free:
> @@ -1289,7 +1297,7 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device
>  
>  static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)
>  {
> -	struct hisi_acc_vf_core_device *hisi_acc_vdev = dev_get_drvdata(&pdev->dev);
> +	struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev);
>  
>  	vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device);
>  	vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device);
> diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c
> index bbec5d288fee97..3391f965abd9f0 100644
> --- a/drivers/vfio/pci/mlx5/main.c
> +++ b/drivers/vfio/pci/mlx5/main.c
> @@ -39,6 +39,14 @@ struct mlx5vf_pci_core_device {
>  	struct mlx5_vf_migration_file *saving_migf;
>  };
>  
> +static struct mlx5vf_pci_core_device *mlx5vf_drvdata(struct pci_dev *pdev)
> +{
> +	struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
> +
> +	return container_of(core_device, struct mlx5vf_pci_core_device,
> +			    core_device);
> +}
> +
>  static struct page *
>  mlx5vf_get_migration_page(struct mlx5_vf_migration_file *migf,
>  			  unsigned long offset)
> @@ -505,7 +513,7 @@ static int mlx5vf_pci_get_device_state(struct vfio_device *vdev,
>  
>  static void mlx5vf_pci_aer_reset_done(struct pci_dev *pdev)
>  {
> -	struct mlx5vf_pci_core_device *mvdev = dev_get_drvdata(&pdev->dev);
> +	struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);
>  
>  	if (!mvdev->migrate_cap)
>  		return;
> @@ -618,7 +626,7 @@ static int mlx5vf_pci_probe(struct pci_dev *pdev,
>  	if (ret)
>  		goto out_free;
>  
> -	dev_set_drvdata(&pdev->dev, mvdev);
> +	dev_set_drvdata(&pdev->dev, &mvdev->core_device);
>  	return 0;
>  
>  out_free:
> @@ -629,7 +637,7 @@ static int mlx5vf_pci_probe(struct pci_dev *pdev,
>  
>  static void mlx5vf_pci_remove(struct pci_dev *pdev)
>  {
> -	struct mlx5vf_pci_core_device *mvdev = dev_get_drvdata(&pdev->dev);
> +	struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);
>  
>  	vfio_pci_core_unregister_device(&mvdev->core_device);
>  	vfio_pci_core_uninit_device(&mvdev->core_device);
> diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
> index 06b6f3594a1316..53ad39d617653d 100644
> --- a/drivers/vfio/pci/vfio_pci_core.c
> +++ b/drivers/vfio/pci/vfio_pci_core.c
> @@ -262,6 +262,10 @@ int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
>  	u16 cmd;
>  	u8 msix_pos;
>  
> +	/* Drivers must set the vfio_pci_core_device to their drvdata */
> +	if (WARN_ON(vdev != dev_get_drvdata(&vdev->pdev->dev)))
> +		return -EINVAL;
> +
>  	vfio_pci_set_power_state(vdev, PCI_D0);
>  
>  	/* Don't allow our initial saved state to include busmaster */