RE: [Patch v4 2/2] PCI: hv: Add support for protocol 1.3 and support PCI_BUS_RELATIONS2

[Long Li]

>Subject: Re: [Patch v4 2/2] PCI: hv: Add support for protocol 1.3 and support
>PCI_BUS_RELATIONS2
>
>On Mon, Jan 13, 2020 at 12:08:21PM -0800, longli@xxxxxxxxxxxxxxxxx wrote:
>> From: Long Li <longli@xxxxxxxxxxxxx>
>>
>> Starting with Hyper-V PCI protocol version 1.3, the host VSP can send
>> PCI_BUS_RELATIONS2 and pass the vNUMA node information for devices
>on
>> the bus. The vNUMA node tells which guest NUMA node this device is on
>> based on guest VM configuration topology and physical device inforamtion.
>
>s/inforamtion/information
>
>Please rebase this series on top of my pci/hv branch, it does not apply, I will
>merge it then.
>
>Thanks,
>Lorenzo

I will send v5.

Thanks,
Long

>
>> Add code to negotiate v1.3 and process PCI_BUS_RELATIONS2.
>>
>> Signed-off-by: Long Li <longli@xxxxxxxxxxxxx>
>> Reviewed-by: Michael Kelley <mikelley@xxxxxxxxxxxxx>
>> ---
>> Changes
>> v2: Changed some spaces to tabs, added put_pcichild() after
>> get_pcichild_wslot(), renamed pci_assign_numa_node() to
>> hv_pci_assign_numa_node()
>> v4: Fixed spelling
>>
>>  drivers/pci/controller/pci-hyperv.c | 109
>> ++++++++++++++++++++++++++++
>>  1 file changed, 109 insertions(+)
>>
>> diff --git a/drivers/pci/controller/pci-hyperv.c
>> b/drivers/pci/controller/pci-hyperv.c
>> index 3b3e1967cf08..147358fae8a2 100644
>> --- a/drivers/pci/controller/pci-hyperv.c
>> +++ b/drivers/pci/controller/pci-hyperv.c
>> @@ -63,6 +63,7 @@
>>  enum pci_protocol_version_t {
>>  	PCI_PROTOCOL_VERSION_1_1 = PCI_MAKE_VERSION(1, 1),	/*
>Win10 */
>>  	PCI_PROTOCOL_VERSION_1_2 = PCI_MAKE_VERSION(1, 2),	/* RS1
>*/
>> +	PCI_PROTOCOL_VERSION_1_3 = PCI_MAKE_VERSION(1, 3),	/*
>Vibranium */
>>  };
>>
>>  #define CPU_AFFINITY_ALL	-1ULL
>> @@ -72,6 +73,7 @@ enum pci_protocol_version_t {
>>   * first.
>>   */
>>  static enum pci_protocol_version_t pci_protocol_versions[] = {
>> +	PCI_PROTOCOL_VERSION_1_3,
>>  	PCI_PROTOCOL_VERSION_1_2,
>>  	PCI_PROTOCOL_VERSION_1_1,
>>  };
>> @@ -124,6 +126,7 @@ enum pci_message_type {
>>  	PCI_RESOURCES_ASSIGNED2		= PCI_MESSAGE_BASE + 0x16,
>>  	PCI_CREATE_INTERRUPT_MESSAGE2	= PCI_MESSAGE_BASE + 0x17,
>>  	PCI_DELETE_INTERRUPT_MESSAGE2	= PCI_MESSAGE_BASE + 0x18,
>/* unused */
>> +	PCI_BUS_RELATIONS2		= PCI_MESSAGE_BASE + 0x19,
>>  	PCI_MESSAGE_MAXIMUM
>>  };
>>
>> @@ -169,6 +172,26 @@ struct pci_function_description {
>>  	u32	ser;	/* serial number */
>>  } __packed;
>>
>> +enum pci_device_description_flags {
>> +	HV_PCI_DEVICE_FLAG_NONE			= 0x0,
>> +	HV_PCI_DEVICE_FLAG_NUMA_AFFINITY	= 0x1,
>> +};
>> +
>> +struct pci_function_description2 {
>> +	u16	v_id;	/* vendor ID */
>> +	u16	d_id;	/* device ID */
>> +	u8	rev;
>> +	u8	prog_intf;
>> +	u8	subclass;
>> +	u8	base_class;
>> +	u32	subsystem_id;
>> +	union	win_slot_encoding win_slot;
>> +	u32	ser;	/* serial number */
>> +	u32	flags;
>> +	u16	virtual_numa_node;
>> +	u16	reserved;
>> +} __packed;
>> +
>>  /**
>>   * struct hv_msi_desc
>>   * @vector:		IDT entry
>> @@ -304,6 +327,12 @@ struct pci_bus_relations {
>>  	struct pci_function_description func[0];  } __packed;
>>
>> +struct pci_bus_relations2 {
>> +	struct pci_incoming_message incoming;
>> +	u32 device_count;
>> +	struct pci_function_description2 func[0]; } __packed;
>> +
>>  struct pci_q_res_req_response {
>>  	struct vmpacket_descriptor hdr;
>>  	s32 status;			/* negative values are failures */
>> @@ -1417,6 +1446,7 @@ static void hv_compose_msi_msg(struct irq_data
>*data, struct msi_msg *msg)
>>  		break;
>>
>>  	case PCI_PROTOCOL_VERSION_1_2:
>> +	case PCI_PROTOCOL_VERSION_1_3:
>>  		size = hv_compose_msi_req_v2(&ctxt.int_pkts.v2,
>>  					dest,
>>  					hpdev->desc.win_slot.slot,
>> @@ -1798,6 +1828,27 @@ static void hv_pci_remove_slots(struct
>hv_pcibus_device *hbus)
>>  	}
>>  }
>>
>> +/*
>> + * Set NUMA node for the devices on the bus  */ static void
>> +hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) {
>> +	struct pci_dev *dev;
>> +	struct pci_bus *bus = hbus->pci_bus;
>> +	struct hv_pci_dev *hv_dev;
>> +
>> +	list_for_each_entry(dev, &bus->devices, bus_list) {
>> +		hv_dev = get_pcichild_wslot(hbus, devfn_to_wslot(dev-
>>devfn));
>> +		if (!hv_dev)
>> +			continue;
>> +
>> +		if (hv_dev->desc.flags &
>HV_PCI_DEVICE_FLAG_NUMA_AFFINITY)
>> +			set_dev_node(&dev->dev, hv_dev-
>>desc.virtual_numa_node);
>> +
>> +		put_pcichild(hv_dev);
>> +	}
>> +}
>> +
>>  /**
>>   * create_root_hv_pci_bus() - Expose a new root PCI bus
>>   * @hbus:	Root PCI bus, as understood by this driver
>> @@ -1820,6 +1871,7 @@ static int create_root_hv_pci_bus(struct
>> hv_pcibus_device *hbus)
>>
>>  	pci_lock_rescan_remove();
>>  	pci_scan_child_bus(hbus->pci_bus);
>> +	hv_pci_assign_numa_node(hbus);
>>  	pci_bus_assign_resources(hbus->pci_bus);
>>  	hv_pci_assign_slots(hbus);
>>  	pci_bus_add_devices(hbus->pci_bus);
>> @@ -2088,6 +2140,7 @@ static void pci_devices_present_work(struct
>work_struct *work)
>>  		 */
>>  		pci_lock_rescan_remove();
>>  		pci_scan_child_bus(hbus->pci_bus);
>> +		hv_pci_assign_numa_node(hbus);
>>  		hv_pci_assign_slots(hbus);
>>  		pci_unlock_rescan_remove();
>>  		break;
>> @@ -2184,6 +2237,46 @@ static void hv_pci_devices_present(struct
>hv_pcibus_device *hbus,
>>  		kfree(dr);
>>  }
>>
>> +/**
>> + * hv_pci_devices_present2() - Handle list of new children
>> + * @hbus:	Root PCI bus, as understood by this driver
>> + * @relations2:	Packet from host listing children
>> + *
>> + * This function is the v2 version of hv_pci_devices_present()  */
>> +static void hv_pci_devices_present2(struct hv_pcibus_device *hbus,
>> +				    struct pci_bus_relations2 *relations) {
>> +	struct hv_dr_state *dr;
>> +	int i;
>> +
>> +	dr = kzalloc(offsetof(struct hv_dr_state, func) +
>> +		     (sizeof(struct hv_pcidev_description) *
>> +		      (relations->device_count)), GFP_NOWAIT);
>> +
>> +	if (!dr)
>> +		return;
>> +
>> +	dr->device_count = relations->device_count;
>> +	for (i = 0; i < dr->device_count; i++) {
>> +		dr->func[i].v_id = relations->func[i].v_id;
>> +		dr->func[i].d_id = relations->func[i].d_id;
>> +		dr->func[i].rev = relations->func[i].rev;
>> +		dr->func[i].prog_intf = relations->func[i].prog_intf;
>> +		dr->func[i].subclass = relations->func[i].subclass;
>> +		dr->func[i].base_class = relations->func[i].base_class;
>> +		dr->func[i].subsystem_id = relations->func[i].subsystem_id;
>> +		dr->func[i].win_slot = relations->func[i].win_slot;
>> +		dr->func[i].ser = relations->func[i].ser;
>> +		dr->func[i].flags = relations->func[i].flags;
>> +		dr->func[i].virtual_numa_node =
>> +			relations->func[i].virtual_numa_node;
>> +	}
>> +
>> +	if (hv_pci_start_relations_work(hbus, dr))
>> +		kfree(dr);
>> +}
>> +
>>  /**
>>   * hv_eject_device_work() - Asynchronously handles ejection
>>   * @work:	Work struct embedded in internal device struct
>> @@ -2289,6 +2382,7 @@ static void hv_pci_onchannelcallback(void
>*context)
>>  	struct pci_response *response;
>>  	struct pci_incoming_message *new_message;
>>  	struct pci_bus_relations *bus_rel;
>> +	struct pci_bus_relations2 *bus_rel2;
>>  	struct pci_dev_inval_block *inval;
>>  	struct pci_dev_incoming *dev_message;
>>  	struct hv_pci_dev *hpdev;
>> @@ -2356,6 +2450,21 @@ static void hv_pci_onchannelcallback(void
>*context)
>>  				hv_pci_devices_present(hbus, bus_rel);
>>  				break;
>>
>> +			case PCI_BUS_RELATIONS2:
>> +
>> +				bus_rel2 = (struct pci_bus_relations2 *)buffer;
>> +				if (bytes_recvd <
>> +				    offsetof(struct pci_bus_relations2, func) +
>> +				    (sizeof(struct pci_function_description2) *
>> +				     (bus_rel2->device_count))) {
>> +					dev_err(&hbus->hdev->device,
>> +						"bus relations v2 too small\n");
>> +					break;
>> +				}
>> +
>> +				hv_pci_devices_present2(hbus, bus_rel2);
>> +				break;
>> +
>>  			case PCI_EJECT:
>>
>>  				dev_message = (struct pci_dev_incoming
>*)buffer;
>> --
>> 2.17.1
>>