Commit eeb386ae authored by Tony Krowiak's avatar Tony Krowiak Committed by Alexander Gordeev
Browse files

s390/vfio-ap: handle config changed and scan complete notification 



This patch implements two new AP driver callbacks:

void (*on_config_changed)(struct ap_config_info *new_config_info,
                  struct ap_config_info *old_config_info);

void (*on_scan_complete)(struct ap_config_info *new_config_info,
                 struct ap_config_info *old_config_info);

The on_config_changed callback is invoked at the start of the AP bus scan
function when it determines that the host AP configuration information
has changed since the previous scan.

The vfio_ap device driver registers a callback function for this callback
that performs the following operations:

1. Unplugs the adapters, domains and control domains removed from the
host's AP configuration from the guests to which they are
assigned in a single operation.

2. Stores bitmaps identifying the adapters, domains and control domains
added to the host's AP configuration with the structure representing
the mediated device. When the vfio_ap device driver's probe callback is
subsequently invoked, the probe function will recognize that the
queue is being probed due to a change in the host's AP configuration
and the plugging of the queue into the guest will be bypassed.

The on_scan_complete callback is invoked after the ap bus scan is
completed if the host AP configuration data has changed. The vfio_ap
device driver registers a callback function for this callback that hot
plugs each queue and control domain added to the AP configuration for each
guest using them in a single hot plug operation.

Signed-off-by: default avatarTony Krowiak <akrowiak@linux.ibm.com>
Reviewed-by: default avatarJason J. Herne <jjherne@linux.ibm.com>
Signed-off-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
parent f7f795c5

drivers/s390/crypto/vfio_ap_drv.c

+2 −0
Original line number Diff line number Diff line
@@ -47,6 +47,8 @@ static struct ap_driver vfio_ap_drv = { 
	.probe = vfio_ap_mdev_probe_queue,

	.remove = vfio_ap_mdev_remove_queue,

	.in_use = vfio_ap_mdev_resource_in_use,

	.on_config_changed = vfio_ap_on_cfg_changed,

	.on_scan_complete = vfio_ap_on_scan_complete,

	.ids = ap_queue_ids,

};

drivers/s390/crypto/vfio_ap_ops.c

+336 −5
Original line number Diff line number Diff line
@@ -646,16 +646,11 @@ static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) 
static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm,

				       struct ap_matrix_mdev *matrix_mdev)

{

	int ret;

	unsigned long apid, apqi, apqn;

	DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);

	DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);

	struct vfio_ap_queue *q;



	ret = ap_qci(&matrix_dev->info);

	if (ret)

		return false;



	bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);

	bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);

	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
@@ -1980,3 +1975,339 @@ int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm) 


	return ret;

}



/**

 * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control

 *				 domains that have been removed from the host's

 *				 AP configuration from a guest.

 *

 * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.

 * @aprem: the adapters that have been removed from the host's AP configuration

 * @aqrem: the domains that have been removed from the host's AP configuration

 * @cdrem: the control domains that have been removed from the host's AP

 *	   configuration.

 */

static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,

					unsigned long *aprem,

					unsigned long *aqrem,

					unsigned long *cdrem)

{

	int do_hotplug = 0;



	if (!bitmap_empty(aprem, AP_DEVICES)) {

		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm,

					    matrix_mdev->shadow_apcb.apm,

					    aprem, AP_DEVICES);

	}



	if (!bitmap_empty(aqrem, AP_DOMAINS)) {

		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm,

					    matrix_mdev->shadow_apcb.aqm,

					    aqrem, AP_DEVICES);

	}



	if (!bitmap_empty(cdrem, AP_DOMAINS))

		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm,

					    matrix_mdev->shadow_apcb.adm,

					    cdrem, AP_DOMAINS);



	if (do_hotplug)

		vfio_ap_mdev_update_guest_apcb(matrix_mdev);

}



/**

 * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,

 *			     domains and control domains that have been removed

 *			     from the host AP configuration and unplugs them

 *			     from those guests.

 *

 * @ap_remove:	bitmap specifying which adapters have been removed from the host

 *		config.

 * @aq_remove:	bitmap specifying which domains have been removed from the host

 *		config.

 * @cd_remove:	bitmap specifying which control domains have been removed from

 *		the host config.

 */

static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,

				    unsigned long *aq_remove,

				    unsigned long *cd_remove)

{

	struct ap_matrix_mdev *matrix_mdev;

	DECLARE_BITMAP(aprem, AP_DEVICES);

	DECLARE_BITMAP(aqrem, AP_DOMAINS);

	DECLARE_BITMAP(cdrem, AP_DOMAINS);

	int do_remove = 0;



	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {

		mutex_lock(&matrix_mdev->kvm->lock);

		mutex_lock(&matrix_dev->mdevs_lock);



		do_remove |= bitmap_and(aprem, ap_remove,

					  matrix_mdev->matrix.apm,

					  AP_DEVICES);

		do_remove |= bitmap_and(aqrem, aq_remove,

					  matrix_mdev->matrix.aqm,

					  AP_DOMAINS);

		do_remove |= bitmap_andnot(cdrem, cd_remove,

					     matrix_mdev->matrix.adm,

					     AP_DOMAINS);



		if (do_remove)

			vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,

						    cdrem);



		mutex_unlock(&matrix_dev->mdevs_lock);

		mutex_unlock(&matrix_mdev->kvm->lock);

	}

}



/**

 * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and

 *				control domains from the host AP configuration

 *				by unplugging them from the guests that are

 *				using them.

 * @cur_config_info: the current host AP configuration information

 * @prev_config_info: the previous host AP configuration information

 */

static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,

				       struct ap_config_info *prev_config_info)

{

	int do_remove;

	DECLARE_BITMAP(aprem, AP_DEVICES);

	DECLARE_BITMAP(aqrem, AP_DOMAINS);

	DECLARE_BITMAP(cdrem, AP_DOMAINS);



	do_remove = bitmap_andnot(aprem,

				  (unsigned long *)prev_config_info->apm,

				  (unsigned long *)cur_config_info->apm,

				  AP_DEVICES);

	do_remove |= bitmap_andnot(aqrem,

				   (unsigned long *)prev_config_info->aqm,

				   (unsigned long *)cur_config_info->aqm,

				   AP_DEVICES);

	do_remove |= bitmap_andnot(cdrem,

				   (unsigned long *)prev_config_info->adm,

				   (unsigned long *)cur_config_info->adm,

				   AP_DEVICES);



	if (do_remove)

		vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);

}



/**

 * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that

 *				 are older than AP type 10 (CEX4).

 * @apm: a bitmap of the APIDs to examine

 * @aqm: a bitmap of the APQIs of the queues to query for the AP type.

 */

static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)

{

	bool apid_cleared;

	struct ap_queue_status status;

	unsigned long apid, apqi, info;

	int qtype, qtype_mask = 0xff000000;



	for_each_set_bit_inv(apid, apm, AP_DEVICES) {

		apid_cleared = false;



		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {

			status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);

			switch (status.response_code) {

			/*

			 * According to the architecture in each case

			 * below, the queue's info should be filled.

			 */

			case AP_RESPONSE_NORMAL:

			case AP_RESPONSE_RESET_IN_PROGRESS:

			case AP_RESPONSE_DECONFIGURED:

			case AP_RESPONSE_CHECKSTOPPED:

			case AP_RESPONSE_BUSY:

				qtype = info & qtype_mask;



				/*

				 * The vfio_ap device driver only

				 * supports CEX4 and newer adapters, so

				 * remove the APID if the adapter is

				 * older than a CEX4.

				 */

				if (qtype < AP_DEVICE_TYPE_CEX4) {

					clear_bit_inv(apid, apm);

					apid_cleared = true;

				}



				break;



			default:

				/*

				 * If we don't know the adapter type,

				 * clear its APID since it can't be

				 * determined whether the vfio_ap

				 * device driver supports it.

				 */

				clear_bit_inv(apid, apm);

				apid_cleared = true;

				break;

			}



			/*

			 * If we've already cleared the APID from the apm, there

			 * is no need to continue examining the remainin AP

			 * queues to determine the type of the adapter.

			 */

			if (apid_cleared)

				continue;

		}

	}

}



/**

 * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and

 *			  control domains that have been added to the host's

 *			  AP configuration for each matrix mdev to which they

 *			  are assigned.

 *

 * @apm_add: a bitmap specifying the adapters that have been added to the AP

 *	     configuration.

 * @aqm_add: a bitmap specifying the domains that have been added to the AP

 *	     configuration.

 * @adm_add: a bitmap specifying the control domains that have been added to the

 *	     AP configuration.

 */

static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,

				 unsigned long *adm_add)

{

	struct ap_matrix_mdev *matrix_mdev;



	if (list_empty(&matrix_dev->mdev_list))

		return;



	vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);



	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {

		bitmap_and(matrix_mdev->apm_add,

			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);

		bitmap_and(matrix_mdev->aqm_add,

			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);

		bitmap_and(matrix_mdev->adm_add,

			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);

	}

}



/**

 * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and

 *			     control domains to the host AP configuration

 *			     by updating the bitmaps that specify what adapters,

 *			     domains and control domains have been added so they

 *			     can be hot plugged into the guest when the AP bus

 *			     scan completes (see vfio_ap_on_scan_complete

 *			     function).

 * @cur_config_info: the current AP configuration information

 * @prev_config_info: the previous AP configuration information

 */

static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,

				    struct ap_config_info *prev_config_info)

{

	bool do_add;

	DECLARE_BITMAP(apm_add, AP_DEVICES);

	DECLARE_BITMAP(aqm_add, AP_DOMAINS);

	DECLARE_BITMAP(adm_add, AP_DOMAINS);



	do_add = bitmap_andnot(apm_add,

			       (unsigned long *)cur_config_info->apm,

			       (unsigned long *)prev_config_info->apm,

			       AP_DEVICES);

	do_add |= bitmap_andnot(aqm_add,

				(unsigned long *)cur_config_info->aqm,

				(unsigned long *)prev_config_info->aqm,

				AP_DOMAINS);

	do_add |= bitmap_andnot(adm_add,

				(unsigned long *)cur_config_info->adm,

				(unsigned long *)prev_config_info->adm,

				AP_DOMAINS);



	if (do_add)

		vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);

}



/**

 * vfio_ap_on_cfg_changed - handles notification of changes to the host AP

 *			    configuration.

 *

 * @cur_cfg_info: the current host AP configuration

 * @prev_cfg_info: the previous host AP configuration

 */

void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,

			    struct ap_config_info *prev_cfg_info)

{

	if (!cur_cfg_info || !prev_cfg_info)

		return;



	mutex_lock(&matrix_dev->guests_lock);



	vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);

	vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);

	memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));



	mutex_unlock(&matrix_dev->guests_lock);

}



static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)

{

	bool do_hotplug = false;

	int filter_domains = 0;

	int filter_adapters = 0;

	DECLARE_BITMAP(apm, AP_DEVICES);

	DECLARE_BITMAP(aqm, AP_DOMAINS);



	mutex_lock(&matrix_mdev->kvm->lock);

	mutex_lock(&matrix_dev->mdevs_lock);



	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,

				     matrix_mdev->apm_add, AP_DEVICES);

	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,

				    matrix_mdev->aqm_add, AP_DOMAINS);



	if (filter_adapters && filter_domains)

		do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev);

	else if (filter_adapters)

		do_hotplug |=

			vfio_ap_mdev_filter_matrix(apm,

						   matrix_mdev->shadow_apcb.aqm,

						   matrix_mdev);

	else

		do_hotplug |=

			vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm,

						   aqm, matrix_mdev);



	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,

			      AP_DOMAINS))

		do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);



	if (do_hotplug)

		vfio_ap_mdev_update_guest_apcb(matrix_mdev);



	mutex_unlock(&matrix_dev->mdevs_lock);

	mutex_unlock(&matrix_mdev->kvm->lock);

}



void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,

			      struct ap_config_info *old_config_info)

{

	struct ap_matrix_mdev *matrix_mdev;



	mutex_lock(&matrix_dev->guests_lock);



	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {

		if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&

		    bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&

		    bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))

			continue;



		vfio_ap_mdev_hot_plug_cfg(matrix_mdev);

		bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);

		bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);

		bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);

	}



	mutex_unlock(&matrix_dev->guests_lock);

}

drivers/s390/crypto/vfio_ap_private.h

+12 −0
Original line number Diff line number Diff line
@@ -105,6 +105,10 @@ struct ap_queue_table { 
 *		PQAP(AQIC) instruction.

 * @mdev:	the mediated device

 * @qtable:	table of queues (struct vfio_ap_queue) assigned to the mdev

 * @apm_add:	bitmap of APIDs added to the host's AP configuration

 * @aqm_add:	bitmap of APQIs added to the host's AP configuration

 * @adm_add:	bitmap of control domain numbers added to the host's AP

 *		configuration

 */

struct ap_matrix_mdev {

	struct vfio_device vdev;
@@ -116,6 +120,9 @@ struct ap_matrix_mdev { 
	crypto_hook pqap_hook;

	struct mdev_device *mdev;

	struct ap_queue_table qtable;

	DECLARE_BITMAP(apm_add, AP_DEVICES);

	DECLARE_BITMAP(aqm_add, AP_DOMAINS);

	DECLARE_BITMAP(adm_add, AP_DOMAINS);

};



/**
@@ -146,4 +153,9 @@ void vfio_ap_mdev_remove_queue(struct ap_device *queue); 


int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm);



void vfio_ap_on_cfg_changed(struct ap_config_info *new_config_info,

			    struct ap_config_info *old_config_info);

void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,

			      struct ap_config_info *old_config_info);



#endif /* _VFIO_AP_PRIVATE_H_ */