Re: [PATCH 3/3 RESEND] Add disk hotswap support to libata

[Lukasz Kosewski]

On 7/26/05, Jeff Garzik <jgarzik@xxxxxxxxx> wrote:
> > +static void pdc2_phy_reset(struct ata_port *ap)
> > +{
> > +     /* As observed on the Promise SATAII150 Tx2 Plus/Tx4, giving the
> > +      * controller a hard reset triggers another hotplug interrupt.  So
> > +      * disable them for the hard reset, and re-enable afterwards.
> > +      *
> > +      * No PATA support here yet
> > +      */
> > +     if (ap->flags & ATA_FLAG_SATA_RESET && ap->flags & ATA_FLAG_SATA) {
> > +             pdc2_disable_channel_hotplug_interrupts(ap);
> > +             pdc_phy_reset(ap);
> > +             pdc2_enable_channel_hotplug_interrupts(ap);
> > +     } else
> > +             pdc_phy_reset(ap);
> > +}
> 
> Don't special case this.  disable/enable hotplug interrupts for all
> controllers.
> 

OK, I've done this in my new patches.

<snip>

> > +     if (mask == 0xffffffff)
> > +             goto try_hotplug;
> 
> you are confusing controller hotplug (mask == 0xffffffff) and device
> hotplug.
> 
> controller hotplug should already be handled correct, by testing
> 0xffffffff and supporting the PCI ->remove hook.
> 

Yes.  Done this as well.

<snip>

> > +     if (plugdata) {
> > +             writeb(plugdata, mmio_base + hotplug_regs);
> > +             for (i = 0; i < host_set->n_ports; ++i) {
> > +                     ap = host_set->ports[i];
> > +                     if (!(ap->flags & ATA_FLAG_SATA))
> > +                             continue;  //No PATA support here... yet
> > +                     // Check unplug flag
> > +                     if (plugdata & 0x01) {
> > +                             ata_hotplug_unplug(ap);
> > +                             handled = 1;
> > +                     } else if ((plugdata >> 4) & 0x01) {  //Check plug flag
> > +                             ata_hotplug_plug(ap);
> > +                             handled = 1;
> > +                     }
> > +                     plugdata >>= 1;
> 
> you probably need a debounce timer; see the SiI email I am about to send.

Keep in mind that I sent three patches:
- 01 - sata_promise properly supporting SATAII150 controllers
- 02 - libata hotswap infrastructure
- 03 - sata_promise supporting the infrastructure

I believe the debounce timer should really go into patch 02.  This is
actually something that's probably needed by every controller... if
you're just plugging drives in/out of a controller, what you're doing
is connecting metal pins, they can be imprecise and trigger a whole
host of interrupts.  Hence, I propose changing patch 02 (attached for
all of your convenience) to be as follows:
- on calling ata_hotplug_unplug:  if a debounce timer is running, stop
it.  Conditionally call ap->ops->unplug_janitor or something like
that, for cases (like the Sil driver, it would seem) that need to do
something special like hard-reset the channel.  Start a new
debounce_timer  (say, with an interval of 500ms, as suggested by the
Sil guys?
- on calling ata_hotplug_plug:  if a debounce_timer is running, stop
it.  Conditionally call ap->ops->plug_janitor, if any driver needs
anything special done on a plug action.  Start a new one.
- when the debounce_timer goes off, the first thing it does is call
ap->ops->hotplug_action, which returns whether it should be plugging
or unplugging.  On a plug, do an ata_bus_probe, an an ata_scsi_plug on
success, ata_scsi_unplug on failure.  On an unplug event,
ata_port_disable and then call ata_scsi_unplug (kill any outstanding
taskfiles).

This means that patch 02 will no longer have a workqueue, but a timer.

Any suggestions/comments on this approach?

Luke Kosewski
21.07.05  Luke Kosewski  <lkosewsk@xxxxxx>

	* A patch to add a general-purpose hotswap framework to libata.  From
	  the point of view of the driver writer, we only have two new API
	  functions; 'ata_hotplug_plug' and 'ata_hotplug_unplug', which have
	  rather self-explanatory functions.
	* A few misc changes are necessary to properly reset flags which are
	  set by devices when they are present to make sure that new devices
	  (for instance, disks with different max UDMA transfer rates, not using
	  LBA48, etc.) being swapped in do not assume values for the old
	  devices.
	* Gratuitous comments here that can probably be removed, so that anyone
	  looking at this can understand this and poke holes in my logic.

Signed-off-by:  Luke Kosewski <lkosewsk@xxxxxx>

--- linux-2.6.13-rc3/drivers/scsi/libata-core.c.old	2005-07-21 13:35:31.609832324 -0400
+++ linux-2.6.13-rc3/drivers/scsi/libata-core.c	2005-07-21 13:42:53.945386060 -0400
@@ -44,7 +44,6 @@
 #include <scsi/scsi_host.h>
 #include <linux/libata.h>
 #include <asm/io.h>
-#include <asm/semaphore.h>
 #include <asm/byteorder.h>
 
 #include "libata.h"
@@ -65,6 +64,7 @@ static void __ata_qc_complete(struct ata
 
 static unsigned int ata_unique_id = 1;
 static struct workqueue_struct *ata_wq;
+static struct workqueue_struct *ata_irq_wq;
 
 MODULE_AUTHOR("Jeff Garzik");
 MODULE_DESCRIPTION("Library module for ATA devices");
@@ -1134,6 +1134,11 @@ static void ata_dev_identify(struct ata_
 		return;
 	}
 
+	/* Necessary if we had an LBA48 drive in, we pulled it out, and put in
+	 * a non-LBA48 drive to replace it.
+	 */
+	dev->flags &= ~ATA_DFLAG_LBA48;
+
 	if (ap->flags & (ATA_FLAG_SRST | ATA_FLAG_SATA_RESET))
 		using_edd = 0;
 	else
@@ -3635,6 +3640,73 @@ idle_irq:
 	return 0;	/* irq not handled */
 }
 
+void ata_check_kill_qc(struct ata_port *ap)
+{
+	struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag);
+
+	if (unlikely(qc)) {
+		/* This is SO bad.  But we can't just run
+		 * ata_qc_complete without doing this, because
+		 * ata_scsi_qc_complete wants to talk to the device,
+		 * and we can't let it do that since it doesn't exist
+		 * anymore.
+		 */
+		ata_scsi_prepare_qc_abort(qc);
+		ata_qc_complete(qc, ATA_ERR);
+	}
+}
+
+static void ata_hotplug_unplug_func(void *_data)
+{
+	struct ata_port *ap = (struct ata_port *)_data;
+	DPRINTK("Got an unplug request on port %d\n", ap->id);
+
+	down(&ap->hotplug_mutex);
+
+	ata_scsi_handle_unplug(ap);
+
+	up(&ap->hotplug_mutex);
+}
+
+static void ata_hotplug_plug_func(void *_data)
+{
+	struct ata_port *ap = (struct ata_port *)_data;
+	DPRINTK("Got a plug request on port %d\n", ap->id);
+
+	down(&ap->hotplug_mutex);
+	/* Pure evil.  Suppose that you have an 'unplug' waiting on your
+	 * queue, and this function executes while it's there (because 
+	 * you unplugged/plugged in a disk on an SMP system VERY FAST).
+	 * REALLY bad news, because when you unplugged your disk, you
+	 * might have had a pending qc which will now sit there and time
+	 * out like the mofo it is.  Check to see if we have one sitting
+	 * around and KILL IT if this is so.
+	 */
+	ata_check_kill_qc(ap);
+	// Observed necessary on some Seagate drives.
+	ap->flags |= ATA_FLAG_SATA_RESET;
+	ap->udma_mask = ap->orig_udma_mask;
+
+	if (ata_bus_probe(ap) /* Does its own locking */)
+		ata_scsi_handle_unplug(ap);  //might be necessary on SMP
+	else
+		ata_scsi_handle_plug(ap);
+	up(&ap->hotplug_mutex);
+}
+
+/* Should be protected by host_set->lock */
+void ata_hotplug_unplug(struct ata_port *ap)
+{
+	ata_port_disable(ap); //disable this NOW, device is gone
+	queue_work(ata_irq_wq, &ap->hotplug_unplug_task);
+}
+
+/* Should be protected by host_set->lock */
+void ata_hotplug_plug(struct ata_port *ap)
+{
+	queue_work(ata_irq_wq, &ap->hotplug_plug_task);
+}
+
 /**
  *	ata_interrupt - Default ATA host interrupt handler
  *	@irq: irq line (unused)
@@ -3860,7 +3932,11 @@ static void ata_host_init(struct ata_por
 	ap->cbl = ATA_CBL_NONE;
 	ap->active_tag = ATA_TAG_POISON;
 	ap->last_ctl = 0xFF;
+	ap->orig_udma_mask = ent->udma_mask;
 
+	init_MUTEX(&ap->hotplug_mutex);
+	INIT_WORK(&ap->hotplug_plug_task, ata_hotplug_plug_func, ap);
+	INIT_WORK(&ap->hotplug_unplug_task, ata_hotplug_unplug_func, ap);
 	INIT_WORK(&ap->packet_task, atapi_packet_task, ap);
 	INIT_WORK(&ap->pio_task, ata_pio_task, ap);
 
@@ -4468,6 +4544,11 @@ static int __init ata_init(void)
 	ata_wq = create_workqueue("ata");
 	if (!ata_wq)
 		return -ENOMEM;
+	ata_irq_wq = create_workqueue("ata_irq");
+	if (!ata_irq_wq) {
+		destroy_workqueue(ata_wq);
+		return -ENOMEM;
+	}
 
 	printk(KERN_DEBUG "libata version " DRV_VERSION " loaded.\n");
 	return 0;
@@ -4476,6 +4557,7 @@ static int __init ata_init(void)
 static void __exit ata_exit(void)
 {
 	destroy_workqueue(ata_wq);
+	destroy_workqueue(ata_irq_wq);
 }
 
 module_init(ata_init);
@@ -4531,6 +4613,8 @@ EXPORT_SYMBOL_GPL(ata_dev_classify);
 EXPORT_SYMBOL_GPL(ata_dev_id_string);
 EXPORT_SYMBOL_GPL(ata_dev_config);
 EXPORT_SYMBOL_GPL(ata_scsi_simulate);
+EXPORT_SYMBOL_GPL(ata_hotplug_unplug);
+EXPORT_SYMBOL_GPL(ata_hotplug_plug);
 
 #ifdef CONFIG_PCI
 EXPORT_SYMBOL_GPL(pci_test_config_bits);
--- linux-2.6.13-rc3/drivers/scsi/libata-scsi.c.old	2005-07-21 13:35:35.622684850 -0400
+++ linux-2.6.13-rc3/drivers/scsi/libata-scsi.c	2005-07-21 13:42:53.950384627 -0400
@@ -1011,6 +1011,53 @@ static int ata_scsi_qc_complete(struct a
 	return 0;
 }
 
+static int ata_scsi_qc_abort(struct ata_queued_cmd *qc, u8 drv_stat)
+{
+	struct scsi_cmnd *cmd = qc->scsicmd;
+
+	cmd->result = SAM_STAT_TASK_ABORTED; //FIXME:  Is this what we want?
+
+	qc->scsidone(cmd);
+
+	return 0;
+}
+
+void ata_scsi_prepare_qc_abort(struct ata_queued_cmd *qc)
+{
+	// For some reason or another, we can't allow a normal scsi_qc_complete
+	if (qc->complete_fn == ata_scsi_qc_complete);
+		qc->complete_fn = ata_scsi_qc_abort;
+}
+
+void ata_scsi_handle_plug(struct ata_port *ap)
+{
+	//Currently SATA only
+	scsi_add_device(ap->host, 0, 0, 0);
+}
+
+void ata_scsi_handle_unplug(struct ata_port *ap)
+{
+	//SATA only, no PATA
+	struct scsi_device *scd = scsi_device_lookup(ap->host, 0, 0, 0);
+	/* scd might not exist; someone did 'echo "scsi remove-single-device
+	 * ... " > /proc/scsi/scsi' or somebody  was turning the key in the
+	 * hotswap bay between on and off really really fast.
+	 */
+	if (scd) {
+		scsi_device_set_state(scd, SDEV_CANCEL);
+		/* We might have a pending qc on I/O to a removed device,
+		 * however, I argue it's impossible unless we have an 'scd'
+		 * because it means we never completed a 'plug' into the system
+		 * (or no device was present on bootup), so either we have no
+		 * possible I/O, or a qc which 'ata_hotplug_plug_func' took
+		 * care of
+		 */
+		ata_check_kill_qc(ap);
+		scsi_remove_device(scd);
+		scsi_device_put(scd);
+	}
+}
+
 /**
  *	ata_scsi_translate - Translate then issue SCSI command to ATA device
  *	@ap: ATA port to which the command is addressed
--- linux-2.6.13-rc3/drivers/scsi/libata.h.old	2005-07-21 13:35:24.217945955 -0400
+++ linux-2.6.13-rc3/drivers/scsi/libata.h	2005-07-21 13:42:53.955383193 -0400
@@ -40,6 +40,7 @@ extern struct ata_queued_cmd *ata_qc_new
 extern void ata_qc_free(struct ata_queued_cmd *qc);
 extern int ata_qc_issue(struct ata_queued_cmd *qc);
 extern int ata_check_atapi_dma(struct ata_queued_cmd *qc);
+extern void ata_check_kill_qc(struct ata_port *ap);
 extern void ata_dev_select(struct ata_port *ap, unsigned int device,
                            unsigned int wait, unsigned int can_sleep);
 extern void ata_tf_to_host_nolock(struct ata_port *ap, struct ata_taskfile *tf);
@@ -76,6 +77,9 @@ extern void ata_scsi_badcmd(struct scsi_
 extern void ata_scsi_rbuf_fill(struct ata_scsi_args *args, 
                         unsigned int (*actor) (struct ata_scsi_args *args,
                                            u8 *rbuf, unsigned int buflen));
+extern void ata_scsi_prepare_qc_abort(struct ata_queued_cmd *qc);
+extern void ata_scsi_handle_plug(struct ata_port *ap);
+extern void ata_scsi_handle_unplug(struct ata_port *ap);
 
 static inline void ata_bad_scsiop(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
 {
--- linux-2.6.13-rc3/include/linux/libata.h.old	2005-07-21 13:35:50.548416576 -0400
+++ linux-2.6.13-rc3/include/linux/libata.h	2005-07-21 13:42:53.960381760 -0400
@@ -29,6 +29,7 @@
 #include <asm/io.h>
 #include <linux/ata.h>
 #include <linux/workqueue.h>
+#include <asm/semaphore.h>
 
 /*
  * compile-time options
@@ -319,6 +320,9 @@ struct ata_port {
 	struct ata_host_stats	stats;
 	struct ata_host_set	*host_set;
 
+	struct semaphore	hotplug_mutex;
+	struct work_struct	hotplug_plug_task;
+	struct work_struct	hotplug_unplug_task;
 	struct work_struct	packet_task;
 
 	struct work_struct	pio_task;
@@ -326,6 +330,8 @@ struct ata_port {
 	unsigned long		pio_task_timeout;
 
 	void			*private_data;
+
+	unsigned int		orig_udma_mask;
 };
 
 struct ata_port_operations {
@@ -402,6 +408,8 @@ extern int ata_scsi_queuecmd(struct scsi
 extern int ata_scsi_error(struct Scsi_Host *host);
 extern int ata_scsi_release(struct Scsi_Host *host);
 extern unsigned int ata_host_intr(struct ata_port *ap, struct ata_queued_cmd *qc);
+extern void ata_hotplug_unplug(struct ata_port *ap);
+extern void ata_hotplug_plug(struct ata_port *ap);
 /*
  * Default driver ops implementations
  */