patch-2.4.4 linux/drivers/usb/uhci.c

• Lines: 2625
• Date: Wed Apr 25 14:10:49 2001
• Orig file: v2.4.3/linux/drivers/usb/uhci.c
• Orig date: Fri Mar 23 11:50:01 2001

diff -u --recursive --new-file v2.4.3/linux/drivers/usb/uhci.c linux/drivers/usb/uhci.c
@@ -1,8 +1,10 @@
 /*
  * Universal Host Controller Interface driver for USB.
  *
+ * Maintainer: Johannes Erdfelt <johannes@erdfelt.com>
+ *
  * (C) Copyright 1999 Linus Torvalds
- * (C) Copyright 1999-2000 Johannes Erdfelt, johannes@erdfelt.com
+ * (C) Copyright 1999-2001 Johannes Erdfelt, johannes@erdfelt.com
  * (C) Copyright 1999 Randy Dunlap
  * (C) Copyright 1999 Georg Acher, acher@in.tum.de
  * (C) Copyright 1999 Deti Fliegl, deti@fliegl.de
@@ -12,7 +14,6 @@
  *               support from usb-ohci.c by Adam Richter, adam@yggdrasil.com).
  * (C) Copyright 1999 Gregory P. Smith (from usb-ohci.c)
  *
- *
  * Intel documents this fairly well, and as far as I know there
  * are no royalties or anything like that, but even so there are
  * people who decided that they want to do the same thing in a
@@ -39,7 +40,12 @@
 #include <linux/unistd.h>
 #include <linux/interrupt.h>
 #include <linux/spinlock.h>
+#include <linux/proc_fs.h>
+#ifdef CONFIG_USB_DEBUG
 #define DEBUG
+#else
+#undef DEBUG
+#endif
 #include <linux/usb.h>
 
 #include <asm/uaccess.h>
@@ -48,23 +54,36 @@
 #include <asm/system.h>
 
 #include "uhci.h"
-#include "uhci-debug.h"
 
 #include <linux/pm.h>
 
+/*
+ * debug = 0, no debugging messages
+ * debug = 1, dump failed URB's except for stalls
+ * debug = 2, dump all failed URB's (including stalls)
+ *            show all queues in /proc/uhci/hc*
+ * debug = 3, show all TD's in URB's when dumping
+ */
+#ifdef DEBUG
 static int debug = 1;
+#else
+static int debug = 0;
+#endif
 MODULE_PARM(debug, "i");
 MODULE_PARM_DESC(debug, "Debug level");
+static char *errbuf;
+#define ERRBUF_LEN    (PAGE_SIZE * 8)
+
+#include "uhci-debug.h"
 
-static kmem_cache_t *uhci_td_cachep;
-static kmem_cache_t *uhci_qh_cachep;
 static kmem_cache_t *uhci_up_cachep;	/* urb_priv */
 
 static int rh_submit_urb(struct urb *urb);
 static int rh_unlink_urb(struct urb *urb);
 static int uhci_get_current_frame_number(struct usb_device *dev);
-static int uhci_unlink_generic(struct urb *urb);
 static int uhci_unlink_urb(struct urb *urb);
+static void uhci_unlink_generic(struct uhci *uhci, struct urb *urb);
+static void uhci_call_completion(struct urb *urb);
 
 static int  ports_active(struct uhci *uhci);
 static void suspend_hc(struct uhci *uhci);
@@ -75,6 +94,8 @@
 /* If a transfer is still active after this much time, turn off FSBR */
 #define IDLE_TIMEOUT	(HZ / 20)	/* 50 ms */
 
+#define MAX_URB_LOOP	2048		/* Maximum number of linked URB's */
+
 /*
  * Only the USB core should call uhci_alloc_dev and uhci_free_dev
  */
@@ -86,80 +107,94 @@
 static int uhci_free_dev(struct usb_device *dev)
 {
 	struct uhci *uhci = (struct uhci *)dev->bus->hcpriv;
-	struct list_head *tmp, *head = &uhci->urb_list;
+	struct list_head list, *tmp, *head;
 	unsigned long flags;
 
 	/* Walk through the entire URB list and forcefully remove any */
 	/*  URBs that are still active for that device */
-	nested_lock(&uhci->urblist_lock, flags);
+
+	/* Two stage unlink so we don't deadlock on urb_list_lock */
+	INIT_LIST_HEAD(&list);
+
+	spin_lock_irqsave(&uhci->urb_list_lock, flags);
+	head = &uhci->urb_list;
 	tmp = head->next;
 	while (tmp != head) {
-		struct urb *u = list_entry(tmp, struct urb, urb_list);
+		struct urb *urb = list_entry(tmp, struct urb, urb_list);
 
 		tmp = tmp->next;
 
-		if (u->dev == dev)
-			uhci_unlink_urb(u);
+		if (urb->dev == dev) {
+			list_del(&urb->urb_list);
+			list_add(&urb->urb_list, &list);
+		}
 	}
-	nested_unlock(&uhci->urblist_lock, flags);
+	spin_unlock_irqrestore(&uhci->urb_list_lock, flags);
 
-	return 0;
-}
+	head = &list;
+	tmp = head->next;
+	while (tmp != head) {
+		struct urb *urb = list_entry(tmp, struct urb, urb_list);
+		tmp = tmp->next;
 
-static void uhci_add_urb_list(struct uhci *uhci, struct urb *urb)
-{
-	unsigned long flags;
+		/* Make sure we block waiting on these to die */
+		urb->transfer_flags &= ~USB_ASYNC_UNLINK;
+
+		/* uhci_unlink_urb will unlink from the temp list */
+		uhci_unlink_urb(urb);
+	}
 
-	nested_lock(&uhci->urblist_lock, flags);
-	list_add(&urb->urb_list, &uhci->urb_list);
-	nested_unlock(&uhci->urblist_lock, flags);
+	return 0;
 }
 
-static void uhci_remove_urb_list(struct uhci *uhci, struct urb *urb)
+static inline void uhci_set_next_interrupt(struct uhci *uhci)
 {
 	unsigned long flags;
 
-	nested_lock(&uhci->urblist_lock, flags);
-	if (!list_empty(&urb->urb_list)) {
-		list_del(&urb->urb_list);
-		INIT_LIST_HEAD(&urb->urb_list);
-	}
-	nested_unlock(&uhci->urblist_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
+	uhci->skel_term_td->status |= TD_CTRL_IOC;
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
-void uhci_set_next_interrupt(struct uhci *uhci)
+static inline void uhci_clear_next_interrupt(struct uhci *uhci)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
-	uhci->skel_term_td.status |= TD_CTRL_IOC;
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
+	uhci->skel_term_td->status &= ~TD_CTRL_IOC;
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
-void uhci_clear_next_interrupt(struct uhci *uhci)
+static inline void uhci_add_complete(struct urb *urb)
 {
+	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
+	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 	unsigned long flags;
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
-	uhci->skel_term_td.status &= ~TD_CTRL_IOC;
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_lock_irqsave(&uhci->complete_list_lock, flags);
+	list_add(&urbp->complete_list, &uhci->complete_list);
+	spin_unlock_irqrestore(&uhci->complete_list_lock, flags);
 }
 
-static struct uhci_td *uhci_alloc_td(struct usb_device *dev)
+static struct uhci_td *uhci_alloc_td(struct uhci *uhci, struct usb_device *dev)
 {
+	dma_addr_t dma_handle;
 	struct uhci_td *td;
 
-	td = kmem_cache_alloc(uhci_td_cachep, in_interrupt() ? SLAB_ATOMIC : SLAB_KERNEL);
+	td = pci_pool_alloc(uhci->td_pool, GFP_DMA | GFP_ATOMIC, &dma_handle);
 	if (!td)
 		return NULL;
 
+	td->dma_handle = dma_handle;
+
 	td->link = UHCI_PTR_TERM;
 	td->buffer = 0;
 
-	td->frameptr = NULL;
-	td->nexttd = td->prevtd = NULL;
+	td->frame = -1;
 	td->dev = dev;
+
 	INIT_LIST_HEAD(&td->list);
+	INIT_LIST_HEAD(&td->fl_list);
 
 	usb_inc_dev_use(dev);
 
@@ -177,20 +212,19 @@
 static void uhci_insert_td(struct uhci *uhci, struct uhci_td *skeltd, struct uhci_td *td)
 {
 	unsigned long flags;
+	struct uhci_td *ltd;
+
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
+	ltd = list_entry(skeltd->fl_list.prev, struct uhci_td, fl_list);
 
-	/* Fix the linked list pointers */
-	td->nexttd = skeltd->nexttd;
-	td->prevtd = skeltd;
-	if (skeltd->nexttd)
-		skeltd->nexttd->prevtd = td;
-	skeltd->nexttd = td;
+	td->link = ltd->link;
+	mb();
+	ltd->link = td->dma_handle;
 
-	td->link = skeltd->link;
-	skeltd->link = virt_to_bus(td);
+	list_add_tail(&td->fl_list, &skeltd->fl_list);
 
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 /*
@@ -200,27 +234,36 @@
  * frame list pointer -> iso td's (if any) ->
  * periodic interrupt td (if frame 0) -> irq td's -> control qh -> bulk qh
  */
-
 static void uhci_insert_td_frame_list(struct uhci *uhci, struct uhci_td *td, unsigned framenum)
 {
 	unsigned long flags;
-	struct uhci_td *nexttd;
 
 	framenum %= UHCI_NUMFRAMES;
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
+
+	td->frame = framenum;
+
+	/* Is there a TD already mapped there? */
+	if (uhci->fl->frame_cpu[framenum]) {
+		struct uhci_td *ftd, *ltd;
+
+		ftd = uhci->fl->frame_cpu[framenum];
+		ltd = list_entry(ftd->fl_list.prev, struct uhci_td, fl_list);
+
+		list_add_tail(&td->fl_list, &ftd->fl_list);
 
-	td->frameptr = &uhci->fl->frame[framenum];
-	td->link = uhci->fl->frame[framenum];
-	if (!(td->link & (UHCI_PTR_TERM | UHCI_PTR_QH))) {
-		nexttd = (struct uhci_td *)uhci_ptr_to_virt(td->link);
-		td->nexttd = nexttd;
-		nexttd->prevtd = td;
-		nexttd->frameptr = NULL;
+		td->link = ltd->link;
+		mb();
+		ltd->link = td->dma_handle;
+	} else {
+		td->link = uhci->fl->frame[framenum];
+		mb();
+		uhci->fl->frame[framenum] = td->dma_handle;
+		uhci->fl->frame_cpu[framenum] = td;
 	}
-	uhci->fl->frame[framenum] = virt_to_bus(td);
 
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 static void uhci_remove_td(struct uhci *uhci, struct uhci_td *td)
@@ -228,29 +271,36 @@
 	unsigned long flags;
 
 	/* If it's not inserted, don't remove it */
-	if (!td->frameptr && !td->prevtd && !td->nexttd)
+	if (td->frame == -1 && list_empty(&td->fl_list))
 		return;
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
-	if (td->frameptr) {
-		*(td->frameptr) = td->link;
-		if (td->nexttd) {
-			td->nexttd->frameptr = td->frameptr;
-			td->nexttd->prevtd = NULL;
-			td->nexttd = NULL;
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
+	if (td->frame != -1 && uhci->fl->frame_cpu[td->frame] == td) {
+		if (list_empty(&td->fl_list)) {
+			uhci->fl->frame[td->frame] = td->link;
+			uhci->fl->frame_cpu[td->frame] = NULL;
+		} else {
+			struct uhci_td *ntd;
+
+			ntd = list_entry(td->fl_list.next, struct uhci_td, fl_list);
+			uhci->fl->frame[td->frame] = ntd->dma_handle;
+			uhci->fl->frame_cpu[td->frame] = ntd;
 		}
-		td->frameptr = NULL;
 	} else {
-		if (td->prevtd) {
-			td->prevtd->nexttd = td->nexttd;
-			td->prevtd->link = td->link;
-		}
-		if (td->nexttd)
-			td->nexttd->prevtd = td->prevtd;
-		td->prevtd = td->nexttd = NULL;
+		struct uhci_td *ptd;
+
+		ptd = list_entry(td->fl_list.prev, struct uhci_td, fl_list);
+		ptd->link = td->link;
 	}
+
+	mb();
 	td->link = UHCI_PTR_TERM;
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+
+	list_del(&td->fl_list);
+	INIT_LIST_HEAD(&td->fl_list);
+	td->frame = -1;
+
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 /*
@@ -260,22 +310,22 @@
 {
 	struct list_head *tmp, *head;
 	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
-	struct uhci_td *td, *prevtd;
+	struct uhci_td *td, *ptd;
 
-	if (!urbp)
+	if (list_empty(&urbp->td_list))
 		return;
 
-	head = &urbp->list;
+	head = &urbp->td_list;
 	tmp = head->next;
-	if (head == tmp)
-		return;
 
+	/* Ordering isn't important here yet since the QH hasn't been */
+	/*  inserted into the schedule yet */
 	td = list_entry(tmp, struct uhci_td, list);
 
 	/* Add the first TD to the QH element pointer */
-	qh->element = virt_to_bus(td) | (breadth ? 0 : UHCI_PTR_DEPTH);
+	qh->element = td->dma_handle | (breadth ? 0 : UHCI_PTR_DEPTH);
 
-	prevtd = td;
+	ptd = td;
 
 	/* Then link the rest of the TD's */
 	tmp = tmp->next;
@@ -284,39 +334,42 @@
 
 		tmp = tmp->next;
 
-		prevtd->link = virt_to_bus(td) | (breadth ? 0 : UHCI_PTR_DEPTH);
+		ptd->link = td->dma_handle | (breadth ? 0 : UHCI_PTR_DEPTH);
 
-		prevtd = td;
+		ptd = td;
 	}
 
-	prevtd->link = UHCI_PTR_TERM;
+	ptd->link = UHCI_PTR_TERM;
 }
 
-static void uhci_free_td(struct uhci_td *td)
+static void uhci_free_td(struct uhci *uhci, struct uhci_td *td)
 {
-	if (!list_empty(&td->list))
+	if (!list_empty(&td->list) || !list_empty(&td->fl_list))
 		dbg("td is still in URB list!");
 
 	if (td->dev)
 		usb_dec_dev_use(td->dev);
 
-	kmem_cache_free(uhci_td_cachep, td);
+	pci_pool_free(uhci->td_pool, td, td->dma_handle);
 }
 
-static struct uhci_qh *uhci_alloc_qh(struct usb_device *dev)
+static struct uhci_qh *uhci_alloc_qh(struct uhci *uhci, struct usb_device *dev)
 {
+	dma_addr_t dma_handle;
 	struct uhci_qh *qh;
 
-	qh = kmem_cache_alloc(uhci_qh_cachep, in_interrupt() ? SLAB_ATOMIC : SLAB_KERNEL);
+	qh = pci_pool_alloc(uhci->qh_pool, GFP_DMA | GFP_ATOMIC, &dma_handle);
 	if (!qh)
 		return NULL;
 
+	qh->dma_handle = dma_handle;
+
 	qh->element = UHCI_PTR_TERM;
 	qh->link = UHCI_PTR_TERM;
 
 	qh->dev = dev;
-	qh->prevqh = qh->nextqh = NULL;
 
+	INIT_LIST_HEAD(&qh->list);
 	INIT_LIST_HEAD(&qh->remove_list);
 
 	usb_inc_dev_use(dev);
@@ -324,181 +377,290 @@
 	return qh;
 }
 
-static void uhci_free_qh(struct uhci_qh *qh)
+static void uhci_free_qh(struct uhci *uhci, struct uhci_qh *qh)
 {
+	if (!list_empty(&qh->list))
+		dbg("qh list not empty!");
+	if (!list_empty(&qh->remove_list))
+		dbg("qh still in remove_list!");
+
 	if (qh->dev)
 		usb_dec_dev_use(qh->dev);
 
-	kmem_cache_free(uhci_qh_cachep, qh);
+	pci_pool_free(uhci->qh_pool, qh, qh->dma_handle);
 }
 
 static void uhci_insert_qh(struct uhci *uhci, struct uhci_qh *skelqh, struct uhci_qh *qh)
 {
+	struct uhci_qh *lqh;
 	unsigned long flags;
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
+
+	/* Grab the last QH */
+	lqh = list_entry(skelqh->list.prev, struct uhci_qh, list);
 
-	/* Fix the linked list pointers */
-	qh->nextqh = skelqh->nextqh;
-	qh->prevqh = skelqh;
-	if (skelqh->nextqh)
-		skelqh->nextqh->prevqh = qh;
-	skelqh->nextqh = qh;
+	qh->link = lqh->link;
+	mb();				/* Ordering is important */
+	lqh->link = qh->dma_handle | UHCI_PTR_QH;
 
-	qh->link = skelqh->link;
-	skelqh->link = virt_to_bus(qh) | UHCI_PTR_QH;
+	list_add_tail(&qh->list, &skelqh->list);
 
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 static void uhci_remove_qh(struct uhci *uhci, struct uhci_qh *qh)
 {
 	unsigned long flags;
-	int delayed;
+	struct uhci_qh *prevqh;
 
-	/* If the QH isn't queued, then we don't need to delay unlink it */
-	delayed = (qh->prevqh || qh->nextqh);
+	/* Only go through the hoops if it's actually linked in */
+	if (list_empty(&qh->list)) {
+		uhci_free_qh(uhci, qh);
+		return;
+	}
+
+	qh->urbp = NULL;
+
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
+
+	prevqh = list_entry(qh->list.prev, struct uhci_qh, list);
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
-	if (qh->prevqh) {
-		qh->prevqh->nextqh = qh->nextqh;
-		qh->prevqh->link = qh->link;
-	}
-	if (qh->nextqh)
-		qh->nextqh->prevqh = qh->prevqh;
-	qh->prevqh = qh->nextqh = NULL;
+	prevqh->link = qh->link;
+	mb();
 	qh->element = qh->link = UHCI_PTR_TERM;
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
 
-	if (delayed) {
-		spin_lock_irqsave(&uhci->qh_remove_lock, flags);
+	list_del(&qh->list);
+	INIT_LIST_HEAD(&qh->list);
 
-		/* Check to see if the remove list is empty */
-		/* Set the IOC bit to force an interrupt so we can remove the QH */
-		if (list_empty(&uhci->qh_remove_list))
-			uhci_set_next_interrupt(uhci);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 
-		/* Add it */
-		list_add(&qh->remove_list, &uhci->qh_remove_list);
+	spin_lock_irqsave(&uhci->qh_remove_list_lock, flags);
 
-		spin_unlock_irqrestore(&uhci->qh_remove_lock, flags);
-	} else
-		uhci_free_qh(qh);
+	/* Check to see if the remove list is empty. Set the IOC bit */
+	/* to force an interrupt so we can remove the QH */
+	if (list_empty(&uhci->qh_remove_list))
+		uhci_set_next_interrupt(uhci);
+
+	list_add(&qh->remove_list, &uhci->qh_remove_list);
+
+	spin_unlock_irqrestore(&uhci->qh_remove_list_lock, flags);
 }
 
-static spinlock_t uhci_append_urb_lock = SPIN_LOCK_UNLOCKED;
+static int uhci_fixup_toggle(struct urb *urb, unsigned int toggle)
+{
+	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
+	struct list_head *head, *tmp;
+
+	head = &urbp->td_list;
+	tmp = head->next;
+	while (head != tmp) {
+		struct uhci_td *td = list_entry(tmp, struct uhci_td, list);
+
+		tmp = tmp->next;
+
+		td->info &= ~(1 << TD_TOKEN_TOGGLE);
+		if (toggle)
+			td->info |= (1 << TD_TOKEN_TOGGLE);
+
+		toggle ^= 1;
+	}
+
+	return toggle;
+}
 
 /* This function will append one URB's QH to another URB's QH. This is for */
-/*  USB_QUEUE_BULK support */
+/*  USB_QUEUE_BULK support for bulk transfers and soon implicitily for */
+/*  control transfers */
 static void uhci_append_queued_urb(struct uhci *uhci, struct urb *eurb, struct urb *urb)
 {
 	struct urb_priv *eurbp, *urbp, *furbp, *lurbp;
 	struct list_head *tmp;
-	struct uhci_td *td, *ltd;
+	struct uhci_td *ftd, *lltd;
 	unsigned long flags;
 
 	eurbp = eurb->hcpriv;
 	urbp = urb->hcpriv;
 
-	spin_lock_irqsave(&uhci_append_urb_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
 
-	/* Find the beginning URB in the queue */
+	/* Find the first URB in the queue */
 	if (eurbp->queued) {
-		struct list_head *head = &eurbp->urb_queue_list;
+		struct list_head *head = &eurbp->queue_list;
 
 		tmp = head->next;
 		while (tmp != head) {
 			struct urb_priv *turbp =
-				list_entry(tmp, struct urb_priv, urb_queue_list);
-
-			tmp = tmp->next;
+				list_entry(tmp, struct urb_priv, queue_list);
 
 			if (!turbp->queued)
 				break;
+
+			tmp = tmp->next;
 		}
 	} else
-		tmp = &eurbp->urb_queue_list;
+		tmp = &eurbp->queue_list;
 
-	furbp = list_entry(tmp, struct urb_priv, urb_queue_list);
+	furbp = list_entry(tmp, struct urb_priv, queue_list);
+	lurbp = list_entry(furbp->queue_list.prev, struct urb_priv, queue_list);
 
-	tmp = furbp->urb_queue_list.prev;
-	lurbp = list_entry(tmp, struct urb_priv, urb_queue_list);
+	lltd = list_entry(lurbp->td_list.prev, struct uhci_td, list);
+	ftd = list_entry(urbp->td_list.next, struct uhci_td, list);
 
-	/* Add this one to the end */
-	list_add_tail(&urbp->urb_queue_list, &furbp->urb_queue_list);
+	uhci_fixup_toggle(urb, uhci_toggle(lltd->info) ^ 1);
 
-	/* Grab the last TD from the last URB */
-	ltd = list_entry(lurbp->list.prev, struct uhci_td, list);
+	mb();			/* Make sure we flush everything */
+	/* Only support bulk right now, so no depth */
+	lltd->link = ftd->dma_handle;
 
-	/* Grab the first TD from the first URB */
-	td = list_entry(urbp->list.next, struct uhci_td, list);
+	list_add_tail(&urbp->queue_list, &furbp->queue_list);
 
-	/* No breadth since this will only be called for bulk transfers */
-	ltd->link = virt_to_bus(td);
+	urbp->queued = 1;
 
-	spin_unlock_irqrestore(&uhci_append_urb_lock, flags);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 static void uhci_delete_queued_urb(struct uhci *uhci, struct urb *urb)
 {
 	struct urb_priv *urbp, *nurbp;
+	struct list_head *head, *tmp;
+	struct urb_priv *purbp;
+	struct uhci_td *pltd;
+	unsigned int toggle;
 	unsigned long flags;
 
 	urbp = urb->hcpriv;
 
-	spin_lock_irqsave(&uhci_append_urb_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
 
-	nurbp = list_entry(urbp->urb_queue_list.next, struct urb_priv,
-			urb_queue_list);
+	if (list_empty(&urbp->queue_list))
+		goto out;
+
+	nurbp = list_entry(urbp->queue_list.next, struct urb_priv, queue_list);
+
+	/* Fix up the toggle for the next URB's */
+	if (!urbp->queued)
+		toggle = usb_gettoggle(urb->dev, usb_pipeendpoint(urb->pipe), usb_pipeout(urb->pipe));
+	else {
+		/* If we're in the middle of the queue, grab the toggle */
+		/*  from the TD previous to us */
+		purbp = list_entry(urbp->queue_list.prev, struct urb_priv,
+				queue_list);
+
+		pltd = list_entry(purbp->td_list.prev, struct uhci_td, list);
+
+		toggle = uhci_toggle(pltd->info) ^ 1;
+	}
+
+	head = &urbp->queue_list;
+	tmp = head->next;
+	while (head != tmp) {
+		struct urb_priv *turbp;
+
+		turbp = list_entry(tmp, struct urb_priv, queue_list);
+
+		tmp = tmp->next;
+
+		if (!turbp->queued)
+			break;
+
+		toggle = uhci_fixup_toggle(turbp->urb, toggle);
+	}
+
+	usb_settoggle(urb->dev, usb_pipeendpoint(urb->pipe),
+		usb_pipeout(urb->pipe), toggle);
 
 	if (!urbp->queued) {
-		/* We're the head, so just insert the QH for the next URB */
-		uhci_insert_qh(uhci, &uhci->skel_bulk_qh, nurbp->qh);
+		int status;
+
+		/*  The HC may continue using the current QH if it finished */
+		/* all of the TD's in this URB and may have started on the */
+		/* next URB's TD's already, so we'll take over ownership */
+		/* of this QH and use it instead. Don't forget to delete */
+		/* the old QH first */
+		uhci_free_qh(uhci, nurbp->qh);
+
+		nurbp->qh = urbp->qh;
+		nurbp->qh->urbp = nurbp;
+		urbp->qh = NULL;
+
+		/* If the last TD from the first (this) urb didn't */
+		/*  complete, reset qh->element to the first TD in the */
+		/*  next urb */
+		pltd = list_entry(urbp->td_list.prev, struct uhci_td, list);
+		status = uhci_status_bits(pltd->status);
+		if ((status & TD_CTRL_ACTIVE) || uhci_actual_length(pltd->status) < uhci_expected_length(pltd->info)) {
+			struct uhci_td *ftd = list_entry(nurbp->td_list.next, struct uhci_td, list);
+			nurbp->qh->element = ftd->dma_handle;
+		}
+
 		nurbp->queued = 0;
 	} else {
-		struct urb_priv *purbp;
-		struct uhci_td *ptd;
-
 		/* We're somewhere in the middle (or end). A bit trickier */
 		/*  than the head scenario */
-		purbp = list_entry(urbp->urb_queue_list.prev, struct urb_priv,
-				urb_queue_list);
+		purbp = list_entry(urbp->queue_list.prev, struct urb_priv,
+				queue_list);
+
+		pltd = list_entry(purbp->td_list.prev, struct uhci_td, list);
+		if (nurbp->queued) {
+			struct uhci_td *nftd;
 
-		ptd = list_entry(purbp->list.prev, struct uhci_td, list);
-		if (nurbp->queued)
 			/* Close the gap between the two */
-			ptd->link = virt_to_bus(list_entry(nurbp->list.next,
-					struct uhci_td, list));
-		else
+			nftd = list_entry(nurbp->td_list.next, struct uhci_td,
+					list);
+			pltd->link = nftd->dma_handle;
+		} else
 			/* The next URB happens to be the beggining, so */
 			/*  we're the last, end the chain */
-			ptd->link = UHCI_PTR_TERM;
-		
+			pltd->link = UHCI_PTR_TERM;
 	}
 
-	list_del(&urbp->urb_queue_list);
+	list_del(&urbp->queue_list);
+	INIT_LIST_HEAD(&urbp->queue_list);
 
-	spin_unlock_irqrestore(&uhci_append_urb_lock, flags);
+out:
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
-struct urb_priv *uhci_alloc_urb_priv(struct urb *urb)
+static struct urb_priv *uhci_alloc_urb_priv(struct uhci *uhci, struct urb *urb)
 {
 	struct urb_priv *urbp;
 
 	urbp = kmem_cache_alloc(uhci_up_cachep, in_interrupt() ? SLAB_ATOMIC : SLAB_KERNEL);
-	if (!urbp)
+	if (!urbp) {
+		err("uhci_alloc_urb_priv: couldn't allocate memory for urb_priv\n");
 		return NULL;
+	}
 
 	memset((void *)urbp, 0, sizeof(*urbp));
 
 	urbp->inserttime = jiffies;
 	urbp->urb = urb;
+	urbp->dev = urb->dev;
 	
-	INIT_LIST_HEAD(&urbp->list);
-	INIT_LIST_HEAD(&urbp->urb_queue_list);
+	INIT_LIST_HEAD(&urbp->td_list);
+	INIT_LIST_HEAD(&urbp->queue_list);
+	INIT_LIST_HEAD(&urbp->complete_list);
 
 	urb->hcpriv = urbp;
 
+	if (urb->transfer_buffer_length) {
+		urbp->transfer_buffer_dma_handle = pci_map_single(uhci->dev,
+			urb->transfer_buffer, urb->transfer_buffer_length,
+			usb_pipein(urb->pipe) ? PCI_DMA_FROMDEVICE :
+			PCI_DMA_TODEVICE);
+		if (!urbp->transfer_buffer_dma_handle)
+			return NULL;
+	}
+
+	if (usb_pipetype(urb->pipe) == PIPE_CONTROL && urb->setup_packet) {
+		urbp->setup_packet_dma_handle = pci_map_single(uhci->dev,
+			urb->setup_packet, sizeof(devrequest),
+			PCI_DMA_TODEVICE);
+		if (!urbp->setup_packet_dma_handle)
+			return NULL;
+	}
+
 	return urbp;
 }
 
@@ -508,13 +670,11 @@
 
 	td->urb = urb;
 
-	list_add_tail(&td->list, &urbp->list);
+	list_add_tail(&td->list, &urbp->td_list);
 }
 
-static void uhci_remove_td_from_urb(struct urb *urb, struct uhci_td *td)
+static void uhci_remove_td_from_urb(struct uhci_td *td)
 {
-	urb = NULL;	/* No warnings */
-
 	if (list_empty(&td->list))
 		return;
 
@@ -526,41 +686,55 @@
 
 static void uhci_destroy_urb_priv(struct urb *urb)
 {
-	struct list_head *tmp, *head;
+	struct list_head *head, *tmp;
 	struct urb_priv *urbp;
 	struct uhci *uhci;
-	struct uhci_td *td;
 	unsigned long flags;
 
 	spin_lock_irqsave(&urb->lock, flags);
 
 	urbp = (struct urb_priv *)urb->hcpriv;
 	if (!urbp)
-		goto unlock;
+		goto out;
 
-	if (!urb->dev || !urb->dev->bus || !urb->dev->bus->hcpriv)
-		goto unlock;
+	if (!urbp->dev || !urbp->dev->bus || !urbp->dev->bus->hcpriv) {
+		warn("uhci_destroy_urb_priv: urb %p belongs to disconnected device or bus?", urb);
+		goto out;
+	}
+
+	if (!list_empty(&urb->urb_list))
+		warn("uhci_destroy_urb_priv: urb %p still on uhci->urb_list or uhci->remove_list", urb);
 
-	uhci = urb->dev->bus->hcpriv;
+	if (!list_empty(&urbp->complete_list))
+		warn("uhci_destroy_urb_priv: urb %p still on uhci->complete_list", urb);
 
-	head = &urbp->list;
+	uhci = urbp->dev->bus->hcpriv;
+
+	head = &urbp->td_list;
 	tmp = head->next;
 	while (tmp != head) {
-		td = list_entry(tmp, struct uhci_td, list);
+		struct uhci_td *td = list_entry(tmp, struct uhci_td, list);
 
 		tmp = tmp->next;
 
-		uhci_remove_td_from_urb(urb, td);
-
+		uhci_remove_td_from_urb(td);
 		uhci_remove_td(uhci, td);
-
-		uhci_free_td(td);
+		uhci_free_td(uhci, td);
 	}
 
+	if (urb->setup_packet)
+		pci_unmap_single(uhci->dev, urbp->setup_packet_dma_handle,
+			sizeof(devrequest), PCI_DMA_TODEVICE);
+
+	if (urb->transfer_buffer_length)
+		pci_unmap_single(uhci->dev, urbp->transfer_buffer_dma_handle,
+			urb->transfer_buffer_length, usb_pipein(urb->pipe) ?
+			PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE);
+
 	urb->hcpriv = NULL;
 	kmem_cache_free(uhci_up_cachep, urbp);
 
-unlock:
+out:
 	spin_unlock_irqrestore(&urb->lock, flags);
 }
 
@@ -569,18 +743,15 @@
 	unsigned long flags;
 	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 
-	if (!urbp)
-		return;
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
 
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
-
-	if ((!(urb->transfer_flags & USB_NO_FSBR)) && (!urbp->fsbr)) {
+	if ((!(urb->transfer_flags & USB_NO_FSBR)) && !urbp->fsbr) {
 		urbp->fsbr = 1;
 		if (!uhci->fsbr++)
-			uhci->skel_term_qh.link = virt_to_bus(&uhci->skel_hs_control_qh) | UHCI_PTR_QH;
+			uhci->skel_term_qh->link = uhci->skel_hs_control_qh->dma_handle | UHCI_PTR_QH;
 	}
 
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 static void uhci_dec_fsbr(struct uhci *uhci, struct urb *urb)
@@ -588,18 +759,15 @@
 	unsigned long flags;
 	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 
-	if (!urbp)
-		return;
-
-	spin_lock_irqsave(&uhci->framelist_lock, flags);
+	spin_lock_irqsave(&uhci->frame_list_lock, flags);
 
 	if ((!(urb->transfer_flags & USB_NO_FSBR)) && urbp->fsbr) {
 		urbp->fsbr = 0;
 		if (!--uhci->fsbr)
-			uhci->skel_term_qh.link = UHCI_PTR_TERM;
+			uhci->skel_term_qh->link = UHCI_PTR_TERM;
 	}
 
-	spin_unlock_irqrestore(&uhci->framelist_lock, flags);
+	spin_unlock_irqrestore(&uhci->frame_list_lock, flags);
 }
 
 /*
@@ -646,7 +814,7 @@
 	unsigned long destination, status;
 	int maxsze = usb_maxpacket(urb->dev, urb->pipe, usb_pipeout(urb->pipe));
 	int len = urb->transfer_buffer_length;
-	unsigned char *data = urb->transfer_buffer;
+	dma_addr_t data = urbp->transfer_buffer_dma_handle;
 
 	/* The "pipe" thing contains the destination in bits 8--18 */
 	destination = (urb->pipe & PIPE_DEVEP_MASK) | USB_PID_SETUP;
@@ -657,13 +825,13 @@
 	/*
 	 * Build the TD for the control request
 	 */
-	td = uhci_alloc_td(urb->dev);
+	td = uhci_alloc_td(uhci, urb->dev);
 	if (!td)
 		return -ENOMEM;
 
 	uhci_add_td_to_urb(urb, td);
 	uhci_fill_td(td, status, destination | (7 << 21),
-		virt_to_bus(urb->setup_packet));
+		urbp->setup_packet_dma_handle);
 
 	/*
 	 * If direction is "send", change the frame from SETUP (0x2D)
@@ -683,7 +851,7 @@
 		if (pktsze > maxsze)
 			pktsze = maxsze;
 
-		td = uhci_alloc_td(urb->dev);
+		td = uhci_alloc_td(uhci, urb->dev);
 		if (!td)
 			return -ENOMEM;
 
@@ -692,7 +860,7 @@
 	
 		uhci_add_td_to_urb(urb, td);
 		uhci_fill_td(td, status, destination | ((pktsze - 1) << 21),
-			virt_to_bus(data));
+			data);
 
 		data += pktsze;
 		len -= pktsze;
@@ -701,7 +869,7 @@
 	/*
 	 * Build the final TD for control status 
 	 */
-	td = uhci_alloc_td(urb->dev);
+	td = uhci_alloc_td(uhci, urb->dev);
 	if (!td)
 		return -ENOMEM;
 
@@ -723,23 +891,22 @@
 	uhci_fill_td(td, status | TD_CTRL_IOC,
 		destination | (UHCI_NULL_DATA_SIZE << 21), 0);
 
-	qh = uhci_alloc_qh(urb->dev);
+	qh = uhci_alloc_qh(uhci, urb->dev);
 	if (!qh)
 		return -ENOMEM;
 
 	/* Low speed or small transfers gets a different queue and treatment */
 	if (urb->pipe & TD_CTRL_LS) {
 		uhci_insert_tds_in_qh(qh, urb, 0);
-		uhci_insert_qh(uhci, &uhci->skel_ls_control_qh, qh);
+		uhci_insert_qh(uhci, uhci->skel_ls_control_qh, qh);
 	} else {
 		uhci_insert_tds_in_qh(qh, urb, 1);
-		uhci_insert_qh(uhci, &uhci->skel_hs_control_qh, qh);
+		uhci_insert_qh(uhci, uhci->skel_hs_control_qh, qh);
 		uhci_inc_fsbr(uhci, urb);
 	}
 
 	urbp->qh = qh;
-
-	uhci_add_urb_list(uhci, urb);
+	qh->urbp = urbp;
 
 	return -EINPROGRESS;
 }
@@ -754,12 +921,10 @@
 	unsigned int status;
 	int ret = 0;
 
-	if (!urbp)
+	if (list_empty(&urbp->td_list))
 		return -EINVAL;
 
-	head = &urbp->list;
-	if (head->next == head)
-		return -EINVAL;
+	head = &urbp->td_list;
 
 	if (urbp->short_control_packet) {
 		tmp = head->prev;
@@ -849,12 +1014,16 @@
 	    			uhci_packetout(td->info));
 
 err:
-	if (debug && ret != -EPIPE) {
+	if ((debug == 1 && ret != -EPIPE) || debug > 1) {
 		/* Some debugging code */
 		dbg("uhci_result_control() failed with status %x", status);
 
-		/* Print the chain for debugging purposes */
-		uhci_show_urb_queue(urb);
+		if (errbuf) {
+			/* Print the chain for debugging purposes */
+			uhci_show_qh(urbp->qh, errbuf, ERRBUF_LEN, 0);
+
+			lprintk(errbuf);
+		}
 	}
 
 	return ret;
@@ -872,34 +1041,34 @@
 	uhci_remove_qh(uhci, urbp->qh);
 
 	/* Delete all of the TD's except for the status TD at the end */
-	head = &urbp->list;
+	head = &urbp->td_list;
 	tmp = head->next;
 	while (tmp != head && tmp->next != head) {
 		struct uhci_td *td = list_entry(tmp, struct uhci_td, list);
 
 		tmp = tmp->next;
 
-		uhci_remove_td_from_urb(urb, td);
-
+		uhci_remove_td_from_urb(td);
 		uhci_remove_td(uhci, td);
-
-		uhci_free_td(td);
+		uhci_free_td(uhci, td);
 	}
 
-	urbp->qh = uhci_alloc_qh(urb->dev);
+	urbp->qh = uhci_alloc_qh(uhci, urb->dev);
 	if (!urbp->qh) {
 		err("unable to allocate new QH for control retrigger");
 		return -ENOMEM;
 	}
 
+	urbp->qh->urbp = urbp;
+
 	/* One TD, who cares about Breadth first? */
 	uhci_insert_tds_in_qh(urbp->qh, urb, 0);
 
 	/* Low speed or small transfers gets a different queue and treatment */
 	if (urb->pipe & TD_CTRL_LS)
-		uhci_insert_qh(uhci, &uhci->skel_ls_control_qh, urbp->qh);
+		uhci_insert_qh(uhci, uhci->skel_ls_control_qh, urbp->qh);
 	else
-		uhci_insert_qh(uhci, &uhci->skel_hs_control_qh, urbp->qh);
+		uhci_insert_qh(uhci, uhci->skel_hs_control_qh, urbp->qh);
 
 	return -EINPROGRESS;
 }
@@ -912,6 +1081,7 @@
 	struct uhci_td *td;
 	unsigned long destination, status;
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
+	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 
 	if (urb->transfer_buffer_length > usb_maxpacket(urb->dev, urb->pipe, usb_pipeout(urb->pipe)))
 		return -EINVAL;
@@ -921,7 +1091,7 @@
 
 	status = (urb->pipe & TD_CTRL_LS) | TD_CTRL_ACTIVE | TD_CTRL_IOC;
 
-	td = uhci_alloc_td(urb->dev);
+	td = uhci_alloc_td(uhci, urb->dev);
 	if (!td)
 		return -ENOMEM;
 
@@ -931,12 +1101,9 @@
 	usb_dotoggle(urb->dev, usb_pipeendpoint(urb->pipe), usb_pipeout(urb->pipe));
 
 	uhci_add_td_to_urb(urb, td);
-	uhci_fill_td(td, status, destination,
-		virt_to_bus(urb->transfer_buffer));
-
-	uhci_insert_td(uhci, &uhci->skeltd[__interval_to_skel(urb->interval)], td);
+	uhci_fill_td(td, status, destination, urbp->transfer_buffer_dma_handle);
 
-	uhci_add_urb_list(uhci, urb);
+	uhci_insert_td(uhci, uhci->skeltd[__interval_to_skel(urb->interval)], td);
 
 	return -EINPROGRESS;
 }
@@ -949,12 +1116,9 @@
 	unsigned int status;
 	int ret = 0;
 
-	if (!urbp)
-		return -EINVAL;
-
 	urb->actual_length = 0;
 
-	head = &urbp->list;
+	head = &urbp->td_list;
 	tmp = head->next;
 	while (tmp != head) {
 		td = list_entry(tmp, struct uhci_td, list);
@@ -1000,16 +1164,20 @@
 	    			uhci_packetout(td->info));
 
 err:
-	if (debug && ret != -EPIPE) {
+	if ((debug == 1 && ret != -EPIPE) || debug > 1) {
 		/* Some debugging code */
 		dbg("uhci_result_interrupt/bulk() failed with status %x",
 			status);
 
-		/* Print the chain for debugging purposes */
-		if (urbp->qh)
-			uhci_show_urb_queue(urb);
-		else
-			uhci_show_td(td);
+		if (errbuf) {
+			/* Print the chain for debugging purposes */
+			if (urbp->qh)
+				uhci_show_qh(urbp->qh, errbuf, ERRBUF_LEN, 0);
+			else
+				uhci_show_td(td, errbuf, ERRBUF_LEN, 0);
+
+			lprintk(errbuf);
+		}
 	}
 
 	return ret;
@@ -1017,24 +1185,28 @@
 
 static void uhci_reset_interrupt(struct urb *urb)
 {
-	struct list_head *tmp;
+	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
 	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 	struct uhci_td *td;
+	unsigned long flags;
 
-	if (!urbp)
-		return;
+	spin_lock_irqsave(&urb->lock, flags);
 
-	tmp = urbp->list.next;
-	td = list_entry(tmp, struct uhci_td, list);
-	if (!td)
-		return;
+	/* Root hub is special */
+	if (urb->dev == uhci->rh.dev)
+		goto out;
+
+	td = list_entry(urbp->td_list.next, struct uhci_td, list);
 
 	td->status = (td->status & 0x2F000000) | TD_CTRL_ACTIVE | TD_CTRL_IOC;
 	td->info &= ~(1 << TD_TOKEN_TOGGLE);
 	td->info |= (usb_gettoggle(urb->dev, usb_pipeendpoint(urb->pipe), usb_pipeout(urb->pipe)) << TD_TOKEN_TOGGLE);
 	usb_dotoggle(urb->dev, usb_pipeendpoint(urb->pipe), usb_pipeout(urb->pipe));
 
+out:
 	urb->status = -EINPROGRESS;
+
+	spin_unlock_irqrestore(&urb->lock, flags);
 }
 
 /*
@@ -1048,8 +1220,8 @@
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
 	int maxsze = usb_maxpacket(urb->dev, urb->pipe, usb_pipeout(urb->pipe));
 	int len = urb->transfer_buffer_length;
-	unsigned char *data = urb->transfer_buffer;
 	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
+	dma_addr_t data = urbp->transfer_buffer_dma_handle;
 
 	if (len < 0)
 		return -EINVAL;
@@ -1076,7 +1248,7 @@
 		if (pktsze > maxsze)
 			pktsze = maxsze;
 
-		td = uhci_alloc_td(urb->dev);
+		td = uhci_alloc_td(uhci, urb->dev);
 		if (!td)
 			return -ENOMEM;
 
@@ -1084,7 +1256,7 @@
 		uhci_fill_td(td, status, destination | ((pktsze - 1) << 21) |
 			(usb_gettoggle(urb->dev, usb_pipeendpoint(urb->pipe),
 			 usb_pipeout(urb->pipe)) << TD_TOKEN_TOGGLE),
-			virt_to_bus(data));
+			data);
 
 		data += pktsze;
 		len -= maxsze;
@@ -1096,22 +1268,20 @@
 			usb_pipeout(urb->pipe));
 	} while (len > 0);
 
-	qh = uhci_alloc_qh(urb->dev);
+	qh = uhci_alloc_qh(uhci, urb->dev);
 	if (!qh)
 		return -ENOMEM;
 
 	urbp->qh = qh;
+	qh->urbp = urbp;
 
-	/* Always assume depth first */
+	/* Always assume breadth first */
 	uhci_insert_tds_in_qh(qh, urb, 1);
 
-	if (urb->transfer_flags & USB_QUEUE_BULK && eurb) {
-		urbp->queued = 1;
+	if (urb->transfer_flags & USB_QUEUE_BULK && eurb)
 		uhci_append_queued_urb(uhci, eurb, urb);
-	} else
-		uhci_insert_qh(uhci, &uhci->skel_bulk_qh, qh);
-
-	uhci_add_urb_list(uhci, urb);
+	else
+		uhci_insert_qh(uhci, uhci->skel_bulk_qh, qh);
 
 	uhci_inc_fsbr(uhci, urb);
 
@@ -1128,11 +1298,12 @@
 {
 	struct urb *last_urb = NULL;
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
-	struct list_head *tmp, *head = &uhci->urb_list;
+	struct list_head *tmp, *head;
 	int ret = 0;
 	unsigned long flags;
 
-	nested_lock(&uhci->urblist_lock, flags);
+	spin_lock_irqsave(&uhci->urb_list_lock, flags);
+	head = &uhci->urb_list;
 	tmp = head->next;
 	while (tmp != head) {
 		struct urb *u = list_entry(tmp, struct urb, urb_list);
@@ -1154,7 +1325,7 @@
 	} else
 		ret = -1;	/* no previous urb found */
 
-	nested_unlock(&uhci->urblist_lock, flags);
+	spin_unlock_irqrestore(&uhci->urb_list_lock, flags);
 
 	return ret;
 }
@@ -1185,12 +1356,16 @@
 	return 0;
 }
 
+/*
+ * Isochronous transfers
+ */
 static int uhci_submit_isochronous(struct urb *urb)
 {
 	struct uhci_td *td;
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
 	int i, ret, framenum;
 	int status, destination;
+	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 
 	status = TD_CTRL_ACTIVE | TD_CTRL_IOS;
 	destination = (urb->pipe & PIPE_DEVEP_MASK) | usb_packetid(urb->pipe);
@@ -1204,13 +1379,13 @@
 		if (!urb->iso_frame_desc[i].length)
 			continue;
 
-		td = uhci_alloc_td(urb->dev);
+		td = uhci_alloc_td(uhci, urb->dev);
 		if (!td)
 			return -ENOMEM;
 
 		uhci_add_td_to_urb(urb, td);
 		uhci_fill_td(td, status, destination | ((urb->iso_frame_desc[i].length - 1) << 21),
-			virt_to_bus(urb->transfer_buffer + urb->iso_frame_desc[i].offset));
+			urbp->transfer_buffer_dma_handle + urb->iso_frame_desc[i].offset);
 
 		if (i + 1 >= urb->number_of_packets)
 			td->status |= TD_CTRL_IOC;
@@ -1218,8 +1393,6 @@
 		uhci_insert_td_frame_list(uhci, td, framenum);
 	}
 
-	uhci_add_urb_list(uhci, urb);
-
 	return -EINPROGRESS;
 }
 
@@ -1230,13 +1403,10 @@
 	int status;
 	int i, ret = 0;
 
-	if (!urbp)
-		return -EINVAL;
-
 	urb->actual_length = 0;
 
 	i = 0;
-	head = &urbp->list;
+	head = &urbp->td_list;
 	tmp = head->next;
 	while (tmp != head) {
 		struct uhci_td *td = list_entry(tmp, struct uhci_td, list);
@@ -1253,7 +1423,7 @@
 
 		status = uhci_map_status(uhci_status_bits(td->status), usb_pipeout(urb->pipe));
 		urb->iso_frame_desc[i].status = status;
-		if (status != 0) {
+		if (status) {
 			urb->error_count++;
 			ret = status;
 		}
@@ -1266,28 +1436,30 @@
 
 static struct urb *uhci_find_urb_ep(struct uhci *uhci, struct urb *urb)
 {
-	struct list_head *tmp, *head = &uhci->urb_list;
+	struct list_head *tmp, *head;
 	unsigned long flags;
 	struct urb *u = NULL;
 
+	/* We don't match Isoc transfers since they are special */
 	if (usb_pipeisoc(urb->pipe))
 		return NULL;
 
-	nested_lock(&uhci->urblist_lock, flags);
+	spin_lock_irqsave(&uhci->urb_list_lock, flags);
+	head = &uhci->urb_list;
 	tmp = head->next;
 	while (tmp != head) {
 		u = list_entry(tmp, struct urb, urb_list);
 
 		tmp = tmp->next;
 
-		if (u->dev == urb->dev &&
-		    u->pipe == urb->pipe)
-			goto found;
+		if (u->dev == urb->dev && u->pipe == urb->pipe &&
+		    u->status == -EINPROGRESS)
+			goto out;
 	}
 	u = NULL;
 
-found:
-	nested_unlock(&uhci->urblist_lock, flags);
+out:
+	spin_unlock_irqrestore(&uhci->urb_list_lock, flags);
 
 	return u;
 }
@@ -1297,33 +1469,52 @@
 	int ret = -EINVAL;
 	struct uhci *uhci;
 	unsigned long flags;
-	struct urb *u;
+	struct urb *eurb;
 	int bustime;
 
 	if (!urb)
 		return -EINVAL;
 
-	if (!urb->dev || !urb->dev->bus || !urb->dev->bus->hcpriv)
+	if (!urb->dev || !urb->dev->bus || !urb->dev->bus->hcpriv) {
+		warn("uhci_submit_urb: urb %p belongs to disconnected device or bus?", urb);
 		return -ENODEV;
+	}
 
 	uhci = (struct uhci *)urb->dev->bus->hcpriv;
 
-	/* Short circuit the virtual root hub */
-	if (usb_pipedevice(urb->pipe) == uhci->rh.devnum)
-		return rh_submit_urb(urb);
-
-	u = uhci_find_urb_ep(uhci, urb);
-	if (u && !(urb->transfer_flags & USB_QUEUE_BULK))
-		return -ENXIO;
-
+	INIT_LIST_HEAD(&urb->urb_list);
 	usb_inc_dev_use(urb->dev);
+
 	spin_lock_irqsave(&urb->lock, flags);
 
-	if (!uhci_alloc_urb_priv(urb)) {
+	if (urb->status == -EINPROGRESS || urb->status == -ECONNRESET ||
+	    urb->status == -ECONNABORTED) {
+		dbg("uhci_submit_urb: urb not available to submit (status = %d)", urb->status);
+		/* Since we can have problems on the out path */
 		spin_unlock_irqrestore(&urb->lock, flags);
 		usb_dec_dev_use(urb->dev);
 
-		return -ENOMEM;
+		return ret;
+	}
+
+	if (!uhci_alloc_urb_priv(uhci, urb)) {
+		ret = -ENOMEM;
+
+		goto out;
+	}
+
+	eurb = uhci_find_urb_ep(uhci, urb);
+	if (eurb && !(urb->transfer_flags & USB_QUEUE_BULK)) {
+		ret = -ENXIO;
+
+		goto out;
+	}
+
+	/* Short circuit the virtual root hub */
+	if (urb->dev == uhci->rh.dev) {
+		ret = rh_submit_urb(urb);
+
+		goto out;
 	}
 
 	switch (usb_pipetype(urb->pipe)) {
@@ -1344,7 +1535,7 @@
 			ret = uhci_submit_interrupt(urb);
 		break;
 	case PIPE_BULK:
-		ret = uhci_submit_bulk(urb, u);
+		ret = uhci_submit_bulk(urb, eurb);
 		break;
 	case PIPE_ISOCHRONOUS:
 		if (urb->bandwidth == 0) {	/* not yet checked/allocated */
@@ -1366,35 +1557,53 @@
 		break;
 	}
 
+out:
 	urb->status = ret;
 
 	spin_unlock_irqrestore(&urb->lock, flags);
 
-	if (ret == -EINPROGRESS)
-		ret = 0;
-	else {
-		uhci_unlink_generic(urb);
-		usb_dec_dev_use(urb->dev);
+	if (ret == -EINPROGRESS) {
+		spin_lock_irqsave(&uhci->urb_list_lock, flags);
+		/* We use _tail to make find_urb_ep more efficient */
+		list_add_tail(&urb->urb_list, &uhci->urb_list);
+		spin_unlock_irqrestore(&uhci->urb_list_lock, flags);
+
+		return 0;
 	}
 
+	uhci_unlink_generic(uhci, urb);
+	uhci_destroy_urb_priv(urb);
+
+	usb_dec_dev_use(urb->dev);
+
 	return ret;
 }
 
 /*
  * Return the result of a transfer
  *
- * Must be called with urblist_lock acquired
+ * Must be called with urb_list_lock acquired
  */
-static void uhci_transfer_result(struct urb *urb)
+static void uhci_transfer_result(struct uhci *uhci, struct urb *urb)
 {
-	struct usb_device *dev = urb->dev;
-	struct urb *turb;
-	int proceed = 0, is_ring = 0;
 	int ret = -EINVAL;
 	unsigned long flags;
+	struct urb_priv *urbp;
+
+	/* The root hub is special */
+	if (urb->dev == uhci->rh.dev)
+		return;
 
 	spin_lock_irqsave(&urb->lock, flags);
 
+	urbp = (struct urb_priv *)urb->hcpriv;
+
+	if (urb->status != -EINPROGRESS) {
+		info("uhci_transfer_result: called for URB %p not in flight?", urb);
+		spin_unlock_irqrestore(&urb->lock, flags);
+		return;
+	}
+
 	switch (usb_pipetype(urb->pipe)) {
 	case PIPE_CONTROL:
 		ret = uhci_result_control(urb);
@@ -1410,7 +1619,7 @@
 		break;
 	}
 
-	urb->status = ret;
+	urbp->status = ret;
 
 	spin_unlock_irqrestore(&urb->lock, flags);
 
@@ -1425,106 +1634,65 @@
 		/* Spinlock needed ? */
 		if (urb->bandwidth)
 			usb_release_bandwidth(urb->dev, urb, 1);
-		uhci_unlink_generic(urb);
+		uhci_unlink_generic(uhci, urb);
 		break;
 	case PIPE_INTERRUPT:
 		/* Interrupts are an exception */
 		if (urb->interval) {
-			urb->complete(urb);
-			uhci_reset_interrupt(urb);
-			return;
+			uhci_add_complete(urb);
+			return;		/* <-- note return */
 		}
 
 		/* Release bandwidth for Interrupt or Isoc. transfers */
 		/* Spinlock needed ? */
 		if (urb->bandwidth)
 			usb_release_bandwidth(urb->dev, urb, 0);
-		uhci_unlink_generic(urb);
+		uhci_unlink_generic(uhci, urb);
 		break;
+	default:
+		info("uhci_transfer_result: unknown pipe type %d for urb %p\n",
+			usb_pipetype(urb->pipe), urb);
 	}
 
-	if (urb->next) {
-		turb = urb->next;
-		do {
-			if (turb->status != -EINPROGRESS) {
-				proceed = 1;
-				break;
-			}
-
-			turb = turb->next;
-		} while (turb && turb != urb && turb != urb->next);
-
-		if (turb == urb || turb == urb->next)
-			is_ring = 1;
-	}
-
-	if (urb->complete && !proceed) {
-		urb->complete(urb);
-		if (!proceed && is_ring)
-			uhci_submit_urb(urb);
-	}
-
-	if (proceed && urb->next) {
-		turb = urb->next;
-		do {
-			if (turb->status != -EINPROGRESS &&
-			    uhci_submit_urb(turb) != 0)
+	list_del(&urb->urb_list);
+	INIT_LIST_HEAD(&urb->urb_list);
 
-			turb = turb->next;
-		} while (turb && turb != urb->next);
-
-		if (urb->complete)
-			urb->complete(urb);
-	}
-
-	/* We decrement the usage count after we're done with everything */
-	usb_dec_dev_use(dev);
+	uhci_add_complete(urb);
 }
 
-static int uhci_unlink_generic(struct urb *urb)
+static void uhci_unlink_generic(struct uhci *uhci, struct urb *urb)
 {
 	struct urb_priv *urbp = urb->hcpriv;
-	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
 
+	/* We can get called when urbp allocation fails, so check */
 	if (!urbp)
-		return -EINVAL;
+		return;
 
 	uhci_dec_fsbr(uhci, urb);	/* Safe since it checks */
 
-	uhci_remove_urb_list(uhci, urb);
+	uhci_delete_queued_urb(uhci, urb);
 
 	if (urbp->qh)
 		/* The interrupt loop will reclaim the QH's */
 		uhci_remove_qh(uhci, urbp->qh);
-
-	if (!list_empty(&urbp->urb_queue_list))
-		uhci_delete_queued_urb(uhci, urb);
-
-	uhci_destroy_urb_priv(urb);
-
-	urb->dev = NULL;
-
-	return 0;
 }
 
+/* FIXME: If we forcefully unlink an urb, we should reset the toggle for */
+/*  that pipe to match what actually completed */
 static int uhci_unlink_urb(struct urb *urb)
 {
 	struct uhci *uhci;
-	int ret = 0;
 	unsigned long flags;
+	struct urb_priv *urbp = urb->hcpriv;
 
 	if (!urb)
 		return -EINVAL;
 
-	if (!urb->dev || !urb->dev->bus)
+	if (!urb->dev || !urb->dev->bus || !urb->dev->bus->hcpriv)
 		return -ENODEV;
 
 	uhci = (struct uhci *)urb->dev->bus->hcpriv;
 
-	/* Short circuit the virtual root hub */
-	if (usb_pipedevice(urb->pipe) == uhci->rh.devnum)
-		return rh_unlink_urb(urb);
-
 	/* Release bandwidth for Interrupt or Isoc. transfers */
 	/* Spinlock needed ? */
 	if (urb->bandwidth) {
@@ -1540,13 +1708,28 @@
 		}
 	}
 
-	if (urb->status == -EINPROGRESS) {
-		uhci_unlink_generic(urb);
+	if (urb->status != -EINPROGRESS)
+		return 0;
+
+	spin_lock_irqsave(&uhci->urb_list_lock, flags);
+	list_del(&urb->urb_list);
+	INIT_LIST_HEAD(&urb->urb_list);
+	spin_unlock_irqrestore(&uhci->urb_list_lock, flags);
+
+	uhci_unlink_generic(uhci, urb);
 
+	/* Short circuit the virtual root hub */
+	if (urb->dev == uhci->rh.dev) {
+		rh_unlink_urb(urb);
+		uhci_call_completion(urb);
+	} else {
 		if (urb->transfer_flags & USB_ASYNC_UNLINK) {
-			urb->status = -ECONNABORTED;
+			/* urb_list is available now since we called */
+			/*  uhci_unlink_generic already */
 
-			spin_lock_irqsave(&uhci->urb_remove_lock, flags);
+			urbp->status = urb->status = -ECONNABORTED;
+
+			spin_lock_irqsave(&uhci->urb_remove_list_lock, flags);
 
 			/* Check to see if the remove list is empty */
 			if (list_empty(&uhci->urb_remove_list))
@@ -1554,7 +1737,7 @@
 			
 			list_add(&urb->urb_list, &uhci->urb_remove_list);
 
-			spin_unlock_irqrestore(&uhci->urb_remove_lock, flags);
+			spin_unlock_irqrestore(&uhci->urb_remove_list_lock, flags);
 		} else {
 			urb->status = -ENOENT;
 
@@ -1567,12 +1750,11 @@
 			} else
 				schedule_timeout(1+1*HZ/1000); 
 
-			if (urb->complete)
-				urb->complete(urb);
+			uhci_call_completion(urb);
 		}
 	}
 
-	return ret;
+	return 0;
 }
 
 static int uhci_fsbr_timeout(struct uhci *uhci, struct urb *urb)
@@ -1588,7 +1770,7 @@
 	/*  and we'd be turning on FSBR next frame anyway, so it's a wash */
 	urbp->fsbr_timeout = 1;
 
-	head = &urbp->list;
+	head = &urbp->td_list;
 	tmp = head->next;
 	while (tmp != head) {
 		struct uhci_td *td = list_entry(tmp, struct uhci_td, list);
@@ -1624,9 +1806,7 @@
 	uhci_unlink_urb
 };
 
-/* -------------------------------------------------------------------
-   Virtual Root Hub
-   ------------------------------------------------------------------- */
+/* Virtual Root Hub */
 
 static __u8 root_hub_dev_des[] =
 {
@@ -1700,7 +1880,6 @@
 	0xff			/*  __u8  PortPwrCtrlMask; *** 7 ports max *** */
 };
 
-/*-------------------------------------------------------------------------*/
 /* prepare Interrupt pipe transaction data; HUB INTERRUPT ENDPOINT */
 static int rh_send_irq(struct urb *urb)
 {
@@ -1708,6 +1887,7 @@
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
 	unsigned int io_addr = uhci->io_addr;
 	__u16 data = 0;
+	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
 
 	for (i = 0; i < uhci->rh.numports; i++) {
 		data |= ((inw(io_addr + USBPORTSC1 + i * 2) & 0xa) > 0 ? (1 << (i + 1)) : 0);
@@ -1716,18 +1896,17 @@
 
 	*(__u16 *) urb->transfer_buffer = cpu_to_le16(data);
 	urb->actual_length = len;
-	urb->status = USB_ST_NOERROR;
+	urbp->status = 0;
 
 	if ((data > 0) && (uhci->rh.send != 0)) {
 		dbg("root-hub INT complete: port1: %x port2: %x data: %x",
 			inw(io_addr + USBPORTSC1), inw(io_addr + USBPORTSC2), data);
-		urb->complete(urb);
+		uhci_call_completion(urb);
 	}
 
-	return USB_ST_NOERROR;
+	return 0;
 }
 
-/*-------------------------------------------------------------------------*/
 /* Virtual Root Hub INTs are polled by this timer every "interval" ms */
 static int rh_init_int_timer(struct urb *urb);
 
@@ -1735,41 +1914,46 @@
 {
 	struct urb *urb = (struct urb *)ptr;
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
-	struct list_head *tmp, *head = &uhci->urb_list;
-	struct urb_priv *urbp;
-	int len;
+	struct list_head list, *tmp, *head;
 	unsigned long flags;
 
-	if (uhci->rh.send) {
-		len = rh_send_irq(urb);
-		if (len > 0) {
-			urb->actual_length = len;
-			if (urb->complete)
-				urb->complete(urb);
+	if (uhci->rh.send)
+		rh_send_irq(urb);
+
+	INIT_LIST_HEAD(&list);
+
+	spin_lock_irqsave(&uhci->urb_list_lock, flags);
+	head = &uhci->urb_list;
+
+	tmp = head->next;
+	while (tmp != head) {
+		struct urb *u = list_entry(tmp, struct urb, urb_list);
+		struct urb_priv *urbp = (struct urb_priv *)u->hcpriv;
+
+		tmp = tmp->next;
+
+		/* Check if the FSBR timed out */
+		if (urbp->fsbr && time_after_eq(jiffies, urbp->inserttime + IDLE_TIMEOUT))
+			uhci_fsbr_timeout(uhci, u);
+
+		/* Check if the URB timed out */
+		if (u->timeout && time_after_eq(jiffies, u->timeout)) {
+			list_del(&u->urb_list);
+			list_add_tail(&u->urb_list, &list);
 		}
 	}
+	spin_unlock_irqrestore(&uhci->urb_list_lock, flags);
 
-	nested_lock(&uhci->urblist_lock, flags);
+	head = &list;
 	tmp = head->next;
 	while (tmp != head) {
-		struct urb *u = list_entry(tmp, urb_t, urb_list);
+		struct urb *u = list_entry(tmp, struct urb, urb_list);
 
 		tmp = tmp->next;
 
-		urbp = (struct urb_priv *)u->hcpriv;
-		if (urbp) {
-			/* Check if the FSBR timed out */
-			if (urbp->fsbr && time_after_eq(jiffies, urbp->inserttime + IDLE_TIMEOUT))
-				uhci_fsbr_timeout(uhci, u);
-
-			/* Check if the URB timed out */
-			if (u->timeout && time_after_eq(jiffies, u->timeout)) {
-				u->transfer_flags |= USB_ASYNC_UNLINK | USB_TIMEOUT_KILLED;
-				uhci_unlink_urb(u);
-			}
-		}
+		u->transfer_flags |= USB_ASYNC_UNLINK | USB_TIMEOUT_KILLED;
+		uhci_unlink_urb(u);
 	}
-	nested_unlock(&uhci->urblist_lock, flags);
 
 	/* enter global suspend if nothing connected */
 	if (!uhci->is_suspended && !ports_active(uhci))
@@ -1778,7 +1962,6 @@
 	rh_init_int_timer(urb);
 }
 
-/*-------------------------------------------------------------------------*/
 /* Root Hub INTs are polled by this timer */
 static int rh_init_int_timer(struct urb *urb)
 {
@@ -1794,7 +1977,6 @@
 	return 0;
 }
 
-/*-------------------------------------------------------------------------*/
 #define OK(x)			len = (x); break
 
 #define CLR_RH_PORTSTAT(x) \
@@ -1808,11 +1990,7 @@
 	outw(status, io_addr + USBPORTSC1 + 2 * (wIndex-1))
 
 
-/*-------------------------------------------------------------------------*/
-/*************************
- ** Root Hub Control Pipe
- *************************/
-
+/* Root Hub Control Pipe */
 static int rh_submit_urb(struct urb *urb)
 {
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
@@ -1822,7 +2000,7 @@
 	int leni = urb->transfer_buffer_length;
 	int len = 0;
 	int status = 0;
-	int stat = USB_ST_NOERROR;
+	int stat = 0;
 	int i;
 	unsigned int io_addr = uhci->io_addr;
 	__u16 cstatus;
@@ -1837,7 +2015,7 @@
 		uhci->rh.interval = urb->interval;
 		rh_init_int_timer(urb);
 
-		return USB_ST_NOERROR;
+		return -EINPROGRESS;
 	}
 
 	bmRType_bReq = cmd->requesttype | cmd->request << 8;
@@ -1987,33 +2165,29 @@
 	}
 
 	urb->actual_length = len;
-	urb->status = stat;
-	if (urb->complete)
-		urb->complete(urb);
 
-	return USB_ST_NOERROR;
+	return stat;
 }
-/*-------------------------------------------------------------------------*/
 
 static int rh_unlink_urb(struct urb *urb)
 {
 	struct uhci *uhci = (struct uhci *)urb->dev->bus->hcpriv;
 
 	if (uhci->rh.urb == urb) {
+		urb->status = -ENOENT;
 		uhci->rh.send = 0;
+		uhci->rh.urb = NULL;
 		del_timer(&uhci->rh.rh_int_timer);
 	}
 	return 0;
 }
-/*-------------------------------------------------------------------*/
 
-void uhci_free_pending_qhs(struct uhci *uhci)
+static void uhci_free_pending_qhs(struct uhci *uhci)
 {
 	struct list_head *tmp, *head;
 	unsigned long flags;
 
-	/* Free any pending QH's */
-	spin_lock_irqsave(&uhci->qh_remove_lock, flags);
+	spin_lock_irqsave(&uhci->qh_remove_list_lock, flags);
 	head = &uhci->qh_remove_list;
 	tmp = head->next;
 	while (tmp != head) {
@@ -2022,10 +2196,128 @@
 		tmp = tmp->next;
 
 		list_del(&qh->remove_list);
+		INIT_LIST_HEAD(&qh->remove_list);
 
-		uhci_free_qh(qh);
+		uhci_free_qh(uhci, qh);
 	}
-	spin_unlock_irqrestore(&uhci->qh_remove_lock, flags);
+	spin_unlock_irqrestore(&uhci->qh_remove_list_lock, flags);
+}
+
+static void uhci_call_completion(struct urb *urb)
+{
+	struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
+	struct usb_device *dev = urb->dev;
+	struct uhci *uhci = (struct uhci *)dev->bus->hcpriv;
+	int is_ring = 0, killed, resubmit_interrupt, status;
+	struct urb *nurb;
+
+	killed = (urb->status == -ENOENT || urb->status == -ECONNABORTED ||
+			urb->status == -ECONNRESET);
+	resubmit_interrupt = (usb_pipetype(urb->pipe) == PIPE_INTERRUPT &&
+			urb->interval && !killed);
+
+	nurb = urb->next;
+	if (nurb && !killed) {
+		int count = 0;
+
+		while (nurb && nurb != urb && count < MAX_URB_LOOP) {
+			if (nurb->status == -ENOENT ||
+			    nurb->status == -ECONNABORTED ||
+			    nurb->status == -ECONNRESET) {
+				killed = 1;
+				break;
+			}
+
+			nurb = nurb->next;
+			count++;
+		}
+
+		if (count == MAX_URB_LOOP)
+			err("uhci_call_completion: too many linked URB's, loop? (first loop)");
+
+		/* Check to see if chain is a ring */
+		is_ring = (nurb == urb);
+	}
+
+	status = urbp->status;
+	if (!resubmit_interrupt)
+		/* We don't need urb_priv anymore */
+		uhci_destroy_urb_priv(urb);
+
+	if (!killed)
+		urb->status = status;
+
+	if (urb->transfer_buffer_length)
+		pci_dma_sync_single(uhci->dev, urbp->transfer_buffer_dma_handle,
+			urb->transfer_buffer_length, usb_pipein(urb->pipe) ?
+			PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE);
+
+	if (usb_pipetype(urb->pipe) == PIPE_CONTROL && urb->setup_packet)
+		pci_dma_sync_single(uhci->dev, urbp->setup_packet_dma_handle,
+			sizeof(devrequest), PCI_DMA_TODEVICE);
+
+	urb->dev = NULL;
+	if (urb->complete)
+		urb->complete(urb);
+
+	if (resubmit_interrupt) {
+		urb->dev = dev;
+		uhci_reset_interrupt(urb);
+	} else {
+		if (is_ring && !killed) {
+			urb->dev = dev;
+			uhci_submit_urb(urb);
+		} else {
+			/* We decrement the usage count after we're done */
+			/*  with everything */
+			usb_dec_dev_use(dev);
+		}
+	}
+}
+
+static void uhci_finish_completion(struct uhci *uhci)
+{
+	struct list_head *tmp, *head;
+	unsigned long flags;
+
+	spin_lock_irqsave(&uhci->complete_list_lock, flags);
+	head = &uhci->complete_list;
+	tmp = head->next;
+	while (tmp != head) {
+		struct urb_priv *urbp = list_entry(tmp, struct urb_priv, complete_list);
+		struct urb *urb = urbp->urb;
+
+		tmp = tmp->next;
+
+		list_del(&urbp->complete_list);
+		INIT_LIST_HEAD(&urbp->complete_list);
+
+		uhci_call_completion(urb);
+	}
+	spin_unlock_irqrestore(&uhci->complete_list_lock, flags);
+}
+
+static void uhci_remove_pending_qhs(struct uhci *uhci)
+{
+	struct list_head *tmp, *head;
+	unsigned long flags;
+
+	spin_lock_irqsave(&uhci->urb_remove_list_lock, flags);
+	head = &uhci->urb_remove_list;
+	tmp = head->next;
+	while (tmp != head) {
+		struct urb *urb = list_entry(tmp, struct urb, urb_list);
+		struct urb_priv *urbp = (struct urb_priv *)urb->hcpriv;
+
+		tmp = tmp->next;
+
+		list_del(&urb->urb_list);
+		INIT_LIST_HEAD(&urb->urb_list);
+
+		urbp->status = urb->status = -ECONNRESET;
+		uhci_call_completion(urb);
+	}
+	spin_unlock_irqrestore(&uhci->urb_remove_list_lock, flags);
 }
 
 static void uhci_interrupt(int irq, void *__uhci, struct pt_regs *regs)
@@ -2033,7 +2325,6 @@
 	struct uhci *uhci = __uhci;
 	unsigned int io_addr = uhci->io_addr;
 	unsigned short status;
-	unsigned long flags;
 	struct list_head *tmp, *head;
 
 	/*
@@ -2043,7 +2334,7 @@
 	status = inw(io_addr + USBSTS);
 	if (!status)	/* shared interrupt, not mine */
 		return;
-	outw(status, io_addr + USBSTS);
+	outw(status, io_addr + USBSTS);		/* Clear it */
 
 	if (status & ~(USBSTS_USBINT | USBSTS_ERROR | USBSTS_RD)) {
 		if (status & USBSTS_HSE)
@@ -2056,31 +2347,17 @@
 		}
 	}
 
-	if (status & USBSTS_RD) {
+	if (status & USBSTS_RD)
 		wakeup_hc(uhci);
-	}
 
 	uhci_free_pending_qhs(uhci);
 
-	spin_lock(&uhci->urb_remove_lock);
-	head = &uhci->urb_remove_list;
-	tmp = head->next;
-	while (tmp != head) {
-		struct urb *urb = list_entry(tmp, struct urb, urb_list);
-
-		tmp = tmp->next;
-
-		list_del(&urb->urb_list);
-
-		if (urb->complete)
-			urb->complete(urb);
-	}
-	spin_unlock(&uhci->urb_remove_lock);
+	uhci_remove_pending_qhs(uhci);
 
 	uhci_clear_next_interrupt(uhci);
 
-	/* Walk the list of pending TD's to see which ones completed */
-	nested_lock(&uhci->urblist_lock, flags);
+	/* Walk the list of pending URB's to see which ones completed */
+	spin_lock(&uhci->urb_list_lock);
 	head = &uhci->urb_list;
 	tmp = head->next;
 	while (tmp != head) {
@@ -2089,9 +2366,11 @@
 		tmp = tmp->next;
 
 		/* Checks the status and does all of the magic necessary */
-		uhci_transfer_result(urb);
+		uhci_transfer_result(uhci, urb);
 	}
-	nested_unlock(&uhci->urblist_lock, flags);
+	spin_unlock(&uhci->urb_list_lock);
+
+	uhci_finish_completion(uhci);
 }
 
 static void reset_hc(struct uhci *uhci)
@@ -2173,12 +2452,43 @@
 
 	/* Start at frame 0 */
 	outw(0, io_addr + USBFRNUM);
-	outl(virt_to_bus(uhci->fl), io_addr + USBFLBASEADD);
+	outl(uhci->fl->dma_handle, io_addr + USBFLBASEADD);
 
 	/* Run and mark it configured with a 64-byte max packet */
 	outw(USBCMD_RS | USBCMD_CF | USBCMD_MAXP, io_addr + USBCMD);
 }
 
+static int uhci_alloc_root_hub(struct uhci *uhci)
+{
+	struct usb_device *dev;
+
+	dev = usb_alloc_dev(NULL, uhci->bus);
+	if (!dev)
+		return -1;
+
+	uhci->bus->root_hub = dev;
+	uhci->rh.dev = dev;
+
+	return 0;
+}
+
+static int uhci_start_root_hub(struct uhci *uhci)
+{
+	usb_connect(uhci->rh.dev);
+
+	if (usb_new_device(uhci->rh.dev) != 0) {
+		usb_free_dev(uhci->rh.dev);
+
+		return -1;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_PROC_FS
+static int uhci_num = 0;
+#endif
+
 /*
  * Allocate a frame list, and then setup the skeleton
  *
@@ -2193,11 +2503,12 @@
  *  - The second queue is the "control queue", split into low and high speed
  *  - The third queue is "bulk data".
  */
-static struct uhci *alloc_uhci(unsigned int io_addr, unsigned int io_size)
+static struct uhci *alloc_uhci(struct pci_dev *dev, unsigned int io_addr, unsigned int io_size)
 {
 	int i, port;
 	struct uhci *uhci;
 	struct usb_bus *bus;
+	dma_addr_t dma_handle;
 
 	uhci = kmalloc(sizeof(*uhci), GFP_KERNEL);
 	if (!uhci)
@@ -2205,30 +2516,54 @@
 
 	memset(uhci, 0, sizeof(*uhci));
 
+	uhci->dev = dev;
 	uhci->irq = -1;
 	uhci->io_addr = io_addr;
 	uhci->io_size = io_size;
 
-	spin_lock_init(&uhci->qh_remove_lock);
+	spin_lock_init(&uhci->qh_remove_list_lock);
 	INIT_LIST_HEAD(&uhci->qh_remove_list);
 
-	spin_lock_init(&uhci->urb_remove_lock);
+	spin_lock_init(&uhci->urb_remove_list_lock);
 	INIT_LIST_HEAD(&uhci->urb_remove_list);
 
-	nested_init(&uhci->urblist_lock);
+	spin_lock_init(&uhci->urb_list_lock);
 	INIT_LIST_HEAD(&uhci->urb_list);
 
-	spin_lock_init(&uhci->framelist_lock);
+	spin_lock_init(&uhci->complete_list_lock);
+	INIT_LIST_HEAD(&uhci->complete_list);
+
+	spin_lock_init(&uhci->frame_list_lock);
 
 	/* We need exactly one page (per UHCI specs), how convenient */
 	/* We assume that one page is atleast 4k (1024 frames * 4 bytes) */
-	uhci->fl = (void *)__get_free_page(GFP_KERNEL);
-	if (!uhci->fl)
-		goto au_free_uhci;
+	uhci->fl = pci_alloc_consistent(uhci->dev, sizeof(*uhci->fl), &dma_handle);
+	if (!uhci->fl) {
+		printk(KERN_ERR "Unable to allocate consistent memory for frame list\n");
+		goto free_uhci;
+	}
+
+	memset((void *)uhci->fl, 0, sizeof(*uhci->fl));
+
+	uhci->fl->dma_handle = dma_handle;
+
+	uhci->td_pool = pci_pool_create("uhci_td", uhci->dev,
+		sizeof(struct uhci_td), 16, 0, GFP_DMA | GFP_ATOMIC);
+	if (!uhci->td_pool) {
+		printk(KERN_ERR "Unable to create td pci_pool\n");
+		goto free_fl;
+	}
+
+	uhci->qh_pool = pci_pool_create("uhci_qh", uhci->dev,
+		sizeof(struct uhci_qh), 16, 0, GFP_DMA | GFP_ATOMIC);
+	if (!uhci->qh_pool) {
+		printk(KERN_ERR "Unable to create qh pci_pool\n");
+		goto free_td_pool;
+	}
 
 	bus = usb_alloc_bus(&uhci_device_operations);
 	if (!bus)
-		goto au_free_fl;
+		goto free_qh_pool;
 
 	uhci->bus = bus;
 	bus->hcpriv = uhci;
@@ -2258,36 +2593,66 @@
 
 	uhci->rh.numports = port;
 
+	if (uhci_alloc_root_hub(uhci)) {
+		err("unable to allocate root hub");
+		goto free_fl;
+	}
+
+	uhci->skeltd[0] = uhci_alloc_td(uhci, uhci->rh.dev);
+	if (!uhci->skeltd[0]) {
+		err("unable to allocate TD 0");
+		goto free_fl;
+	}
+
 	/*
 	 * 9 Interrupt queues; link int2 to int1, int4 to int2, etc
 	 * then link int1 to control and control to bulk
 	 */
 	for (i = 1; i < 9; i++) {
-		struct uhci_td *td = &uhci->skeltd[i];
+		struct uhci_td *td;
+
+		td = uhci->skeltd[i] = uhci_alloc_td(uhci, uhci->rh.dev);
+		if (!td) {
+			err("unable to allocate TD %d", i);
+			goto free_tds;
+		}
 
 		uhci_fill_td(td, 0, (UHCI_NULL_DATA_SIZE << 21) | (0x7f << 8) | USB_PID_IN, 0);
-		td->link = virt_to_bus(&uhci->skeltd[i - 1]);
+		td->link = uhci->skeltd[i - 1]->dma_handle;
 	}
 
+	uhci->skel_term_td = uhci_alloc_td(uhci, uhci->rh.dev);
+	if (!uhci->skel_term_td) {
+		err("unable to allocate TD 0");
+		goto free_fl;
+	}
 
-	uhci_fill_td(&uhci->skel_int1_td, 0, (UHCI_NULL_DATA_SIZE << 21) | (0x7f << 8) | USB_PID_IN, 0);
-	uhci->skel_int1_td.link = virt_to_bus(&uhci->skel_ls_control_qh) | UHCI_PTR_QH;
+	for (i = 0; i < UHCI_NUM_SKELQH; i++) {
+		uhci->skelqh[i] = uhci_alloc_qh(uhci, uhci->rh.dev);
+		if (!uhci->skelqh[i]) {
+			err("unable to allocate QH %d", i);
+			goto free_qhs;
+		}
+	}
+
+	uhci_fill_td(uhci->skel_int1_td, 0, (UHCI_NULL_DATA_SIZE << 21) | (0x7f << 8) | USB_PID_IN, 0);
+	uhci->skel_int1_td->link = uhci->skel_ls_control_qh->dma_handle | UHCI_PTR_QH;
 
-	uhci->skel_ls_control_qh.link = virt_to_bus(&uhci->skel_hs_control_qh) | UHCI_PTR_QH;
-	uhci->skel_ls_control_qh.element = UHCI_PTR_TERM;
+	uhci->skel_ls_control_qh->link = uhci->skel_hs_control_qh->dma_handle | UHCI_PTR_QH;
+	uhci->skel_ls_control_qh->element = UHCI_PTR_TERM;
 
-	uhci->skel_hs_control_qh.link = virt_to_bus(&uhci->skel_bulk_qh) | UHCI_PTR_QH;
-	uhci->skel_hs_control_qh.element = UHCI_PTR_TERM;
+	uhci->skel_hs_control_qh->link = uhci->skel_bulk_qh->dma_handle | UHCI_PTR_QH;
+	uhci->skel_hs_control_qh->element = UHCI_PTR_TERM;
 
-	uhci->skel_bulk_qh.link = virt_to_bus(&uhci->skel_term_qh) | UHCI_PTR_QH;
-	uhci->skel_bulk_qh.element = UHCI_PTR_TERM;
+	uhci->skel_bulk_qh->link = uhci->skel_term_qh->dma_handle | UHCI_PTR_QH;
+	uhci->skel_bulk_qh->element = UHCI_PTR_TERM;
 
 	/* This dummy TD is to work around a bug in Intel PIIX controllers */
-	uhci_fill_td(&uhci->skel_term_td, 0, (UHCI_NULL_DATA_SIZE << 21) | (0x7f << 8) | USB_PID_IN, 0);
-	uhci->skel_term_td.link = UHCI_PTR_TERM;
+	uhci_fill_td(uhci->skel_term_td, 0, (UHCI_NULL_DATA_SIZE << 21) | (0x7f << 8) | USB_PID_IN, 0);
+	uhci->skel_term_td->link = uhci->skel_term_td->dma_handle;
 
-	uhci->skel_term_qh.link = UHCI_PTR_TERM;
-	uhci->skel_term_qh.element = virt_to_bus(&uhci->skel_term_td);
+	uhci->skel_term_qh->link = UHCI_PTR_TERM;
+	uhci->skel_term_qh->element = uhci->skel_term_td->dma_handle;
 
 	/*
 	 * Fill the frame list: make all entries point to
@@ -2297,8 +2662,8 @@
 	 * scatter the interrupt queues in a way that gives
 	 * us a reasonable dynamic range for irq latencies.
 	 */
-	for (i = 0; i < 1024; i++) {
-		struct uhci_td *irq = &uhci->skel_int1_td;
+	for (i = 0; i < UHCI_NUMFRAMES; i++) {
+		int irq = 0;
 
 		if (i & 1) {
 			irq++;
@@ -2322,7 +2687,7 @@
 		}
 
 		/* Only place we don't use the frame list routines */
-		uhci->fl->frame[i] =  virt_to_bus(irq);
+		uhci->fl->frame[i] =  uhci->skeltd[irq]->dma_handle;
 	}
 
 	return uhci;
@@ -2330,9 +2695,30 @@
 /*
  * error exits:
  */
-au_free_fl:
-	free_page((unsigned long)uhci->fl);
-au_free_uhci:
+free_qhs:
+	for (i = 0; i < UHCI_NUM_SKELQH; i++)
+		if (uhci->skelqh[i]) {
+			uhci_free_qh(uhci, uhci->skelqh[i]);
+			uhci->skelqh[i] = NULL;
+		}
+
+free_tds:
+	for (i = 0; i < UHCI_NUM_SKELTD; i++)
+		if (uhci->skeltd[i]) {
+			uhci_free_td(uhci, uhci->skeltd[i]);
+			uhci->skeltd[i] = NULL;
+		}
+
+free_qh_pool:
+	pci_pool_destroy(uhci->qh_pool);
+
+free_td_pool:
+	pci_pool_destroy(uhci->td_pool);
+
+free_fl:
+	pci_free_consistent(uhci->dev, sizeof(*uhci->fl), uhci->fl, uhci->fl->dma_handle);
+
+free_uhci:
 	kfree(uhci);
 
 	return NULL;
@@ -2343,49 +2729,66 @@
  */
 static void release_uhci(struct uhci *uhci)
 {
+	int i;
+#ifdef CONFIG_PROC_FS
+	char buf[8];
+#endif
+
 	if (uhci->irq >= 0) {
 		free_irq(uhci->irq, uhci);
 		uhci->irq = -1;
 	}
 
+	for (i = 0; i < UHCI_NUM_SKELQH; i++)
+		if (uhci->skelqh[i]) {
+			uhci_free_qh(uhci, uhci->skelqh[i]);
+			uhci->skelqh[i] = NULL;
+		}
+
+	for (i = 0; i < UHCI_NUM_SKELTD; i++)
+		if (uhci->skeltd[i]) {
+			uhci_free_td(uhci, uhci->skeltd[i]);
+			uhci->skeltd[i] = NULL;
+		}
+
+	if (uhci->qh_pool) {
+		pci_pool_destroy(uhci->qh_pool);
+		uhci->qh_pool = NULL;
+	}
+
+	if (uhci->td_pool) {
+		pci_pool_destroy(uhci->td_pool);
+		uhci->td_pool = NULL;
+	}
+
 	if (uhci->fl) {
-		free_page((unsigned long)uhci->fl);
+		pci_free_consistent(uhci->dev, sizeof(*uhci->fl), uhci->fl, uhci->fl->dma_handle);
 		uhci->fl = NULL;
 	}
 
 	usb_free_bus(uhci->bus);
-	kfree(uhci);
-}
-
-int uhci_start_root_hub(struct uhci *uhci)
-{
-	struct usb_device *dev;
-
-	dev = usb_alloc_dev(NULL, uhci->bus);
-	if (!dev)
-		return -1;
-
-	uhci->bus->root_hub = dev;
-	usb_connect(dev);
 
-	if (usb_new_device(dev) != 0) {
-		usb_free_dev(dev);
+#ifdef CONFIG_PROC_FS
+	sprintf(buf, "hc%d", uhci->num);
 
-		return -1;
-	}
+	remove_proc_entry(buf, uhci_proc_root);
+	uhci->proc_entry = NULL;
+#endif
 
-	return 0;
+	kfree(uhci);
 }
 
 /*
- * If we've successfully found a UHCI, now is the time to increment the
- * module usage count, and return success..
+ * If we've successfully found a UHCI, now is the time to return success..
  */
 static int setup_uhci(struct pci_dev *dev, int irq, unsigned int io_addr, unsigned int io_size)
 {
 	int retval;
 	struct uhci *uhci;
 	char buf[8], *bufp = buf;
+#ifdef CONFIG_PROC_FS
+	struct proc_dir_entry *ent;
+#endif
 
 #ifndef __sparc__
 	sprintf(buf, "%d", irq);
@@ -2395,11 +2798,27 @@
 	printk(KERN_INFO __FILE__ ": USB UHCI at I/O 0x%x, IRQ %s\n",
 		io_addr, bufp);
 
-	uhci = alloc_uhci(io_addr, io_size);
+	uhci = alloc_uhci(dev, io_addr, io_size);
 	if (!uhci)
 		return -ENOMEM;
+
 	dev->driver_data = uhci;
 
+#ifdef CONFIG_PROC_FS
+	uhci->num = uhci_num++;
+
+	sprintf(buf, "hc%d", uhci->num);
+
+	ent = create_proc_entry(buf, S_IFREG|S_IRUGO|S_IWUSR, uhci_proc_root);
+	if (!ent)
+		return -ENOMEM;
+
+	ent->data = uhci;
+	ent->proc_fops = &uhci_proc_operations;
+	ent->size = 0;
+	uhci->proc_entry = ent;
+#endif
+
 	request_region(uhci->io_addr, io_size, "usb-uhci");
 
 	reset_hc(uhci);
@@ -2430,6 +2849,12 @@
 {
 	int i;
 
+	if (!pci_dma_supported(dev, 0xFFFFFFFF)) {
+		err("PCI subsystem doesn't support 32 bit addressing?");
+		return -ENODEV;
+	}
+	dev->dma_mask = 0xFFFFFFFF;
+
 	/* disable legacy emulation */
 	pci_write_config_word(dev, USBLEGSUP, 0);
 
@@ -2470,6 +2895,13 @@
 
 	usb_deregister_bus(uhci->bus);
 
+	/*
+	 * At this point, we're guaranteed that no new connects can be made
+	 * to this bus since there are no more parents
+	 */
+	uhci_free_pending_qhs(uhci);
+	uhci_remove_pending_qhs(uhci);
+
 	reset_hc(uhci);
 	release_region(uhci->io_addr, uhci->io_size);
 
@@ -2489,9 +2921,7 @@
 	start_hc((struct uhci *) dev->driver_data);
 }
 
-/*-------------------------------------------------------------------------*/
-
-static const struct pci_device_id __devinitdata uhci_pci_ids [] = { {
+static const struct pci_device_id __devinitdata uhci_pci_ids[] = { {
 
 	/* handle any USB UHCI controller */
 	class: 		((PCI_CLASS_SERIAL_USB << 8) | 0x00),
@@ -2524,34 +2954,26 @@
  
 static int __init uhci_hcd_init(void)
 {
-	int retval;
-
-	retval = -ENOMEM;
+	int retval = -ENOMEM;
 
-	/* We throw all of the TD's and QH's into a kmem cache */
-	/* TD's and QH's need to be 16 byte aligned and SLAB_HWCACHE_ALIGN */
-	/*  does this for us */
-	uhci_td_cachep = kmem_cache_create("uhci_td",
-		sizeof(struct uhci_td), 0,
-		SLAB_HWCACHE_ALIGN, NULL, NULL);
-
-	if (!uhci_td_cachep)
-		goto td_failed;
-
-	uhci_qh_cachep = kmem_cache_create("uhci_qh",
-		sizeof(struct uhci_qh), 0,
-		SLAB_HWCACHE_ALIGN, NULL, NULL);
+	if (debug) {
+		errbuf = kmalloc(ERRBUF_LEN, GFP_KERNEL);
+		if (!errbuf)
+			goto errbuf_failed;
+	}
 
-	if (!uhci_qh_cachep)
-		goto qh_failed;
+#ifdef CONFIG_PROC_FS
+	uhci_proc_root = create_proc_entry("driver/uhci", S_IFDIR, 0);
+	if (!uhci_proc_root)
+		goto proc_failed;
+#endif
 
 	uhci_up_cachep = kmem_cache_create("uhci_urb_priv",
 		sizeof(struct urb_priv), 0, 0, NULL, NULL);
-
 	if (!uhci_up_cachep)
 		goto up_failed;
 
-	retval = pci_module_init (&uhci_pci_driver);
+	retval = pci_module_init(&uhci_pci_driver);
 	if (retval)
 		goto init_failed;
 
@@ -2562,29 +2984,33 @@
 		printk(KERN_INFO "uhci: not all urb_priv's were freed\n");
 
 up_failed:
-	if (kmem_cache_destroy(uhci_qh_cachep))
-		printk(KERN_INFO "uhci: not all QH's were freed\n");
 
-qh_failed:
-	if (kmem_cache_destroy(uhci_td_cachep))
-		printk(KERN_INFO "uhci: not all TD's were freed\n");
+#ifdef CONFIG_PROC_FS
+	remove_proc_entry("uhci", 0);
+
+proc_failed:
+#endif
+	if (errbuf)
+		kfree(errbuf);
+
+errbuf_failed:
 
-td_failed:
 	return retval;
 }
 
 static void __exit uhci_hcd_cleanup (void) 
 {
-	pci_unregister_driver (&uhci_pci_driver);
+	pci_unregister_driver(&uhci_pci_driver);
 	
 	if (kmem_cache_destroy(uhci_up_cachep))
 		printk(KERN_INFO "uhci: not all urb_priv's were freed\n");
 
-	if (kmem_cache_destroy(uhci_qh_cachep))
-		printk(KERN_INFO "uhci: not all QH's were freed\n");
+#ifdef CONFIG_PROC_FS
+	remove_proc_entry("uhci", 0);
+#endif
 
-	if (kmem_cache_destroy(uhci_td_cachep))
-		printk(KERN_INFO "uhci: not all TD's were freed\n");
+	if (errbuf)
+		kfree(errbuf);
 }
 
 module_init(uhci_hcd_init);