diff -ru a/drivers/usb/host/ehci.h b/drivers/usb/host/ehci.h --- a/drivers/usb/host/ehci.h 2006-08-07 00:18:54.000000000 -0400 +++ b/drivers/usb/host/ehci.h 2006-09-25 22:19:32.000000000 -0400 @@ -72,11 +72,19 @@ int next_uframe; /* scan periodic, start here */ unsigned periodic_sched; /* periodic activity count */ + kmem_cache_t *budget_pool; /* Pool for shadow budget */ + struct ehci_shadow_budget **budget; /* pointer to the shadow budget + of bandwidth placeholders */ + + struct ehci_fstn *periodic_restore_fstn; + struct ehci_fstn **periodic_save_fstns; /* [PERIODIC_QH_MAX_PERIOD] */ + /* per root hub port */ unsigned long reset_done [EHCI_MAX_ROOT_PORTS]; /* per-HC memory pools (could be per-bus, but ...) */ struct dma_pool *qh_pool; /* qh per active urb */ + struct dma_pool *fstn_pool; /* a qh has [up to] two fstns */ struct dma_pool *qtd_pool; /* one or more per qh */ struct dma_pool *itd_pool; /* itd per iso urb */ struct dma_pool *sitd_pool; /* sitd per split iso urb */ @@ -155,8 +163,8 @@ // to go OFF neither can matter, and afterwards the IO // watchdog stops unless there's still periodic traffic. if (action != TIMER_IAA_WATCHDOG - && t > ehci->watchdog.expires - && timer_pending (&ehci->watchdog)) + && t > ehci->watchdog.expires + && timer_pending (&ehci->watchdog)) return; mod_timer (&ehci->watchdog, t); } @@ -354,6 +362,7 @@ /* next async queue entry, or pointer to interrupt/periodic QH */ #define QH_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_QH) +#define FSTN_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_FSTN) /* for periodic/async schedules and qtd lists, mark end of list */ #define EHCI_LIST_END __constant_cpu_to_le32(1) /* "null pointer" to hw */ @@ -377,6 +386,56 @@ /*-------------------------------------------------------------------------*/ +/* the shadow budget is a means of reserving bandwidth throughout the + periodic schedule (the hardware schedule is currently used as a + sliding window by the current transaction code). The shadow budget + follows the same list-then-tree structure as the hardware schedule. */ + +/* must be equal to or smaller than PERIODIC_QH_MAX_PERIOD */ +#define BUDGET_SLOTS 32 +#define BUDGET_SLOT_MASK (BUDGET_SLOTS-1) + +/* Because EHCI cannot issue ssplits in uframe 7 and USB 2.0 does not + allow ssplits in uframe 6, EHCI can only generate an efficient FS + frame by scheduling according to best-case (not worst-case) bit + stuffing. Thus we purposely 'overcommit' the FS frame budget + within the buffering ability of the TT to buffer, and within the + limits of the 90/10 rule. The TT will catch up in the microframes + where we cannot issue start splits. +*/ + +#define BUDGET_WRAP_P(b) ((b)->cmask & (~0xff)) + +struct ehci_shadow_budget { + struct ehci_shadow_budget *next; + + u16 s_usecs; /* highspeed usecs */ + u16 c_usecs; /* highspeed completion usecs (FS/LS)*/ + u16 tt_bytes; /* bytes to budget at TT (FS/LS)*/ + u16 cmask; /* 16 bits: a 'too big' mask indicates the need for a + frame wrap (FSTN or dummy SITD) */ + u8 smask; + +#define BUDGET_TYPE_ITD 1 /* ITD or SITD; budgets early */ +#define BUDGET_TYPE_QH 2 /* QH; budgets late */ + u8 type; + + /* QHs and FSTNs are persistent in the periodic schedule, so they + own their own budget placeholders. [s]ITD placeholders are owned + by an ehci_iso_stream. */ + union { + struct ehci_qh *qh; /* needed to extract + period and specific + TT */ + struct ehci_iso_stream *iso; /* needed to extract + period and specific + TT */ + void *ptr; /* used as an ID */ + } owner; +}; + +/*-------------------------------------------------------------------------*/ + /* * EHCI Specification 0.95 Section 3.6 * QH: describes control/bulk/interrupt endpoints @@ -385,6 +444,13 @@ * These appear in both the async and (for interrupt) periodic schedules. */ +/* as per ehci spec guidelines, replicate QH tree linkage of a maximum + size that is less than the full periodic schedule size. This is + also necessary to limit memory usage of FSTN support. */ +/* must be equal to or larger than BUDGET_SLOTS*8, otherwise the budget + will overcommit */ +#define PERIODIC_QH_MAX_PERIOD (BUDGET_SLOTS*8) + struct ehci_qh { /* first part defined by EHCI spec */ __le32 hw_next; /* see EHCI 3.6.1 */ @@ -425,13 +491,11 @@ /* periodic schedule info */ u8 usecs; /* intr bandwidth */ - u8 gap_uf; /* uframes split/csplit gap */ u8 c_usecs; /* ... split completion bw */ - u16 tt_usecs; /* tt downstream bandwidth */ - unsigned short period; /* polling interval */ - unsigned short start; /* where polling starts */ -#define NO_FRAME ((unsigned short)~0) /* pick new start */ + u16 tt_bytes; /* tt downstream bandwidth */ + unsigned short period; /* polling interval; uFrame */ struct usb_device *dev; /* access to TT */ + struct ehci_shadow_budget *budget; /* pointer to budget placeholder */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ @@ -475,19 +539,16 @@ struct usb_host_endpoint *ep; /* output of (re)scheduling */ - unsigned long start; /* jiffies */ - unsigned long rescheduled; int next_uframe; - __le32 splits; + int budget_state; /* the rest is derived from the endpoint descriptor, * trusting urb->interval == f(epdesc->bInterval) and * including the extra info for hw_bufp[0..2] */ - u8 interval; + u16 interval; u8 usecs, c_usecs; - u16 tt_usecs; - u16 maxp; + u16 tt_bytes; u16 raw_mask; unsigned bandwidth; @@ -535,8 +596,9 @@ /* any/all hw_transactions here may be used by that urb */ unsigned frame; /* where scheduled */ unsigned pg; - unsigned index[8]; /* in urb->iso_frame_desc */ + int index[8]; /* in urb->iso_frame_desc */ u8 usecs[8]; + struct ehci_shadow_budget *budget; /* pointer to budget placeholder */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ @@ -566,6 +628,7 @@ #define SITD_STS_STS (1 << 1) /* split transaction state */ #define SITD_ACTIVE __constant_cpu_to_le32(SITD_STS_ACTIVE) +#define SITD_DOCOMPLETE __constant_cpu_to_le32(SITD_STS_STS) __le32 hw_buf [2]; /* EHCI table 3-12 */ __le32 hw_backpointer; /* EHCI table 3-13 */ @@ -579,7 +642,8 @@ struct ehci_iso_stream *stream; /* endpoint's queue */ struct list_head sitd_list; /* list of stream's sitds */ unsigned frame; - unsigned index; + int index; + struct ehci_shadow_budget *budget; /* pointer to budget placeholder */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ @@ -600,10 +664,19 @@ /* the rest is HCD-private */ dma_addr_t fstn_dma; union ehci_shadow fstn_next; /* ptr to periodic q entry */ + struct ehci_qh *fstn_prev; /* ptr to backlinked qh */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ +#define EHCI_EP_QH(x) (((x)->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) \ + != USB_ENDPOINT_XFER_ISOC ? (x)->hcpriv : NULL) +#define EHCI_EP_STREAM(x) (((x)->desc.bmAttributes & \ + USB_ENDPOINT_XFERTYPE_MASK) == \ + USB_ENDPOINT_XFER_ISOC ? (x)->hcpriv : NULL) + +/*-------------------------------------------------------------------------*/ + #ifdef CONFIG_USB_EHCI_ROOT_HUB_TT /* diff -ru a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c --- a/drivers/usb/host/ehci-hcd.c 2006-08-09 22:12:43.000000000 -0400 +++ b/drivers/usb/host/ehci-hcd.c 2006-09-25 22:19:32.000000000 -0400 @@ -255,6 +255,8 @@ /*-------------------------------------------------------------------------*/ static void ehci_work(struct ehci_hcd *ehci, struct pt_regs *regs); +static void budget_unlink_entries_by_owner(struct ehci_hcd *ehci,void *owner); +static unsigned sched_verbose = 0; #include "ehci-hub.c" #include "ehci-mem.c" @@ -425,6 +427,11 @@ if ((retval = ehci_mem_init(ehci, GFP_KERNEL)) < 0) return retval; + /* a periodic schedule that supports FSTNs always has a single + static restore FSTN that matches all possible save + FSTNs. */ + periodic_init_link_restore_fstn(ehci); + /* controllers may cache some of the periodic schedule ... */ hcc_params = readl(&ehci->caps->hcc_params); if (HCC_ISOC_CACHE(hcc_params)) // full frame cache @@ -764,7 +771,7 @@ break; switch (qh->qh_state) { case QH_STATE_LINKED: - intr_deschedule (ehci, qh); + periodic_qh_deschedule (ehci, qh); /* FALL THROUGH */ case QH_STATE_IDLE: qh_completions (ehci, qh, NULL); @@ -780,7 +787,7 @@ && HC_IS_RUNNING (hcd->state)) { int status; - status = qh_schedule (ehci, qh); + status = periodic_qh_schedule (ehci, qh); spin_unlock_irqrestore (&ehci->lock, flags); if (status != 0) { @@ -815,62 +822,79 @@ struct ehci_hcd *ehci = hcd_to_ehci (hcd); unsigned long flags; struct ehci_qh *qh, *tmp; + struct ehci_iso_stream *iso; /* ASSERT: any requests/urbs are being unlinked */ /* ASSERT: nobody can be submitting urbs for this any more */ -rescan: spin_lock_irqsave (&ehci->lock, flags); - qh = ep->hcpriv; - if (!qh) - goto done; - - /* endpoints can be iso streams. for now, we don't - * accelerate iso completions ... so spin a while. - */ - if (qh->hw_info1 == 0) { - ehci_vdbg (ehci, "iso delay\n"); - goto idle_timeout; - } + iso = EHCI_EP_STREAM(ep); - if (!HC_IS_RUNNING (hcd->state)) - qh->qh_state = QH_STATE_IDLE; - switch (qh->qh_state) { - case QH_STATE_LINKED: - for (tmp = ehci->async->qh_next.qh; - tmp && tmp != qh; - tmp = tmp->qh_next.qh) - continue; - /* periodic qh self-unlinks on empty */ - if (!tmp) - goto nogood; - unlink_async (ehci, qh); - /* FALL THROUGH */ - case QH_STATE_UNLINK: /* wait for hw to finish? */ -idle_timeout: + if (iso){ + /* for now, we don't accelerate iso completions ... so spin + a while. */ + + while(iso->refcount>1){ + spin_unlock_irqrestore (&ehci->lock, flags); + schedule_timeout_uninterruptible(1); + spin_lock_irqsave (&ehci->lock, flags); + } + + /* we want to be sure completions deref the stream to 1, + then we finally pull the plug here */ + iso_stream_put(ehci,iso); + ep->hcpriv = NULL; spin_unlock_irqrestore (&ehci->lock, flags); - schedule_timeout_uninterruptible(1); - goto rescan; - case QH_STATE_IDLE: /* fully unlinked */ - if (list_empty (&qh->qtd_list)) { - qh_put (qh); + return; + } + + while ( (qh = EHCI_EP_QH(ep)) ){ + + if (!HC_IS_RUNNING (hcd->state)) + qh->qh_state = QH_STATE_IDLE; + switch (qh->qh_state) { + case QH_STATE_LINKED: + for (tmp = ehci->async->qh_next.qh; + tmp && tmp != qh; + tmp = tmp->qh_next.qh) + continue; + + /* periodic qh self-unlinks on empty */ + if (!tmp) goto error; + unlink_async (ehci, qh); + /* FALL THROUGH */ + + case QH_STATE_UNLINK: /* wait for hw to finish? */ + + spin_unlock_irqrestore (&ehci->lock, flags); + schedule_timeout_uninterruptible(1); + spin_lock_irqsave (&ehci->lock, flags); break; + + case QH_STATE_IDLE: /* fully unlinked */ + + if (list_empty (&qh->qtd_list)) { + qh_put (qh); + ep->hcpriv = NULL; + break; + } + /* else FALL THROUGH */ + default: + goto error; } - /* else FALL THROUGH */ - default: -nogood: - /* caller was supposed to have unlinked any requests; - * that's not our job. just leak this memory. - */ - ehci_err (ehci, "qh %p (#%02x) state %d%s\n", - qh, ep->desc.bEndpointAddress, qh->qh_state, - list_empty (&qh->qtd_list) ? "" : "(has tds)"); - break; } - ep->hcpriv = NULL; -done: + spin_unlock_irqrestore (&ehci->lock, flags); return; + +error: + /* caller was supposed to have unlinked any requests; + * that's not our job. just leak this memory. + */ + ehci_err (ehci, "qh %p (#%02x) state %d%s\n", + qh, ep->desc.bEndpointAddress, qh->qh_state, + list_empty (&qh->qtd_list) ? "" : "(has tds)"); + ep->hcpriv = NULL; } static int ehci_get_frame (struct usb_hcd *hcd) diff -ru a/drivers/usb/host/ehci-mem.c b/drivers/usb/host/ehci-mem.c --- a/drivers/usb/host/ehci-mem.c 2006-08-07 00:18:54.000000000 -0400 +++ b/drivers/usb/host/ehci-mem.c 2006-09-25 23:46:11.000000000 -0400 @@ -62,6 +62,26 @@ dma_pool_free (ehci->qtd_pool, qtd, qtd->qtd_dma); } +static struct ehci_fstn *ehci_fstn_alloc (struct ehci_hcd *ehci, gfp_t flags) +{ + struct ehci_fstn *fstn = NULL; + dma_addr_t dma; + + if(ehci->fstn_pool){ /* alloced only if HC actually supports fstn */ + fstn = dma_pool_alloc (ehci->fstn_pool, flags, &dma); + if (fstn != NULL) { + memset (fstn, 0, sizeof *fstn); + fstn->fstn_dma = dma; + } + } + return fstn; +} + +static inline void ehci_fstn_free (struct ehci_hcd *ehci, + struct ehci_fstn *fstn) +{ + dma_pool_free (ehci->fstn_pool, fstn, fstn->fstn_dma); +} static void qh_destroy (struct kref *kref) { @@ -73,6 +93,14 @@ ehci_dbg (ehci, "unused qh not empty!\n"); BUG (); } + + /* remove from shadow budget */ + if(sched_verbose) + ehci_info(ehci, "Removing QH %p from budget:\n", qh); + + budget_unlink_entries_by_owner(ehci,qh); + qh->budget = NULL; + if (qh->dummy) ehci_qtd_free (ehci, qh->dummy); dma_pool_free (ehci->qh_pool, qh, qh->qh_dma); @@ -82,7 +110,7 @@ { struct ehci_qh *qh; dma_addr_t dma; - + qh = (struct ehci_qh *) dma_pool_alloc (ehci->qh_pool, flags, &dma); if (!qh) @@ -135,6 +163,10 @@ dma_pool_destroy (ehci->qtd_pool); ehci->qtd_pool = NULL; + if (ehci->fstn_pool) + dma_pool_destroy (ehci->fstn_pool); + ehci->fstn_pool = NULL; + if (ehci->qh_pool) { dma_pool_destroy (ehci->qh_pool); ehci->qh_pool = NULL; @@ -150,12 +182,22 @@ if (ehci->periodic) dma_free_coherent (ehci_to_hcd(ehci)->self.controller, - ehci->periodic_size * sizeof (u32), - ehci->periodic, ehci->periodic_dma); + ehci->periodic_size * sizeof (u32), + ehci->periodic, ehci->periodic_dma); ehci->periodic = NULL; + if(ehci->periodic_save_fstns) + kfree(ehci->periodic_save_fstns); + if(ehci->budget) + kfree(ehci->budget); + ehci->budget = NULL; + if(ehci->budget_pool) + kmem_cache_destroy(ehci->budget_pool); + ehci->budget_pool = NULL; + /* shadow periodic table */ - kfree(ehci->pshadow); + if(ehci->pshadow) + kfree(ehci->pshadow); ehci->pshadow = NULL; } @@ -165,45 +207,70 @@ int i; /* QTDs for control/bulk/intr transfers */ - ehci->qtd_pool = dma_pool_create ("ehci_qtd", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_qtd), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->qtd_pool = + dma_pool_create ("ehci_qtd", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_qtd), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->qtd_pool) { goto fail; } + /* does this HC support FSTNs? Don't allocate FSTN structs if not. */ + if( HC_VERSION(readl (&ehci->caps->hc_capbase)) >= 96){ + /* EHCI 0.96+ */ + ehci->fstn_pool = + dma_pool_create ("ehci_fstn", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_fstn), + 32, + 4096); + /* do not give up on failure; + scheduling is merely constrained */ + if(ehci->fstn_pool) + ehci->periodic_save_fstns = + kcalloc(PERIODIC_QH_MAX_PERIOD, + sizeof(*ehci->periodic_save_fstns), + flags); + /* lazy-alloc the rest */ + + } + /* QHs for control/bulk/intr transfers */ - ehci->qh_pool = dma_pool_create ("ehci_qh", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_qh), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->qh_pool = + dma_pool_create ("ehci_qh", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_qh), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->qh_pool) { goto fail; } + ehci->async = ehci_qh_alloc (ehci, flags); if (!ehci->async) { goto fail; } /* ITD for high speed ISO transfers */ - ehci->itd_pool = dma_pool_create ("ehci_itd", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_itd), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->itd_pool = + dma_pool_create ("ehci_itd", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_itd), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->itd_pool) { goto fail; } /* SITD for full/low speed split ISO transfers */ - ehci->sitd_pool = dma_pool_create ("ehci_sitd", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_sitd), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->sitd_pool = + dma_pool_create ("ehci_sitd", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_sitd), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->sitd_pool) { goto fail; } @@ -211,14 +278,27 @@ /* Hardware periodic table */ ehci->periodic = (__le32 *) dma_alloc_coherent (ehci_to_hcd(ehci)->self.controller, - ehci->periodic_size * sizeof(__le32), - &ehci->periodic_dma, 0); + ehci->periodic_size * sizeof(__le32), + &ehci->periodic_dma, 0); if (ehci->periodic == NULL) { goto fail; } for (i = 0; i < ehci->periodic_size; i++) ehci->periodic [i] = EHCI_LIST_END; + /* budgeting placeholders */ + ehci->budget = kcalloc(BUDGET_SLOTS, sizeof(*ehci->budget), flags); + if (ehci->budget == NULL){ + goto fail; + } + ehci->budget_pool = + kmem_cache_create ("ehci_budget", + sizeof(struct ehci_shadow_budget), + 0,0,NULL,NULL); + if (!ehci->budget_pool) { + goto fail; + } + /* software shadow of hardware table */ ehci->pshadow = kcalloc(ehci->periodic_size, sizeof(void *), flags); if (ehci->pshadow != NULL) diff -ru a/drivers/usb/host/ehci-q.c b/drivers/usb/host/ehci-q.c --- a/drivers/usb/host/ehci-q.c 2006-08-07 00:18:54.000000000 -0400 +++ b/drivers/usb/host/ehci-q.c 2006-09-25 23:36:12.000000000 -0400 @@ -44,7 +44,7 @@ static int qtd_fill (struct ehci_qtd *qtd, dma_addr_t buf, size_t len, - int token, int maxpacket) + int token, int maxpacket) { int i, count; u64 addr = buf; @@ -269,8 +269,8 @@ static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh); static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh); -static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh); -static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh); +static void periodic_qh_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh); +static int periodic_qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh); /* * Process and free completed qtds for a qh, returning URBs to drivers. @@ -430,8 +430,12 @@ */ if ((__constant_cpu_to_le32 (QH_SMASK) & qh->hw_info2) != 0) { - intr_deschedule (ehci, qh); - (void) qh_schedule (ehci, qh); + + qh_get(qh); + periodic_qh_deschedule (ehci, qh); + periodic_qh_schedule (ehci, qh); + qh_put(qh); + } else unlink_async (ehci, qh); break; @@ -631,9 +635,9 @@ struct urb *urb, gfp_t flags ) { + int is_input, type = usb_pipetype (urb->pipe); struct ehci_qh *qh = ehci_qh_alloc (ehci, flags); u32 info1 = 0, info2 = 0; - int is_input, type; int maxp = 0; if (!qh) @@ -646,7 +650,6 @@ info1 |= usb_pipedevice (urb->pipe) << 0; is_input = usb_pipein (urb->pipe); - type = usb_pipetype (urb->pipe); maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input); /* Compute interrupt scheduling parameters just once, and save. @@ -660,31 +663,34 @@ if (type == PIPE_INTERRUPT) { qh->usecs = NS_TO_US (usb_calc_bus_time (USB_SPEED_HIGH, is_input, 0, hb_mult (maxp) * max_packet (maxp))); - qh->start = NO_FRAME; if (urb->dev->speed == USB_SPEED_HIGH) { qh->c_usecs = 0; - qh->gap_uf = 0; + qh->tt_bytes = 0; + qh->period = urb->interval; /* uFrame */ - qh->period = urb->interval >> 3; - if (qh->period == 0 && urb->interval != 1) { - /* NOTE interval 2 or 4 uframes could work. - * But interval 1 scheduling is simpler, and - * includes high bandwidth. - */ - dbg ("intr period %d uframes, NYET!", - urb->interval); - goto done; - } } else { struct usb_tt *tt = urb->dev->tt; - int think_time; + int think_time, think_bytes; - /* gap is f(FS/LS transfer times) */ - qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed, - is_input, 0, maxp) / (125 * 1000); + think_time = tt ? tt->think_time : 0; + think_bytes = (think_time+665)/666; + + if(urb->dev->speed == USB_SPEED_FULL) + /* full speed bytes + + think time [TT host delay] + + FS non-iso protocol overhead */ + qh->tt_bytes = think_bytes + maxp + 14; + else + /* low speed bytes + + think time [TT host delay] + + low speed protocol overhead */ + /* expressed in full speed bytes */ + qh->tt_bytes = think_bytes + maxp*8 + 98; /* FIXME this just approximates SPLIT/CSPLIT times */ + /* can't fix without going to ns granularity; + this is at least safe */ if (is_input) { // SPLIT, gap, CSPLIT+DATA qh->c_usecs = qh->usecs + HS_USECS (0); qh->usecs = HS_USECS (1); @@ -693,11 +699,7 @@ qh->c_usecs = HS_USECS (0); } - think_time = tt ? tt->think_time : 0; - qh->tt_usecs = NS_TO_US (think_time + - usb_calc_bus_time (urb->dev->speed, - is_input, 0, max_packet (maxp))); - qh->period = urb->interval; + qh->period = urb->interval << 3; /* uFrame */ } } @@ -760,7 +762,6 @@ break; default: dbg ("bogus dev %p speed %d", urb->dev, urb->dev->speed); -done: qh_put (qh); return NULL; } diff -ru a/drivers/usb/host/ehci-sched.c b/drivers/usb/host/ehci-sched.c --- a/drivers/usb/host/ehci-sched.c 2006-08-09 22:12:43.000000000 -0400 +++ b/drivers/usb/host/ehci-sched.c 2006-09-25 22:19:32.000000000 -0400 @@ -1,6 +1,7 @@ /* * Copyright (c) 2001-2004 by David Brownell * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers + * Copyright (c) 2006 Monty (budgeting code) * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -22,24 +23,167 @@ /*-------------------------------------------------------------------------*/ /* - * EHCI scheduled transaction support: interrupt, iso, split iso - * These are called "periodic" transactions in the EHCI spec. + * EHCI scheduled transaction support: interrupt, iso, split + * iso. These are called "periodic" transactions in the EHCI spec. It + * should be noted that the Linux USB subsystem has no high-level + * notion of periodically serviced transfers; bandwidth is reserved + * periodically [the shadow budget], but transfers are still entirely + * single-request/single-response. To avoid over/underruns, periodic + * streaming relies on upper layers to make a new request before the + * next schedule slot passes. Any such xruns are reported as + * -EL2NSYNC if they occur. * - * Note that for interrupt transfers, the QH/QTD manipulation is shared - * with the "asynchronous" transaction support (control/bulk transfers). - * The only real difference is in how interrupt transfers are scheduled. + * The shadow budget consists of placeholders for reserving bandwidth + * throughout the periodic schedule for any given active endpoint. + * This avoids both underallocation errors (spurious ENOSPC during a + * transfer that began successfully) and overallocation errors (lost + * frames or HC/hub/irq errors and resets during transfers) due to + * basing bandwidth allocation on the always-incomplete hardware + * schedule. Although it would be conceptually simpler to place + * inactive [s]ITDs and QHs into the actual hardware budget (to be + * activated when a transfer makes use of that segment of the periodic + * schedule), the additional memory overhead of populating the + * complete 1024 frame periodic schedule is not insignificant. + * Therefore, a relatively small shadow budget (BUDGET_SLOTS) is + * projected ('translated') onto the full (1024 slot) hardware + * schedule. Budgeting is fully computed upon the first transfer + * attempt with an endpoint and remains in the shadow budget until the + * endpoint is closed [disabled]. Transfer descriptors are linked + * into the hardware schedule on demand using the precomputed + * information in the shadow budget. + * + * Interrupt transfers share the QH/QTD manipulation with the + * "asynchronous" transaction support (control/bulk transfers) in + * ehci-q.c, although all toplevel interrupt control originates here. + * The only real difference is in how interrupt transfers are + * scheduled. + * + * For isochronous, an "iso_stream" head serves the same role as a QH. + * It keeps track of every ITD (or SITD) that's scheduled. + * + * This scheduler supports and aggressively uses both FSTNs and sITD + * backpointers in order to allow interrupt QH and sITD transfers to + * span H-frame boundaries. This support is necessary for efficient + * FS bus bandwidth usage through a 2.0 hub's TT mechanism. + * + * The transfer ordering in the shadow schedule looks like: + * + * . | Level N | N/2 | ... | Level 1 + * . | | | | + * . | | | | + * Frame 2: [iso] + * \_ [save FSTN]; [QHs] | | | + * / \_ [QHs] | | + * Frame 1: [iso] ... _/ \_ ... _ | + * ... _/ \_ [restore FSTN]; [QHs] + * ... _/ + * Frame 0: [iso]... + * + * Each [iso] block is ordered [dummy sITDs]; [sITDs/ITDs] + * + * The shadow budget is structured like the shadow schedule, however + * it is both smaller (to save memory) and simpler, containing only + * QHs and normal [s]ITDs. dummy sITDs and FSTNs are intuited in + * budgeting calculations and do not appear explicitly in the shadow + * budget: + * + * . |Level N| N/2 | ... |Level 1 + * . | | | | + * . append | | | | + * ---------> |prepend| | | + * Frame 2: [sITDs/ITDs] | <---- |prepend| | + * \_ [QHs] | <---- | | + * / \_ [QHs] | |prepend + * Frame 1: [sITDs/ITDs] ... _/ \_ ... _ | <---- + * ... _/ \_ [QHs] + * ... _/ + * Frame 0: [sITDs/ITDs]... + * + * + * Transaction ordering is determined by the budgeting code and set in + * the budget. Transactions are linked into the actual hardware + * schedule in the order specified by the budget. FSTN positioning is + * static; it follows the recommendations of the EHCI 1.0 spec and + * places one save-state FSTN at the lowest level in each branch of + * the INTR tree and one restore FSTN at the head of level one. Dummy + * sITDs are positioned in the order of the original sITDs of the + * preceeding frame. + * + * The scheduler does not currently implement any inter- or + * intra-frame rebalancing; that is, the budget is not adjusted to + * fill in 'holes' or shuffle transfer slots to optimize remaining + * bandwidth when endpoints are removed. QHs will reuse holes at the + * moment, isoch transfers will not. This can also lead to starving + * QHs depending on the order in which the QHs are budgeted because + * the tree arrangement of the QHs places ordering restrictions upon + * QHs that do not exist for isoch transfers. Starvation is unlikely + * (and far less likely than for the old scheduler), but it's still + * possible to be unable to schedule a new QH before actually running + * out of FS bandwidth. * - * For ISO, we make an "iso_stream" head to serve the same role as a QH. - * It keeps track of every ITD (or SITD) that's linked, and holds enough - * pre-calculated schedule data to make appending to the queue be quick. */ -static int ehci_get_frame (struct usb_hcd *hcd); + +/* enable/disable shedule-specific debugging output */ +module_param (sched_verbose, uint, S_IRUGO); +MODULE_PARM_DESC (sched_verbose, + "schedule verbose: dump additional scheduling-specific " + "debugging information to syslog. Incurs large latencies in" + " near-realtime code; default is 0 (off)"); + +/* enable/disable use of FSTNs */ +static unsigned use_fstn = 1; +module_param (use_fstn, uint, S_IRUGO); +MODULE_PARM_DESC (use_fstn, + "use FSTNs: allow use of FSTN to schedule full speed INTR " + "transfers across frame boundaries; default is 1 (on)"); + +/* enable/disable use of dummy sITDs */ +static unsigned use_dummy_sitds = 1; +module_param (use_dummy_sitds, uint, S_IRUGO); +MODULE_PARM_DESC (use_dummy_sitds, + "use dummy sITDs: allow use of 'dummy' sITDs to " + "schedule schedule full speed ISOCH transfers across frame " + "boundaries; default is 1 (on)"); + +/* set limit on periodic load per TT uframe */ +static unsigned fs_bytes_per_uframe = 188; +module_param (fs_bytes_per_uframe, uint, S_IRUGO); +MODULE_PARM_DESC (fs_bytes_per_uframe, + "full-speed bytes per high-speed uframe: set the " + "maximum number of periodic full-speed bytes to schedule " + "per TT per microframe; default [spec] is 188"); + +/* set limit on periodic load per TT frame */ +static unsigned fs_bytes_per_fullframe = 1157; +module_param (fs_bytes_per_fullframe, uint, S_IRUGO); +MODULE_PARM_DESC (fs_bytes_per_uframe, + "full-speed bytes per full-speed frame: set the " + "maximum number of periodic full-speed bytes to schedule " + "per frame; default [spec] is 1157"); + +/* set limit on periodic load per high-speed uframe */ +static unsigned hs_usec_per_uframe = 100; +module_param (hs_usec_per_uframe, uint, S_IRUGO); +MODULE_PARM_DESC (hs_usec_per_uframe, + "high-speed usec per uframe: set the maximum " + "number of usec to use for periodic high-speed transfers; " + "default [spec] is 100"); + +/* set limit on unique endpoints per uframe */ +static unsigned endpoints_per_uframe = 16; +module_param (endpoints_per_uframe, uint, S_IRUGO); +MODULE_PARM_DESC (endpoints_per_uframe, + "endpoints per uframe: maximum number of unique " + "transfers to attemps during a microframe; default " + "[spec] is 16"); /*-------------------------------------------------------------------------*/ +/* shadow schedule positioning, as this is needed throughout */ /* * periodic_next_shadow - return "next" pointer on shadow list + * * @periodic: host pointer to qh/itd/sitd * @tag: hardware tag for type of this record */ @@ -53,502 +197,1584 @@ return &periodic->fstn->fstn_next; case Q_TYPE_ITD: return &periodic->itd->itd_next; - // case Q_TYPE_SITD: + // case Q_TYPE_SITD: default: return &periodic->sitd->sitd_next; } } -/* caller must hold ehci->lock */ -static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr) -{ - union ehci_shadow *prev_p = &ehci->pshadow [frame]; - __le32 *hw_p = &ehci->periodic [frame]; - union ehci_shadow here = *prev_p; - - /* find predecessor of "ptr"; hw and shadow lists are in sync */ - while (here.ptr && here.ptr != ptr) { - prev_p = periodic_next_shadow (prev_p, Q_NEXT_TYPE (*hw_p)); - hw_p = here.hw_next; - here = *prev_p; +/* the following are merely frame-structure dumpers to aid in + debugging. Be careful of their use; they will introduce extreme + latencies in what is [essentially] realtime code. */ +static void print_budget_frame (struct ehci_hcd *ehci, int frame, + struct ehci_shadow_budget *insert, + void *owner) +{ + struct ehci_shadow_budget *here = + ehci->budget [frame & BUDGET_SLOT_MASK]; + printk(KERN_INFO " slot %d: ",frame & BUDGET_SLOT_MASK); + + while(here){ + if(here == insert || here->owner.ptr == owner) + printk(">>"); + + switch(here->type){ + case BUDGET_TYPE_ITD: + if(here->tt_bytes) + printk("[SITD %p 0x%x 0x%x]", + here, + (unsigned)here->cmask, + (unsigned)here->smask); + else + printk("[ITD %p 0x%x 0x%x]", + here, + (unsigned)here->cmask, + (unsigned)here->smask); + break; + case BUDGET_TYPE_QH: + printk("[QH (%s) %p %d 0x%x 0x%x]", + (here->tt_bytes?"FS/LS":"HS"), + here, + here->owner.qh->period, + (unsigned)here->cmask, + (unsigned)here->smask); + break; + } + + if(here == insert || here->owner.ptr == owner) + printk("<<"); + + if(here == here->next){ + printk("\nERROR: schedule entry linked " + "to itself!\n"); + return; + } + + here=here->next; + } - /* an interrupt entry (at list end) could have been shared */ - if (!here.ptr) - return; + printk("\n"); +} - /* update shadow and hardware lists ... the old "next" pointers - * from ptr may still be in use, the caller updates them. - */ - *prev_p = *periodic_next_shadow (&here, Q_NEXT_TYPE (*hw_p)); - *hw_p = *here.hw_next; +static void print_budget (struct ehci_hcd *ehci, + struct ehci_shadow_budget *insert, + void *owner) +{ + int i; + for(i=0; iperiodic [frame]; - union ehci_shadow *q = &ehci->pshadow [frame]; - unsigned usecs = 0; + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 *hw_p = &ehci->periodic [frame & (ehci->periodic_size-1)]; + __le32 type = Q_NEXT_TYPE (*hw_p); + printk(KERN_INFO "%s @ %d: ",pre,frame & (ehci->periodic_size-1)); + while(here && here->ptr){ + if(here->ptr == insert) + printk(">>"); - while (q->ptr) { - switch (Q_NEXT_TYPE (*hw_p)) { - case Q_TYPE_QH: - /* is it in the S-mask? */ - if (q->qh->hw_info2 & cpu_to_le32 (1 << uframe)) - usecs += q->qh->usecs; - /* ... or C-mask? */ - if (q->qh->hw_info2 & cpu_to_le32 (1 << (8 + uframe))) - usecs += q->qh->c_usecs; - hw_p = &q->qh->hw_next; - q = &q->qh->qh_next; - break; - // case Q_TYPE_FSTN: - default: - /* for "save place" FSTNs, count the relevant INTR - * bandwidth from the previous frame - */ - if (q->fstn->hw_prev != EHCI_LIST_END) { - ehci_dbg (ehci, "ignoring FSTN cost ...\n"); - } - hw_p = &q->fstn->hw_next; - q = &q->fstn->fstn_next; - break; + switch(type){ case Q_TYPE_ITD: - usecs += q->itd->usecs [uframe]; - hw_p = &q->itd->hw_next; - q = &q->itd->itd_next; - break; + { + struct ehci_itd *itd = here->itd; + printk("[ITD 0x%p : 0x%x]", + itd, + (unsigned)itd->hw_next); + } + break; case Q_TYPE_SITD: - /* is it in the S-mask? (count SPLIT, DATA) */ - if (q->sitd->hw_uframe & cpu_to_le32 (1 << uframe)) { - if (q->sitd->hw_fullspeed_ep & - __constant_cpu_to_le32 (1<<31)) - usecs += q->sitd->stream->usecs; - else /* worst case for OUT start-split */ - usecs += HS_USECS_ISO (188); - } - - /* ... C-mask? (count CSPLIT, DATA) */ - if (q->sitd->hw_uframe & - cpu_to_le32 (1 << (8 + uframe))) { - /* worst case for IN complete-split */ - usecs += q->sitd->stream->c_usecs; - } - - hw_p = &q->sitd->hw_next; - q = &q->sitd->sitd_next; - break; + { + struct ehci_sitd *sitd = here->sitd; + if(sitd->hw_backpointer != EHCI_LIST_END) + printk("[DUMMY 0x%p]", + sitd); + else + printk("[SITD 0x%p]", + sitd); + } + break; + case Q_TYPE_FSTN: + { + struct ehci_fstn *fstn = here->fstn; + if(fstn->hw_prev == EHCI_LIST_END) + printk("[FSTN restore 0x%p]",fstn); + else + printk("[FSTN save 0x%p <- 0x%p]", + fstn, + fstn->fstn_prev); } + break; + case Q_TYPE_QH: + { + struct ehci_qh *qh = here->qh; + printk("[QH %d 0x%p]", + qh->period,qh); + } + break; + } + + if(here->ptr == insert) + printk("<<"); + + if(here == periodic_next_shadow (here, type)){ + printk("\nERROR: periodic schedule entry " + "linked to itself!\n"); + return; + } + + hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (*hw_p); + } -#ifdef DEBUG - if (usecs > 100) - ehci_err (ehci, "uframe %d sched overrun: %d usecs\n", - frame * 8 + uframe, usecs); -#endif - return usecs; + printk("\n"); } +/* find position of a specific entry in the periodic schedule (ie, + * returns pointers such that we can update the predecessor's + * linkage); here->ptr == NULL indicates the find failed. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number of the shadow/hardware schedule to search + * @ptr: entry for which to search + * @hw_p: return pointer to hw_next field of preceeding entry + */ +static union ehci_shadow * +periodic_find_entry(struct ehci_hcd *ehci, + unsigned frame, + void *ptr, + __le32 **hw_p) +{ + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 type; + + *hw_p = &ehci->periodic [frame & (ehci->periodic_size-1)]; + type = Q_NEXT_TYPE (**hw_p); + + while (here->ptr && here->ptr != ptr) { + *hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (**hw_p); + } + return here; +} + /*-------------------------------------------------------------------------*/ +/* shadow budget implementation: low-level manipulation machinery */ -static int same_tt (struct usb_device *dev1, struct usb_device *dev2) +/* covert a QH's period [expressed in uFrames] to tree level. + * + * @period: period (uFrames) of a QH + */ +static int _period_to_level(int period) { - if (!dev1->tt || !dev2->tt) - return 0; - if (dev1->tt != dev2->tt) - return 0; - if (dev1->tt->multi) - return dev1->ttport == dev2->ttport; - else - return 1; + if(period<8)return 1; + if(period > PERIODIC_QH_MAX_PERIOD){ + return (PERIODIC_QH_MAX_PERIOD >> 3); + } + return period>>3; } -#ifdef CONFIG_USB_EHCI_TT_NEWSCHED - -/* Which uframe does the low/fullspeed transfer start in? +/* budget_position_new_itd - determine and return proper insertion position + * in the shadow budget for a new itd or sitd (both position into the + * same segment of the frame) * - * The parameter is the mask of ssplits in "H-frame" terms - * and this returns the transfer start uframe in "B-frame" terms, - * which allows both to match, e.g. a ssplit in "H-frame" uframe 0 - * will cause a transfer in "B-frame" uframe 0. "B-frames" lag - * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7. + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow/hardware schedule into which new + * transaction is to be inserted. */ -static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __le32 mask) +static struct ehci_shadow_budget ** +budget_position_new_itd(struct ehci_hcd *ehci, int frame) { - unsigned char smask = QH_SMASK & le32_to_cpu(mask); - if (!smask) { - ehci_err(ehci, "invalid empty smask!\n"); - /* uframe 7 can't have bw so this will indicate failure */ - return 7; + struct ehci_shadow_budget **here = + &ehci->budget [(frame & BUDGET_SLOT_MASK)]; + + /* skip to the end of any dummies or [s]ITDs at list head */ + while ( (*here) && + (*here)->type == BUDGET_TYPE_ITD ){ + here = &(*here)->next; } - return ffs(smask) - 1; + + return here; } -static const unsigned char -max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 }; - -/* carryover low/fullspeed bandwidth that crosses uframe boundries */ -static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8]) +/* budget_position_new_qh - determine and return proper insertion position + * in the shadow budget for a newly queued qh + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow/hardware schedule into which new + * transaction is to be inserted. + * @period: the endpoint period (in uFrames) + */ +static struct ehci_shadow_budget ** +budget_position_new_qh(struct ehci_hcd *ehci, int frame, int period) { - int i; - for (i=0; i<7; i++) { - if (max_tt_usecs[i] < tt_usecs[i]) { - tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i]; - tt_usecs[i] = max_tt_usecs[i]; - } + struct ehci_shadow_budget **here = + budget_position_new_itd(ehci, frame); + + while (*here){ + + /* skip past higher (but not same) period QH entries */ + /* > 8 to left of restore FSTN, <= 8 to right */ + if ( (*here)->type == BUDGET_TYPE_QH && + period >= (*here)->owner.qh->period) break; + + here = &(*here)->next; } + return here; } -/* How many of the tt's periodic downstream 1000 usecs are allocated? - * - * While this measures the bandwidth in terms of usecs/uframe, - * the low/fullspeed bus has no notion of uframes, so any particular - * low/fullspeed transfer can "carry over" from one uframe to the next, - * since the TT just performs downstream transfers in sequence. +/* budget_position_new - determine and return proper insertion position + * in the shadow budget for a new budget entry * - * For example two seperate 100 usec transfers can start in the same uframe, - * and the second one would "carry over" 75 usecs into the next uframe. + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow/hardware schedule into which new + * transaction is to be inserted. + * @new: new,populated budget entry to insert */ -static void -periodic_tt_usecs ( - struct ehci_hcd *ehci, - struct usb_device *dev, - unsigned frame, - unsigned short tt_usecs[8] -) -{ - __le32 *hw_p = &ehci->periodic [frame]; - union ehci_shadow *q = &ehci->pshadow [frame]; - unsigned char uf; +static struct ehci_shadow_budget ** +budget_position_new(struct ehci_hcd *ehci, + int frame, + struct ehci_shadow_budget *new) +{ + switch(new->type){ + case BUDGET_TYPE_QH: + return budget_position_new_qh(ehci,frame, + new->owner.qh->period); + + case BUDGET_TYPE_ITD: + return budget_position_new_itd(ehci,frame); - memset(tt_usecs, 0, 16); + default: + return NULL; - while (q->ptr) { - switch (Q_NEXT_TYPE(*hw_p)) { - case Q_TYPE_ITD: - hw_p = &q->itd->hw_next; - q = &q->itd->itd_next; - continue; - case Q_TYPE_QH: - if (same_tt(dev, q->qh->dev)) { - uf = tt_start_uframe(ehci, q->qh->hw_info2); - tt_usecs[uf] += q->qh->tt_usecs; + } +} + +/* budget_find_entry_by_owner - find position of a specific entry in + * the shadow budget by looking for the single entry in the frame + * belonging to the passed in owner. return == NULL indicates the + * entry was not found. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow budget to search; masked to + * prevent overflow of the shadow budget + * @owner: the owner of the placeholder (qh or ehci_iso_stream) + */ +static struct ehci_shadow_budget * +budget_find_entry_by_owner(struct ehci_hcd *ehci, + unsigned frame, + void *owner) +{ + struct ehci_shadow_budget *here = + ehci->budget [(frame & BUDGET_SLOT_MASK)]; + + /* invariant: any one budget frame will contain at most a single entry + for a given owner; the budget does not explicitly contain spanning + elements, it only declares them in the transaction's cmask. Thus + searching by owner is unique. */ + while (here){ + if(here->owner.ptr == owner) break; + here = here->next; + } + return here; +} + +static void *periodic_shadow_owner(struct ehci_hcd *ehci, + union ehci_shadow *ptr, + __le32 type){ + switch(type){ + case Q_TYPE_FSTN: + return NULL; + case Q_TYPE_QH: + return ptr; + case Q_TYPE_ITD: + return ((struct ehci_itd *)ptr)->stream; + case Q_TYPE_SITD: + return ((struct ehci_sitd *)ptr)->stream; + default: + return NULL; + } +} + +static union ehci_shadow * +periodic_translate_budget_dummy (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_shadow_budget *translate, + __le32 **hw_p) +{ + struct ehci_shadow_budget *budget_prev = + ehci->budget [((frame-1) & BUDGET_SLOT_MASK)]; + union ehci_shadow *shadow = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 type; + + *hw_p = &ehci->periodic [frame & (ehci->periodic_size-1)]; + type = Q_NEXT_TYPE (**hw_p); + + /* Everything in the shadow schedule will be there in the budget + [except for FSTNs] */ + while (shadow->ptr){ + struct ehci_sitd *sitd; + if(type != Q_TYPE_SITD) return shadow; + + sitd = shadow->sitd; + if(sitd->hw_backpointer != EHCI_LIST_END) return shadow; + + /* advance budget until the current budget + placeholder is the one belonging to current + shadow entry or until we hit the stopping + point. */ + + while(budget_prev && + budget_prev != translate && + budget_prev->owner.ptr != + periodic_shadow_owner(ehci,shadow->ptr,type)){ + budget_prev = budget_prev->next; + } + if(budget_prev == translate){ + return shadow; /* a match! */ + } + + /* falling off the end of the budget indicates + * an internal logic error */ + if(budget_prev == NULL){ + ehci_err(ehci, + "schedule contains " + "unbudgeted transaction\n"); + return NULL; + } + + /* advance shadow and hardware schedule by one, then + loop to test this position */ + *hw_p = shadow->hw_next; + shadow = periodic_next_shadow (shadow, type); + type = Q_NEXT_TYPE (**hw_p); + + } + + /* nothing queued or fell off the end looking for stopping point. + Append to end. */ + return shadow; + +} + +/* periodic_translate_budget - Finds the position in the + * hardware/shadow schedule to insert a new hardware schedule entry + * based on the position in the shadow budget of the passed in budget + * entry. This handles positioning non-dummy sITD, ITD and QH + * entries; FSTN position has nothing to do with the budget and is + * handled by code specific to FSTNs. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the hardware schedule to search; masked to + * prevent overflow + * @translate: budget entry to translate to hardware schedule + * @hw_p: return a pointer to the predecessor's hw_next field + */ +static union ehci_shadow * +periodic_translate_budget (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_shadow_budget *translate, + __le32 **hw_p) +{ + struct ehci_shadow_budget *budget_curr = + ehci->budget [(frame & BUDGET_SLOT_MASK)]; + union ehci_shadow *shadow = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 type; + + /* Complete/Inactive entries that have not yet been processed + by scan_periodic may yet be present, that will not affect + this algorithm (they will have entries present in the + budget, even waiting for final urb completion). */ + + *hw_p = &ehci->periodic [frame & (ehci->periodic_size-1)]; + type = Q_NEXT_TYPE (**hw_p); + + /* Everything in the shadow schedule will be there in the budget + [except for FSTNs] */ + while (shadow->ptr){ + + if(type != Q_TYPE_FSTN){ + + /* advance past all dummy sITDs before doing anything else. */ + if(type == Q_TYPE_SITD && + shadow->sitd->hw_backpointer != EHCI_LIST_END) + goto advance; + + while(budget_curr && + budget_curr != translate && + budget_curr->owner.ptr != + periodic_shadow_owner(ehci,shadow->ptr,type)){ + budget_curr = budget_curr->next; } - hw_p = &q->qh->hw_next; - q = &q->qh->qh_next; - continue; - case Q_TYPE_SITD: - if (same_tt(dev, q->sitd->urb->dev)) { - uf = tt_start_uframe(ehci, q->sitd->hw_uframe); - tt_usecs[uf] += q->sitd->stream->tt_usecs; + if(budget_curr == translate){ + return shadow; /* a match! */ } - hw_p = &q->sitd->hw_next; - q = &q->sitd->sitd_next; - continue; - // case Q_TYPE_FSTN: - default: - ehci_dbg(ehci, - "ignoring periodic frame %d FSTN\n", frame); - hw_p = &q->fstn->hw_next; - q = &q->fstn->fstn_next; + + /* falling off the end of the budget indicates + * an internal logic error */ + if(budget_curr == NULL){ + ehci_err(ehci, + "schedule contains " + "unbudgeted transaction\n"); + return NULL; + } + }else{ + struct ehci_fstn *fstn = shadow->fstn; + /* restore fstn: + ITD/sITD must go before + QH of period > 8 must go before + QH of period <=8 must go after + */ + /* save fstn: + all ITD/sITD must go before + all QH must go after + */ + + if(translate->type == BUDGET_TYPE_ITD){ + return shadow; + } + + if(fstn->hw_prev == EHCI_LIST_END) + if(translate->owner.qh->period > 8) + return shadow; + + } + + advance: + /* advance shadow and hardware schedule by one, then + loop to test this position */ + *hw_p = shadow->hw_next; + shadow = periodic_next_shadow (shadow, type); + type = Q_NEXT_TYPE (**hw_p); + + } + + /* nothing queued or fell off the end looking for stopping point. + Append to end. */ + return shadow; + +} + +/* budget_unlink_entries_by_owner - scan the complete shadow budget + * and unlink/free all entries owned by passed in owner ptr. + * + * @ehci: pointer to ehci host controller device structure. + * @owner: pointer to owner + */ +static void budget_unlink_entries_by_owner(struct ehci_hcd *ehci,void *owner) +{ + struct ehci_shadow_budget *qh = NULL; + int frame; + + if(sched_verbose){ + ehci_info(ehci, "Releasing bandwidth for owner %p\n", + owner); + ehci_info(ehci, " Budget before: \n"); + print_budget(ehci,NULL,owner); + } + + for(frame=0; framebudget[frame]; + + while(*budget){ + if((*budget)->owner.ptr == owner){ + struct ehci_shadow_budget *next = + ((*budget)->next); + + switch((*budget)->type){ + default: + case BUDGET_TYPE_ITD: + + kmem_cache_free(ehci->budget_pool, + *budget); + *budget = next; + + break; + case BUDGET_TYPE_QH: + if(!qh || (*budget)==qh){ + qh = *budget; + *budget = next; + break; + } + + /* another qh with the same + owner is an internal error; + not fatal, but shouldn't + happen. We will leak the + memory rather than risk a + double-free. */ + ehci_err(ehci, + "multiple unique QH budget " + "entries with same endpoint " + "owner.\n"); + + break; + } + /* do not advance; need to process the + replacement which is now *budget */ + }else + budget = &((*budget)->next); + } } + + if(qh)kmem_cache_free(ehci->budget_pool,qh); + + if(sched_verbose){ + ehci_info(ehci, " Budget after: \n"); + print_budget(ehci,NULL,owner); + } - carryover_tt_bandwidth(tt_usecs); - - if (max_tt_usecs[7] < tt_usecs[7]) - ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n", - frame, tt_usecs[7] - max_tt_usecs[7]); } -/* - * Return true if the device's tt's downstream bus is available for a - * periodic transfer of the specified length (usecs), starting at the - * specified frame/uframe. Note that (as summarized in section 11.19 - * of the usb 2.0 spec) TTs can buffer multiple transactions for each - * uframe. - * - * The uframe parameter is when the fullspeed/lowspeed transfer - * should be executed in "B-frame" terms, which is the same as the - * highspeed ssplit's uframe (which is in "H-frame" terms). For example - * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0. - * See the EHCI spec sec 4.5 and fig 4.7. - * - * This checks if the full/lowspeed bus, at the specified starting uframe, - * has the specified bandwidth available, according to rules listed - * in USB 2.0 spec section 11.18.1 fig 11-60. - * - * This does not check if the transfer would exceed the max ssplit - * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4, - * since proper scheduling limits ssplits to less than 16 per uframe. +/*-------------------------------------------------------------------------*/ +/* Shadow budget constraint/metric calculation routines */ + +/* determine the TT relevant to this shadow budget entry + * + * @b: pointer to shadow budget entry */ -static int tt_available ( - struct ehci_hcd *ehci, - unsigned period, - struct usb_device *dev, - unsigned frame, - unsigned uframe, - u16 usecs -) +static int +budget_same_tt (struct ehci_shadow_budget *b1,struct ehci_shadow_budget *b2) { - if ((period == 0) || (uframe >= 7)) /* error */ - return 0; + struct usb_device *dev1; + struct usb_device *dev2; - for (; frame < ehci->periodic_size; frame += period) { - unsigned short tt_usecs[8]; + switch (b1->type) { + case BUDGET_TYPE_QH: + dev1 = b1->owner.qh->dev; + break; + case BUDGET_TYPE_ITD: + dev1 = b1->owner.iso->udev; + break; + default: + return 0; + } - periodic_tt_usecs (ehci, dev, frame, tt_usecs); + switch (b2->type) { + case BUDGET_TYPE_QH: + dev2 = b2->owner.qh->dev; + break; + case BUDGET_TYPE_ITD: + dev2 = b2->owner.iso->udev; + break; + default: + return 0; + } - ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in" - " schedule %d/%d/%d/%d/%d/%d/%d/%d\n", - frame, usecs, uframe, - tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3], - tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]); - - if (max_tt_usecs[uframe] <= tt_usecs[uframe]) { - ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n", - frame, uframe); - return 0; - } - - /* special case for isoc transfers larger than 125us: - * the first and each subsequent fully used uframe - * must be empty, so as to not illegally delay - * already scheduled transactions - */ - if (125 < usecs) { - int ufs = (usecs / 125) - 1; - int i; - for (i = uframe; i < (uframe + ufs) && i < 8; i++) - if (0 < tt_usecs[i]) { - ehci_vdbg(ehci, - "multi-uframe xfer can't fit " - "in frame %d uframe %d\n", - frame, i); - return 0; - } + if(dev1->tt != dev2->tt) + return 0; + if (dev1->tt->multi) + return dev1->ttport == dev2->ttport; + else + return 1; + +} + +/* _calc_hs_uframe - count usecs used in the HS schedule of the + * specified frame/uFrame. *new is a new proposed shadow budget entry + * to be treated as if it is already linked into the shadow budget at + * position **insert. Returns abolute usec count, not metric or + * remaining available usecs. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame position in the shadow budget (masked to shadow budget size) + * @uFrame: uFrame position within full frame; may be 8 or 9 to access + * spanning c_mask + * @insert: position of *new [NULL if new is NULL] + * @new: a proposed shadow entry to be treated as appearing at position + * *insert; may be NULL + */ +static int _calc_hs_uframe(struct ehci_hcd *ehci, + int frame, + int uFrame, + struct ehci_shadow_budget **insert, + struct ehci_shadow_budget *new) +{ + struct ehci_shadow_budget **budget = + &ehci->budget [(frame & BUDGET_SLOT_MASK)]; + int usec = 0; + if(budget == insert) budget = &new; + + while(*budget){ + struct ehci_shadow_budget *budgetp = *budget; + + /* is it in the S-mask? */ + if (budgetp->smask & (1 << uFrame)) + usec += budgetp->s_usecs; + + /* ... or C-mask? */ + if (budgetp->cmask & (1 << uFrame)) + usec += budgetp->c_usecs; + + /* advance */ + if(budget == &new) + budget = insert; + else{ + budget = &(budgetp->next); + if(budget == insert) budget = &new; } + } - tt_usecs[uframe] += usecs; + return usec; +} + +/* budget_calc_hs_frame - returns the budget 'load' metric for a given + * high speed frame, range -1 (invalid schedule), 0 (lowest good + * score) to 100 (best possible score). HS scheduling uses this to + * choose between possbile slots. FS/LS scheduling uses it as a check + * to make sure the FS budgeting choice is valid in the context of the + * HS schedule. *new is a new proposed shadow budget entry + * to be treated as if it is already linked into the shadow budget at + * position **insert. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame position in the shadow budget (masked to shadow budget size) + * @insert: position to use for *new; in other functions that take an + * insertion point as a marker, *insert=NULL indicates insert at end + * of list. budget_calc_hs_frame checks more than one frame (because + * of transactions that can span frames), so a simple NULL would be + * ambiguous. Thus the pointer to the insertion point, which would + * uniquely point to only one frame's list, even in the case of + * insertion. + * @new: a proposed shadow entry to be treated as appearing at position + * *insert; + * may be NULL + */ - carryover_tt_bandwidth(tt_usecs); +static int budget_calc_hs_frame(struct ehci_hcd *ehci, + int frame, + struct ehci_shadow_budget **insert, + struct ehci_shadow_budget *new) +{ + int max=0, uFrame; + for(uFrame=0; uFrame<8; uFrame++){ + + /* first verify we've not overcommitted any uFrames */ + /* If uFrame < 2, count the cmask contribution of + * spanning from prior frame */ + int val = _calc_hs_uframe(ehci,frame,uFrame,insert,new); + if(uFrame<2) + val += _calc_hs_uframe(ehci,frame-1, + uFrame+8,insert,new); + + if(val>max)max=val; + if(max>hs_usec_per_uframe){ + return -1; + } + } + return 100-(max*100/hs_usec_per_uframe); +} + +/* periodic_budget_fs_frame - test a proposed new addition to the + * periodic schedule of a given frame. Returns <0 if the frame is + * unsuitable, else a positive value indicating the relative space + * remaining in that frame after the new transaction would be + * scheduled; 0 is lowest score, 100 is best possible score. This + * return value is a metric used for breadth-first allocation of + * bandwidth. As a side efect, determine and set appropriate s_mask + * and c_mask for the passed in *new entry. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame position in the shadow budget (masked to shadow + * budget size) + * @earliest_uFrame: prevent *new from scheduling into any uframe + * earlier than this + * @insert: position of *new + * @new: a proposed shadow entry to be treated as appearing at + * position *insert + */ +static int budget_calc_fs_frame(struct ehci_hcd *ehci, + int frame, + unsigned earliest_uFrame, + struct ehci_shadow_budget *insert, + struct ehci_shadow_budget *new) +{ + struct ehci_shadow_budget *budget = + ehci->budget [(frame & BUDGET_SLOT_MASK)]; + + /* Scheduling through a TT is done by byte counting, not usec + counting. Because we cannot issue a FS/LS start split in + uFrame 6 or 7, the only way to get anywhere near a full 90% + of a FS frame for periodic transactions is to 'overcommit' + the FS scheduling of each microframe [when considered by + usecs needed for the transfer]. This is not actually + overcommitting as the TT will buffer the 'excess' from any + uFrame and schedule it to transmit in a later uFrame. USB + 2.0 11.18.4: "Scheduling of split transactions is done by + the host (typically in software) based on a best-case + estimate of how the full-/low-speed transacitons can be run + on the downstream facing bus." */ + + /* per uFrame best-case byte budget; USB 2.0 11.18.1 */ + int remaining_tt_bytes = fs_bytes_per_uframe; + + /* max full-frame bytes for enforcing worst-case 90/10 */ + int remaining_frame_bytes = fs_bytes_per_fullframe; + + /* track max ssplits per uframe per TT */ + int remaining_tt_transactions = endpoints_per_uframe; + + /* track the amount of frame spanned (likely > used) by ISO */ + int load_frame_bytes = 0; + + int uFrame=0; + int load = remaining_frame_bytes; + int spanned_gap = 0; + + if(budget == insert) budget = new; + earliest_uFrame &= 0x7; + + while(budget){ + int start_frame = uFrame; + u8 smask=0; + u16 cmask=0; + + /* exclude other TTs as well as HS transfers, which + have no bearing on FS sched. */ + if(!budget_same_tt(new,budget)) + goto advance; + + if(budget != new){ + + /* For a variety of reasons, the preexisting + FS budget may contain holes where something + could be scheduled but nothing is. In this + case, the current budget placeholder's + smask will exist entirely in the future. + Test for this case and advance the counters + if true. */ + if( ((budget->smask >> (uFrame+1)) << (uFrame+1)) == + budget->smask){ + remaining_tt_bytes = + fs_bytes_per_uframe; + remaining_tt_transactions = + endpoints_per_uframe; + uFrame++; + continue; + } + + /* For now, the scheduler holds invariant that + transactions appear in a frame in smask + order; that is, that FS transactions appear + in the frame budget in the same order that + they are run by the host controller. + + Thus, the current transaction must not have + already been scheduled into the hardware + schedule to start before now. */ + + if( ((budget->smask >> uFrame) << uFrame) != + budget->smask) + return -1; + + + }else{ + if(uFrame < earliest_uFrame){ + remaining_tt_bytes = + fs_bytes_per_uframe; + remaining_tt_transactions = + endpoints_per_uframe; + uFrame++; + continue; + } + } - /* fail if the carryover pushed bw past the last uframe's limit */ - if (max_tt_usecs[7] < tt_usecs[7]) { - ehci_vdbg(ehci, - "tt unavailable usecs %d frame %d uframe %d\n", - usecs, frame, uframe); - return 0; + if(!spanned_gap && budget->type == BUDGET_TYPE_QH){ + load = fs_bytes_per_uframe * (uFrame+1) - + load_frame_bytes; + + if(load<1)load=1; + spanned_gap=1; + } + + remaining_frame_bytes -= budget->tt_bytes; + remaining_tt_bytes -= budget->tt_bytes; + remaining_tt_transactions--; + + if(remaining_tt_transactions<0 || remaining_frame_bytes<0) + return -1; + + while(remaining_tt_bytes <= 0){ + remaining_tt_bytes += fs_bytes_per_uframe; + remaining_tt_transactions = endpoints_per_uframe; + uFrame++; + } + + if(!spanned_gap){ + load_frame_bytes = + fs_bytes_per_uframe * (uFrame+1) - + remaining_tt_bytes; + load = fs_bytes_per_fullframe - + load_frame_bytes; + } + + if(budget != new) goto advance; + + /* Generate smask/cmask for new entry */ + + if(budget->type == BUDGET_TYPE_QH){ + smask = (0x01 << start_frame); + /* cmask purposely overflows the frame by some + amount if FSTN is needed for frame + spanning */ + /* USB 2.0 11.18.4.3.b */ + cmask = (0x1c << start_frame); + + /* if we need an FSTN are they even + available? */ + if(cmask&(~0xff)){ + if(use_fstn==0) + return -1; + if(ehci->fstn_pool==NULL) + return -1; + } + }else{ + /* sitd */ + if(budget->owner.iso->bEndpointAddress & USB_DIR_IN){ + /* cmask purposely overflows the frame + by some amount if dummy SITD is + needed for frame spanning */ + smask = (0x01 << start_frame); + cmask = ((1<<(uFrame-start_frame+3)) - 1) << + (2 + start_frame); + }else{ + /* ISO OUT */ + smask = ((1 << (uFrame-start_frame+1)) - 1) << + start_frame; + cmask = 0; + } + + /* have we disallowed frame spanning? */ + if(cmask&(~0xff)){ + if(use_dummy_sitds==0) + return -1; + } + } + + /* FS/LS ssplit in uFrame 6 or 7 is illegal */ + if(smask & 0xffc0) + return -1; + + new->smask=smask; + new->cmask=cmask; + + advance: + + if(budget == new) + budget = insert; + else{ + budget = budget->next; + if(budget == insert) budget = new; } } + + return (load * 100 / fs_bytes_per_fullframe); +} - return 1; +/* _make_hs_smask - convenience function for generating a HS smask + * + * @period: trnsaction interval (in uFrames) + * @uFrame: intraframe offset + */ +static unsigned _make_hs_smask(int period, int uFrame) +{ + switch(period){ + case 1: + return 0xff; + case 2: + return 0x55 << (uFrame & 0x1); + case 4: + return 0x11 << (uFrame & 0x3); + default: + return 0x01 << (uFrame & 0x7); + } } -#else +/*-------------------------------------------------------------------------*/ +/* Top-level shadow budget query and controlling logic */ -/* return true iff the device's transaction translator is available - * for a periodic transfer starting at the specified frame, using - * all the uframes in the mask. +/* budget_add_endpoint - create a new shadow budget entry for a + * transaction of passed in type, owner and interval, and add it to + * the shadow budget (or return nonzero if the request is + * impossible,eg, no room in the bandwidth budget). + * + * @ehci: pointer to ehci host controller device structure. + * @owner: pointer to transaction owner (ehci_qh or ehci_iso_stream) + * @type: BUDGET_TYPE_QH (intr) or BUDGET_TYPE_ITD (itd or sitd) + * @interval: endpoint interval (always in uFrames) */ -static int tt_no_collision ( - struct ehci_hcd *ehci, - unsigned period, - struct usb_device *dev, - unsigned frame, - u32 uf_mask -) -{ - if (period == 0) /* error */ - return 0; +static int budget_add_endpoint (struct ehci_hcd *ehci, + void *owner, + u8 type, + int interval) +{ + int status = 0; + int frame,uFrame; + int best_choice = -1; + int best_metric = -1; + int period = (interval ? + (interval > BUDGET_SLOTS*8 ? BUDGET_SLOTS*8 : interval) : + 1); + int period_8 = ( (period>>3) ? (period>>3) : 1); // always <= + // _period_to_level + struct ehci_shadow_budget new, *new_p = NULL; + memset(&new,0,sizeof(new)); + new.owner.ptr = owner; + new.type = type; + + switch (type){ + case BUDGET_TYPE_ITD: + { + struct ehci_iso_stream *stream = + (struct ehci_iso_stream *)owner; + new.s_usecs = stream->usecs; + new.c_usecs = stream->c_usecs; + new.tt_bytes = stream->tt_bytes; + if(sched_verbose){ + ehci_info(ehci, "Budgeting new [s]ITD: owner %p, " + "timing %u,%u,%u, interval %d\n", + owner,new.s_usecs,new.c_usecs,new.tt_bytes, + interval); + } + } + break; + case BUDGET_TYPE_QH: + { + struct ehci_qh *qh = (struct ehci_qh *)owner; + new.s_usecs = qh->usecs; + new.c_usecs = qh->c_usecs; + new.tt_bytes = qh->tt_bytes; + if(sched_verbose){ + ehci_info(ehci, "Budgeting new QH: owner %p, " + "timing %u,%u,%u, interval %d\n", + owner,new.s_usecs,new.c_usecs,new.tt_bytes, + interval); + } + } + break; + default: + status = -ENOSYS; + goto out; + } + + if(sched_verbose){ + if(new.tt_bytes) + printk(KERN_INFO " FS/LS strategy, Metrics:\n"); + else + printk(KERN_INFO " HS strategy, Metrics:\n"); + } + + /* determine best 'slot' */ + for(uFrame = 0; uFrame < period; uFrame++){ + int frame = (uFrame >> 3); + int min_metric = 100; /* percent */ + + if(sched_verbose) + printk(KERN_INFO " uFrame slot %d : ",uFrame); + + /* scan entirety of slot */ + for(; frameperiodic_size; frame += period) { - union ehci_shadow here; - __le32 type; - - here = ehci->pshadow [frame]; - type = Q_NEXT_TYPE (ehci->periodic [frame]); - while (here.ptr) { - switch (type) { - case Q_TYPE_ITD: - type = Q_NEXT_TYPE (here.itd->hw_next); - here = here.itd->itd_next; - continue; - case Q_TYPE_QH: - if (same_tt (dev, here.qh->dev)) { - u32 mask; - - mask = le32_to_cpu (here.qh->hw_info2); - /* "knows" no gap is needed */ - mask |= mask >> 8; - if (mask & uf_mask) - break; - } - type = Q_NEXT_TYPE (here.qh->hw_next); - here = here.qh->qh_next; - continue; - case Q_TYPE_SITD: - if (same_tt (dev, here.sitd->urb->dev)) { - u16 mask; - - mask = le32_to_cpu (here.sitd - ->hw_uframe); - /* FIXME assumes no gap for IN! */ - mask |= mask >> 8; - if (mask & uf_mask) - break; - } - type = Q_NEXT_TYPE (here.sitd->hw_next); - here = here.sitd->sitd_next; - continue; - // case Q_TYPE_FSTN: - default: - ehci_dbg (ehci, - "periodic frame %d bogus type %d\n", - frame, type); + if(sched_verbose){ + printk(" %d:%d|%d", + frame, fs_metric,hs_metric); + } + + + /* If the c_mask 'overflowed', this indicates + a frame wrap either FSTN or dummy sITD. + Verify it will fit into the next frame's HS + budget. */ + if(BUDGET_WRAP_P(&new)){ + int span_metric = budget_calc_hs_frame( + ehci, frame+1, insert, &new); + if(span_metric < 0) min_metric = -1; + if(sched_verbose) + printk("|%d",span_metric); + + } + + if(min_metric <= best_metric) + break; // already no better than + // previous best; no reason to + // keep looking + + } + if(sched_verbose) + printk("\n"); + + if(min_metric>best_metric){ + best_metric = min_metric; + best_choice = uFrame; + } + + if(best_metric == 100) + break; // best possible result; no reason to + // keep looking + } + + if(sched_verbose) + ehci_info(ehci," Budgeting slot choice: %d\n",best_choice); + + if(best_choice < 0){ + status = -ENOSPC; + goto out; + } + + /* All checks pass. Add necessary budget placeholder[s] to + shadow budget */ + uFrame = best_choice; + frame = (uFrame >> 3); + for(; framebudget_pool, + GFP_ATOMIC); + if(new_p == NULL){ + /* clean up; remove partially complete + * budgeting and error out*/ + budget_unlink_entries_by_owner(ehci,owner); + status = -ENOMEM; + goto out; + } + memcpy(new_p,&new,sizeof(new)); + + /* reconstruct best budget masks and positioning + (saves on storage not to save all the possibilities + from above) */ + insert = budget_position_new(ehci,frame,new_p); + if(*insert != new_p){ + if(new.tt_bytes){ + /* FS/LS */ + budget_calc_fs_frame(ehci, frame, uFrame, + *insert, new_p); + }else{ + /* HS */ + new_p->smask = _make_hs_smask(period,uFrame); } + + if(type == BUDGET_TYPE_QH) + ((struct ehci_qh *)owner)->budget = new_p; + + /* link new entry */ + new_p->next = *insert; + *insert = new_p; + + }else + ehci_err(ehci,"Should not have seen same QH " + "while linking in budget tree\n"); + } +out: + if(sched_verbose){ + ehci_info(ehci, " Budgeting status: %d\n", + status); + ehci_info(ehci, " Budget after: \n"); + print_budget(ehci,NULL,owner); + } - /* collision or error */ - return 0; + return status; +} + +/* budget_schedule_next - Return the proper frame number and position + * in the shadow schedule of the next transaction belonging to owner + * in the shadow budget. If the transaction is already in the + * shadow/hard schedule, here->ptr will point to it. + * + * @ehci: pointer to ehci host controller device structure. + * @startframe: frame number in the shadow schedule at which to begin search + * @owner: pointer to transaction owner (ehci_qh or ehci_iso_stream) + * @b: returns pointer to matching shadow budget entry (NULL if none) + * @here: returns pointer to insertion position in shadow schedule + * @hw_p: returns pointer to insertion position in hardware schedule + */ +static int budget_schedule_next (struct ehci_hcd *ehci, + int startframe, + void *owner, + struct ehci_shadow_budget **b, + union ehci_shadow **here, + __le32 **hw_p) +{ + int i; + for(i=startframe; i < startframe+BUDGET_SLOTS; i++){ + int pframe = i & (ehci->periodic_size-1); + + /* search for it in this frame's budget; we're only + searching for sITD, ITD and QH. Finding a spanning + element reuses the discovered *b elsewhere */ + *b = budget_find_entry_by_owner(ehci, i, owner); + if(*b){ + *here = periodic_translate_budget(ehci,pframe,*b,hw_p); + if(*here){ + return i; + } } } + + *here = NULL; + *b = NULL; + ehci_err(ehci,"tried to schedule an endpoint that doesn't " + "appear in the budget\n"); + return -1; +} - /* no collision */ - return 1; +/* budget_schedule_next - Return the number of the next frame for + * which there's a budgeted transaction for owner. + * + * @ehci: pointer to ehci host controller device structure. + * @startframe: frame number in the shadow schedule at which to begin search + * @owner: pointer to transaction owner (ehci_qh or ehci_iso_stream) + */ +static int budget_schedule_next_frame (struct ehci_hcd *ehci, + int startframe, + void *owner) +{ + int i; + for(i=startframe; i < startframe+BUDGET_SLOTS; i++){ + struct ehci_shadow_budget *b; + int bframe = i & BUDGET_SLOT_MASK; + b = budget_find_entry_by_owner(ehci, bframe, owner); + if(b){ + return i; + } + } + return -1; } -#endif /* CONFIG_USB_EHCI_TT_NEWSCHED */ +/* end of shadow budget implementation */ /*-------------------------------------------------------------------------*/ +/* enable_periodic - Activate the periodic schedule + * + * @ehci: pointer to ehci host controller device structure. + */ static int enable_periodic (struct ehci_hcd *ehci) { u32 cmd; int status; - /* did clearing PSE did take effect yet? - * takes effect only at frame boundaries... - */ - status = handshake (&ehci->regs->status, STS_PSS, 0, 9 * 125); - if (status != 0) { - ehci_to_hcd(ehci)->state = HC_STATE_HALT; - return status; - } + if(!ehci->periodic_sched){ + + if(sched_verbose) + printk(KERN_INFO "ehci: enabling periodic schedule\n"); + + /* did clearing PSE did take effect yet? + * takes effect only at frame boundaries... + */ + status = handshake (&ehci->regs->status, STS_PSS, 0, 9 * 125); + if (status != 0) { + ehci_to_hcd(ehci)->state = HC_STATE_HALT; + return status; + } + + cmd = readl (&ehci->regs->command) | CMD_PSE; + writel (cmd, &ehci->regs->command); + /* posted write ... PSS happens later */ + ehci_to_hcd(ehci)->state = HC_STATE_RUNNING; + + /* make sure ehci_work scans these */ + ehci->next_uframe = readl (&ehci->regs->frame_index) + & ((ehci->periodic_size << 3)-1); + } + + if(ehci->periodic_sched<2) + ehci->periodic_sched=2; + + ehci->periodic_sched++; - cmd = readl (&ehci->regs->command) | CMD_PSE; - writel (cmd, &ehci->regs->command); - /* posted write ... PSS happens later */ - ehci_to_hcd(ehci)->state = HC_STATE_RUNNING; - - /* make sure ehci_work scans these */ - ehci->next_uframe = readl (&ehci->regs->frame_index) - % (ehci->periodic_size << 3); return 0; } +/* deref_periodic - decrement refcount of periodic schedule use; will + * only take the schedule down to 'idle but running'; it is the + * responsibility of scan_periodic to actually halt a running schedule + * after it has been idle for one full sweep. + * + * @ehci: pointer to ehci host controller device structure. + */ +static void deref_periodic (struct ehci_hcd *ehci) +{ + /* deref as far as 'idle' */ + if(ehci->periodic_sched>2) + ehci->periodic_sched--; + + /* should *never* happen. Watch anyway. */ + if(ehci->periodic_sched<2){ + ehci_err(ehci,"periodic schedule refcount incorrect; " + "is %d, should be 2. Setting to 2.\n", + ehci->periodic_sched); + ehci->periodic_sched=2; + } +} + +/* disable_periodic - halt execution of the periodic schedule. Note + * that the periodic schedule is now disabled after a full pass + * through the periodic schedule in which it is unused. It's not + * immediately disabled when the last active transfer completes. + * + * @ehci: pointer to ehci host controller device structure. + */ static int disable_periodic (struct ehci_hcd *ehci) { u32 cmd; int status; + + if(ehci->periodic_sched == 1){ /* idle, marked to shutdown */ - /* did setting PSE not take effect yet? - * takes effect only at frame boundaries... - */ - status = handshake (&ehci->regs->status, STS_PSS, STS_PSS, 9 * 125); - if (status != 0) { - ehci_to_hcd(ehci)->state = HC_STATE_HALT; - return status; - } - - cmd = readl (&ehci->regs->command) & ~CMD_PSE; - writel (cmd, &ehci->regs->command); - /* posted write ... */ + if(sched_verbose) + printk(KERN_INFO "ehci: disabling periodic schedule\n"); - ehci->next_uframe = -1; + /* did setting PSE not take effect yet? + * takes effect only at frame boundaries... + */ + status = handshake (&ehci->regs->status, + STS_PSS, STS_PSS, 9 * 125); + if (status != 0) { + ehci_to_hcd(ehci)->state = HC_STATE_HALT; + return status; + } + + cmd = readl (&ehci->regs->command) & ~CMD_PSE; + writel (cmd, &ehci->regs->command); + /* posted write ... */ + + ehci->next_uframe = -1; + ehci->periodic_sched = 0; /* shut down */ + } + return 0; } /*-------------------------------------------------------------------------*/ +/* FSTN machinery + +FSTN decision making and manipulation is self contained and has +nothing to do with the budget, only the schedule. The master restore +FSTN is set up along with the ehci initialization; save-place FSTNs +are manipulated and adjusted after any changes to the QH tree in the +hardware schedule. */ + +/* periodic_init_link_restore_fstn - Place one restore FSTN at the + * beginning of node level 1 of the hardware schedule interrupt tree + * [EHCI 1.0 4.12.2.2.2]. Called once by the ehci initialization code + * and not again. + * + * @ehci: pointer to ehci host controller device structure. + */ +static void periodic_init_link_restore_fstn(struct ehci_hcd *ehci) +{ + int i; + + if(use_fstn){ + ehci->periodic_restore_fstn = + ehci_fstn_alloc (ehci, GFP_ATOMIC); + + if(ehci->periodic_restore_fstn){ + ehci->periodic_restore_fstn->hw_next = EHCI_LIST_END; + ehci->periodic_restore_fstn->hw_prev = EHCI_LIST_END; + + for(i=0; iperiodic_size; i++){ + union ehci_shadow *here = &ehci->pshadow [i]; + __le32 *hw_p = &ehci->periodic [i]; + + here->fstn = ehci->periodic_restore_fstn; + *hw_p = FSTN_NEXT (here->fstn->fstn_dma); + wmb (); + + } + } + } +} -/* periodic schedule slots have iso tds (normal or split) first, then a - * sparse tree for active interrupt transfers. +/* periodic_position_save_fstn - determine the insertion point within + * the specified frame for a save-place FSTN. Any given frame + * contains only one save-state FSTN, which sits at the beginning of + * the lowest level (highest interval) of the hardware schedule + * interrupt tree. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @hw_p: returns pointer to insertion point in hardware schedule + */ +static union ehci_shadow * +periodic_position_save_fstn(struct ehci_hcd *ehci, + unsigned frame, + __le32 **hw_p) +{ + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 type; + + *hw_p = &ehci->periodic [frame & (ehci->periodic_size-1)]; + type = Q_NEXT_TYPE (**hw_p); + + while (here->ptr){ + + if(type == Q_TYPE_FSTN || type == Q_TYPE_QH) + break; + + *hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (**hw_p); + } + + return here; +} + +/* periodic_find_save_fstn - search a frame in the shadow/hardware + * schedule for a save FSTN, returning the insertion point if found or + * NULL if not found. Any given frame will contain at most a single + * save-state FSTN. * - * this just links in a qh; caller guarantees uframe masks are set right. - * no FSTN support (yet; ehci 0.96+) + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @hw_p: returns pointer to insertion point in hardware schedule */ -static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) +static union ehci_shadow * +periodic_find_save_fstn(struct ehci_hcd *ehci, + unsigned frame, + __le32 **hw_p) +{ + union ehci_shadow *here; + int fframe = frame & (PERIODIC_QH_MAX_PERIOD-1); + struct ehci_fstn *fstn; + + if(ehci->periodic_save_fstns == NULL) + goto out; /* no FSTN support in HC */ + fstn = ehci->periodic_save_fstns[fframe]; + if(fstn == NULL) + goto out; /* this FSTN not yet alloced, couldn't be linked */ + + here = periodic_position_save_fstn(ehci, frame, hw_p); + if(here->ptr == fstn){ + return here; + } + +out: + return NULL; +} + +/* _periodic_unlink_fstn - unconditionally unlinks the save-state FSTN + * from the specified frame. noop if the frame does not have an + * active save-state FSTN. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + */ +static void _periodic_unlink_fstn (struct ehci_hcd *ehci, unsigned frame) { - unsigned i; - unsigned period = qh->period; + __le32 *hw_p; + union ehci_shadow *here = periodic_find_save_fstn(ehci, frame, &hw_p); - dev_dbg (&qh->dev->dev, - "link qh%d-%04x/%p start %d [%d/%d us]\n", - period, le32_to_cpup (&qh->hw_info2) & (QH_CMASK | QH_SMASK), - qh, qh->start, qh->usecs, qh->c_usecs); - - /* high bandwidth, or otherwise every microframe */ - if (period == 0) - period = 1; - - for (i = qh->start; i < ehci->periodic_size; i += period) { - union ehci_shadow *prev = &ehci->pshadow [i]; - __le32 *hw_p = &ehci->periodic [i]; - union ehci_shadow here = *prev; - __le32 type = 0; + if (!here) + return; + + /* inactivate */ + here->fstn->hw_prev = EHCI_LIST_END; + here->fstn->fstn_prev = NULL; - /* skip the iso nodes at list head */ - while (here.ptr) { - type = Q_NEXT_TYPE (*hw_p); - if (type == Q_TYPE_QH) - break; - prev = periodic_next_shadow (prev, type); - hw_p = &here.qh->hw_next; - here = *prev; - } + /* unlink */ + *hw_p = *here->hw_next; + *here = here->fstn->fstn_next; - /* sorting each branch by period (slow-->fast) - * enables sharing interior tree nodes - */ - while (here.ptr && qh != here.qh) { - if (qh->period > here.qh->period) - break; - prev = &here.qh->qh_next; - hw_p = &here.qh->hw_next; - here = *prev; - } - /* link in this qh, unless some earlier pass did that */ - if (qh != here.qh) { - qh->qh_next = here; - if (here.qh) - qh->hw_next = *hw_p; +} + +/* _periodic_link_fstn - allocs [if needed] and links save-state + * fstn into frame in appropraite position, also patching FSTN + * backpointer. If this frame already contains an active FSTN, only + * patches backpointer. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @back_qh: pointer to qh which the FSTN's backpointer should reference + */ +static void _periodic_link_fstn (struct ehci_hcd *ehci, + int frame, + struct ehci_qh *back_qh) +{ + + if(ehci->periodic_save_fstns){ + int fframe = frame & (PERIODIC_QH_MAX_PERIOD-1); + struct ehci_fstn *fstn = ehci->periodic_save_fstns[fframe]; + __le32 *hw_p; + union ehci_shadow *here = + periodic_position_save_fstn(ehci, frame, &hw_p); + + /* is the FSTN already alloced? */ + if(!fstn){ + fstn = ehci_fstn_alloc (ehci, GFP_ATOMIC); + if(!fstn){ + ehci_err(ehci, + "Unable to allocate FSTN\n"); + return; + } + + ehci->periodic_save_fstns[fframe] = fstn; + } + + /* patch the FSTN */ + fstn->hw_prev = QH_NEXT(back_qh->qh_dma); + fstn->fstn_prev = back_qh; + + /* already linked? */ + if(here->ptr != fstn){ + /* splice right to save-place fstn */ + fstn->fstn_next = *here; + fstn->hw_next = *hw_p; wmb (); - prev->qh = qh; - *hw_p = QH_NEXT (qh->qh_dma); + + /* splice it to left; it's now live */ + here->fstn = fstn; + *hw_p = FSTN_NEXT (fstn->fstn_dma); } + wmb (); } - qh->qh_state = QH_STATE_LINKED; - qh_get (qh); +} - /* update per-qh bandwidth for usbfs */ - ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period - ? ((qh->usecs + qh->c_usecs) / qh->period) - : (qh->usecs * 8); +/* _periodic_inspect_fstn_frame - toplevel machinery for managing all + * changes to FSTN state in the schedule; called whenever changes are + * made to the QH structure of a frame, automagically updates FSTN + * state in the hardware/shadow schedule to correctly reflect QH frame + * spanning. Links/unlinks/patches FSTNs in the following frame as + * needed. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @insert: if changes are to be affected to FSTN preceeding linking + * new QH into live schedule, *insert designates new QH + * insertion point + * @new_qh: points to new qh to be linked into schedule following FSTN updates; + * allows valid FSTN to preceed activating the QH; NULL if this is an + * unlink update. + */ +static void _periodic_inspect_fstn_frame (struct ehci_hcd *ehci, + int frame, + union ehci_shadow *insert, + union ehci_shadow *new_qh) +{ + __le32 type = + Q_NEXT_TYPE (ehci->periodic [frame & (ehci->periodic_size-1)]); + __le32 save_type = type; + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + + if(here == insert){ + here = new_qh; + type = Q_TYPE_QH; + } + + /* search frame for first spanning QH */ + while (here->ptr){ + if(type == Q_TYPE_QH){ + if(BUDGET_WRAP_P(here->qh->budget)){ + _periodic_link_fstn (ehci, frame+1, here->qh); + return; + } + } + + /* advance */ + if(here == new_qh){ + here = insert; + type = save_type; + }else{ + save_type = Q_NEXT_TYPE (*here->hw_next); + here = periodic_next_shadow (here, type); + if(here == insert){ + here = new_qh; + type = Q_TYPE_QH; + }else + type = save_type; + } + } + + /* no QH found, make sure there's no FSTN in following frame */ + _periodic_unlink_fstn (ehci, frame+1); +} - /* maybe enable periodic schedule processing */ - if (!ehci->periodic_sched++) - return enable_periodic (ehci); +/*-------------------------------------------------------------------------*/ +/* Periodic interrupt (QH) endpoint machinery */ - return 0; -} +/* periodic_qh_unlink_frame - unlink the passed in QH from the + * specified frame and update frame FSTN state + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @qh: QH to unlink + */ +static void periodic_qh_unlink_frame (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_qh *qh) +{ + __le32 *hw_p; + union ehci_shadow *here = periodic_find_entry(ehci, frame, qh, &hw_p); + + if (here->ptr){ + + *hw_p = *here->hw_next; + *here = here->qh->qh_next; + wmb (); + + /*... the old "next" pointers from ptr (and if a qh, the + fstns) may still be in use, the caller updates them. */ + } + + _periodic_inspect_fstn_frame(ehci,frame,NULL,NULL); + +} + +/* periodic_qh_deschedule - toplevel entry point for removing a given + * interrupt endpoint (rather, its QH) from the shadow/hardware + * schedule. Does not touch the budget; that is handled in the + * destructor. + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to remove + */ -static void qh_unlink_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) +static void periodic_qh_deschedule(struct ehci_hcd *ehci, struct ehci_qh *qh) { unsigned i; - unsigned period; + unsigned wait; // FIXME: // IF this isn't high speed @@ -557,40 +1783,26 @@ // THEN // qh->hw_info1 |= __constant_cpu_to_le32 (1 << 7 /* "ignore" */); - /* high bandwidth, or otherwise part of every microframe */ - if ((period = qh->period) == 0) - period = 1; + if(sched_verbose) + ehci_info(ehci, "Removing QH %p from schedule:\n", qh); - for (i = qh->start; i < ehci->periodic_size; i += period) - periodic_unlink (ehci, i, qh); + /* remove from hardware schedule */ + for (i=0; i < ehci->periodic_size; i++) + periodic_qh_unlink_frame (ehci, i, qh); /* update per-qh bandwidth for usbfs */ - ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period - ? ((qh->usecs + qh->c_usecs) / qh->period) - : (qh->usecs * 8); - - dev_dbg (&qh->dev->dev, - "unlink qh%d-%04x/%p start %d [%d/%d us]\n", - qh->period, - le32_to_cpup (&qh->hw_info2) & (QH_CMASK | QH_SMASK), - qh, qh->start, qh->usecs, qh->c_usecs); - + /* XXX Hm... now that we have the budget, this should be moved + to where the budget is released */ + ehci_to_hcd(ehci)->self.bandwidth_allocated -= + (qh->usecs + qh->c_usecs) / qh->period; + /* qh->qh_next still "live" to HC */ qh->qh_state = QH_STATE_UNLINK; qh->qh_next.ptr = NULL; qh_put (qh); /* maybe turn off periodic schedule */ - ehci->periodic_sched--; - if (!ehci->periodic_sched) - (void) disable_periodic (ehci); -} - -static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh) -{ - unsigned wait; - - qh_unlink_periodic (ehci, qh); + deref_periodic (ehci); /* simple/paranoid: always delay, expecting the HC needs to read * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and @@ -598,8 +1810,8 @@ * active high speed queues may need bigger delays... */ if (list_empty (&qh->qtd_list) - || (__constant_cpu_to_le32 (QH_CMASK) - & qh->hw_info2) != 0) + || (__constant_cpu_to_le32 (QH_CMASK) + & qh->hw_info2) != 0) wait = 2; else wait = 55; /* worst case: 3 * 1024 */ @@ -607,195 +1819,136 @@ udelay (wait); qh->qh_state = QH_STATE_IDLE; qh->hw_next = EHCI_LIST_END; - wmb (); -} - -/*-------------------------------------------------------------------------*/ - -static int check_period ( - struct ehci_hcd *ehci, - unsigned frame, - unsigned uframe, - unsigned period, - unsigned usecs -) { - int claimed; - - /* complete split running into next frame? - * given FSTN support, we could sometimes check... - */ - if (uframe >= 8) - return 0; - - /* - * 80% periodic == 100 usec/uframe available - * convert "usecs we need" to "max already claimed" - */ - usecs = 100 - usecs; - - /* we "know" 2 and 4 uframe intervals were rejected; so - * for period 0, check _every_ microframe in the schedule. - */ - if (unlikely (period == 0)) { - do { - for (uframe = 0; uframe < 7; uframe++) { - claimed = periodic_usecs (ehci, frame, uframe); - if (claimed > usecs) - return 0; - } - } while ((frame += 1) < ehci->periodic_size); - /* just check the specified uframe, at that period */ - } else { - do { - claimed = periodic_usecs (ehci, frame, uframe); - if (claimed > usecs) - return 0; - } while ((frame += period) < ehci->periodic_size); - } - - // success! - return 1; + wmb (); } -static int check_intr_schedule ( - struct ehci_hcd *ehci, - unsigned frame, - unsigned uframe, - const struct ehci_qh *qh, - __le32 *c_maskp -) +/* periodic_qh_link - link the passed in QH into all relevant frames of the + * hardware schedule and update FSTN state; assumes QH is already budgeted. + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to link + */ +static int periodic_qh_link (struct ehci_hcd *ehci, struct ehci_qh *qh) { - int retval = -ENOSPC; - u8 mask = 0; - - if (qh->c_usecs && uframe >= 6) /* FSTN territory? */ - goto done; + unsigned i=0; + unsigned level = _period_to_level(qh->period); - if (!check_period (ehci, frame, uframe, qh->period, qh->usecs)) - goto done; - if (!qh->c_usecs) { - retval = 0; - *c_maskp = 0; - goto done; - } + /* set masks */ + qh->hw_info2 &= + __constant_cpu_to_le32(~(QH_CMASK | QH_SMASK)); + qh->hw_info2 |= + qh->budget->smask | + ((qh->budget->cmask&0xff)<<8) | + (((qh->budget->cmask>>8)&0xff)<<8); + + while(1){ + __le32 *hw_p; + union ehci_shadow *here; + struct ehci_shadow_budget *budget; + + i = budget_schedule_next (ehci, i, qh, &budget, &here, &hw_p); + + if(i >= ehci->periodic_size)break; + + if(i<0){ + ehci_err(ehci, + "QH slot not in budget where expected"); + i += level; + continue; + } + + /* perform FSTN inspection/adjustment before qh is + * linked */ + { + union ehci_shadow temp; + temp.qh=qh; + _periodic_inspect_fstn_frame (ehci, i, here, &temp); + } + + /* already linked in? */ + if(here->ptr != qh){ + + /* link right */ + qh->qh_next = *here; + qh->hw_next = *hw_p; + wmb (); -#ifdef CONFIG_USB_EHCI_TT_NEWSCHED - if (tt_available (ehci, qh->period, qh->dev, frame, uframe, - qh->tt_usecs)) { - unsigned i; - - /* TODO : this may need FSTN for SSPLIT in uframe 5. */ - for (i=uframe+1; i<8 && iperiod, qh->c_usecs)) - goto done; - else - mask |= 1 << i; + /* link left; fully live */ + here->qh = qh; + *hw_p = QH_NEXT (qh->qh_dma); - retval = 0; + } - *c_maskp = cpu_to_le32 (mask << 8); + i += level; } -#else - /* Make sure this tt's buffer is also available for CSPLITs. - * We pessimize a bit; probably the typical full speed case - * doesn't need the second CSPLIT. - * - * NOTE: both SPLIT and CSPLIT could be checked in just - * one smart pass... - */ - mask = 0x03 << (uframe + qh->gap_uf); - *c_maskp = cpu_to_le32 (mask << 8); - mask |= 1 << uframe; - if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) { - if (!check_period (ehci, frame, uframe + qh->gap_uf + 1, - qh->period, qh->c_usecs)) - goto done; - if (!check_period (ehci, frame, uframe + qh->gap_uf, - qh->period, qh->c_usecs)) - goto done; - retval = 0; - } -#endif -done: - return retval; + qh->qh_state = QH_STATE_LINKED; + qh_get (qh); + + /* update per-qh bandwidth for usbfs */ + ehci_to_hcd(ehci)->self.bandwidth_allocated += + (qh->usecs + qh->c_usecs) / qh->period; + + /* maybe enable periodic schedule processing */ + return enable_periodic (ehci); } -/* "first fit" scheduling policy used the first time through, - * or when the previous schedule slot can't be re-used. +/* periodic_qh_schedule - budget the passed in QH if it has not + * already been budgeted, then add QH to the hardware schedule + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to link */ -static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh) +static int periodic_qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh) { int status; - unsigned uframe; - __le32 c_mask; - unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */ qh_refresh(ehci, qh); qh->hw_next = EHCI_LIST_END; - frame = qh->start; - /* reuse the previous schedule slots, if we can */ - if (frame < qh->period) { - uframe = ffs (le32_to_cpup (&qh->hw_info2) & QH_SMASK); - status = check_intr_schedule (ehci, frame, --uframe, - qh, &c_mask); - } else { - uframe = 0; - c_mask = 0; - status = -ENOSPC; + /* budget the qh if not already budgeted */ + if(!qh->budget){ + + status = budget_add_endpoint (ehci, qh, BUDGET_TYPE_QH, + qh->period); + if(status) + return status; + } + + /* schedule the qh */ + if(sched_verbose){ + ehci_info(ehci, "Adding new QH %p to schedule:\n", + qh); + } + + status = periodic_qh_link (ehci, qh); + + + if(status){ + budget_unlink_entries_by_owner(ehci,qh); + qh->budget = NULL; } - - /* else scan the schedule to find a group of slots such that all - * uframes have enough periodic bandwidth available. - */ - if (status) { - /* "normal" case, uframing flexible except with splits */ - if (qh->period) { - frame = qh->period - 1; - do { - for (uframe = 0; uframe < 8; uframe++) { - status = check_intr_schedule (ehci, - frame, uframe, qh, - &c_mask); - if (status == 0) - break; - } - } while (status && frame--); - - /* qh->period == 0 means every uframe */ - } else { - frame = 0; - status = check_intr_schedule (ehci, 0, 0, qh, &c_mask); - } - if (status) - goto done; - qh->start = frame; - - /* reset S-frame and (maybe) C-frame masks */ - qh->hw_info2 &= __constant_cpu_to_le32(~(QH_CMASK | QH_SMASK)); - qh->hw_info2 |= qh->period - ? cpu_to_le32 (1 << uframe) - : __constant_cpu_to_le32 (QH_SMASK); - qh->hw_info2 |= c_mask; - } else - ehci_dbg (ehci, "reused qh %p schedule\n", qh); - - /* stuff into the periodic schedule */ - status = qh_link_periodic (ehci, qh); -done: return status; } +/* intr_submit - toplevel entry point for submitting an interrupt + * endpoint request. If needed, performs initial creation, budgeting + * and scheduling of a QH for this endpoint. + * + * @ehci: + * @ep: + * @urb: + * @qtd_list: + * @mem_flags + */ static int intr_submit ( struct ehci_hcd *ehci, struct usb_host_endpoint *ep, struct urb *urb, struct list_head *qtd_list, - gfp_t mem_flags -) { + gfp_t mem_flags) +{ unsigned epnum; unsigned long flags; struct ehci_qh *qh; @@ -821,7 +1974,7 @@ goto done; } if (qh->qh_state == QH_STATE_IDLE) { - if ((status = qh_schedule (ehci, qh)) != 0) + if ((status = periodic_qh_schedule (ehci, qh)) != 0) goto done; } @@ -841,9 +1994,45 @@ } /*-------------------------------------------------------------------------*/ +/* ISO scheduling temporary structure machinery necessitated by lazy + alloc of the iso_stream structure */ + +static struct ehci_iso_sched * +iso_sched_alloc (unsigned packets, gfp_t mem_flags) +{ + struct ehci_iso_sched *iso_sched; + int size = sizeof *iso_sched; + + size += packets * sizeof (struct ehci_iso_packet); + iso_sched = kzalloc(size, mem_flags); + if (likely (iso_sched != NULL)) { + INIT_LIST_HEAD (&iso_sched->td_list); + } + return iso_sched; +} + +static void +iso_sched_free (struct ehci_iso_stream *stream, + struct ehci_iso_sched *iso_sched) +{ + if (!iso_sched){ + return; + } + // caller must hold ehci->lock! + list_splice (&iso_sched->td_list, &stream->free_list); + kfree (iso_sched); +} -/* ehci_iso_stream ops work with both ITD and SITD */ +/*-------------------------------------------------------------------------*/ +/* ISO stream generic machinery; tracks the ITDs and sITDs associated + with an isochronous endpoint */ +/* iso_stream_alloc - allocate a new iso stream structure; and iso + * stream performs a role similar to QHs for ISO endpoints (both ITD + * and SITD) + * + * @mem_flags: flags for memory allocation + */ static struct ehci_iso_stream * iso_stream_alloc (gfp_t mem_flags) { @@ -859,6 +2048,14 @@ return stream; } +/* iso_stream_init - initialize fields of an ehci_iso_stream structure + * + * @ehci: pointer to ehci host controller device structure + * @stream: ehci_iso_stream structure to initialize + * @dev: endpoint usb device + * @pipe: pipe handle + * @interval: bInterval of endpoint; frames for FS/LS, uFrames for HS + */ static void iso_stream_init ( struct ehci_hcd *ehci, @@ -866,10 +2063,8 @@ struct usb_device *dev, int pipe, unsigned interval -) + ) { - static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f }; - u32 buf1; unsigned epnum, maxp; int is_input; @@ -906,65 +2101,92 @@ * when transfers on this endpoint are scheduled ... */ stream->usecs = HS_USECS_ISO (maxp); + stream->interval = interval; bandwidth = stream->usecs * 8; bandwidth /= 1 << (interval - 1); } else { u32 addr; - int think_time; - int hs_transfers; + int think_time, think_bytes; addr = dev->ttport << 24; if (!ehci_is_TDI(ehci) - || (dev->tt->hub != - ehci_to_hcd(ehci)->self.root_hub)) + || (dev->tt->hub != + ehci_to_hcd(ehci)->self.root_hub)) addr |= dev->tt->hub->devnum << 16; addr |= epnum << 8; addr |= dev->devnum; stream->usecs = HS_USECS_ISO (maxp); + think_time = dev->tt ? dev->tt->think_time : 0; - stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time ( - dev->speed, is_input, 1, maxp)); - hs_transfers = max (1u, (maxp + 187) / 188); - if (is_input) { - u32 tmp; + think_bytes = (think_time+665)/666; + if (is_input) { addr |= 1 << 31; - stream->c_usecs = stream->usecs; + + stream->c_usecs = stream->usecs + HS_USECS_ISO (1); stream->usecs = HS_USECS_ISO (1); - stream->raw_mask = 1; - /* c-mask as specified in USB 2.0 11.18.4 3.c */ - tmp = (1 << (hs_transfers + 2)) - 1; - stream->raw_mask |= tmp << (8 + 2); - } else - stream->raw_mask = smask_out [hs_transfers - 1]; + if(dev->speed == USB_SPEED_FULL){ + /* full speed bytes + think time [TT + * host delay] + FS non-iso protocol + * overhead */ + stream->tt_bytes = think_bytes + maxp + 11; + }else{ + /* low speed bytes + think time [TT + * host delay] + low speed protocol + * overhead */ + /* expressed in full speed bytes */ + stream->tt_bytes = think_bytes + maxp*8 + 98; + } + } else { + /* out */ + + stream->usecs += HS_USECS_ISO (1); + + if(dev->speed == USB_SPEED_FULL){ + /* full speed bytes + think time [TT + * host delay] + FS non-iso protocol + * overhead */ + stream->tt_bytes = think_bytes + maxp + 10; + }else{ + /* low speed bytes + think time [TT + * host delay] + low speed protocol + * overhead */ + /* expressed in full speed bytes */ + stream->tt_bytes = think_bytes + maxp*8 + 98; + } + } bandwidth = stream->usecs + stream->c_usecs; bandwidth /= 1 << (interval + 2); - - /* stream->splits gets created from raw_mask later */ stream->address = cpu_to_le32 (addr); + stream->interval = interval<<3; } - stream->bandwidth = bandwidth; + stream->bandwidth = bandwidth; stream->udev = dev; - stream->bEndpointAddress = is_input | epnum; - stream->interval = interval; - stream->maxp = maxp; } +/* iso_stream_put - decrement refcount of passed in ehci_iso_stream + * structure, reaping it if count has fallen to zero. + * + * @ehci: pointer to ehci host controller device structure + * @stream: ehci_iso_stream structure + */ static void iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream) { stream->refcount--; - /* free whenever just a dev->ep reference remains. - * not like a QH -- no persistent state (toggle, halt) - */ - if (stream->refcount == 1) { - int is_in; - + /* don't free on last descriptor; free when endpoint disable + finally releases last refcount. Although it is technically + broken for an endpoint driver to submit its streaming + descriptors such that a new one appears after the old one + ends, it is only punishing the users to insist on breaking + these drivers when it's not necessary to do so. This saves + substantial overhead in that case.*/ + if (stream->refcount == 0) { // BUG_ON (!list_empty(&stream->td_list)); while (!list_empty (&stream->free_list)) { @@ -978,36 +2200,41 @@ struct ehci_itd *itd; itd = list_entry (entry, struct ehci_itd, - itd_list); + itd_list); dma_pool_free (ehci->itd_pool, itd, - itd->itd_dma); + itd->itd_dma); } else { struct ehci_sitd *sitd; sitd = list_entry (entry, struct ehci_sitd, - sitd_list); + sitd_list); dma_pool_free (ehci->sitd_pool, sitd, - sitd->sitd_dma); + sitd->sitd_dma); } } - - is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0; stream->bEndpointAddress &= 0x0f; stream->ep->hcpriv = NULL; - if (stream->rescheduled) { - ehci_info (ehci, "ep%d%s-iso rescheduled " - "%lu times in %lu seconds\n", - stream->bEndpointAddress, is_in ? "in" : "out", - stream->rescheduled, - ((jiffies - stream->start)/HZ) - ); - } + /* eliminate this stream from the shadow budget */ + if(sched_verbose) + ehci_info(ehci, "Releasing bandwidth for ISO %p\n", + stream); + budget_unlink_entries_by_owner(ehci,stream); + stream->budget_state = 0; + + /* potentially shut off periodic schedule */ + deref_periodic(ehci); + /* free the stream */ kfree(stream); } } +/* iso_stream_get - increment refcount of passed in ehci_iso_stream + * structure + * + * @stream: ehci_iso_stream structure + */ static inline struct ehci_iso_stream * iso_stream_get (struct ehci_iso_stream *stream) { @@ -1016,6 +2243,14 @@ return stream; } +/* iso_stream_find - returns ehci_iso_stream structure matching the + * passed in urb; if no match is found, a new ehci_iso_stream is + * allocated and initialized to match the urb. This function is thus + * used both for looup and lazy alloc. + * + * @ehci: pointer to ehci host controller device structure + * @urb: usb request block + */ static struct ehci_iso_stream * iso_stream_find (struct ehci_hcd *ehci, struct urb *urb) { @@ -1043,11 +2278,11 @@ urb->interval); } - /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */ + /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */ } else if (unlikely (stream->hw_info1 != 0)) { ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n", - urb->dev->devpath, epnum, - usb_pipein(urb->pipe) ? "in" : "out"); + urb->dev->devpath, epnum, + usb_pipein(urb->pipe) ? "in" : "out"); stream = NULL; } @@ -1058,30 +2293,141 @@ return stream; } -/*-------------------------------------------------------------------------*/ - -/* ehci_iso_sched ops can be ITD-only or SITD-only */ +/* Too large now? Budgeting is already done by the time we compute + using the slop */ +#define SCHEDULE_SLOP 10 /* frames */ -static struct ehci_iso_sched * -iso_sched_alloc (unsigned packets, gfp_t mem_flags) +/* iso_stream_schedule - schedules an iso request transaction into the + * periodic schedule, budgeting persistent periodic bandwidth for this + * stream if that has not yet been done. + * + * @ehci: pointer to ehci host controller device structure + * @urb: request + * @stream: persistent iso stream + */ +static int iso_stream_schedule ( + struct ehci_hcd *ehci, + struct urb *urb, + struct ehci_iso_stream *stream + ) { - struct ehci_iso_sched *iso_sched; - int size = sizeof *iso_sched; + int status; + unsigned mod = ehci->periodic_size << 3; + struct ehci_iso_sched *sched = urb->hcpriv; - size += packets * sizeof (struct ehci_iso_packet); - iso_sched = kzalloc(size, mem_flags); - if (likely (iso_sched != NULL)) { - INIT_LIST_HEAD (&iso_sched->td_list); + if (sched->span > (mod - 8 * SCHEDULE_SLOP)) { + ehci_dbg (ehci, "iso request %p too long\n", urb); + status = -EFBIG; + goto fail; } - return iso_sched; + + if ((stream->depth + sched->span) > mod) { + ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n", + urb, stream->depth, sched->span, mod); + status = -EFBIG; + goto fail; + } + + /* has this endpoint already been submitted into the shadow budget? */ + if(!stream->budget_state){ + + /* no, budget this endpoint */ + + status = budget_add_endpoint(ehci, stream, BUDGET_TYPE_ITD, + stream->interval); + + if(status)goto fail; + + /* potentially turn on the periodic hardware */ + status=enable_periodic (ehci); + if(status)goto fail; + + stream->next_uframe = -1; + stream->budget_state = 1; + } + + if(stream->next_uframe == -1){ + int now, start; + + /* initial schedule; when's the next (u)frame we could + * start? this is bigger than ehci->i_thresh allows; + * scheduling itself isn't free, the slop should handle + * reasonably slow cpus. it can also help high bandwidth if + * the dma and irq loads don't jump until after the queue is + * primed. + */ + + now = readl (&ehci->regs->frame_index) & (mod-1); + start = ((now + SCHEDULE_SLOP * 8) & (mod-1)) >> 3; + + stream->next_uframe = + (budget_schedule_next_frame (ehci, start, stream) & + (ehci->periodic_size-1))* 8; + + if(stream->next_uframe == -1){ + /* should be impossible */ + ehci_err(ehci,"Couldn't find budgeted slot " + "after scheduling\n"); + status = -ENOSPC; + goto fail; + } + + /* report high speed start in uframes; full speed, in frames */ + urb->start_frame = stream->next_uframe; + if (!stream->highspeed) + urb->start_frame >>= 3; + + }else{ + + /* use the next uframe field as already marked */ + /* did we fall behind? Throw a sync error if so. */ + /* in sync == tail < now < head */ + + int now = (readl (&ehci->regs->frame_index) & (mod-1)); + int tail = stream->next_uframe - stream->depth - + stream->interval - SCHEDULE_SLOP * 8; + int head = stream->next_uframe; + + if(tail < 0){ + tail += mod; + head += mod; + if(now < tail) now += mod; + } + + if (now < tail || now >= head){ + if(sched_verbose) + ehci_err(ehci,"Loss of sync!\n"); + stream->next_uframe = -1; + status = -EL2NSYNC; + goto fail; + } + } + + return 0; + +fail: + iso_sched_free (stream, sched); + urb->hcpriv = NULL; + return status; + } +/*-------------------------------------------------------------------------*/ + +/* itd_sched_init - ITD-specific initialization of an ehci_iso_sched + * structure to hold the specifics of the current requested + * transaction during the various steps of budgeting and scheduling. + * + * @iso_sched: structure to initialize + * @stream: iso stream + * @urb: current request + */ static inline void itd_sched_init ( struct ehci_iso_sched *iso_sched, struct ehci_iso_stream *stream, struct urb *urb -) + ) { unsigned i; dma_addr_t dma = urb->transfer_dma; @@ -1104,7 +2450,7 @@ trans = EHCI_ISOC_ACTIVE; trans |= buf & 0x0fff; if (unlikely (((i + 1) == urb->number_of_packets)) - && !(urb->transfer_flags & URB_NO_INTERRUPT)) + && !(urb->transfer_flags & URB_NO_INTERRUPT)) trans |= EHCI_ITD_IOC; trans |= length << 16; uframe->transaction = cpu_to_le32 (trans); @@ -1117,26 +2463,22 @@ } } -static void -iso_sched_free ( - struct ehci_iso_stream *stream, - struct ehci_iso_sched *iso_sched -) -{ - if (!iso_sched) - return; - // caller must hold ehci->lock! - list_splice (&iso_sched->td_list, &stream->free_list); - kfree (iso_sched); -} - +/* itd_urb_transaction - sets up memory and relevant structures to + * process current transaction; initializes the iso_sched structure + * and preallocates transfer descriptors + * + * @stream: iso stream + * @ehci: pointer to ehci host controller device structure + * @urb: current request + * @mem_flags: flags to use for in-kernel memory allocation + */ static int itd_urb_transaction ( struct ehci_iso_stream *stream, struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags -) + ) { struct ehci_itd *itd; dma_addr_t itd_dma; @@ -1167,7 +2509,7 @@ /* prefer previously-allocated itds */ if (likely (!list_empty(&stream->free_list))) { itd = list_entry (stream->free_list.prev, - struct ehci_itd, itd_list); + struct ehci_itd, itd_list); list_del (&itd->itd_list); itd_dma = itd->itd_dma; } else @@ -1176,7 +2518,7 @@ if (!itd) { spin_unlock_irqrestore (&ehci->lock, flags); itd = dma_pool_alloc (ehci->itd_pool, mem_flags, - &itd_dma); + &itd_dma); spin_lock_irqsave (&ehci->lock, flags); } @@ -1197,223 +2539,12 @@ return 0; } -/*-------------------------------------------------------------------------*/ - -static inline int -itd_slot_ok ( - struct ehci_hcd *ehci, - u32 mod, - u32 uframe, - u8 usecs, - u32 period -) -{ - uframe %= period; - do { - /* can't commit more than 80% periodic == 100 usec */ - if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7) - > (100 - usecs)) - return 0; - - /* we know urb->interval is 2^N uframes */ - uframe += period; - } while (uframe < mod); - return 1; -} - -static inline int -sitd_slot_ok ( - struct ehci_hcd *ehci, - u32 mod, - struct ehci_iso_stream *stream, - u32 uframe, - struct ehci_iso_sched *sched, - u32 period_uframes -) -{ - u32 mask, tmp; - u32 frame, uf; - - mask = stream->raw_mask << (uframe & 7); - - /* for IN, don't wrap CSPLIT into the next frame */ - if (mask & ~0xffff) - return 0; - - /* this multi-pass logic is simple, but performance may - * suffer when the schedule data isn't cached. - */ - - /* check bandwidth */ - uframe %= period_uframes; - do { - u32 max_used; - - frame = uframe >> 3; - uf = uframe & 7; - -#ifdef CONFIG_USB_EHCI_TT_NEWSCHED - /* The tt's fullspeed bus bandwidth must be available. - * tt_available scheduling guarantees 10+% for control/bulk. - */ - if (!tt_available (ehci, period_uframes << 3, - stream->udev, frame, uf, stream->tt_usecs)) - return 0; -#else - /* tt must be idle for start(s), any gap, and csplit. - * assume scheduling slop leaves 10+% for control/bulk. - */ - if (!tt_no_collision (ehci, period_uframes << 3, - stream->udev, frame, mask)) - return 0; -#endif - - /* check starts (OUT uses more than one) */ - max_used = 100 - stream->usecs; - for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) { - if (periodic_usecs (ehci, frame, uf) > max_used) - return 0; - } - - /* for IN, check CSPLIT */ - if (stream->c_usecs) { - uf = uframe & 7; - max_used = 100 - stream->c_usecs; - do { - tmp = 1 << uf; - tmp <<= 8; - if ((stream->raw_mask & tmp) == 0) - continue; - if (periodic_usecs (ehci, frame, uf) - > max_used) - return 0; - } while (++uf < 8); - } - - /* we know urb->interval is 2^N uframes */ - uframe += period_uframes; - } while (uframe < mod); - - stream->splits = cpu_to_le32(stream->raw_mask << (uframe & 7)); - return 1; -} - -/* - * This scheduler plans almost as far into the future as it has actual - * periodic schedule slots. (Affected by TUNE_FLS, which defaults to - * "as small as possible" to be cache-friendlier.) That limits the size - * transfers you can stream reliably; avoid more than 64 msec per urb. - * Also avoid queue depths of less than ehci's worst irq latency (affected - * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter, - * and other factors); or more than about 230 msec total (for portability, - * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler! +/* itd_init - performs master initilization of the ITD fields that are + * drawn from the stream structure. + * + * @stream: iso stream + * @itd: New ITD that is to be added to the shadow/hardware schedule */ - -#define SCHEDULE_SLOP 10 /* frames */ - -static int -iso_stream_schedule ( - struct ehci_hcd *ehci, - struct urb *urb, - struct ehci_iso_stream *stream -) -{ - u32 now, start, max, period; - int status; - unsigned mod = ehci->periodic_size << 3; - struct ehci_iso_sched *sched = urb->hcpriv; - - if (sched->span > (mod - 8 * SCHEDULE_SLOP)) { - ehci_dbg (ehci, "iso request %p too long\n", urb); - status = -EFBIG; - goto fail; - } - - if ((stream->depth + sched->span) > mod) { - ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n", - urb, stream->depth, sched->span, mod); - status = -EFBIG; - goto fail; - } - - now = readl (&ehci->regs->frame_index) % mod; - - /* when's the last uframe this urb could start? */ - max = now + mod; - - /* typical case: reuse current schedule. stream is still active, - * and no gaps from host falling behind (irq delays etc) - */ - if (likely (!list_empty (&stream->td_list))) { - start = stream->next_uframe; - if (start < now) - start += mod; - if (likely ((start + sched->span) < max)) - goto ready; - /* else fell behind; someday, try to reschedule */ - status = -EL2NSYNC; - goto fail; - } - - /* need to schedule; when's the next (u)frame we could start? - * this is bigger than ehci->i_thresh allows; scheduling itself - * isn't free, the slop should handle reasonably slow cpus. it - * can also help high bandwidth if the dma and irq loads don't - * jump until after the queue is primed. - */ - start = SCHEDULE_SLOP * 8 + (now & ~0x07); - start %= mod; - stream->next_uframe = start; - - /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */ - - period = urb->interval; - if (!stream->highspeed) - period <<= 3; - - /* find a uframe slot with enough bandwidth */ - for (; start < (stream->next_uframe + period); start++) { - int enough_space; - - /* check schedule: enough space? */ - if (stream->highspeed) - enough_space = itd_slot_ok (ehci, mod, start, - stream->usecs, period); - else { - if ((start % 8) >= 6) - continue; - enough_space = sitd_slot_ok (ehci, mod, stream, - start, sched, period); - } - - /* schedule it here if there's enough bandwidth */ - if (enough_space) { - stream->next_uframe = start % mod; - goto ready; - } - } - - /* no room in the schedule */ - ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n", - list_empty (&stream->td_list) ? "" : "re", - urb, now, max); - status = -ENOSPC; - -fail: - iso_sched_free (stream, sched); - urb->hcpriv = NULL; - return status; - -ready: - /* report high speed start in uframes; full speed, in frames */ - urb->start_frame = stream->next_uframe; - if (!stream->highspeed) - urb->start_frame >>= 3; - return 0; -} - -/*-------------------------------------------------------------------------*/ - static inline void itd_init (struct ehci_iso_stream *stream, struct ehci_itd *itd) { @@ -1431,13 +2562,20 @@ /* All other fields are filled when scheduling */ } +/* itd_patch - patches additional transfer descriptors into an existing ITD. + * + * @itd: New ITD that is to be added to the shadow/hardware schedule + * @iso_sched: ehci_iso_sched holding transfer descriptors + * @index: packet number to reference within the ehci_iso_sched + * @uframe: uframe in which to schedule specified packet + */ static inline void itd_patch ( struct ehci_itd *itd, struct ehci_iso_sched *iso_sched, unsigned index, u16 uframe -) + ) { struct ehci_iso_packet *uf = &iso_sched->packet [index]; unsigned pg = itd->pg; @@ -1461,44 +2599,64 @@ } } -static inline void -itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) -{ - /* always prepend ITD/SITD ... only QH tree is order-sensitive */ - itd->itd_next = ehci->pshadow [frame]; - itd->hw_next = ehci->periodic [frame]; - ehci->pshadow [frame].itd = itd; +/* itd_link - place one completed itd into the shadow/hardware schedule + * + * @ehci: pointer to ehci host controller device structure + * @frame: shadow/hardware schedule frame + * @stream: stream to which ITD belongs + * @itd: ITD to link into schedule + */ +static inline void itd_link (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_iso_stream *stream, + struct ehci_itd *itd) +{ + __le32 *hw_p; + struct ehci_shadow_budget *b; + union ehci_shadow *here; + + if( budget_schedule_next(ehci,frame,stream,&b,&here,&hw_p) != frame){ + /* should be impossible unless we've hit a bug or + * hardware failure */ + ehci_err(ehci,"itd not where it should be " + "in the periodic budget"); + return; + } + itd->frame = frame; + itd->itd_next = *here; + itd->hw_next = *hw_p; + here->itd = itd; wmb (); - ehci->periodic [frame] = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD; + *hw_p = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD; } -/* fit urb's itds into the selected schedule slot; activate as needed */ +/* itd_link_urb - link urb's ITDs into the hardware schedule acording + * to shadow budget + * + * @ehci: pointer to ehci host controller device structure + * @urb: current USB request + * @mod: size of periodic hardware schedule [in uFrames] + * @stream: stream into which to queue request + */ static int itd_link_urb ( struct ehci_hcd *ehci, struct urb *urb, unsigned mod, struct ehci_iso_stream *stream -) + ) { int packet; unsigned next_uframe, uframe, frame; struct ehci_iso_sched *iso_sched = urb->hcpriv; struct ehci_itd *itd; - next_uframe = stream->next_uframe % mod; + next_uframe = stream->next_uframe & (mod-1); if (unlikely (list_empty(&stream->td_list))) { ehci_to_hcd(ehci)->self.bandwidth_allocated - += stream->bandwidth; - ehci_vdbg (ehci, - "schedule devp %s ep%d%s-iso period %d start %d.%d\n", - urb->dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", - urb->interval, - next_uframe >> 3, next_uframe & 0x7); - stream->start = jiffies; + += stream->bandwidth; } ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; @@ -1511,7 +2669,7 @@ /* ASSERT: no itds for this endpoint in this uframe */ itd = list_entry (iso_sched->td_list.next, - struct ehci_itd, itd_list); + struct ehci_itd, itd_list); list_move_tail (&itd->itd_list, &stream->td_list); itd->stream = iso_stream_get (stream); itd->urb = usb_get_urb (urb); @@ -1526,16 +2684,18 @@ next_uframe += stream->interval; stream->depth += stream->interval; - next_uframe %= mod; + next_uframe &= (mod-1); packet++; /* link completed itds into the schedule */ if (((next_uframe >> 3) != frame) - || packet == urb->number_of_packets) { - itd_link (ehci, frame % ehci->periodic_size, itd); + || packet == urb->number_of_packets) { + itd_link (ehci, frame & (ehci->periodic_size-1), + stream, itd); itd = NULL; } } + stream->next_uframe = next_uframe; /* don't need that schedule data any more */ @@ -1543,19 +2703,25 @@ urb->hcpriv = NULL; timer_action (ehci, TIMER_IO_WATCHDOG); - if (unlikely (!ehci->periodic_sched++)) - return enable_periodic (ehci); return 0; } #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) -static unsigned -itd_complete ( - struct ehci_hcd *ehci, +/* itd_complete - process a single complete ITD that has been removed + * from the periodic schedule and handed to us by scan_periodic. If + * this is the last ITD that needed to be completed for a URB, hand + * the URB back to the upper level driver. + * + * @ehci: pointer to ehci host controller device structure + * @itd: completed ITD + * @regs: ptrace registers + */ +static unsigned itd_complete ( + struct ehci_hcd *ehci, struct ehci_itd *itd, - struct pt_regs *regs -) { + struct pt_regs *regs) +{ struct urb *urb = itd->urb; struct usb_iso_packet_descriptor *desc; u32 t; @@ -1563,7 +2729,7 @@ int urb_index = -1; struct ehci_iso_stream *stream = itd->stream; struct usb_device *dev; - + /* for each uframe with a packet */ for (uframe = 0; uframe < 8; uframe++) { if (likely (itd->index[uframe] == -1)) @@ -1605,40 +2771,44 @@ /* handle completion now? */ if (likely ((urb_index + 1) != urb->number_of_packets)) return 0; - + /* ASSERT: it's really the last itd for this urb - list_for_each_entry (itd, &stream->td_list, itd_list) - BUG_ON (itd->urb == urb); - */ + list_for_each_entry (itd, &stream->td_list, itd_list) + BUG_ON (itd->urb == urb); + */ /* give urb back to the driver ... can be out-of-order */ dev = urb->dev; ehci_urb_done (ehci, urb, regs); urb = NULL; - /* defer stopping schedule; completion can submit */ - ehci->periodic_sched--; - if (unlikely (!ehci->periodic_sched)) - (void) disable_periodic (ehci); ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; + /* XXX Hm... now that we have the budget, this should be moved + to where the budget is released */ if (unlikely (list_empty (&stream->td_list))) { ehci_to_hcd(ehci)->self.bandwidth_allocated - -= stream->bandwidth; + -= stream->bandwidth; ehci_vdbg (ehci, - "deschedule devp %s ep%d%s-iso\n", - dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); + "deschedule devp %s ep%d%s-iso\n", + dev->devpath, stream->bEndpointAddress & 0x0f, + (stream->bEndpointAddress & USB_DIR_IN) ? + "in" : "out"); } iso_stream_put (ehci, stream); return 1; } -/*-------------------------------------------------------------------------*/ - +/* itd_submit - top level submission point for a HS isochronous + * endpoint request. + * + * @ehci: pointer to ehci host controller device structure + * @urb: new USB request + * @mem_flags: flags for memory allocation + */ static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, - gfp_t mem_flags) + gfp_t mem_flags) { int status = -EINVAL; unsigned long flags; @@ -1652,21 +2822,10 @@ } if (unlikely (urb->interval != stream->interval)) { ehci_dbg (ehci, "can't change iso interval %d --> %d\n", - stream->interval, urb->interval); + stream->interval, urb->interval); goto done; } -#ifdef EHCI_URB_TRACE - ehci_dbg (ehci, - "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n", - __FUNCTION__, urb->dev->devpath, urb, - usb_pipeendpoint (urb->pipe), - usb_pipein (urb->pipe) ? "in" : "out", - urb->transfer_buffer_length, - urb->number_of_packets, urb->interval, - stream); -#endif - /* allocate ITDs w/o locking anything */ status = itd_urb_transaction (stream, ehci, urb, mem_flags); if (unlikely (status < 0)) { @@ -1691,26 +2850,29 @@ return status; } -#ifdef CONFIG_USB_EHCI_SPLIT_ISO - /*-------------------------------------------------------------------------*/ +/* Split-ISO transfer machinery; used for FS isoch transfers through the + * TTs in USB 2.0 hubs. */ -/* - * "Split ISO TDs" ... used for USB 1.1 devices going through the - * TTs in USB 2.0 hubs. These need microframe scheduling. +/* sitd_sched_init - sITD-specific initialization of an ehci_iso_sched + * structure to hold the specifics of the current requested + * transaction during the various steps of budgeting and scheduling. + * + * @iso_sched: structure to initialize + * @stream: iso stream + * @urb: current request */ - static inline void sitd_sched_init ( struct ehci_iso_sched *iso_sched, struct ehci_iso_stream *stream, struct urb *urb -) + ) { unsigned i; dma_addr_t dma = urb->transfer_dma; - /* how many frames are needed for these transfers */ + /* how many uframes are needed for these transfers */ iso_sched->span = urb->number_of_packets * stream->interval; /* figure out per-frame sitd fields that we'll need later @@ -1727,7 +2889,7 @@ trans = SITD_STS_ACTIVE; if (((i + 1) == urb->number_of_packets) - && !(urb->transfer_flags & URB_NO_INTERRUPT)) + && !(urb->transfer_flags & URB_NO_INTERRUPT)) trans |= SITD_IOC; trans |= length << 16; packet->transaction = cpu_to_le32 (trans); @@ -1748,13 +2910,22 @@ } } +/* sitd_urb_transaction - sets up memory and relevant structures to + * process current transaction; initializes the iso_sched structure + * and preallocates transfer descriptors + * + * @stream: iso stream + * @ehci: pointer to ehci host controller device structure + * @urb: current request + * @mem_flags: flags to use for in-kernel memory allocation + */ static int sitd_urb_transaction ( struct ehci_iso_stream *stream, struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags -) + ) { struct ehci_sitd *sitd; dma_addr_t sitd_dma; @@ -1770,12 +2941,10 @@ /* allocate/init sITDs */ spin_lock_irqsave (&ehci->lock, flags); - for (i = 0; i < urb->number_of_packets; i++) { - /* NOTE: for now, we don't try to handle wraparound cases - * for IN (using sitd->hw_backpointer, like a FSTN), which - * means we never need two sitds for full speed packets. - */ + /* note: frame spanning may require two SITDs per transaction; + allocate all we may need, and free the ones we don't later. */ + for (i = 0; i < urb->number_of_packets * 2; i++) { /* free_list.next might be cache-hot ... but maybe * the HC caches it too. avoid that issue for now. @@ -1784,7 +2953,7 @@ /* prefer previously-allocated sitds */ if (!list_empty(&stream->free_list)) { sitd = list_entry (stream->free_list.prev, - struct ehci_sitd, sitd_list); + struct ehci_sitd, sitd_list); list_del (&sitd->sitd_list); sitd_dma = sitd->sitd_dma; } else @@ -1793,7 +2962,7 @@ if (!sitd) { spin_unlock_irqrestore (&ehci->lock, flags); sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags, - &sitd_dma); + &sitd_dma); spin_lock_irqsave (&ehci->lock, flags); } @@ -1815,26 +2984,45 @@ return 0; } -/*-------------------------------------------------------------------------*/ - +/* sitd_patch - finishes filling in the hardware fields of a sITD + * schedule prior to it being linked into the hardware/shadow + * schedule. + * + * @stream: stream into which this request will be queued + * @b: shadow budget entry holding this sITD's bandwidth allocation + * information + * @sitd: New sITD that is to be added to the shadow/hardware schedule + * @uf: iso stream packet + * @back: hw pointer (cpu order) pointing back to original sITD + * if this is a spanning dummy sITD + */ static inline void sitd_patch ( struct ehci_iso_stream *stream, + struct ehci_shadow_budget *b, struct ehci_sitd *sitd, - struct ehci_iso_sched *iso_sched, - unsigned index -) + struct ehci_iso_packet *uf, + dma_addr_t back + ) { - struct ehci_iso_packet *uf = &iso_sched->packet [index]; u64 bufp = uf->bufp; sitd->hw_next = EHCI_LIST_END; sitd->hw_fullspeed_ep = stream->address; - sitd->hw_uframe = stream->splits; sitd->hw_results = uf->transaction; - sitd->hw_backpointer = EHCI_LIST_END; - bufp = uf->bufp; + if(back){ + /* spanning completion only */ + sitd->hw_uframe = cpu_to_le32( ((b->cmask>>8)&0xff)<<8 ); + /* set dummy's completion state to DoComplete (ehci + 4.12.3.3.x) */ + sitd->hw_results |= SITD_DOCOMPLETE; + sitd->hw_backpointer = cpu_to_le32 (back); + }else{ + sitd->hw_uframe = cpu_to_le32(b->smask | ((b->cmask&0xff)<<8)); + sitd->hw_backpointer = EHCI_LIST_END; + } + sitd->hw_buf [0] = cpu_to_le32 (bufp); sitd->hw_buf_hi [0] = cpu_to_le32 (bufp >> 32); @@ -1842,101 +3030,169 @@ if (uf->cross) bufp += 4096; sitd->hw_buf_hi [1] = cpu_to_le32 (bufp >> 32); - sitd->index = index; } -static inline void -sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd) -{ - /* note: sitd ordering could matter (CSPLIT then SSPLIT) */ - sitd->sitd_next = ehci->pshadow [frame]; - sitd->hw_next = ehci->periodic [frame]; - ehci->pshadow [frame].sitd = sitd; +/* sitd_link - fill in and link one sITD into the shadow/hardware + * schedule; if this sITD requires a spanning dummy in the next frame, + * a spanning dummy is implicitly filled in and linked as well. + * + * @ehci: pointer to ehci host controller device structure + * @urb: new USB request + * @frame: shadow/hardware schedule frame + * @stream: stream to which ITD belongs + * @sched: ehci_iso_sched structure holding packets + * @index: index of packet to use from sched + */ +static inline void sitd_link (struct ehci_hcd *ehci, + struct urb *urb, + unsigned frame, + struct ehci_iso_stream *stream, + struct ehci_iso_sched *sched, + int index) +{ + struct ehci_sitd *sitd, *dummy; + __le32 *hw_p; + struct ehci_shadow_budget *b; + union ehci_shadow *here; + + /* get SITD from already allocated list */ + BUG_ON (list_empty (&sched->td_list)); + sitd = list_entry (sched->td_list.next, + struct ehci_sitd, sitd_list); + list_move_tail (&sitd->sitd_list, &stream->td_list); + sitd->stream = iso_stream_get (stream); + sitd->urb = usb_get_urb (urb); + + /* find the proper insertion point */ + if( budget_schedule_next(ehci,frame,stream,&b,&here,&hw_p) + != frame){ + /* should be impossible unless we've hit a bug or + * hardware failure */ + ehci_err(ehci,"sitd not where it should be in " + "the periodic budget"); + return; + } + + /* set the sitd fields */ + sitd_patch (stream, b, sitd, &sched->packet [index], 0); + sitd->frame = frame; + sitd->index = index; + sitd->sitd_next = *here; + sitd->hw_next = *hw_p; + sitd->budget = b; + + here->sitd = sitd; wmb (); - ehci->periodic [frame] = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD; + + /* is this a wrap case? if so, have the dummy SITD fully in place + before the real SITD is reachable from the hardware schedule */ + + if(BUDGET_WRAP_P(b)){ + __le32 *d_hw_p; + union ehci_shadow *d_here; + + /* get SITD from already allocated list */ + BUG_ON (list_empty (&sched->td_list)); + dummy = list_entry (sched->td_list.next, + struct ehci_sitd, sitd_list); + list_move_tail (&dummy->sitd_list, &stream->td_list); + dummy->stream = iso_stream_get (stream); + dummy->urb = usb_get_urb (urb); + dummy->index = index; + dummy->frame = frame+1; + + /* find the proper insertion point */ + d_here = periodic_translate_budget_dummy (ehci, frame+1, b, &d_hw_p); + + /* set the sitd fields */ + sitd_patch (stream, b, dummy, &sched->packet [index], + sitd->sitd_dma); + dummy->budget = b; + + dummy->sitd_next = *d_here; + dummy->hw_next = *d_hw_p; + + d_here->sitd = dummy; + wmb (); + *d_hw_p = cpu_to_le32 (dummy->sitd_dma) | Q_TYPE_SITD; + + } + + /* make sitd reacahable from periodic schedule */ + *hw_p = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD; } -/* fit urb's sitds into the selected schedule slot; activate as needed */ +/* sitd_link_urb - link urb's sITDs into the hardware schedule + * acording to shadow budget + * + * @ehci: pointer to ehci host controller device structure + * @urb: current USB request + * @mod: size of periodic hardware schedule [in uFrames] + * @stream: stream into which to queue request + */ static int sitd_link_urb ( struct ehci_hcd *ehci, struct urb *urb, unsigned mod, struct ehci_iso_stream *stream -) + ) { int packet; unsigned next_uframe; struct ehci_iso_sched *sched = urb->hcpriv; - struct ehci_sitd *sitd; next_uframe = stream->next_uframe; if (list_empty(&stream->td_list)) { /* usbfs ignores TT bandwidth */ ehci_to_hcd(ehci)->self.bandwidth_allocated - += stream->bandwidth; - ehci_vdbg (ehci, - "sched devp %s ep%d%s-iso [%d] %dms/%04x\n", - urb->dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", - (next_uframe >> 3) % ehci->periodic_size, - stream->interval, le32_to_cpu (stream->splits)); - stream->start = jiffies; + += stream->bandwidth; } ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; /* fill sITDs frame by frame */ - for (packet = 0, sitd = NULL; - packet < urb->number_of_packets; - packet++) { - - /* ASSERT: we have all necessary sitds */ - BUG_ON (list_empty (&sched->td_list)); - - /* ASSERT: no itds for this endpoint in this frame */ - - sitd = list_entry (sched->td_list.next, - struct ehci_sitd, sitd_list); - list_move_tail (&sitd->sitd_list, &stream->td_list); - sitd->stream = iso_stream_get (stream); - sitd->urb = usb_get_urb (urb); - - sitd_patch (stream, sitd, sched, packet); - sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size, - sitd); - - next_uframe += stream->interval << 3; - stream->depth += stream->interval << 3; + for (packet = 0; packet < urb->number_of_packets; packet++) { + sitd_link (ehci, urb, ((next_uframe & (mod-1)) >> 3), + stream, sched, packet); + next_uframe += stream->interval; + stream->depth += stream->interval; } - stream->next_uframe = next_uframe % mod; + + stream->next_uframe = next_uframe & (mod-1); /* don't need that schedule data any more */ + /* also releases unused sITDs that weren't needed for spanning */ iso_sched_free (stream, sched); urb->hcpriv = NULL; timer_action (ehci, TIMER_IO_WATCHDOG); - if (!ehci->periodic_sched++) - return enable_periodic (ehci); return 0; } -/*-------------------------------------------------------------------------*/ - -#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ - | SITD_STS_XACT | SITD_STS_MMF) +#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ + | SITD_STS_XACT | SITD_STS_MMF) +/* sitd_complete - process a single complete sITD that has been removed + * from the periodic schedule and handed to us by scan_periodic. If + * this is the last sITD that needed to be completed for a URB, hand + * the URB back to the upper level driver. + * + * @ehci: pointer to ehci host controller device structure + * @sitd: completed sITD + * @regs: ptrace registers + */ static unsigned sitd_complete ( struct ehci_hcd *ehci, struct ehci_sitd *sitd, struct pt_regs *regs -) { + ) { struct urb *urb = sitd->urb; struct usb_iso_packet_descriptor *desc; u32 t; - int urb_index = -1; + int urb_index; struct ehci_iso_stream *stream = sitd->stream; struct usb_device *dev; @@ -1964,7 +3220,7 @@ sitd->urb = NULL; sitd->stream = NULL; list_move (&sitd->sitd_list, &stream->free_list); - stream->depth -= stream->interval << 3; + stream->depth -= stream->interval; iso_stream_put (ehci, stream); /* handle completion now? */ @@ -1972,28 +3228,27 @@ return 0; /* ASSERT: it's really the last sitd for this urb - list_for_each_entry (sitd, &stream->td_list, sitd_list) - BUG_ON (sitd->urb == urb); - */ + list_for_each_entry (sitd, &stream->td_list, sitd_list) + BUG_ON (sitd->urb == urb); + */ /* give urb back to the driver */ dev = urb->dev; ehci_urb_done (ehci, urb, regs); urb = NULL; - /* defer stopping schedule; completion can submit */ - ehci->periodic_sched--; - if (!ehci->periodic_sched) - (void) disable_periodic (ehci); ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; + /* XXX Hm... now that we have the budget, this should be moved + to where the budget is released */ if (list_empty (&stream->td_list)) { ehci_to_hcd(ehci)->self.bandwidth_allocated - -= stream->bandwidth; + -= stream->bandwidth; ehci_vdbg (ehci, - "deschedule devp %s ep%d%s-iso\n", - dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); + "deschedule devp %s ep%d%s-iso\n", + dev->devpath, stream->bEndpointAddress & 0x0f, + (stream->bEndpointAddress & USB_DIR_IN) ? + "in" : "out"); } iso_stream_put (ehci, stream); @@ -2001,8 +3256,15 @@ } +/* sitd_submit - top level submission point for a FS split isochronous + * endpoint request. + * + * @ehci: pointer to ehci host controller device structure + * @urb: new USB request + * @mem_flags: flags for memory allocation + */ static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb, - gfp_t mem_flags) + gfp_t mem_flags) { int status = -EINVAL; unsigned long flags; @@ -2014,21 +3276,12 @@ ehci_dbg (ehci, "can't get iso stream\n"); return -ENOMEM; } - if (urb->interval != stream->interval) { + if (urb->interval != ((stream->interval+7)>>3)) { ehci_dbg (ehci, "can't change iso interval %d --> %d\n", - stream->interval, urb->interval); + ((stream->interval+7)>>3), urb->interval); goto done; } -#ifdef EHCI_URB_TRACE - ehci_dbg (ehci, - "submit %p dev%s ep%d%s-iso len %d\n", - urb, urb->dev->devpath, - usb_pipeendpoint (urb->pipe), - usb_pipein (urb->pipe) ? "in" : "out", - urb->transfer_buffer_length); -#endif - /* allocate SITDs */ status = sitd_urb_transaction (stream, ehci, urb, mem_flags); if (status < 0) { @@ -2053,34 +3306,207 @@ return status; } -#else +/*-------------------------------------------------------------------------*/ + +static void unlink_dummy(struct ehci_hcd *ehci, struct ehci_sitd *sitd, + struct pt_regs *regs, int frame){ -static inline int -sitd_submit (struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags) -{ - ehci_dbg (ehci, "split iso support is disabled\n"); - return -ENOSYS; + /* there is a dummy; search for it */ + + __le32 *hw_p = + &ehci->periodic [frame & (ehci->periodic_size-1)]; + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 type = Q_NEXT_TYPE (*hw_p); + + while (here->ptr) { + if(type == Q_TYPE_SITD){ + if(here->sitd->budget == sitd->budget){ + struct ehci_sitd *temp = here->sitd; + *hw_p = here->sitd->hw_next; + *here = here->sitd->sitd_next; + wmb(); + + iso_stream_put (ehci, temp->stream); + + break; /* only one dummy will match */ + } + if(here->sitd->hw_backpointer == EHCI_LIST_END) + break; /* all dummies come first */ + + }else + break; /* all dummies come first */ + + hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (*hw_p); + } } -static inline unsigned -sitd_complete ( - struct ehci_hcd *ehci, - struct ehci_sitd *sitd, - struct pt_regs *regs -) { - ehci_err (ehci, "sitd_complete %p?\n", sitd); +static int scan_frame(struct ehci_hcd *ehci, struct pt_regs *regs, + int frame, int uframes, int rescan){ + + unsigned modified = 0; + union ehci_shadow q, *q_p; + __le32 type, *hw_p; + + frame = frame & (ehci->periodic_size-1); + + q_p = &ehci->pshadow [frame]; + hw_p = &ehci->periodic [frame]; + + q.ptr = q_p->ptr; + type = Q_NEXT_TYPE (*hw_p); + + while (q.ptr != NULL) { + unsigned uf; + union ehci_shadow temp; + int live; + + live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state); + switch (type) { + case Q_TYPE_QH: + /* handle any completions */ + temp.qh = qh_get (q.qh); + q_p = &q.qh->qh_next; + hw_p = &q.qh->hw_next; + type = Q_NEXT_TYPE (*hw_p); + + q = *q_p; + + /* process completions for this frame only if + * we're certain all completions for a + * preceeding spanning frame have first been + * processed; that is true iff clock was past + * uframe 1 when we started */ + if(uframes>2){ + modified = qh_completions (ehci, temp.qh, regs); + if (unlikely (list_empty (&temp.qh->qtd_list))) + periodic_qh_deschedule (ehci, temp.qh); + } + + qh_put (temp.qh); + break; + case Q_TYPE_FSTN: + + /* for save-place FSTNs, inspect all QH + * entries in the previous frame for + * completions, but not if we're already in + * recovery mode. */ + + if (q.fstn->hw_prev != EHCI_LIST_END && !rescan) { + modified = scan_frame (ehci,regs,frame-1,8,1); + rescan = 1; + } + + q_p = &q.fstn->fstn_next; + hw_p = &q.fstn->hw_next; + type = Q_NEXT_TYPE (*hw_p); + q = *q_p; + + break; + + case Q_TYPE_ITD: + /* skip itds for later in the frame */ + rmb (); + for (uf = live ? uframes : 8; uf < 8; uf++) { + if (0 == (q.itd->hw_transaction [uf] + & ITD_ACTIVE)) + continue; + q_p = &q.itd->itd_next; + hw_p = &q.itd->hw_next; + type = Q_NEXT_TYPE (*hw_p); + q = *q_p; + break; + } + if (uf != 8) + break; + + /* this one's ready ... HC won't cache the + * pointer for much longer, if at all. + */ + *q_p = q.itd->itd_next; + *hw_p = q.itd->hw_next; + type = Q_NEXT_TYPE (*hw_p); + wmb(); + modified = itd_complete (ehci, q.itd, regs); + q = *q_p; + break; + + case Q_TYPE_SITD: + + /* is this a spanning dummy? */ + if (q.sitd->hw_backpointer != EHCI_LIST_END){ + q_p = &q.sitd->sitd_next; + hw_p = &q.sitd->hw_next; + type = Q_NEXT_TYPE (*hw_p); + q = *q_p; + + if(rescan) break; + + /* note that the original sITD's + completion will unlink the dummy */ + modified = scan_frame(ehci,regs, + frame-1,8,1); + rescan = 1; + + }else{ + /* process completions for this frame only if + * we're certain all completions for a + * preceeding spanning frame have first been + * processed; that is true iff clock was past + * uframe 1 when we started */ + if ( ((q.sitd->hw_results & SITD_ACTIVE) || + uframes < 3) + && live) { + q_p = &q.sitd->sitd_next; + hw_p = &q.sitd->hw_next; + type = Q_NEXT_TYPE (*hw_p); + q = *q_p; + break; + } + + /* unlink the sITD */ + *q_p = q.sitd->sitd_next; + *hw_p = q.sitd->hw_next; + type = Q_NEXT_TYPE (*hw_p); + wmb(); + + /* unlink the dummy (if any) */ + if(BUDGET_WRAP_P(q.sitd->budget)) + unlink_dummy(ehci,q.sitd,regs,frame+1); + + modified = sitd_complete (ehci, q.sitd, regs); + + q = *q_p; + } + break; + default: + dbg ("corrupt type %d frame %d shadow %p", + type, frame, q.ptr); + // BUG (); + q.ptr = NULL; + } + + /* assume completion callbacks modify the queue */ + if (unlikely (modified)) return 1; + } return 0; } -#endif /* USB_EHCI_SPLIT_ISO */ - -/*-------------------------------------------------------------------------*/ - +/* scan_periodic - completion worker; called out of interrupt (or + * poll) handler to finish processing of transactions that have been + * completed by the host controller. Also now has the responsibility + * of halting the hardware periodic schedule after a complete pass + * through the schedule with nothing to do. + * + * @ehci: pointer to ehci host controller device structure + * @regs: ptrace registers + */ static void scan_periodic (struct ehci_hcd *ehci, struct pt_regs *regs) { unsigned frame, clock, now_uframe, mod; - unsigned modified; mod = ehci->periodic_size << 3; @@ -2094,12 +3520,11 @@ clock = readl (&ehci->regs->frame_index); else clock = now_uframe + mod - 1; - clock %= mod; + clock &= (mod-1); for (;;) { - union ehci_shadow q, *q_p; - __le32 type, *hw_p; unsigned uframes; + int repeat = 1; /* don't scan past the live uframe */ frame = now_uframe >> 3; @@ -2111,94 +3536,10 @@ uframes = 8; } -restart: - /* scan each element in frame's queue for completions */ - q_p = &ehci->pshadow [frame]; - hw_p = &ehci->periodic [frame]; - q.ptr = q_p->ptr; - type = Q_NEXT_TYPE (*hw_p); - modified = 0; - - while (q.ptr != NULL) { - unsigned uf; - union ehci_shadow temp; - int live; - - live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state); - switch (type) { - case Q_TYPE_QH: - /* handle any completions */ - temp.qh = qh_get (q.qh); - type = Q_NEXT_TYPE (q.qh->hw_next); - q = q.qh->qh_next; - modified = qh_completions (ehci, temp.qh, regs); - if (unlikely (list_empty (&temp.qh->qtd_list))) - intr_deschedule (ehci, temp.qh); - qh_put (temp.qh); - break; - case Q_TYPE_FSTN: - /* for "save place" FSTNs, look at QH entries - * in the previous frame for completions. - */ - if (q.fstn->hw_prev != EHCI_LIST_END) { - dbg ("ignoring completions from FSTNs"); - } - type = Q_NEXT_TYPE (q.fstn->hw_next); - q = q.fstn->fstn_next; - break; - case Q_TYPE_ITD: - /* skip itds for later in the frame */ - rmb (); - for (uf = live ? uframes : 8; uf < 8; uf++) { - if (0 == (q.itd->hw_transaction [uf] - & ITD_ACTIVE)) - continue; - q_p = &q.itd->itd_next; - hw_p = &q.itd->hw_next; - type = Q_NEXT_TYPE (q.itd->hw_next); - q = *q_p; - break; - } - if (uf != 8) - break; - - /* this one's ready ... HC won't cache the - * pointer for much longer, if at all. - */ - *q_p = q.itd->itd_next; - *hw_p = q.itd->hw_next; - type = Q_NEXT_TYPE (q.itd->hw_next); - wmb(); - modified = itd_complete (ehci, q.itd, regs); - q = *q_p; - break; - case Q_TYPE_SITD: - if ((q.sitd->hw_results & SITD_ACTIVE) - && live) { - q_p = &q.sitd->sitd_next; - hw_p = &q.sitd->hw_next; - type = Q_NEXT_TYPE (q.sitd->hw_next); - q = *q_p; - break; - } - *q_p = q.sitd->sitd_next; - *hw_p = q.sitd->hw_next; - type = Q_NEXT_TYPE (q.sitd->hw_next); - wmb(); - modified = sitd_complete (ehci, q.sitd, regs); - q = *q_p; - break; - default: - dbg ("corrupt type %d frame %d shadow %p", - type, frame, q.ptr); - // BUG (); - q.ptr = NULL; - } - - /* assume completion callbacks modify the queue */ - if (unlikely (modified)) - goto restart; - } + /* scan each element in frame's queue for completions */ + while(repeat) + repeat = scan_frame(ehci, regs, frame, uframes, 0); + /* stop when we catch up to the HC */ @@ -2215,8 +3556,9 @@ if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)) break; + ehci->next_uframe = now_uframe; - now = readl (&ehci->regs->frame_index) % mod; + now = readl (&ehci->regs->frame_index) & (mod-1); if (now_uframe == now) break; @@ -2224,7 +3566,26 @@ clock = now; } else { now_uframe++; - now_uframe %= mod; + now_uframe &= (mod-1); + + if(now_uframe == 0){ + + /* if the periodic schedule is still marked + to be disabled after a complete pass, + disable it now */ + disable_periodic(ehci); + + /* if the periodic schedule is running + but currently empty, do not disable + it immediately; that may hurt + streaming isoch (as Linux is not + using a true periodic schedule). + Mark the refcount so that it will + be disabled after this pass if no + new transactions are enabled */ + if(ehci->periodic_sched == 2) + ehci->periodic_sched = 1; + } } } } diff -ru a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig --- a/drivers/usb/host/Kconfig 2006-08-09 22:12:43.000000000 -0400 +++ b/drivers/usb/host/Kconfig 2006-09-25 22:19:32.000000000 -0400 @@ -29,15 +29,6 @@ To compile this driver as a module, choose M here: the module will be called ehci-hcd. -config USB_EHCI_SPLIT_ISO - bool "Full speed ISO transactions (EXPERIMENTAL)" - depends on USB_EHCI_HCD && EXPERIMENTAL - default n - ---help--- - This code is new and hasn't been used with many different - EHCI or USB 2.0 transaction translator implementations. - It should work for ISO-OUT transfers, like audio. - config USB_EHCI_ROOT_HUB_TT bool "Root Hub Transaction Translators (EXPERIMENTAL)" depends on USB_EHCI_HCD && EXPERIMENTAL @@ -50,23 +41,6 @@ This supports the EHCI implementation that's originally from ARC, and has since changed hands a few times. -config USB_EHCI_TT_NEWSCHED - bool "Improved Transaction Translator scheduling (EXPERIMENTAL)" - depends on USB_EHCI_HCD && EXPERIMENTAL - ---help--- - This changes the periodic scheduling code to fill more of the low - and full speed bandwidth available from the Transaction Translator - (TT) in USB 2.0 hubs. Without this, only one transfer will be - issued in each microframe, significantly reducing the number of - periodic low/fullspeed transfers possible. - - If you have multiple periodic low/fullspeed devices connected to a - highspeed USB hub which is connected to a highspeed USB Host - Controller, and some of those devices will not work correctly - (possibly due to "ENOSPC" or "-28" errors), say Y. - - If unsure, say N. - config USB_ISP116X_HCD tristate "ISP116X HCD support" depends on USB