diff -rub --minimal a/drivers/usb/host/ehci.h b/drivers/usb/host/ehci.h --- a/drivers/usb/host/ehci.h 2006-04-26 22:19:25.000000000 -0400 +++ b/drivers/usb/host/ehci.h 2006-08-09 19:49:30.000000000 -0400 @@ -72,11 +72,19 @@ int next_uframe; /* scan periodic, start here */ unsigned periodic_sched; /* periodic activity count */ + kmem_cache_t *budget_pool; /* Pool for shadow budget placeholder entries */ + struct ehci_shadow_budget **budget; /* pointer to the shadow budget + of bandwidth placeholders */ + + struct ehci_fstn *periodic_restore_fstn; + struct ehci_fstn **periodic_save_fstns; /* [PERIODIC_QH_MAX_PERIOD] */ + /* per root hub port */ unsigned long reset_done [EHCI_MAX_ROOT_PORTS]; /* per-HC memory pools (could be per-bus, but ...) */ struct dma_pool *qh_pool; /* qh per active urb */ + struct dma_pool *fstn_pool; /* a qh has [up to] two fstns */ struct dma_pool *qtd_pool; /* one or more per qh */ struct dma_pool *itd_pool; /* itd per iso urb */ struct dma_pool *sitd_pool; /* sitd per split iso urb */ @@ -354,6 +362,7 @@ /* next async queue entry, or pointer to interrupt/periodic QH */ #define QH_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_QH) +#define FSTN_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_FSTN) /* for periodic/async schedules and qtd lists, mark end of list */ #define EHCI_LIST_END __constant_cpu_to_le32(1) /* "null pointer" to hw */ @@ -377,6 +386,55 @@ /*-------------------------------------------------------------------------*/ +/* the shadow budget is a means of reserving bandwidth throughout the + periodic schedule (the hardware schedule is currently used as a + sliding window by the current transaction code). The shadow budget + follows the same list-then-tree structure as the hardware schedule. */ + +/* must be equal to or smaller than PERIODIC_QH_MAX_PERIOD */ +#define BUDGET_SLOTS 32 +#define BUDGET_SLOT_MASK (BUDGET_SLOTS-1) + +#define BUDGET_TT_BYTES_PER_UFRAME 188 +#define BUDGET_BYTES_PER_FRAME 1157 + +/* Because EHCI cannot issue ssplits in uframe 7 and USB 2.0 does not + allow ssplits in uframe 6, EHCI can only generate an efficient FS + frame by scheduling according to best-case (not worst-case) bit + stuffing. Thus we purposely 'overcommit' the FS frame budget + within the buffering ability of the TT to buffer, and within the + limits of the 90/10 rule. The TT will catch up in the microframes + where we cannot issue start splits. +*/ + +#define BUDGET_WRAP_P(b) ((b)->cmask & (~0xff)) + +struct ehci_shadow_budget { + struct ehci_shadow_budget *next; + + u16 s_usecs; /* highspeed usecs */ + u16 c_usecs; /* highspeed completion usecs (FS/LS)*/ + u16 tt_bytes; /* bytes to budget at TT (FS/LS)*/ + u16 cmask; /* 16 bits: a 'too big' mask indicates the need for a + frame wrap (FSTN or dummy SITD) */ + u8 smask; + +#define BUDGET_TYPE_ITD 1 /* ITD or SITD; budgets early */ +#define BUDGET_TYPE_QH 2 /* QH; budgets late */ + u8 type; + + /* QHs and FSTNs are persistent in the periodic schedule, so they + own their own budget placeholders. [s]ITD placeholders are owned + by an ehci_iso_stream. */ + union { + struct ehci_qh *qh; /* needed to extract period and specific TT */ + struct ehci_iso_stream *iso; /* needed to extract period and specific TT */ + void *ptr; /* used as an ID */ + } owner; +}; + +/*-------------------------------------------------------------------------*/ + /* * EHCI Specification 0.95 Section 3.6 * QH: describes control/bulk/interrupt endpoints @@ -385,6 +443,13 @@ * These appear in both the async and (for interrupt) periodic schedules. */ +/* as per ehci spec guidelines, replicate QH tree linkage of a maximum + size that is less than the full periodic schedule size. This is + also necessary to limit memory usage of FSTN support. */ +/* must be equal to or larger than BUDGET_SLOTS*8, otherwise the budget + will overcommit */ +#define PERIODIC_QH_MAX_PERIOD (BUDGET_SLOTS*8) + struct ehci_qh { /* first part defined by EHCI spec */ __le32 hw_next; /* see EHCI 3.6.1 */ @@ -425,13 +490,11 @@ /* periodic schedule info */ u8 usecs; /* intr bandwidth */ - u8 gap_uf; /* uframes split/csplit gap */ u8 c_usecs; /* ... split completion bw */ - u16 tt_usecs; /* tt downstream bandwidth */ - unsigned short period; /* polling interval */ - unsigned short start; /* where polling starts */ -#define NO_FRAME ((unsigned short)~0) /* pick new start */ + u16 tt_bytes; /* tt downstream bandwidth */ + unsigned short period; /* polling interval; uFrame granularity */ struct usb_device *dev; /* access to TT */ + struct ehci_shadow_budget *budget; /* pointer to budget placeholder */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ @@ -475,21 +538,19 @@ struct usb_host_endpoint *ep; /* output of (re)scheduling */ - unsigned long start; /* jiffies */ - unsigned long rescheduled; int next_uframe; - __le32 splits; /* the rest is derived from the endpoint descriptor, * trusting urb->interval == f(epdesc->bInterval) and * including the extra info for hw_bufp[0..2] */ - u8 interval; + u16 interval; u8 usecs, c_usecs; - u16 tt_usecs; + u16 tt_bytes; u16 maxp; u16 raw_mask; unsigned bandwidth; + int budget_state; /* This is used to initialize iTD's hw_bufp fields */ __le32 buf0; @@ -537,6 +598,7 @@ unsigned pg; unsigned index[8]; /* in urb->iso_frame_desc */ u8 usecs[8]; + struct ehci_shadow_budget *budget; /* pointer to budget placeholder */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ @@ -566,6 +628,7 @@ #define SITD_STS_STS (1 << 1) /* split transaction state */ #define SITD_ACTIVE __constant_cpu_to_le32(SITD_STS_ACTIVE) +#define SITD_DOCOMPLETE __constant_cpu_to_le32(SITD_STS_STS) __le32 hw_buf [2]; /* EHCI table 3-12 */ __le32 hw_backpointer; /* EHCI table 3-13 */ @@ -580,6 +643,7 @@ struct list_head sitd_list; /* list of stream's sitds */ unsigned frame; unsigned index; + struct ehci_shadow_budget *budget; /* pointer to budget placeholder */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ @@ -600,10 +664,16 @@ /* the rest is HCD-private */ dma_addr_t fstn_dma; union ehci_shadow fstn_next; /* ptr to periodic q entry */ + struct ehci_qh *fstn_prev; /* ptr to backlinked qh */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/ +#define EHCI_EP_QH(x) (((x)->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) != USB_ENDPOINT_XFER_ISOC ? (x)->hcpriv : NULL) +#define EHCI_EP_STREAM(x) (((x)->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) == USB_ENDPOINT_XFER_ISOC ? (x)->hcpriv : NULL) + +/*-------------------------------------------------------------------------*/ + #ifdef CONFIG_USB_EHCI_ROOT_HUB_TT /* diff -rub --minimal a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c --- a/drivers/usb/host/ehci-hcd.c 2006-04-26 22:19:25.000000000 -0400 +++ b/drivers/usb/host/ehci-hcd.c 2006-08-04 19:51:54.000000000 -0400 @@ -426,6 +426,11 @@ if ((retval = ehci_mem_init(ehci, GFP_KERNEL)) < 0) return retval; + /* a periodic schedule that supports FSTNs always has a single + static restore FSTN that matches all possible save + FSTNs. */ + periodic_init_link_restore_fstn(ehci); + /* controllers may cache some of the periodic schedule ... */ hcc_params = readl(&ehci->caps->hcc_params); if (HCC_ISOC_CACHE(hcc_params)) // full frame cache @@ -780,7 +785,7 @@ && HC_IS_RUNNING (hcd->state)) { int status; - status = qh_schedule (ehci, qh); + status = periodic_qh_schedule (ehci, qh); spin_unlock_irqrestore (&ehci->lock, flags); if (status != 0) { @@ -815,24 +820,32 @@ struct ehci_hcd *ehci = hcd_to_ehci (hcd); unsigned long flags; struct ehci_qh *qh, *tmp; + struct ehci_iso_stream *iso; /* ASSERT: any requests/urbs are being unlinked */ /* ASSERT: nobody can be submitting urbs for this any more */ -rescan: spin_lock_irqsave (&ehci->lock, flags); - qh = ep->hcpriv; - if (!qh) - goto done; + iso = EHCI_EP_STREAM(ep); - /* endpoints can be iso streams. for now, we don't - * accelerate iso completions ... so spin a while. - */ - if (qh->hw_info1 == 0) { - ehci_vdbg (ehci, "iso delay\n"); - goto idle_timeout; + if (iso){ + /* for now, we don't accelerate iso completions ... so spin + a while. */ + + while(iso->refcount>1){ + spin_unlock_irqrestore (&ehci->lock, flags); + schedule_timeout_uninterruptible(1); + spin_lock_irqsave (&ehci->lock, flags); } + /* we want to be sure completions deref the stream to 1, + then we finally pull the plug here */ + iso_stream_put(ehci,iso); + ep->hcpriv = NULL; + }else{ + + while ( (qh = EHCI_EP_QH(ep)) ){ + if (!HC_IS_RUNNING (hcd->state)) qh->qh_state = QH_STATE_IDLE; switch (qh->qh_state) { @@ -841,34 +854,41 @@ tmp && tmp != qh; tmp = tmp->qh_next.qh) continue; + /* periodic qh self-unlinks on empty */ - if (!tmp) - goto nogood; + if (!tmp) goto error; unlink_async (ehci, qh); /* FALL THROUGH */ + case QH_STATE_UNLINK: /* wait for hw to finish? */ -idle_timeout: + spin_unlock_irqrestore (&ehci->lock, flags); schedule_timeout_uninterruptible(1); - goto rescan; + spin_lock_irqsave (&ehci->lock, flags); + break; + case QH_STATE_IDLE: /* fully unlinked */ + if (list_empty (&qh->qtd_list)) { qh_put (qh); + ep->hcpriv = NULL; break; } /* else FALL THROUGH */ default: -nogood: +error: /* caller was supposed to have unlinked any requests; * that's not our job. just leak this memory. */ ehci_err (ehci, "qh %p (#%02x) state %d%s\n", qh, ep->desc.bEndpointAddress, qh->qh_state, list_empty (&qh->qtd_list) ? "" : "(has tds)"); + ep->hcpriv = NULL; break; } - ep->hcpriv = NULL; -done: + } + } + spin_unlock_irqrestore (&ehci->lock, flags); return; } diff -rub --minimal a/drivers/usb/host/ehci-mem.c b/drivers/usb/host/ehci-mem.c --- a/drivers/usb/host/ehci-mem.c 2006-04-26 22:19:25.000000000 -0400 +++ b/drivers/usb/host/ehci-mem.c 2006-08-09 19:33:06.000000000 -0400 @@ -62,6 +62,26 @@ dma_pool_free (ehci->qtd_pool, qtd, qtd->qtd_dma); } +static struct ehci_fstn *ehci_fstn_alloc (struct ehci_hcd *ehci, gfp_t flags) +{ + struct ehci_fstn *fstn = NULL; + dma_addr_t dma; + + if(ehci->fstn_pool){ /* alloced only if this HC actually supports fstn */ + fstn = dma_pool_alloc (ehci->fstn_pool, flags, &dma); + if (fstn != NULL) { + memset (fstn, 0, sizeof *fstn); + fstn->fstn_dma = dma; + } + } + return fstn; +} + +static inline void ehci_fstn_free (struct ehci_hcd *ehci, struct ehci_fstn *fstn) +{ + dma_pool_free (ehci->fstn_pool, fstn, fstn->fstn_dma); +} + static void qh_destroy (struct kref *kref) { @@ -135,6 +155,10 @@ dma_pool_destroy (ehci->qtd_pool); ehci->qtd_pool = NULL; + if (ehci->fstn_pool) + dma_pool_destroy (ehci->fstn_pool); + ehci->fstn_pool = NULL; + if (ehci->qh_pool) { dma_pool_destroy (ehci->qh_pool); ehci->qh_pool = NULL; @@ -154,7 +178,17 @@ ehci->periodic, ehci->periodic_dma); ehci->periodic = NULL; + if(ehci->periodic_save_fstns) + kfree(ehci->periodic_save_fstns); + if(ehci->budget) + kfree(ehci->budget); + ehci->budget = NULL; + if(ehci->budget_pool) + kmem_cache_destroy(ehci->budget_pool); + ehci->budget_pool = NULL; + /* shadow periodic table */ + if(ehci->pshadow) kfree(ehci->pshadow); ehci->pshadow = NULL; } @@ -174,6 +208,23 @@ goto fail; } + /* does this HC support FSTNs? Don't allocate FSTN structs if not. */ + if( HC_VERSION(readl (&ehci->caps->hc_capbase)) >= 96){ + /* EHCI 0.96+ */ + ehci->fstn_pool = dma_pool_create ("ehci_fstn", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_fstn), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); + /* do not give up on failure; scheduling is merely constrained */ + if(ehci->fstn_pool) + ehci->periodic_save_fstns = + kcalloc(PERIODIC_QH_MAX_PERIOD, + sizeof(*ehci->periodic_save_fstns), flags); + /* lazy-alloc the rest */ + + } + /* QHs for control/bulk/intr transfers */ ehci->qh_pool = dma_pool_create ("ehci_qh", ehci_to_hcd(ehci)->self.controller, @@ -183,6 +234,7 @@ if (!ehci->qh_pool) { goto fail; } + ehci->async = ehci_qh_alloc (ehci, flags); if (!ehci->async) { goto fail; @@ -219,6 +271,19 @@ for (i = 0; i < ehci->periodic_size; i++) ehci->periodic [i] = EHCI_LIST_END; + /* budgeting placeholders */ + ehci->budget = kcalloc(BUDGET_SLOTS, sizeof(*ehci->budget), flags); + if (ehci->budget == NULL){ + goto fail; + } + ehci->budget_pool = + kmem_cache_create ("ehci_budget", + sizeof(struct ehci_shadow_budget), + 0,0,NULL,NULL); + if (!ehci->budget_pool) { + goto fail; + } + /* software shadow of hardware table */ ehci->pshadow = kcalloc(ehci->periodic_size, sizeof(void *), flags); if (ehci->pshadow != NULL) diff -rub --minimal a/drivers/usb/host/ehci-q.c b/drivers/usb/host/ehci-q.c --- a/drivers/usb/host/ehci-q.c 2006-04-26 22:19:25.000000000 -0400 +++ b/drivers/usb/host/ehci-q.c 2006-08-09 19:36:15.000000000 -0400 @@ -269,8 +269,8 @@ static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh); static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh); -static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh); -static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh); +static void periodic_qh_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh); +static int periodic_qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh); /* * Process and free completed qtds for a qh, returning URBs to drivers. @@ -430,8 +430,12 @@ */ if ((__constant_cpu_to_le32 (QH_SMASK) & qh->hw_info2) != 0) { - intr_deschedule (ehci, qh); - (void) qh_schedule (ehci, qh); + + qh_get(qh); + periodic_qh_deschedule (ehci, qh); + (void) periodic_qh_schedule (ehci, qh); + qh_put(qh); + } else unlink_async (ehci, qh); break; @@ -631,9 +635,9 @@ struct urb *urb, gfp_t flags ) { + int is_input, type = usb_pipetype (urb->pipe); struct ehci_qh *qh = ehci_qh_alloc (ehci, flags); u32 info1 = 0, info2 = 0; - int is_input, type; int maxp = 0; if (!qh) @@ -646,7 +650,6 @@ info1 |= usb_pipedevice (urb->pipe) << 0; is_input = usb_pipein (urb->pipe); - type = usb_pipetype (urb->pipe); maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input); /* Compute interrupt scheduling parameters just once, and save. @@ -660,31 +663,29 @@ if (type == PIPE_INTERRUPT) { qh->usecs = NS_TO_US (usb_calc_bus_time (USB_SPEED_HIGH, is_input, 0, hb_mult (maxp) * max_packet (maxp))); - qh->start = NO_FRAME; if (urb->dev->speed == USB_SPEED_HIGH) { qh->c_usecs = 0; - qh->gap_uf = 0; + qh->tt_bytes = 0; + qh->period = urb->interval; - qh->period = urb->interval >> 3; - if (qh->period == 0 && urb->interval != 1) { - /* NOTE interval 2 or 4 uframes could work. - * But interval 1 scheduling is simpler, and - * includes high bandwidth. - */ - dbg ("intr period %d uframes, NYET!", - urb->interval); - goto done; - } } else { struct usb_tt *tt = urb->dev->tt; - int think_time; + int think_time, think_bytes; - /* gap is f(FS/LS transfer times) */ - qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed, - is_input, 0, maxp) / (125 * 1000); + think_time = tt ? tt->think_time : 0; + think_bytes = (think_time+665)/666; + + if(urb->dev->speed == USB_SPEED_FULL) + /* full speed bytes + think time [TT host delay] + FS non-iso protocol overhead */ + qh->tt_bytes = think_bytes + maxp + 14; + else + /* low speed bytes + think time [TT host delay] + low speed protocol overhead */ + /* expressed in full speed bytes */ + qh->tt_bytes = think_bytes + maxp*8 + 98; /* FIXME this just approximates SPLIT/CSPLIT times */ + /* can't fix without going to ns granularity; this is at least safe */ if (is_input) { // SPLIT, gap, CSPLIT+DATA qh->c_usecs = qh->usecs + HS_USECS (0); qh->usecs = HS_USECS (1); @@ -693,11 +694,7 @@ qh->c_usecs = HS_USECS (0); } - think_time = tt ? tt->think_time : 0; - qh->tt_usecs = NS_TO_US (think_time + - usb_calc_bus_time (urb->dev->speed, - is_input, 0, max_packet (maxp))); - qh->period = urb->interval; + qh->period = urb->interval << 3; /* uFrame granularity */ } } @@ -760,7 +757,7 @@ break; default: dbg ("bogus dev %p speed %d", urb->dev, urb->dev->speed); -done: +/* done: */ qh_put (qh); return NULL; } diff -rub --minimal a/drivers/usb/host/ehci-sched.c b/drivers/usb/host/ehci-sched.c --- a/drivers/usb/host/ehci-sched.c 2006-04-26 22:19:25.000000000 -0400 +++ b/drivers/usb/host/ehci-sched.c 2006-08-09 19:51:33.000000000 -0400 @@ -1,6 +1,7 @@ /* * Copyright (c) 2001-2004 by David Brownell * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers + * Copyright (c) 2006 Monty, for budgeting code * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -22,24 +23,114 @@ /*-------------------------------------------------------------------------*/ /* - * EHCI scheduled transaction support: interrupt, iso, split iso - * These are called "periodic" transactions in the EHCI spec. + * EHCI scheduled transaction support: interrupt, iso, split + * iso. These are called "periodic" transactions in the EHCI spec. It + * should be noted that the Linux USB subsystem has no high-level + * notion of periodically serviced transfers; bandwidth is reserved + * periodically [the shadow budget], but transfers are still entirely + * single-request/single-response. To avoid over/underruns, periodic + * streaming relies on upper layers to make a new request before the + * next schedule slot passes. Any such xruns are reported as + * -EL2NSYNC if they occur. * - * Note that for interrupt transfers, the QH/QTD manipulation is shared - * with the "asynchronous" transaction support (control/bulk transfers). - * The only real difference is in how interrupt transfers are scheduled. + * The shadow budget consists of placeholders for reserving bandwidth + * throughout the periodic schedule for any given active endpoint. + * This avoids both underallocation errors (spurious ENOSPC during a + * transfer that began successfully) and overallocation errors (lost + * frames or HC/hub/irq errors and resets during transfers) due to + * basing bandwidth allocation on the always-incomplete hardware + * schedule. Although it would be conceptually simpler to place + * inactive [s]ITDs and QHs into the actual hardware budget (to be + * activated when a transfer makes use of that segment of the periodic + * schedule), the additional memory overhead of populating the + * complete 1024 frame periodic schedule is not insignificant. + * Therefore, a relatively small shadow budget (BUDGET_SLOTS) is + * projected ('translated') onto the full (1024 slot) hardware + * schedule. Budgeting is fully computed upon the first transfer + * attempt with an endpoint and remains in the shadow budget until the + * endpoint is closed [disabled]. Transfer descriptors are linked + * into the hardware schedule on demand using the precomputed + * information in the shadow budget. + * + * Interrupt transfers share the QH/QTD manipulation with the + * "asynchronous" transaction support (control/bulk transfers) in + * ehci-q.c, although all toplevel interrupt control originates here. + * The only real difference is in how interrupt transfers are + * scheduled. + * + * For isochronous, an "iso_stream" head serves the same role as a QH. + * It keeps track of every ITD (or SITD) that's scheduled. + * + * This scheduler supports and aggressively uses both FSTNs and sITD + * backpointers in order to allow interrupt QH and sITD transfers to + * span H-frame boundaries. This support is necessary for efficient + * FS bus bandwidth usage through a 2.0 hub's TT mechanism. + * + * The transfer ordering in the shadow schedule looks like: + * + * . | Level N | N/2 | ... | Level 1 + * . | | | | + * . | | | | + * Frame 2: [iso] + * \_ [save FSTN]; [QHs] | | | + * / \_ [QHs] | | + * Frame 1: [iso] ... _/ \_ ... _ | + * ... _/ \_ [restore FSTN]; [QHs] + * ... _/ + * Frame 0: [iso]... + * + * Each [iso] block is ordered [dummy sITDs]; [sITDs/ITDs] + * + * The shadow budget is structured like the shadow schedule, however + * it is both smaller (to save memory) and simpler, containing only + * QHs and normal [s]ITDs. dummy sITDs and FSTNs are intuited in + * budgeting calculations and do not appear explicitly in the shadow + * budget: + * + * . |Level N| N/2 | ... |Level 1 + * . | | | | + * . append | | | | + * ---------> |prepend| | | + * Frame 2: [sITDs/ITDs] | <---- |prepend| | + * \_ [QHs] | <---- | | + * / \_ [QHs] | |prepend + * Frame 1: [sITDs/ITDs] ... _/ \_ ... _ | <---- + * ... _/ \_ [QHs] + * ... _/ + * Frame 0: [sITDs/ITDs]... + * + * + * Transaction ordering is determined by the budgeting code and set in + * the budget. Transactions are linked into the actual hardware + * schedule in the order specified by the budget. FSTN positioning is + * static; it follows the recommendations of the EHCI 1.0 spec and + * places one save-state FSTN at the lowest level in each branch of + * the INTR tree and one restore FSTN at the head of level one. Dummy + * sITDs are positioned in the order of the original sITDs of the + * preceeding frame. + * + * The scheduler does not currently implement any inter- or + * intra-frame rebalancing; that is, the budget is not adjusted to + * fill in 'holes' or shuffle transfer slots to optimize remaining + * bandwidth when endpoints are removed. QHs will reuse holes at the + * moment, isoch transfers will not. This can also lead to starving + * QHs depending on the order in which the QHs are budgeted because + * the tree arrangement of the QHs places ordering restrictions upon + * QHs that do not exist for isoch transfers. Starvation is unlikely + * (and far less likely than for the old scheduler), but it's still + * possible to be unable to schedule a new QH before actually running + * out of FS bandwidth. * - * For ISO, we make an "iso_stream" head to serve the same role as a QH. - * It keeps track of every ITD (or SITD) that's linked, and holds enough - * pre-calculated schedule data to make appending to the queue be quick. */ static int ehci_get_frame (struct usb_hcd *hcd); /*-------------------------------------------------------------------------*/ +/* shadow schedule positioning, as this is needed throughout */ /* * periodic_next_shadow - return "next" pointer on shadow list + * * @periodic: host pointer to qh/itd/sitd * @tag: hardware tag for type of this record */ @@ -59,191 +150,1098 @@ } } -/* caller must hold ehci->lock */ -static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr) +/* the following are merely frame-structure dumpers to aid in + debugging. Be careful of their use; they will introduce extreme + latencies in what is [essentially] realtime code. */ +static void print_budget_frame (struct ehci_hcd *ehci, int frame, + struct ehci_shadow_budget *insert, + void *owner) { - union ehci_shadow *prev_p = &ehci->pshadow [frame]; - __le32 *hw_p = &ehci->periodic [frame]; - union ehci_shadow here = *prev_p; + struct ehci_shadow_budget *here = + ehci->budget [frame & BUDGET_SLOT_MASK]; + printk(KERN_INFO " slot %d: ",frame & BUDGET_SLOT_MASK); - /* find predecessor of "ptr"; hw and shadow lists are in sync */ - while (here.ptr && here.ptr != ptr) { - prev_p = periodic_next_shadow (prev_p, Q_NEXT_TYPE (*hw_p)); - hw_p = here.hw_next; - here = *prev_p; + while(here){ + if(here == insert || here->owner.ptr == owner) + printk(">>"); + + switch(here->type){ + case BUDGET_TYPE_ITD: + if(here->tt_bytes) + printk("[SITD %p 0x%x 0x%x]", + here, + (unsigned)here->cmask, + (unsigned)here->smask); + else + printk("[ITD %p 0x%x 0x%x]", + here, + (unsigned)here->cmask, + (unsigned)here->smask); + break; + case BUDGET_TYPE_QH: + printk("[QH (%s) %p %d 0x%x 0x%x]", + (here->tt_bytes?"FS/LS":"HS"), + here, + here->owner.qh->period, + (unsigned)here->cmask, + (unsigned)here->smask); + break; } - /* an interrupt entry (at list end) could have been shared */ - if (!here.ptr) + + if(here == insert || here->owner.ptr == owner) + printk("<<"); + + if(here == here->next){ + printk("\nERROR: schedule entry linked " + "to itself!\n"); return; + } - /* update shadow and hardware lists ... the old "next" pointers - * from ptr may still be in use, the caller updates them. - */ - *prev_p = *periodic_next_shadow (&here, Q_NEXT_TYPE (*hw_p)); - *hw_p = *here.hw_next; + here=here->next; + + } + printk("\n"); } -/* how many of the uframe's 125 usecs are allocated? */ -static unsigned short -periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe) +static void print_budget (struct ehci_hcd *ehci, + struct ehci_shadow_budget *insert, + void *owner) { - __le32 *hw_p = &ehci->periodic [frame]; - union ehci_shadow *q = &ehci->pshadow [frame]; - unsigned usecs = 0; + int i; + for(i=0; iptr) { - switch (Q_NEXT_TYPE (*hw_p)) { +static void print_schedule_frame (struct ehci_hcd *ehci, int frame, + void *insert) +{ + union ehci_shadow *here = + &ehci->pshadow [frame % ehci->periodic_size]; + __le32 *hw_p = &ehci->periodic [frame % ehci->periodic_size]; + __le32 type = Q_NEXT_TYPE (*hw_p); + printk(KERN_INFO " frame %d: ",frame % ehci->periodic_size); + while(here && here->ptr){ + if(here->ptr == insert) + printk(">>"); + + switch(type){ + case Q_TYPE_ITD: + { + struct ehci_itd *itd = here->itd; + printk("[ITD 0x%p : 0x%x]", + itd, + (unsigned)itd->hw_next); + } + break; + case Q_TYPE_SITD: + { + struct ehci_sitd *sitd = here->sitd; + if(sitd->hw_backpointer != EHCI_LIST_END) + printk("[DUMMY 0x%p]", + sitd); + else + printk("[SITD 0x%p]", + sitd); + } + break; + case Q_TYPE_FSTN: + { + struct ehci_fstn *fstn = here->fstn; + if(fstn->hw_prev == EHCI_LIST_END) + printk("[FSTN restore 0x%p]",fstn); + else + printk("[FSTN save 0x%p <- 0x%p]", + fstn, + fstn->fstn_prev); + } + break; case Q_TYPE_QH: - /* is it in the S-mask? */ - if (q->qh->hw_info2 & cpu_to_le32 (1 << uframe)) - usecs += q->qh->usecs; - /* ... or C-mask? */ - if (q->qh->hw_info2 & cpu_to_le32 (1 << (8 + uframe))) - usecs += q->qh->c_usecs; - hw_p = &q->qh->hw_next; - q = &q->qh->qh_next; + { + struct ehci_qh *qh = here->qh; + printk("[QH %d 0x%p]", + qh->period,qh); + } break; - // case Q_TYPE_FSTN: + } + + if(here->ptr == insert) + printk("<<"); + + if(here == periodic_next_shadow (here, type)){ + printk("\nERROR: periodic schedule entry " + "linked to itself!\n"); + return; + } + + hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (*hw_p); + + } + printk("\n"); +} + +static void print_schedule(struct ehci_hcd *ehci,void *insert) +{ + int i; + for(i=0; iperiodic_size; i++) + print_schedule_frame(ehci,i,insert); +} + +/* find position of a specific entry in the periodic schedule (ie, + * returns pointers such that we can update the predecessor's + * linkage); here->ptr == NULL indicates the find failed. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number of the shadow/hardware schedule to search + * @ptr: entry for which to search + * @hw_p: return pointer to hw_next field of preceeding entry + */ +static union ehci_shadow * +periodic_find_entry(struct ehci_hcd *ehci, + unsigned frame, + void *ptr, + __le32 **hw_p) +{ + union ehci_shadow *here = &ehci->pshadow [frame % ehci->periodic_size]; + __le32 type; + + *hw_p = &ehci->periodic [frame % ehci->periodic_size]; + type = Q_NEXT_TYPE (**hw_p); + + while (here->ptr && here->ptr != ptr) { + *hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (**hw_p); + } + return here; +} + +/*-------------------------------------------------------------------------*/ +/* shadow budget implementation */ + +/* determine the TT relevant to this shadow budget entry + * + * @b: pointer to shadow budget entry + */ +static struct usb_tt * +budget_tt (struct ehci_shadow_budget *b) +{ + switch (b->type) { + case BUDGET_TYPE_QH: + return b->owner.qh->dev->tt; + case BUDGET_TYPE_ITD: + return b->owner.iso->udev->tt; default: - /* for "save place" FSTNs, count the relevant INTR - * bandwidth from the previous frame + return NULL; + } +} + +/* covert a QH's period [expressed in uFrames] to tree level. + * + * @period: period (uFrames) of a QH */ - if (q->fstn->hw_prev != EHCI_LIST_END) { - ehci_dbg (ehci, "ignoring FSTN cost ...\n"); +static int _period_to_level(int period) +{ + if(period<8)return 1; + if(period > PERIODIC_QH_MAX_PERIOD){ + return (PERIODIC_QH_MAX_PERIOD >> 3); } - hw_p = &q->fstn->hw_next; - q = &q->fstn->fstn_next; + return period>>3; +} + +/* budget_position_new_itd - determine and return proper insertion position + * in the shadow budget for a new itd or sitd (both position into the + * same segment of the frame) + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow/hardware schedule into which new + * transaction is to be inserted. + */ +static struct ehci_shadow_budget ** +budget_position_new_itd(struct ehci_hcd *ehci, int frame) +{ + struct ehci_shadow_budget **here = + &ehci->budget [(frame & BUDGET_SLOT_MASK)]; + + /* skip to the end of any dummies or [s]ITDs at list head */ + while ( (*here) && + (*here)->type == BUDGET_TYPE_ITD ){ + here = &(*here)->next; + } + + return here; +} + +/* budget_position_new_qh - determine and return proper insertion position + * in the shadow budget for a newly queued qh + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow/hardware schedule into which new + * transaction is to be inserted. + * @period: the endpoint period (in uFrames) + */ +static struct ehci_shadow_budget ** +budget_position_new_qh(struct ehci_hcd *ehci, int frame, int period) +{ + struct ehci_shadow_budget **here = + budget_position_new_itd(ehci, frame); + + while (*here){ + + /* skip past higher (but not same) period QH entries */ + /* > 8 to left of restore FSTN, <= 8 to right */ + if ( (*here)->type == BUDGET_TYPE_QH && + period >= (*here)->owner.qh->period) break; + + here = &(*here)->next; + } + return here; +} + +/* budget_position_new - determine and return proper insertion position + * in the shadow budget for a new budget entry + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow/hardware schedule into which new + * transaction is to be inserted. + * @new: new,populated budget entry to insert + */ +static struct ehci_shadow_budget ** +budget_position_new(struct ehci_hcd *ehci, + int frame, + struct ehci_shadow_budget *new) +{ + switch(new->type){ + case BUDGET_TYPE_QH: + return budget_position_new_qh(ehci,frame, + new->owner.qh->period); + + case BUDGET_TYPE_ITD: + return budget_position_new_itd(ehci,frame); + + default: + return NULL; + + } +} + +/* budget_find_entry_by_owner - find position of a specific entry in + * the shadow budget by looking for the single entry in the frame + * belonging to the passed in owner. return == NULL indicates the + * entry was not found. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the shadow budget to search; masked to + * prevent overflow of the shadow budget + * @owner: the owner of the placeholder (qh or ehci_iso_stream) + */ +static struct ehci_shadow_budget * +budget_find_entry_by_owner(struct ehci_hcd *ehci, + unsigned frame, + void *owner) +{ + struct ehci_shadow_budget *here = + ehci->budget [(frame & BUDGET_SLOT_MASK)]; + + /* invariant: any one budget frame will contain at most a single entry + for a given owner; the budget does not explicitly contain spanning + elements, it only declares them in the transaction's cmask. Thus + searching by owner is unique. */ + while (here){ + if(here->owner.ptr == owner) break; + here = here->next; + } + return here; +} + +/* preiodic_translate_budget - Finds the position in the + * hardware/shadow schedule to insert a new hardware schedule entry + * based on the position in the shadow budget of the passed in budget + * entry. This handles positioning dummy sITD, sITD, ITD and QH + * entries; FSTN position has nothing to do with the budget and is + * handled by code specific to FSTNs. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame number in the hardware schedule to search; masked to + * prevent overflow + * @translate: budget entry to translate to hardware schedule + * @hw_p: return a pointer to the predecessor's hw_next field + */ +static union ehci_shadow * +periodic_translate_budget (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_shadow_budget *translate, + __le32 **hw_p) +{ + struct ehci_shadow_budget *budget_prev = + ehci->budget [((frame-1) & BUDGET_SLOT_MASK)]; + struct ehci_shadow_budget *budget_curr = + ehci->budget [(frame & BUDGET_SLOT_MASK)]; + union ehci_shadow *shadow = + &ehci->pshadow [frame % ehci->periodic_size]; + __le32 type; + + /* transaction search order: + [spanning previous frame itd/sitd] -> + [current frame itd/sitd] -> + [qh] ... + thus the search begins against previous frame's budget but this + frame's shadow schedule. */ + /* Complete/Inactive entries that have not yet been processed + by scan_periodic may yet be present, that will not affect + this algorithm (they will have entries present in the + budget, even waiting for final urb completion). */ + + *hw_p = &ehci->periodic [frame % ehci->periodic_size]; + type = Q_NEXT_TYPE (**hw_p); + + /* Everything in the shadow schedule will be there in the budget + [except for FSTNs] */ + while (shadow->ptr){ + if(type != Q_TYPE_FSTN){ + + /* advance budget until the current budget + placeholder is the one belonging to current + shadow entry or until we hit the stopping + point. */ + while(budget_prev && + budget_prev != translate && + budget_prev->owner.ptr != shadow->ptr){ + budget_prev = budget_prev->next; + } + if(budget_prev == translate){ + return shadow; /* a match! */ + } + + /* If we've already fully searched the + preceding frame, search the current + frame */ + while(!budget_prev && + budget_curr && + budget_curr != translate && + budget_curr->owner.ptr != shadow->ptr){ + budget_curr = budget_curr->next; + } + if(budget_curr == translate){ + return shadow; /* a match! */ + } + + /* falling off the end of the budget indicates + * an internal logic error */ + if(budget_prev == NULL && budget_curr == NULL){ + ehci_err(ehci, + "schedule contains " + "unbudgeted transaction\n"); + return NULL; + } + }else{ + struct ehci_fstn *fstn = shadow->fstn; + /* restore fstn: + ITD/sITD must go before + QH of period > 8 must go before + QH of period <=8 must go after + */ + /* save fstn: + all ITD/sITD must go before + all QH must go after + */ + + if(translate->type == BUDGET_TYPE_ITD){ + return shadow; + } + + if(fstn->hw_prev == EHCI_LIST_END) + if(translate->owner.qh->period > 8){ + return shadow; + } + + } + + /* advance shadow and hardware schedule by one, then + loop to test this position */ + *hw_p = shadow->hw_next; + shadow = periodic_next_shadow (shadow, type); + type = Q_NEXT_TYPE (**hw_p); + + } + + /* nothing queued or fell off the end looking for stopping point. + Append to end. */ + return shadow; + +} + +/* budget_unlink_entries_by_owner - scan the complete shadow budget + * and unlink/free all entries owned by passed in owner ptr. + * + * @ehci: pointer to ehci host controller device structure. + * @owner: pointer to owner + */ +static void budget_unlink_entries_by_owner(struct ehci_hcd *ehci,void *owner) +{ + struct ehci_shadow_budget *qh = NULL; + int frame; + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Releasing bandwidth for owner %p\n", + owner); + ehci_info(ehci, " Budget before: \n"); + print_budget(ehci,NULL,owner); +#endif + + for(frame=0; framebudget[frame]; + + while(*budget){ + if((*budget)->owner.ptr == owner){ + struct ehci_shadow_budget *next = + ((*budget)->next); + + switch((*budget)->type){ + default: + case BUDGET_TYPE_ITD: + + kmem_cache_free(ehci->budget_pool, + *budget); + *budget = next; + break; - case Q_TYPE_ITD: - usecs += q->itd->usecs [uframe]; - hw_p = &q->itd->hw_next; - q = &q->itd->itd_next; + case BUDGET_TYPE_QH: + if(!qh || (*budget)==qh){ + qh = *budget; + *budget = next; break; - case Q_TYPE_SITD: - /* is it in the S-mask? (count SPLIT, DATA) */ - if (q->sitd->hw_uframe & cpu_to_le32 (1 << uframe)) { - if (q->sitd->hw_fullspeed_ep & - __constant_cpu_to_le32 (1<<31)) - usecs += q->sitd->stream->usecs; - else /* worst case for OUT start-split */ - usecs += HS_USECS_ISO (188); } - /* ... C-mask? (count CSPLIT, DATA) */ - if (q->sitd->hw_uframe & - cpu_to_le32 (1 << (8 + uframe))) { - /* worst case for IN complete-split */ - usecs += q->sitd->stream->c_usecs; - } + /* another qh with the same + owner is an internal error; + not fatal, but shouldn't + happen. We will leak the + memory rather than risk a + double-free. */ + ehci_err(ehci, + "multiple unique QH budget " + "entries with same endpoint " + "owner.\n"); - hw_p = &q->sitd->hw_next; - q = &q->sitd->sitd_next; break; } + /* do not advance; need to process the + replacement which is now *budget */ + }else + budget = &((*budget)->next); + + } } -#ifdef DEBUG - if (usecs > 100) - ehci_err (ehci, "uframe %d sched overrun: %d usecs\n", - frame * 8 + uframe, usecs); + + if(qh)kmem_cache_free(ehci->budget_pool,qh); + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, " Budget after: \n"); + print_budget(ehci,NULL,owner); #endif - return usecs; + } /*-------------------------------------------------------------------------*/ +/* Shadow budget controlling logic */ -static int same_tt (struct usb_device *dev1, struct usb_device *dev2) +/* _calc_hs_uframe - count usecs used in the HS schedule of the + * specified frame/uFrame. *new is a new proposed shadow budget entry + * to be treated as if it is already linked into the shadow budget at + * position **insert. Returns abolute usec count, not metric or + * remaining available usecs. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame position in the shadow budget (masked to shadow budget size) + * @uFrame: uFrame position within full frame; may be 8 or 9 to access + * spanning c_mask + * @insert: position of *new [NULL if new is NULL] + * @new: a proposed shadow entry to be treated as appearing at position + * *insert; may be NULL + */ +static int _calc_hs_uframe(struct ehci_hcd *ehci, + int frame, + int uFrame, + struct ehci_shadow_budget **insert, + struct ehci_shadow_budget *new) { - if (!dev1->tt || !dev2->tt) - return 0; - if (dev1->tt != dev2->tt) - return 0; - if (dev1->tt->multi) - return dev1->ttport == dev2->ttport; - else - return 1; + struct ehci_shadow_budget **budget = + &ehci->budget [(frame & BUDGET_SLOT_MASK)]; + int usec = 0; + if(budget == insert) budget = &new; + + while(*budget){ + struct ehci_shadow_budget *budgetp = *budget; + + /* is it in the S-mask? */ + if (budgetp->smask & (1 << uFrame)) + usec += budgetp->s_usecs; + + /* ... or C-mask? */ + if (budgetp->cmask & (1 << uFrame)) + usec += budgetp->c_usecs; + + /* advance */ + if(budget == &new) + budget = insert; + else{ + budget = &(budgetp->next); + if(budget == insert) budget = &new; + } + } + + return usec; } -/* return true iff the device's transaction translator is available - * for a periodic transfer starting at the specified frame, using - * all the uframes in the mask. +/* budget_calc_hs_frame - returns the budget 'load' metric for a given + * high speed frame, range -1 (invalid schedule), 0 (lowest good + * score) to 100 (best possible score). HS scheduling uses this to + * choose between possbile slots. FS/LS scheduling uses it as a check + * to make sure the FS budgeting choice is valid in the context of the + * HS schedule. *new is a new proposed shadow budget entry + * to be treated as if it is already linked into the shadow budget at + * position **insert. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame position in the shadow budget (masked to shadow budget size) + * @insert: position to use for *new; in other functions that take an + * insertion point as a marker, *insert=NULL indicates insert at end + * of list. budget_calc_hs_frame checks more than one frame (because + * of transactions that can span frames), so a simple NULL would be + * ambiguous. Thus the pointer to the insertion point, which would + * uniquely point to only one frame's list, even in the case of + * insertion. + * @new: a proposed shadow entry to be treated as appearing at position + * *insert; + * may be NULL */ -static int tt_no_collision ( - struct ehci_hcd *ehci, - unsigned period, - struct usb_device *dev, - unsigned frame, - u32 uf_mask -) + +static int budget_calc_hs_frame(struct ehci_hcd *ehci, + int frame, + struct ehci_shadow_budget **insert, + struct ehci_shadow_budget *new) { - if (period == 0) /* error */ - return 0; + int max=0, uFrame; + for(uFrame=0; uFrame<8; uFrame++){ - /* note bandwidth wastage: split never follows csplit - * (different dev or endpoint) until the next uframe. - * calling convention doesn't make that distinction. + /* first verify we've not overcommitted any uFrames */ + /* If uFrame < 2, count the cmask contribution of + * spanning from prior frame */ + int val = _calc_hs_uframe(ehci,frame,uFrame,insert,new); + if(uFrame<2) + val += _calc_hs_uframe(ehci,frame-1, + uFrame+8,insert,new); + + if(val>max)max=val; + if(max>100){ + return -1; + } + } + return 100-max; +} + +/* periodic_budget_fs_frame - test a proposed new addition to the + * periodic schedule of a given frame. Returns <0 if the frame is + * unsuitable, else a positive value indicating the relative space + * remaining in that frame after the new transaction would be + * scheduled; 0 is lowest score, 100 is best possible score. This + * return value is a metric used for breadth-first allocation of + * bandwidth. As a side efect, determine and set appropriate s_mask + * and c_mask for the passed in *new entry. + * + * @ehci: pointer to ehci host controller device structure. + * @frame: frame position in the shadow budget (masked to shadow + * budget size) + * @earliest_uFrame: prevent *new from scheduling into any uframe + * earlier than this + * @insert: position of *new + * @new: a proposed shadow entry to be treated as appearing at + * position *insert */ - for (; frame < ehci->periodic_size; frame += period) { - union ehci_shadow here; - __le32 type; +static int budget_calc_fs_frame(struct ehci_hcd *ehci, + int frame, + unsigned earliest_uFrame, + struct ehci_shadow_budget *insert, + struct ehci_shadow_budget *new) +{ + struct ehci_shadow_budget *budget = + ehci->budget [(frame & BUDGET_SLOT_MASK)]; - here = ehci->pshadow [frame]; - type = Q_NEXT_TYPE (ehci->periodic [frame]); - while (here.ptr) { - switch (type) { - case Q_TYPE_ITD: - type = Q_NEXT_TYPE (here.itd->hw_next); - here = here.itd->itd_next; - continue; - case Q_TYPE_QH: - if (same_tt (dev, here.qh->dev)) { - u32 mask; + /* Scheduling through a TT is done by byte counting, not usec + counting. Because we cannot issue a FS/LS start split in + uFrame 6 or 7, the only way to get anywhere near a full 90% + of a FS frame for periodic transactions is to 'overcommit' + the FS scheduling of each microframe [when considered by + usecs needed for the transfer]. This is not actually + overcommitting as the TT will buffer the 'excess' from any + uFrame and schedule it to transmit in a later uFrame. USB + 2.0 11.18.4: "Scheduling of split transactions is done by + the host (typically in software) based on a best-case + estimate of how the full-/low-speed transacitons can be run + on the downstream facing bus." */ - mask = le32_to_cpu (here.qh->hw_info2); - /* "knows" no gap is needed */ - mask |= mask >> 8; - if (mask & uf_mask) - break; + int remaining_tt_bytes = + BUDGET_TT_BYTES_PER_UFRAME; /* per uFrame best-case + byte budget; USB 2.0 + 11.18.1 */ + int remaining_frame_bytes = + BUDGET_BYTES_PER_FRAME; /* max full-frame bytes + for enforcing worst-case + 90/10 */ + int remaining_tt_transactions = 16; + int uFrame=0; + int load = remaining_frame_bytes; + int spanned_gap = 0; + struct usb_tt *tt = budget_tt(new); + + if(budget == insert) budget = new; + earliest_uFrame &= 0x7; + + while(budget){ + int start_frame = uFrame; + u8 smask=0; + u16 cmask=0; + + /* exclude other TTs as well as HS transfers, which + have no bearing on FS sched. */ + if(budget_tt(budget) != tt) + goto advance; + + if(budget != new){ + + /* For a variety of reasons, the preexisting + FS budget may contain holes where something + could be scheduled but nothing is. In this + case, the current budget placeholder's + smask will exist entirely in the future. + Test for this case and advance the counters + if true. */ + if( ((budget->smask >> (uFrame+1)) << (uFrame+1)) == + budget->smask){ + remaining_tt_bytes = + BUDGET_TT_BYTES_PER_UFRAME; + remaining_tt_transactions = 16; + uFrame++; + continue; } - type = Q_NEXT_TYPE (here.qh->hw_next); - here = here.qh->qh_next; + + /* For now, the scheduler holds invariant that + transactions appear in a frame in smask + order; that is, that FS transactions appear + in the frame budget in the same order that + they are run by the host controller. + + Thus, the current transaction must not have + already been scheduled into the hardware + schedule to start before now. */ + + if( ((budget->smask >> uFrame) << uFrame) != + budget->smask) + return -1; + + + }else{ + if(uFrame < earliest_uFrame){ + remaining_tt_bytes = + BUDGET_TT_BYTES_PER_UFRAME; + remaining_tt_transactions = 16; + uFrame++; continue; - case Q_TYPE_SITD: - if (same_tt (dev, here.sitd->urb->dev)) { - u16 mask; + } + } - mask = le32_to_cpu (here.sitd - ->hw_uframe); - /* FIXME assumes no gap for IN! */ - mask |= mask >> 8; - if (mask & uf_mask) + if(!spanned_gap && budget->type == BUDGET_TYPE_QH){ + load = remaining_frame_bytes - + BUDGET_BYTES_PER_FRAME + + BUDGET_TT_BYTES_PER_UFRAME * uFrame; + if(load<1)load=1; + spanned_gap=1; + } + + remaining_frame_bytes -= budget->tt_bytes; + remaining_tt_bytes -= budget->tt_bytes; + remaining_tt_transactions--; + + if(remaining_tt_transactions<0 || remaining_frame_bytes<0) + return -1; + + while(remaining_tt_bytes <= 0){ + remaining_tt_bytes += + BUDGET_TT_BYTES_PER_UFRAME; + remaining_tt_transactions = 16; + uFrame++; + } + + if(budget->type == BUDGET_TYPE_ITD) + load = remaining_frame_bytes; + + if(budget != new) goto advance; + + /* Generate smask/cmask for new entry */ + + if(budget->type == BUDGET_TYPE_QH){ + smask = (0x01 << start_frame); + /* cmask purposely overflows the frame by some + amount if FSTN is needed for frame + spanning */ + /* USB 2.0 11.18.4.3.b */ + cmask = (0x1c << start_frame); + + /* if we need an FSTN are they even + available? */ + if((cmask&(~0xff)) && ehci->fstn_pool==NULL) + return -1; + }else{ + /* sitd */ + if(budget->owner.iso->bEndpointAddress & USB_DIR_IN){ + /* cmask purposely overflows the frame + by some amount if dummy SITD is + needed for frame spanning */ + smask = (0x01 << start_frame); + cmask = ((1<<(uFrame-start_frame+3)) - 1) << 2; + }else{ + /* ISO OUT */ + smask = ((1 << (uFrame-start_frame+1)) - 1); + cmask = 0; + } + } + + /* FS/LS ssplit in uFrame 6 or 7 is illegal */ + if(smask & 0xc0){ + return -1; + } + new->smask=smask; + new->cmask=cmask; + + advance: + + if(budget == new) + budget = insert; + else{ + budget = budget->next; + if(budget == insert) budget = new; + } + } + + return (load * 100 / BUDGET_BYTES_PER_FRAME); +} + +/* _make_hs_smask - convenience function for generating a HS smask + * + * @period: trnsaction interval (in uFrames) + * @uFrame: intraframe offset + */ +static unsigned _make_hs_smask(int period, int uFrame) +{ + switch(period){ + case 1: + return 0xff; + case 2: + return 0x55 << (uFrame & 0x1); + case 4: + return 0x11 << (uFrame & 0x3); + default: + return 0x01 << (uFrame & 0x7); + } +} + +/* budget_add_endpoint - create a new shadow budget entry for a + * transaction of passed in type, owner and interval, and add it to + * the shadow budget (or return nonzero if the request is + * impossible,eg, no room in the bandwidth budget). + * + * @ehci: pointer to ehci host controller device structure. + * @owner: pointer to transaction owner (ehci_qh or ehci_iso_stream) + * @type: BUDGET_TYPE_QH (intr) or BUDGET_TYPE_ITD (itd or sitd) + * @interval: endpoint interval (always in uFrames) + */ +static int budget_add_endpoint (struct ehci_hcd *ehci, + void *owner, + u8 type, + int interval) +{ + int status = 0; + int frame,uFrame; + int best_choice = -1; + int best_metric = -1; + int period = (interval ? + (interval > BUDGET_SLOTS*8 ? BUDGET_SLOTS*8 : interval) : + 1); + int period_8 = ( (period>>3) ? (period>>3) : 1); // always <= + // _period_to_level + struct ehci_shadow_budget new, *new_p = NULL; + memset(&new,0,sizeof(new)); + new.owner.ptr = owner; + new.type = type; + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Budgeting bandwidth for new owner %p\n", + owner); + ehci_info(ehci, " Budget before: \n"); + print_budget(ehci,NULL,owner); +#endif + + + switch (type){ + case BUDGET_TYPE_ITD: + { + struct ehci_iso_stream *stream = + (struct ehci_iso_stream *)owner; + new.s_usecs = stream->usecs; + new.c_usecs = stream->c_usecs; + new.tt_bytes = stream->tt_bytes; + } break; + case BUDGET_TYPE_QH: + { + struct ehci_qh *qh = (struct ehci_qh *)owner; + new.s_usecs = qh->usecs; + new.c_usecs = qh->c_usecs; + new.tt_bytes = qh->tt_bytes; } - type = Q_NEXT_TYPE (here.sitd->hw_next); - here = here.sitd->sitd_next; - continue; - // case Q_TYPE_FSTN: + break; default: - ehci_dbg (ehci, - "periodic frame %d bogus type %d\n", - frame, type); + status = -ENOSYS; + goto out; } - /* collision or error */ - return 0; +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + if(new.tt_bytes) + printk(KERN_INFO " FS/LS strategy, Metrics:\n"); + else + printk(KERN_INFO " HS strategy, Metrics:\n"); +#endif + + /* determine best 'slot' */ + for(uFrame = 0; uFrame < period; uFrame++){ + int frame = (uFrame >> 3); + int min_metric = 100; /* percent */ + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + printk(KERN_INFO " uFrame slot %d : ",uFrame); +#endif + + /* scan entirety of slot */ + for(; framebest_metric){ + best_metric = min_metric; + best_choice = uFrame; + } + + if(best_metric == 100) + break; // best possible result; no reason to + // keep looking + } + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci," Budgeting slot choice: %d\n",best_choice); +#endif + + if(best_choice < 0){ + status = -ENOSPC; + goto out; + } + + /* All checks pass. Add necessary budget placeholder[s] to + shadow budget */ + uFrame = best_choice; + frame = (uFrame >> 3); + for(; framebudget_pool, + GFP_ATOMIC); + if(new_p == NULL){ + /* clean up; remove partially complete + * budgeting and error out*/ + budget_unlink_entries_by_owner(ehci,owner); + status = -ENOMEM; + goto out; + } + memcpy(new_p,&new,sizeof(new)); + + /* reconstruct best budget masks and positioning + (saves on storage not to save all the possibilities + from above) */ + insert = budget_position_new(ehci,frame,new_p); + if(*insert != new_p){ + if(new.tt_bytes){ + /* FS/LS */ + budget_calc_fs_frame(ehci, frame, uFrame, + *insert, new_p); + }else{ + /* HS */ + new_p->smask = _make_hs_smask(period,uFrame); + } + + if(type == BUDGET_TYPE_QH) + ((struct ehci_qh *)owner)->budget = new_p; + + /* link new entry */ + new_p->next = *insert; + *insert = new_p; + + }else + ehci_err(ehci,"Should not have seen same QH " + "while linking in budget tree\n"); + } +out: +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, " Budgeting status: %d\n", + status); + ehci_info(ehci, " Budget after: \n"); + print_budget(ehci,NULL,owner); +#endif + + return status; } +/* budget_schedule_next - Return the proper frame number and position + * in the shadow schedule of the next transaction belonging to owner + * in the shadow budget. If the transaction is already in the + * shadow/hard schedule, here->ptr will point to it. + * + * @ehci: pointer to ehci host controller device structure. + * @startframe: frame number in the shadow schedule at which to begin search + * @owner: pointer to transaction owner (ehci_qh or ehci_iso_stream) + * @b: returns pointer to matching shadow budget entry (NULL if none) + * @here: returns pointer to insertion position in shadow schedule + * @hw_p: returns pointer to insertion position in hardware schedule + */ +static int budget_schedule_next (struct ehci_hcd *ehci, + int startframe, + void *owner, + struct ehci_shadow_budget **b, + union ehci_shadow **here, + __le32 **hw_p) +{ + int i; + for(i=startframe; i < startframe+BUDGET_SLOTS; i++){ + int pframe = i % ehci->periodic_size; + + /* search for it in this frame's budget; we're only + searching for sITD, ITD and QH. Finding a spanning + element reuses the discovered *b elsewhere */ + *b = budget_find_entry_by_owner(ehci, i, owner); + if(*b){ + *here = periodic_translate_budget(ehci,pframe,*b,hw_p); + if(*here){ + return i; + } + } + } + + *here = NULL; + *b = NULL; + ehci_err(ehci,"tried to schedule an endpoint that doesn't " + "appear in the budget\n"); + return -1; +} + +/* budget_schedule_next - Return the number of the next frame for + * which there's a budgeted transaction for owner. + * + * @ehci: pointer to ehci host controller device structure. + * @startframe: frame number in the shadow schedule at which to begin search + * @owner: pointer to transaction owner (ehci_qh or ehci_iso_stream) + */ +static int budget_schedule_next_frame (struct ehci_hcd *ehci, + int startframe, + void *owner) +{ + int i; + for(i=startframe; i < startframe+BUDGET_SLOTS; i++){ + struct ehci_shadow_budget *b; + int bframe = i & BUDGET_SLOT_MASK; + b = budget_find_entry_by_owner(ehci, bframe, owner); + if(b){ + return i; + } + } + return -1; +} + +/* end of shadow budget implementation */ + /*-------------------------------------------------------------------------*/ +/* enable_periodic - Activate the periodic schedule + * + * @ehci: pointer to ehci host controller device structure. + */ static int enable_periodic (struct ehci_hcd *ehci) { u32 cmd; int status; + if(ehci->periodic_sched<=0){ + ehci->periodic_sched=0; + /* did clearing PSE did take effect yet? * takes effect only at frame boundaries... */ @@ -261,18 +1259,38 @@ /* make sure ehci_work scans these */ ehci->next_uframe = readl (&ehci->regs->frame_index) % (ehci->periodic_size << 3); + } + + ehci->periodic_sched++; return 0; } +/* disable_periodic - halt execution of the periodic schedule. Note + * that the periodic schedule is now disabled after a full pass + * through the periodic schedule in which it is unused. It's not + * immediately disabled when the last active transfer completes as + * [especially] streaming code may be working in a 'just in time' mode + * where the next transaction comes in soon after [not before] the + * last finishes. Now that the loss-of-sync detection is working + * properly, this is not treated as an error unless streaming really + * is lost. + * + * @ehci: pointer to ehci host controller device structure. + */ static int disable_periodic (struct ehci_hcd *ehci) { u32 cmd; int status; + if(ehci->periodic_sched>0){ + ehci->periodic_sched--; + + if(ehci->periodic_sched==0){ /* did setting PSE not take effect yet? * takes effect only at frame boundaries... */ - status = handshake (&ehci->regs->status, STS_PSS, STS_PSS, 9 * 125); + status = handshake (&ehci->regs->status, + STS_PSS, STS_PSS, 9 * 125); if (status != 0) { ehci_to_hcd(ehci)->state = HC_STATE_HALT; return status; @@ -283,86 +1301,295 @@ /* posted write ... */ ehci->next_uframe = -1; + } + } + return 0; } /*-------------------------------------------------------------------------*/ +/* FSTN machinery -/* periodic schedule slots have iso tds (normal or split) first, then a - * sparse tree for active interrupt transfers. +FSTN decision making and manipulation is self contained and has +nothing to do with the budget, only the schedule. The master restore +FSTN is set up along with the ehci initialization; save-place FSTNs +are manipulated and adjusted after any changes to the QH tree in +the hardware schedule. */ + +/* periodic_init_link_restore_fstn - Place one restore FSTN at the + * beginning of node level 1 of the hardware schedule interrupt tree + * [EHCI 1.0 4.12.2.2.2]. Called once by the ehci initialization code + * and not again. * - * this just links in a qh; caller guarantees uframe masks are set right. - * no FSTN support (yet; ehci 0.96+) + * @ehci: pointer to ehci host controller device structure. */ -static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) +static void periodic_init_link_restore_fstn(struct ehci_hcd *ehci) { - unsigned i; - unsigned period = qh->period; + int i; - dev_dbg (&qh->dev->dev, - "link qh%d-%04x/%p start %d [%d/%d us]\n", - period, le32_to_cpup (&qh->hw_info2) & (QH_CMASK | QH_SMASK), - qh, qh->start, qh->usecs, qh->c_usecs); + ehci->periodic_restore_fstn = ehci_fstn_alloc (ehci, GFP_ATOMIC); - /* high bandwidth, or otherwise every microframe */ - if (period == 0) - period = 1; + if(ehci->periodic_restore_fstn){ + ehci->periodic_restore_fstn->hw_next = EHCI_LIST_END; + ehci->periodic_restore_fstn->hw_prev = EHCI_LIST_END; - for (i = qh->start; i < ehci->periodic_size; i += period) { - union ehci_shadow *prev = &ehci->pshadow [i]; + for(i=0; iperiodic_size; i++){ + union ehci_shadow *here = &ehci->pshadow [i]; __le32 *hw_p = &ehci->periodic [i]; - union ehci_shadow here = *prev; - __le32 type = 0; - /* skip the iso nodes at list head */ - while (here.ptr) { - type = Q_NEXT_TYPE (*hw_p); - if (type == Q_TYPE_QH) - break; - prev = periodic_next_shadow (prev, type); - hw_p = &here.qh->hw_next; - here = *prev; + here->fstn = ehci->periodic_restore_fstn; + *hw_p = FSTN_NEXT (here->fstn->fstn_dma); + wmb (); + + } } +} - /* sorting each branch by period (slow-->fast) - * enables sharing interior tree nodes +/* periodic_position_save_fstn - determine the insertion point within + * the specified frame for a save-place FSTN. Any given frame + * contains only one save-state FSTN, which sits at the beginning of + * the lowest level (highest interval) of the hardware schedule + * interrupt tree. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @hw_p: returns pointer to insertion point in hardware schedule */ - while (here.ptr && qh != here.qh) { - if (qh->period > here.qh->period) +static union ehci_shadow * +periodic_position_save_fstn(struct ehci_hcd *ehci, + unsigned frame, + __le32 **hw_p) +{ + union ehci_shadow *here = &ehci->pshadow [frame % ehci->periodic_size]; + __le32 type; + + *hw_p = &ehci->periodic [frame % ehci->periodic_size]; + type = Q_NEXT_TYPE (**hw_p); + + while (here->ptr){ + + if(type == Q_TYPE_FSTN || type == Q_TYPE_QH) break; - prev = &here.qh->qh_next; - hw_p = &here.qh->hw_next; - here = *prev; + + *hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (**hw_p); } - /* link in this qh, unless some earlier pass did that */ - if (qh != here.qh) { - qh->qh_next = here; - if (here.qh) - qh->hw_next = *hw_p; + + return here; +} + +/* periodic_find_save_fstn - search a frame in the shadow/hardware + * schedule for a save FSTN, returning the insertion point if found or + * NULL if not found. Any given frame will contain at most a single + * save-state FSTN. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @hw_p: returns pointer to insertion point in hardware schedule + */ +static union ehci_shadow * +periodic_find_save_fstn(struct ehci_hcd *ehci, + unsigned frame, + __le32 **hw_p) +{ + union ehci_shadow *here; + int fframe = frame % PERIODIC_QH_MAX_PERIOD; + struct ehci_fstn *fstn; + + if(ehci->periodic_save_fstns == NULL) + goto out; /* no FSTN support in HC */ + fstn = ehci->periodic_save_fstns[fframe]; + if(fstn == NULL) + goto out; /* this FSTN not yet alloced, couldn't be linked */ + + here = periodic_position_save_fstn(ehci, frame, hw_p); + if(here->ptr == fstn){ + return here; + } + +out: + return NULL; +} + +/* _periodic_unlink_fstn - unconditionally unlinks the save-state FSTN + * from the specified frame. noop if the frame does not have an + * active save-state FSTN. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + */ +static void _periodic_unlink_fstn (struct ehci_hcd *ehci, unsigned frame) +{ + __le32 *hw_p; + union ehci_shadow *here = periodic_find_save_fstn(ehci, frame, &hw_p); + + if (!here) + return; + + /* inactivate */ + here->fstn->hw_prev = EHCI_LIST_END; + here->fstn->fstn_prev = NULL; + + /* unlink */ + *hw_p = *here->hw_next; + *here = here->fstn->fstn_next; +} + +/* _periodic_link_fstn - allocs [if needed] and links save-state + * fstn into frame in appropraite position, also patching FSTN + * backpointer. If this frame already contains an active FSTN, only + * patches backpointer. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @back_qh: pointer to qh which the FSTN's backpointer should reference + */ +static void _periodic_link_fstn (struct ehci_hcd *ehci, + int frame, + struct ehci_qh *back_qh) +{ + + if(ehci->periodic_save_fstns){ + int fframe = frame % PERIODIC_QH_MAX_PERIOD; + struct ehci_fstn *fstn = ehci->periodic_save_fstns[fframe]; + __le32 *hw_p; + union ehci_shadow *here = + periodic_position_save_fstn(ehci, frame, &hw_p); + + /* is the FSTN already alloced? */ + if(!fstn){ + fstn = ehci_fstn_alloc (ehci, GFP_ATOMIC); + if(!fstn){ + ehci_err(ehci, + "Unable to allocate FSTN\n"); + return; + } + + ehci->periodic_save_fstns[fframe] = fstn; + } + + /* patch the FSTN */ + fstn->hw_prev = QH_NEXT(back_qh->qh_dma); + fstn->fstn_prev = back_qh; + + /* already linked? */ + if(here->ptr != fstn){ + /* splice right to save-place fstn */ + fstn->fstn_next = *here; + fstn->hw_next = *hw_p; wmb (); - prev->qh = qh; - *hw_p = QH_NEXT (qh->qh_dma); + + /* splice it to left; it's now live */ + here->fstn = fstn; + *hw_p = FSTN_NEXT (fstn->fstn_dma); } + wmb (); } - qh->qh_state = QH_STATE_LINKED; - qh_get (qh); +} - /* update per-qh bandwidth for usbfs */ - ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period - ? ((qh->usecs + qh->c_usecs) / qh->period) - : (qh->usecs * 8); +/* _periodic_inspect_fstn_frame - toplevel machinery for managing all + * changes to FSTN state in the schedule; called whenever changes are + * made to the QH structure of a frame, automagically updates FSTN + * state in the hardware/shadow schedule to correctly reflect QH frame + * spanning. Links/unlinks/patches FSTNs in the following frame as + * needed. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @insert: if changes are to be affected to FSTN preceeding linking + * new QH into live schedule, *insert designates new QH + * insertion point + * @new_qh: points to new qh to be linked into schedule following FSTN updates; + * allows valid FSTN to preceed activating the QH; NULL if this is an + * unlink update. + */ +static void _periodic_inspect_fstn_frame (struct ehci_hcd *ehci, + int frame, + union ehci_shadow *insert, + union ehci_shadow *new_qh) +{ + __le32 type = + Q_NEXT_TYPE (ehci->periodic [frame % ehci->periodic_size]); + __le32 save_type = type; + union ehci_shadow *here = &ehci->pshadow [frame % ehci->periodic_size]; - /* maybe enable periodic schedule processing */ - if (!ehci->periodic_sched++) - return enable_periodic (ehci); + if(here == insert){ + here = new_qh; + type = Q_TYPE_QH; + } - return 0; + /* search frame for first spanning QH */ + while (here->ptr){ + if(type == Q_TYPE_QH){ + if(BUDGET_WRAP_P(here->qh->budget)){ + _periodic_link_fstn (ehci, frame+1, here->qh); + return; + } + } + + /* advance */ + if(here == new_qh){ + here = insert; + type = save_type; + }else{ + save_type = Q_NEXT_TYPE (*here->hw_next); + here = periodic_next_shadow (here, type); + if(here == insert){ + here = new_qh; + type = Q_TYPE_QH; + }else + type = save_type; + } + } + + /* no QH found, make sure there's no FSTN in following frame */ + _periodic_unlink_fstn (ehci, frame+1); } -static void qh_unlink_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) +/*-------------------------------------------------------------------------*/ +/* Periodic interrupt (QH) endpoint machinery */ + +/* periodic_qh_unlink_frame - unlink the passed in QH from the + * specified frame and update frame FSTN state + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @qh: QH to unlink + */ +static void periodic_qh_unlink_frame (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_qh *qh) +{ + __le32 *hw_p; + union ehci_shadow *here = periodic_find_entry(ehci, frame, qh, &hw_p); + + if (here->ptr){ + + *hw_p = *here->hw_next; + *here = here->qh->qh_next; + + /*... the old "next" pointers from ptr (and if a qh, the + fstns) may still be in use, the caller updates them. */ + + } + + _periodic_inspect_fstn_frame(ehci,frame,NULL,NULL); + +} + +/* periodic_qh_deschedule - unlink the passed in QH from the entire + * hardware schedule and update FSTN state; does not remove the QH + * from the shadow budget. + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to unlink + */ +#include +static void periodic_qh_deschedule(struct ehci_hcd *ehci, struct ehci_qh *qh) { unsigned i; - unsigned period; + unsigned wait; // FIXME: // IF this isn't high speed @@ -371,40 +1598,39 @@ // THEN // qh->hw_info1 |= __constant_cpu_to_le32 (1 << 7 /* "ignore" */); - /* high bandwidth, or otherwise part of every microframe */ - if ((period = qh->period) == 0) - period = 1; +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Removing QH %p from schedule:\n", qh); + ehci_info(ehci, " Schedule before: \n"); + print_schedule(ehci,qh); +#endif - for (i = qh->start; i < ehci->periodic_size; i += period) - periodic_unlink (ehci, i, qh); + /* remove from hardware schedule */ + for (i=0; i < ehci->periodic_size; i++) + periodic_qh_unlink_frame (ehci, i, qh); /* update per-qh bandwidth for usbfs */ - ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period - ? ((qh->usecs + qh->c_usecs) / qh->period) - : (qh->usecs * 8); + /* XXX Hm... now that we have the budget, this should be moved + to where the budget is released */ + ehci_to_hcd(ehci)->self.bandwidth_allocated -= + (qh->usecs + qh->c_usecs) / qh->period; + + /* dev_dbg (&qh->dev->dev, "unlink qh%d-%04x/%p start %d [%d/%d us]\n", qh->period, le32_to_cpup (&qh->hw_info2) & (QH_CMASK | QH_SMASK), qh, qh->start, qh->usecs, qh->c_usecs); + */ /* qh->qh_next still "live" to HC */ qh->qh_state = QH_STATE_UNLINK; qh->qh_next.ptr = NULL; + qh_put (qh); /* maybe turn off periodic schedule */ - ehci->periodic_sched--; - if (!ehci->periodic_sched) - (void) disable_periodic (ehci); -} - -static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh) -{ - unsigned wait; - - qh_unlink_periodic (ehci, qh); + disable_periodic(ehci); /* simple/paranoid: always delay, expecting the HC needs to read * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and @@ -421,176 +1647,152 @@ udelay (wait); qh->qh_state = QH_STATE_IDLE; qh->hw_next = EHCI_LIST_END; + wmb (); + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, " Schedule after: \n"); + print_schedule(ehci,qh); +#endif } -/*-------------------------------------------------------------------------*/ +/* periodic_qh_link - link the passed in QH into all relevant frames of the + * hardware schedule and update FSTN state; assumes QH is already budgeted. + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to link + */ +static int periodic_qh_link (struct ehci_hcd *ehci, struct ehci_qh *qh) +{ + unsigned i=0; + unsigned level = _period_to_level(qh->period); -static int check_period ( - struct ehci_hcd *ehci, - unsigned frame, - unsigned uframe, - unsigned period, - unsigned usecs -) { - int claimed; + /* set masks */ + qh->hw_info2 &= + __constant_cpu_to_le32(~(QH_CMASK | QH_SMASK)); + qh->hw_info2 |= + qh->budget->smask | + ((qh->budget->cmask&0xff)<<8) | + (((qh->budget->cmask>>8)&0xff)<<8); - /* complete split running into next frame? - * given FSTN support, we could sometimes check... - */ - if (uframe >= 8) - return 0; + while(1){ + __le32 *hw_p; + union ehci_shadow *here; + struct ehci_shadow_budget *budget; - /* - * 80% periodic == 100 usec/uframe available - * convert "usecs we need" to "max already claimed" - */ - usecs = 100 - usecs; + i = budget_schedule_next (ehci, i, qh, &budget, &here, &hw_p); - /* we "know" 2 and 4 uframe intervals were rejected; so - * for period 0, check _every_ microframe in the schedule. - */ - if (unlikely (period == 0)) { - do { - for (uframe = 0; uframe < 7; uframe++) { - claimed = periodic_usecs (ehci, frame, uframe); - if (claimed > usecs) - return 0; + if(i >= ehci->periodic_size)break; + + if(i<0){ + ehci_err(ehci, + "QH slot not in budget where expected"); + i += level; + continue; } - } while ((frame += 1) < ehci->periodic_size); - /* just check the specified uframe, at that period */ - } else { - do { - claimed = periodic_usecs (ehci, frame, uframe); - if (claimed > usecs) - return 0; - } while ((frame += period) < ehci->periodic_size); + /* perform FSTN inspection/adjustment before qh is + * linked */ + { + union ehci_shadow temp; + temp.qh=qh; + _periodic_inspect_fstn_frame (ehci, i, here, &temp); } - // success! - return 1; -} + /* already linked in? */ + if(here->ptr != qh){ -static int check_intr_schedule ( - struct ehci_hcd *ehci, - unsigned frame, - unsigned uframe, - const struct ehci_qh *qh, - __le32 *c_maskp -) -{ - int retval = -ENOSPC; - u8 mask; + /* link right */ + qh->qh_next = *here; + qh->hw_next = *hw_p; + wmb (); - if (qh->c_usecs && uframe >= 6) /* FSTN territory? */ - goto done; + /* link left; fully live */ + here->qh = qh; + *hw_p = QH_NEXT (qh->qh_dma); - if (!check_period (ehci, frame, uframe, qh->period, qh->usecs)) - goto done; - if (!qh->c_usecs) { - retval = 0; - *c_maskp = 0; - goto done; + } + i += level; } - /* Make sure this tt's buffer is also available for CSPLITs. - * We pessimize a bit; probably the typical full speed case - * doesn't need the second CSPLIT. - * - * NOTE: both SPLIT and CSPLIT could be checked in just - * one smart pass... - */ - mask = 0x03 << (uframe + qh->gap_uf); - *c_maskp = cpu_to_le32 (mask << 8); + qh->qh_state = QH_STATE_LINKED; + qh_get (qh); - mask |= 1 << uframe; - if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) { - if (!check_period (ehci, frame, uframe + qh->gap_uf + 1, - qh->period, qh->c_usecs)) - goto done; - if (!check_period (ehci, frame, uframe + qh->gap_uf, - qh->period, qh->c_usecs)) - goto done; - retval = 0; - } -done: - return retval; + /* update per-qh bandwidth for usbfs */ + ehci_to_hcd(ehci)->self.bandwidth_allocated += + (qh->usecs + qh->c_usecs) / qh->period; + + /* maybe enable periodic schedule processing */ + return enable_periodic (ehci); } -/* "first fit" scheduling policy used the first time through, - * or when the previous schedule slot can't be re-used. +/* periodic_qh_schedule - budget the passed in QH if it has not + * already been budgeted, then add QH to the hardware schedule and + * update FSTN state. + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to link */ -static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh) +static int periodic_qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh) { int status; - unsigned uframe; - __le32 c_mask; - unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */ qh_refresh(ehci, qh); qh->hw_next = EHCI_LIST_END; - frame = qh->start; - /* reuse the previous schedule slots, if we can */ - if (frame < qh->period) { - uframe = ffs (le32_to_cpup (&qh->hw_info2) & QH_SMASK); - status = check_intr_schedule (ehci, frame, --uframe, - qh, &c_mask); - } else { - uframe = 0; - c_mask = 0; - status = -ENOSPC; - } + /* budget the qh, but only if not already budgeted. Fault recovery + code in ehci-q likes to de- and re- schedule a QH into the + hardware schedule; not sure what that actually achieves (perhaps + it's really looking for a side effect?) but we should not repeat + budgeting overhead. */ + if(!qh->budget){ - /* else scan the schedule to find a group of slots such that all - * uframes have enough periodic bandwidth available. - */ - if (status) { - /* "normal" case, uframing flexible except with splits */ - if (qh->period) { - frame = qh->period - 1; - do { - for (uframe = 0; uframe < 8; uframe++) { - status = check_intr_schedule (ehci, - frame, uframe, qh, - &c_mask); - if (status == 0) - break; - } - } while (status && frame--); +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Budgeting new QH %p\n",qh); +#endif - /* qh->period == 0 means every uframe */ - } else { - frame = 0; - status = check_intr_schedule (ehci, 0, 0, qh, &c_mask); + status = budget_add_endpoint (ehci, qh, BUDGET_TYPE_QH, + qh->period); + if(status){ + return status; + } } - if (status) - goto done; - qh->start = frame; - /* reset S-frame and (maybe) C-frame masks */ - qh->hw_info2 &= __constant_cpu_to_le32(~(QH_CMASK | QH_SMASK)); - qh->hw_info2 |= qh->period - ? cpu_to_le32 (1 << uframe) - : __constant_cpu_to_le32 (QH_SMASK); - qh->hw_info2 |= c_mask; - } else - ehci_dbg (ehci, "reused qh %p schedule\n", qh); + /* schedule the qh */ +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Adding new QH %p to schedule:\n", + qh); + ehci_info(ehci, " Schedule before: \n"); + print_schedule(ehci,qh); +#endif - /* stuff into the periodic schedule */ - status = qh_link_periodic (ehci, qh); -done: + status = periodic_qh_link (ehci, qh); +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, " Schedule after: \n"); + print_schedule(ehci,qh); +#endif + + if(status) + budget_unlink_entries_by_owner(ehci,qh); return status; } -static int intr_submit ( - struct ehci_hcd *ehci, +/* intr_submit - toplevel entry point for submitting an interrupt + * endpoint request. If needed, performs initial creation, budgeting + * and scheduling of a QH for this endpoint. + * + * @ehci: + * @ep: + * @urb: + * @qtd_list: + * @mem_flags + */ +static int intr_submit (struct ehci_hcd *ehci, struct usb_host_endpoint *ep, struct urb *urb, struct list_head *qtd_list, - gfp_t mem_flags -) { + gfp_t mem_flags) +{ unsigned epnum; unsigned long flags; struct ehci_qh *qh; @@ -616,7 +1818,9 @@ goto done; } if (qh->qh_state == QH_STATE_IDLE) { - if ((status = qh_schedule (ehci, qh)) != 0) + /* the QH is not in the shadow budget or hardware + * schedule. Budget and schedule now */ + if ((status = periodic_qh_schedule (ehci, qh)) != 0) goto done; } @@ -635,10 +1839,76 @@ return status; } +/* intr_descedule - toplevel entry point for removing a given + * interrupt endpoint (rather, its QH) from the shadow/hardware + * schedule and the shadow budget. + * + * @ehci: pointer to ehci host controller device structure + * @qh: QH to remove + */ +static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh) +{ + + qh_get(qh); + + /* remove from schedule */ + periodic_qh_deschedule (ehci, qh); + + /* remove from shadow budget; must happen after as deschedule + references budget when manipulating FSTNs */ +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Removing QH %p from budget:\n", + qh); +#endif + + budget_unlink_entries_by_owner(ehci,qh); + + qh_put(qh); +} + + /*-------------------------------------------------------------------------*/ +/* ISO scheduling temporary structure machinery necessitated by lazy + alloc of the iso_stream structure */ +/* ehci_iso_sched ops can be ITD-only or SITD-only */ -/* ehci_iso_stream ops work with both ITD and SITD */ +static struct ehci_iso_sched * +iso_sched_alloc (unsigned packets, gfp_t mem_flags) +{ + struct ehci_iso_sched *iso_sched; + int size = sizeof *iso_sched; + + size += packets * sizeof (struct ehci_iso_packet); + iso_sched = kzalloc(size, mem_flags); + if (likely (iso_sched != NULL)) { + INIT_LIST_HEAD (&iso_sched->td_list); + } + return iso_sched; +} +static void +iso_sched_free (struct ehci_iso_stream *stream, + struct ehci_iso_sched *iso_sched) +{ + if (!iso_sched){ + return; + } + // caller must hold ehci->lock! + list_splice (&iso_sched->td_list, &stream->free_list); + kfree (iso_sched); + +} + +/*-------------------------------------------------------------------------*/ +/* ISO stream generic machinery; tracks the ITDs and sITDs associated + with an isochronous endpoint */ + +/* iso_stream_alloc - allocate a new iso stream structure; and iso + * stream performs a role similar to QHs for ISO endpoints (both ITD + * and SITD) + * + * @mem_flags: flags for memory allocation + */ static struct ehci_iso_stream * iso_stream_alloc (gfp_t mem_flags) { @@ -654,6 +1924,14 @@ return stream; } +/* iso_stream_init - initialize fields of an ehci_iso_stream structure + * + * @ehci: pointer to ehci host controller device structure + * @stream: ehci_iso_stream structure to initialize + * @dev: endpoint usb device + * @pipe: pipe handle + * @interval: bInterval of endpoint; frames for FS/LS, uFrames for HS + */ static void iso_stream_init ( struct ehci_hcd *ehci, @@ -661,10 +1939,8 @@ struct usb_device *dev, int pipe, unsigned interval -) + ) { - static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f }; - u32 buf1; unsigned epnum, maxp; int is_input; @@ -701,12 +1977,13 @@ * when transfers on this endpoint are scheduled ... */ stream->usecs = HS_USECS_ISO (maxp); + stream->interval = interval; bandwidth = stream->usecs * 8; bandwidth /= 1 << (interval - 1); } else { u32 addr; - int think_time; + int think_time, think_bytes; int hs_transfers; addr = dev->ttport << 24; @@ -716,50 +1993,87 @@ addr |= dev->tt->hub->devnum << 16; addr |= epnum << 8; addr |= dev->devnum; + hs_transfers = max (1u, (maxp + 187) / 188); stream->usecs = HS_USECS_ISO (maxp); + think_time = dev->tt ? dev->tt->think_time : 0; - stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time ( - dev->speed, is_input, 1, maxp)); - hs_transfers = max (1u, (maxp + 187) / 188); - if (is_input) { - u32 tmp; + think_bytes = (think_time+665)/666; + if (is_input) { addr |= 1 << 31; - stream->c_usecs = stream->usecs; + + /* FIXME this just approximates SPLIT/CSPLIT times */ + /* can't fix without going to ns granularity; + * this is at least safe */ + stream->c_usecs = stream->usecs + HS_USECS_ISO (1); stream->usecs = HS_USECS_ISO (1); - stream->raw_mask = 1; - /* c-mask as specified in USB 2.0 11.18.4 3.c */ - tmp = (1 << (hs_transfers + 2)) - 1; - stream->raw_mask |= tmp << (8 + 2); - } else - stream->raw_mask = smask_out [hs_transfers - 1]; + if(dev->speed == USB_SPEED_FULL){ + /* full speed bytes + think time [TT + * host delay] + FS non-iso protocol + * overhead */ + stream->tt_bytes = think_bytes + maxp + 11; + }else{ + /* low speed bytes + think time [TT + * host delay] + low speed protocol + * overhead */ + /* expressed in full speed bytes */ + stream->tt_bytes = think_bytes + maxp*8 + 98; + } + } else { + /* out */ + + /* FIXME this just approximates SPLIT/CSPLIT times */ + /* can't fix without going to ns granularity; + * this is at least safe */ + stream->usecs += HS_USECS_ISO (1); + + if(dev->speed == USB_SPEED_FULL){ + /* full speed bytes + think time [TT + * host delay] + FS non-iso protocol + * overhead */ + stream->tt_bytes = think_bytes + maxp + 10; + }else{ + /* low speed bytes + think time [TT + * host delay] + low speed protocol + * overhead */ + /* expressed in full speed bytes */ + stream->tt_bytes = think_bytes + maxp*8 + 98; + } + } + bandwidth = stream->usecs + stream->c_usecs; bandwidth /= 1 << (interval + 2); - - /* stream->splits gets created from raw_mask later */ stream->address = cpu_to_le32 (addr); + stream->interval = interval<<3; } - stream->bandwidth = bandwidth; + stream->bandwidth = bandwidth; stream->udev = dev; - stream->bEndpointAddress = is_input | epnum; - stream->interval = interval; + stream->maxp = maxp; } +/* iso_stream_put - decrement refcount of passed in ehci_iso_stream + * structure, reaping it if count has fallen to zero. + * + * @ehci: pointer to ehci host controller device structure + * @stream: ehci_iso_stream structure + */ static void iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream) { stream->refcount--; - /* free whenever just a dev->ep reference remains. - * not like a QH -- no persistent state (toggle, halt) - */ - if (stream->refcount == 1) { - int is_in; - + /* don't free on last descriptor; free when endpoint disable + finally releases last refcount. Although it is technically + broken for an endpoint driver to submit its streaming + descriptors such that a new one appears after the old one + ends, it is only punishing the users to insist on breaking + these drivers when it's not necessary to do so. This saves + substantial re-budgeting overhead in that case.*/ + if (stream->refcount == 0) { // BUG_ON (!list_empty(&stream->td_list)); while (!list_empty (&stream->free_list)) { @@ -785,24 +2099,39 @@ sitd->sitd_dma); } } - - is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0; stream->bEndpointAddress &= 0x0f; - stream->ep->hcpriv = NULL; + stream->ep->hcpriv=NULL; - if (stream->rescheduled) { - ehci_info (ehci, "ep%d%s-iso rescheduled " - "%lu times in %lu seconds\n", - stream->bEndpointAddress, is_in ? "in" : "out", - stream->rescheduled, - ((jiffies - stream->start)/HZ) - ); - } + /* eliminate this stream from the shadow budget */ + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Releasing bandwidth for ISO %p\n", + stream); + ehci_info(ehci, " Budget before: \n"); + print_budget(ehci,NULL,stream); +#endif + budget_unlink_entries_by_owner(ehci,stream); + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, " Budget after: \n"); + print_budget(ehci,NULL,stream); +#endif + + /* potentially shut off periodic schedule */ + disable_periodic(ehci); + /* free the stream */ kfree(stream); + } } +/* iso_stream_get - increment refcount of passed in ehci_iso_stream + * structure + * + * @ehci: pointer to ehci host controller device structure + * @stream: ehci_iso_stream structure + */ static inline struct ehci_iso_stream * iso_stream_get (struct ehci_iso_stream *stream) { @@ -811,6 +2140,14 @@ return stream; } +/* iso_stream_find - returns ehci_iso_stream structure matching the + * passed in urb; if no match is found, a new ehci_iso_stream is + * allocated and initialized to match the urb. This function is thus + * used both for looup and lazy alloc. + * + * @ehci: pointer to ehci host controller device structure + * @urb: usb request block + */ static struct ehci_iso_stream * iso_stream_find (struct ehci_hcd *ehci, struct urb *urb) { @@ -853,30 +2190,152 @@ return stream; } -/*-------------------------------------------------------------------------*/ - -/* ehci_iso_sched ops can be ITD-only or SITD-only */ +/* too large now; budgeting is already done by the time we compute + using the slop */ +#define SCHEDULE_SLOP 10 /* frames */ -static struct ehci_iso_sched * -iso_sched_alloc (unsigned packets, gfp_t mem_flags) +/* iso_stream_schedule - schedules an iso request transaction into the + * periodic schedule, budgeting persistent periodic bandwidth for this + * stream if that has not yet been done. + * + * @ehci: pointer to ehci host controller device structure + * @urb: request + * @stream: persistent iso stream + */ +static int iso_stream_schedule ( + struct ehci_hcd *ehci, + struct urb *urb, + struct ehci_iso_stream *stream + ) { - struct ehci_iso_sched *iso_sched; - int size = sizeof *iso_sched; + int status; + unsigned mod = ehci->periodic_size << 3; + struct ehci_iso_sched *sched = urb->hcpriv; - size += packets * sizeof (struct ehci_iso_packet); - iso_sched = kzalloc(size, mem_flags); - if (likely (iso_sched != NULL)) { - INIT_LIST_HEAD (&iso_sched->td_list); + if (sched->span > (mod - 8 * SCHEDULE_SLOP)) { + ehci_dbg(ehci, "iso request %p too long\n", urb); + status = -EFBIG; + goto fail; } - return iso_sched; + + if ((stream->depth + sched->span) > mod) { + ehci_dbg(ehci, "request %p would overflow " + "(%d+%d>%d)\n", + urb, stream->depth, sched->span, mod); + status = -EFBIG; + goto fail; + } + + /* has this endpoint already been submitted into the shadow budget? */ + if(!stream->budget_state){ + + /* no, budget this endpoint */ + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, "Budgeting bandwidth for new ISO %p\n", + stream); + ehci_info(ehci, " Budget before: \n"); + print_budget(ehci,NULL,stream); +#endif + + status = budget_add_endpoint(ehci, stream, BUDGET_TYPE_ITD, + stream->interval); + +#if defined(CONFIG_USB_EHCI_BUDGET_DEBUG) + ehci_info(ehci, " Budget after: \n"); + print_budget(ehci,NULL,stream); +#endif + + if(status)goto fail; + + /* potentially turn on the periodic hardware */ + status=enable_periodic (ehci); + if(status)goto fail; + + stream->next_uframe = -1; + stream->budget_state = 1; + } + + if(stream->next_uframe == -1){ + int now, start; + + /* initial schedule; when's the next (u)frame we could + * start? this is bigger than ehci->i_thresh allows; + * scheduling itself isn't free, the slop should handle + * reasonably slow cpus. it can also help high bandwidth if + * the dma and irq loads don't jump until after the queue is + * primed. + */ + + now = readl (&ehci->regs->frame_index) % mod; + start = ((now + SCHEDULE_SLOP * 8) % mod) >> 3; + + stream->next_uframe = + (budget_schedule_next_frame (ehci, start, stream) % + ehci->periodic_size)* 8; + + if(stream->next_uframe == -1){ + /* should be impossible */ + ehci_err(ehci,"Couldn't find budgeted slot " + "after scheduling\n"); + status = -ENOSPC; + goto fail; + } + + /* report high speed start in uframes; full speed, in frames */ + urb->start_frame = stream->next_uframe; + if (!stream->highspeed) + urb->start_frame >>= 3; + + }else{ + + /* use the next uframe field as already marked */ + /* did we fall behind? Throw a sync error if so. */ + /* in sync == tail < now < head */ + + int now = (readl (&ehci->regs->frame_index) % mod); + int tail = stream->next_uframe - stream->depth - + stream->interval - SCHEDULE_SLOP * 8; + int head = stream->next_uframe; + + if(tail < 0){ + tail += mod; + head += mod; + if(now < tail) now += mod; + } + + if (now < tail || now >= head){ + stream->next_uframe = -1; + status = -EL2NSYNC; + goto fail; + } + } + + return 0; + +fail: + iso_sched_free (stream, sched); + urb->hcpriv = NULL; + return status; + } +/*-------------------------------------------------------------------------*/ + +/* itd_sched_init - ITD-specific initialization of an ehci_iso_sched + * structure to hold the specifics of the current requested + * transaction during the various steps of budgeting and scheduling. + * + * @iso_sched: structure to initialize + * @stream: iso stream + * @urb: current request + */ static inline void itd_sched_init ( struct ehci_iso_sched *iso_sched, struct ehci_iso_stream *stream, struct urb *urb -) + ) { unsigned i; dma_addr_t dma = urb->transfer_dma; @@ -912,26 +2371,22 @@ } } -static void -iso_sched_free ( - struct ehci_iso_stream *stream, - struct ehci_iso_sched *iso_sched -) -{ - if (!iso_sched) - return; - // caller must hold ehci->lock! - list_splice (&iso_sched->td_list, &stream->free_list); - kfree (iso_sched); -} - +/* itd_urb_transaction - sets up memory and relevant structures to + * process current transaction; initializes the iso_sched structure + * and preallocates transfer descriptors + * + * @stream: iso stream + * @ehci: pointer to ehci host controller device structure + * @urb: current request + * mem_flags: flags to use for in-kernel memory allocation + */ static int itd_urb_transaction ( struct ehci_iso_stream *stream, struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags -) + ) { struct ehci_itd *itd; dma_addr_t itd_dma; @@ -941,8 +2396,9 @@ unsigned long flags; sched = iso_sched_alloc (urb->number_of_packets, mem_flags); - if (unlikely (sched == NULL)) + if (unlikely (sched == NULL)){ return -ENOMEM; + } itd_sched_init (sched, stream, urb); @@ -992,214 +2448,12 @@ return 0; } -/*-------------------------------------------------------------------------*/ - -static inline int -itd_slot_ok ( - struct ehci_hcd *ehci, - u32 mod, - u32 uframe, - u8 usecs, - u32 period -) -{ - uframe %= period; - do { - /* can't commit more than 80% periodic == 100 usec */ - if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7) - > (100 - usecs)) - return 0; - - /* we know urb->interval is 2^N uframes */ - uframe += period; - } while (uframe < mod); - return 1; -} - -static inline int -sitd_slot_ok ( - struct ehci_hcd *ehci, - u32 mod, - struct ehci_iso_stream *stream, - u32 uframe, - struct ehci_iso_sched *sched, - u32 period_uframes -) -{ - u32 mask, tmp; - u32 frame, uf; - - mask = stream->raw_mask << (uframe & 7); - - /* for IN, don't wrap CSPLIT into the next frame */ - if (mask & ~0xffff) - return 0; - - /* this multi-pass logic is simple, but performance may - * suffer when the schedule data isn't cached. - */ - - /* check bandwidth */ - uframe %= period_uframes; - do { - u32 max_used; - - frame = uframe >> 3; - uf = uframe & 7; - - /* tt must be idle for start(s), any gap, and csplit. - * assume scheduling slop leaves 10+% for control/bulk. - */ - if (!tt_no_collision (ehci, period_uframes << 3, - stream->udev, frame, mask)) - return 0; - - /* check starts (OUT uses more than one) */ - max_used = 100 - stream->usecs; - for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) { - if (periodic_usecs (ehci, frame, uf) > max_used) - return 0; - } - - /* for IN, check CSPLIT */ - if (stream->c_usecs) { - uf = uframe & 7; - max_used = 100 - stream->c_usecs; - do { - tmp = 1 << uf; - tmp <<= 8; - if ((stream->raw_mask & tmp) == 0) - continue; - if (periodic_usecs (ehci, frame, uf) - > max_used) - return 0; - } while (++uf < 8); - } - - /* we know urb->interval is 2^N uframes */ - uframe += period_uframes; - } while (uframe < mod); - - stream->splits = cpu_to_le32(stream->raw_mask << (uframe & 7)); - return 1; -} - -/* - * This scheduler plans almost as far into the future as it has actual - * periodic schedule slots. (Affected by TUNE_FLS, which defaults to - * "as small as possible" to be cache-friendlier.) That limits the size - * transfers you can stream reliably; avoid more than 64 msec per urb. - * Also avoid queue depths of less than ehci's worst irq latency (affected - * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter, - * and other factors); or more than about 230 msec total (for portability, - * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler! - */ - -#define SCHEDULE_SLOP 10 /* frames */ - -static int -iso_stream_schedule ( - struct ehci_hcd *ehci, - struct urb *urb, - struct ehci_iso_stream *stream -) -{ - u32 now, start, max, period; - int status; - unsigned mod = ehci->periodic_size << 3; - struct ehci_iso_sched *sched = urb->hcpriv; - - if (sched->span > (mod - 8 * SCHEDULE_SLOP)) { - ehci_dbg (ehci, "iso request %p too long\n", urb); - status = -EFBIG; - goto fail; - } - - if ((stream->depth + sched->span) > mod) { - ehci_dbg (ehci, "request %p would overflow (%d+%d>%d)\n", - urb, stream->depth, sched->span, mod); - status = -EFBIG; - goto fail; - } - - now = readl (&ehci->regs->frame_index) % mod; - - /* when's the last uframe this urb could start? */ - max = now + mod; - - /* typical case: reuse current schedule. stream is still active, - * and no gaps from host falling behind (irq delays etc) - */ - if (likely (!list_empty (&stream->td_list))) { - start = stream->next_uframe; - if (start < now) - start += mod; - if (likely ((start + sched->span) < max)) - goto ready; - /* else fell behind; someday, try to reschedule */ - status = -EL2NSYNC; - goto fail; - } - - /* need to schedule; when's the next (u)frame we could start? - * this is bigger than ehci->i_thresh allows; scheduling itself - * isn't free, the slop should handle reasonably slow cpus. it - * can also help high bandwidth if the dma and irq loads don't - * jump until after the queue is primed. +/* itd_init - performs master initilization of the ITD fields that are + * drawn from the stream structure. + * + * @stream: iso stream + * @itd: New ITD that is to be added to the shadow/hardware schedule */ - start = SCHEDULE_SLOP * 8 + (now & ~0x07); - start %= mod; - stream->next_uframe = start; - - /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */ - - period = urb->interval; - if (!stream->highspeed) - period <<= 3; - - /* find a uframe slot with enough bandwidth */ - for (; start < (stream->next_uframe + period); start++) { - int enough_space; - - /* check schedule: enough space? */ - if (stream->highspeed) - enough_space = itd_slot_ok (ehci, mod, start, - stream->usecs, period); - else { - if ((start % 8) >= 6) - continue; - enough_space = sitd_slot_ok (ehci, mod, stream, - start, sched, period); - } - - /* schedule it here if there's enough bandwidth */ - if (enough_space) { - stream->next_uframe = start % mod; - goto ready; - } - } - - /* no room in the schedule */ - ehci_dbg (ehci, "iso %ssched full %p (now %d max %d)\n", - list_empty (&stream->td_list) ? "" : "re", - urb, now, max); - status = -ENOSPC; - -fail: - iso_sched_free (stream, sched); - urb->hcpriv = NULL; - return status; - -ready: - /* report high speed start in uframes; full speed, in frames */ - urb->start_frame = stream->next_uframe; - if (!stream->highspeed) - urb->start_frame >>= 3; - return 0; -} - -/*-------------------------------------------------------------------------*/ - static inline void itd_init (struct ehci_iso_stream *stream, struct ehci_itd *itd) { @@ -1215,15 +2469,23 @@ itd->index[i] = -1; /* All other fields are filled when scheduling */ + } +/* itd_patch - patches additional transfer descriptors into an existing ITD. + * + * @itd: New ITD that is to be added to the shadow/hardware schedule + * @iso_sched: ehci_iso_sched holding transfer descriptors + * @index: packet number to reference within the ehci_iso_sched + * @uframe: uframe in which to schedule specified packet + */ static inline void itd_patch ( struct ehci_itd *itd, struct ehci_iso_sched *iso_sched, unsigned index, u16 uframe -) + ) { struct ehci_iso_packet *uf = &iso_sched->packet [index]; unsigned pg = itd->pg; @@ -1247,44 +2509,64 @@ } } -static inline void -itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) +/* itd_link - place one completed itd into the shadow/hardware schedule + * + * @ehci: pointer to ehci host controller device structure + * @frame: shadow/hardware schedule frame + * @stream: stream to which ITD belongs + * @itd: ITD to link into schedule + */ +static inline void itd_link (struct ehci_hcd *ehci, + unsigned frame, + struct ehci_iso_stream *stream, + struct ehci_itd *itd) { - /* always prepend ITD/SITD ... only QH tree is order-sensitive */ - itd->itd_next = ehci->pshadow [frame]; - itd->hw_next = ehci->periodic [frame]; - ehci->pshadow [frame].itd = itd; + __le32 *hw_p; + struct ehci_shadow_budget *b; + union ehci_shadow *here; + + if( budget_schedule_next(ehci,frame,stream,&b,&here,&hw_p) != frame){ + /* should be impossible unless we've hit a bug or + * hardware failure */ + ehci_err(ehci,"itd not where it should be " + "in the periodic budget"); + return; + } + itd->frame = frame; + itd->itd_next = *here; + itd->hw_next = *hw_p; + here->itd = itd; wmb (); - ehci->periodic [frame] = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD; + *hw_p = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD; } -/* fit urb's itds into the selected schedule slot; activate as needed */ +/* itd_link_urb - link urb's ITDs into the hardware schedule acording + * to shadow budget + * + * @ehci: pointer to ehci host controller device structure + * @urb: current USB request + * @mod: size of periodic hardware schedule [in uFrames] + * @stream: stream into which to queue request + */ static int itd_link_urb ( struct ehci_hcd *ehci, struct urb *urb, unsigned mod, struct ehci_iso_stream *stream -) + ) { int packet; unsigned next_uframe, uframe, frame; struct ehci_iso_sched *iso_sched = urb->hcpriv; struct ehci_itd *itd; - next_uframe = stream->next_uframe % mod; + next_uframe = stream->next_uframe; if (unlikely (list_empty(&stream->td_list))) { ehci_to_hcd(ehci)->self.bandwidth_allocated += stream->bandwidth; - ehci_vdbg (ehci, - "schedule devp %s ep%d%s-iso period %d start %d.%d\n", - urb->dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", - urb->interval, - next_uframe >> 3, next_uframe & 0x7); - stream->start = jiffies; } ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; @@ -1318,10 +2600,12 @@ /* link completed itds into the schedule */ if (((next_uframe >> 3) != frame) || packet == urb->number_of_packets) { - itd_link (ehci, frame % ehci->periodic_size, itd); + itd_link (ehci, frame % ehci->periodic_size, + stream, itd); itd = NULL; } } + stream->next_uframe = next_uframe; /* don't need that schedule data any more */ @@ -1329,19 +2613,24 @@ urb->hcpriv = NULL; timer_action (ehci, TIMER_IO_WATCHDOG); - if (unlikely (!ehci->periodic_sched++)) - return enable_periodic (ehci); return 0; } #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) -static unsigned -itd_complete ( - struct ehci_hcd *ehci, +/* itd_complete - process a single complete ITD that has been removed + * from the periodic schedule and handed to us by scan_periodic. If + * this is the last ITD that needed to be completed for a URB, hand + * the URB back to the upper level driver. + * + * @ehci: pointer to ehci host controller device structure + * @itd: completed ITD + * @regs: ptrace registers + */ +static unsigned itd_complete (struct ehci_hcd *ehci, struct ehci_itd *itd, - struct pt_regs *regs -) { + struct pt_regs *regs) +{ struct urb *urb = itd->urb; struct usb_iso_packet_descriptor *desc; u32 t; @@ -1349,6 +2638,7 @@ int urb_index = -1; struct ehci_iso_stream *stream = itd->stream; struct usb_device *dev; + int period = stream->interval; /* for each uframe with a packet */ for (uframe = 0; uframe < 8; uframe++) { @@ -1359,7 +2649,7 @@ t = le32_to_cpup (&itd->hw_transaction [uframe]); itd->hw_transaction [uframe] = 0; - stream->depth -= stream->interval; + stream->depth -= period; /* report transfer status */ if (unlikely (t & ISO_ERRS)) { @@ -1389,9 +2679,9 @@ iso_stream_put (ehci, stream); /* handle completion now? */ - if (likely ((urb_index + 1) != urb->number_of_packets)) + if (likely ((urb_index + 1) != urb->number_of_packets)){ return 0; - + } /* ASSERT: it's really the last itd for this urb list_for_each_entry (itd, &stream->td_list, itd_list) BUG_ON (itd->urb == urb); @@ -1403,26 +2693,30 @@ urb = NULL; /* defer stopping schedule; completion can submit */ - ehci->periodic_sched--; - if (unlikely (!ehci->periodic_sched)) - (void) disable_periodic (ehci); ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; + /* XXX Hm... now that we have the budget, this should be moved + to where the budget is released */ if (unlikely (list_empty (&stream->td_list))) { ehci_to_hcd(ehci)->self.bandwidth_allocated -= stream->bandwidth; ehci_vdbg (ehci, "deschedule devp %s ep%d%s-iso\n", dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); + (stream->bEndpointAddress & USB_DIR_IN) ? + "in" : "out"); } iso_stream_put (ehci, stream); - return 1; } -/*-------------------------------------------------------------------------*/ - +/* itd_submit - top level submission point for a HS isochronous + * endpoint request. + * + * @ehci: pointer to ehci host controller device structure + * @urb: new USB request + * @mem_flags: flags for memory allocation + */ static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags) { @@ -1433,30 +2727,20 @@ /* Get iso_stream head */ stream = iso_stream_find (ehci, urb); if (unlikely (stream == NULL)) { - ehci_dbg (ehci, "can't get iso stream\n"); + ehci_dbg(ehci,"can't get iso stream\n"); return -ENOMEM; } + if (unlikely (urb->interval != stream->interval)) { - ehci_dbg (ehci, "can't change iso interval %d --> %d\n", + ehci_dbg(ehci,"can't change iso interval %d --> %d\n", stream->interval, urb->interval); goto done; } -#ifdef EHCI_URB_TRACE - ehci_dbg (ehci, - "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n", - __FUNCTION__, urb->dev->devpath, urb, - usb_pipeendpoint (urb->pipe), - usb_pipein (urb->pipe) ? "in" : "out", - urb->transfer_buffer_length, - urb->number_of_packets, urb->interval, - stream); -#endif - /* allocate ITDs w/o locking anything */ status = itd_urb_transaction (stream, ehci, urb, mem_flags); if (unlikely (status < 0)) { - ehci_dbg (ehci, "can't init itds\n"); + ehci_dbg(ehci,"can't init itds\n"); goto done; } @@ -1474,29 +2758,35 @@ done: if (unlikely (status < 0)) iso_stream_put (ehci, stream); + return status; } -#ifdef CONFIG_USB_EHCI_SPLIT_ISO - /*-------------------------------------------------------------------------*/ +/* Split-ISO transfer machinery; used for FS isoch transfers through the + * TTs in USB 2.0 hubs. */ -/* - * "Split ISO TDs" ... used for USB 1.1 devices going through the - * TTs in USB 2.0 hubs. These need microframe scheduling. +/* sitd_sched_init - sITD-specific initialization of an ehci_iso_sched + * structure to hold the specifics of the current requested + * transaction during the various steps of budgeting and scheduling. + * + * @iso_sched: structure to initialize + * @ehci: pointer to ehci host controller device structure + * @stream: iso stream + * @urb: current request */ - static inline void sitd_sched_init ( struct ehci_iso_sched *iso_sched, + struct ehci_hcd *ehci, struct ehci_iso_stream *stream, struct urb *urb -) + ) { unsigned i; dma_addr_t dma = urb->transfer_dma; - /* how many frames are needed for these transfers */ + /* how many uframes are needed for these transfers */ iso_sched->span = urb->number_of_packets * stream->interval; /* figure out per-frame sitd fields that we'll need later @@ -1534,13 +2824,22 @@ } } +/* sitd_urb_transaction - sets up memory and relevant structures to + * process current transaction; initializes the iso_sched structure + * and preallocates transfer descriptors + * + * @stream: iso stream + * @ehci: pointer to ehci host controller device structure + * @urb: current request + * mem_flags: flags to use for in-kernel memory allocation + */ static int sitd_urb_transaction ( struct ehci_iso_stream *stream, struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags -) + ) { struct ehci_sitd *sitd; dma_addr_t sitd_dma; @@ -1549,19 +2848,18 @@ unsigned long flags; iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags); - if (iso_sched == NULL) + if (iso_sched == NULL){ return -ENOMEM; + } - sitd_sched_init (iso_sched, stream, urb); + sitd_sched_init (iso_sched, ehci,stream, urb); /* allocate/init sITDs */ spin_lock_irqsave (&ehci->lock, flags); - for (i = 0; i < urb->number_of_packets; i++) { - /* NOTE: for now, we don't try to handle wraparound cases - * for IN (using sitd->hw_backpointer, like a FSTN), which - * means we never need two sitds for full speed packets. - */ + /* note: frame spanning may require two SITDs per transaction; + allocate all we may need, and free the ones we don't later. */ + for (i = 0; i < urb->number_of_packets * 2; i++) { /* free_list.next might be cache-hot ... but maybe * the HC caches it too. avoid that issue for now. @@ -1601,26 +2899,48 @@ return 0; } -/*-------------------------------------------------------------------------*/ - +/* sitd_patch - finishes filling in the hardware fields of a sITD + * schedule prior to it being linked into the hardware/shadow + * schedule. + * + * @ehci: pointer to ehci host controller device structure + * @stream: stream into which this request will be queued + * @b: shadow budget entry holding this sITD's bandwidth allocation + * information + * @sitd: New sITD that is to be added to the shadow/hardware schedule + * @iso_sched: ehci_iso_sched holding transfer descriptors + * @uf: iso stream packet + * @back: hw pointer (cpu order) pointing back to original sITD + * if this is a spanning dummy sITD + */ static inline void sitd_patch ( + struct ehci_hcd *ehci, struct ehci_iso_stream *stream, + struct ehci_shadow_budget *b, struct ehci_sitd *sitd, - struct ehci_iso_sched *iso_sched, - unsigned index -) + struct ehci_iso_packet *uf, + dma_addr_t back + ) { - struct ehci_iso_packet *uf = &iso_sched->packet [index]; u64 bufp = uf->bufp; sitd->hw_next = EHCI_LIST_END; sitd->hw_fullspeed_ep = stream->address; - sitd->hw_uframe = stream->splits; sitd->hw_results = uf->transaction; + + if(back){ + /* spanning completion only */ + sitd->hw_uframe = cpu_to_le32( ((b->cmask>>8)&0xff)<<8 ); + /* set dummy's completion state to DoComplete (ehci + 4.12.3.3.x) */ + sitd->hw_results |= SITD_DOCOMPLETE; + sitd->hw_backpointer = cpu_to_le32 (back); + }else{ + sitd->hw_uframe = cpu_to_le32(b->smask | ((b->cmask&0xff)<<8)); sitd->hw_backpointer = EHCI_LIST_END; + } - bufp = uf->bufp; sitd->hw_buf [0] = cpu_to_le32 (bufp); sitd->hw_buf_hi [0] = cpu_to_le32 (bufp >> 32); @@ -1628,34 +2948,120 @@ if (uf->cross) bufp += 4096; sitd->hw_buf_hi [1] = cpu_to_le32 (bufp >> 32); - sitd->index = index; } -static inline void -sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd) +/* sitd_link - fill in and link one sITD into the shadow/hardware + * schedule; if this sITD requires a spanning dummy in the next frame, + * a spanning dummy is implicitly filled in and linked as well. + * + * @ehci: pointer to ehci host controller device structure + * @urb: new USB request + * @frame: shadow/hardware schedule frame + * @stream: stream to which ITD belongs + * @sched: ehci_iso_sched structure holding packets + * @index: index of packet to use from sched + */ +static inline void sitd_link (struct ehci_hcd *ehci, + struct urb *urb, + unsigned frame, + struct ehci_iso_stream *stream, + struct ehci_iso_sched *sched, + int index) { - /* note: sitd ordering could matter (CSPLIT then SSPLIT) */ - sitd->sitd_next = ehci->pshadow [frame]; - sitd->hw_next = ehci->periodic [frame]; - ehci->pshadow [frame].sitd = sitd; - sitd->frame = frame; + struct ehci_sitd *sitd, *dummy; + __le32 *hw_p; + struct ehci_shadow_budget *b; + union ehci_shadow *here; + + /* get SITD from already allocated list */ + BUG_ON (list_empty (&sched->td_list)); + sitd = list_entry (sched->td_list.next, + struct ehci_sitd, sitd_list); + list_move_tail (&sitd->sitd_list, &stream->td_list); + sitd->stream = iso_stream_get (stream); + sitd->urb = usb_get_urb (urb); + + /* find the proper insertion point */ + if( budget_schedule_next(ehci,frame,stream,&b,&here,&hw_p) + != frame){ + /* should be impossible unless we've hit a bug or + * hardware failure */ + ehci_err(ehci,"sitd not where it should be in " + "the periodic budget"); + return; + } + + /* set the sitd fields */ + sitd_patch (ehci,stream, b, sitd, &sched->packet [index], 0); + + sitd->frame = frame % ehci->periodic_size; + sitd->index = index; + sitd->sitd_next = *here; + sitd->hw_next = *hw_p; + here->sitd = sitd; wmb (); - ehci->periodic [frame] = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD; + + /* is this a wrap case? if so, have the dummy SITD fully in place + before the real SITD is reachable from the hardware schedule */ + + if(BUDGET_WRAP_P(b)){ + __le32 *d_hw_p; + union ehci_shadow *d_here; + + /* get SITD from already allocated list */ + BUG_ON (list_empty (&sched->td_list)); + dummy = list_entry (sched->td_list.next, + struct ehci_sitd, sitd_list); + list_move_tail (&dummy->sitd_list, &stream->td_list); + dummy->stream = iso_stream_get (stream); + dummy->urb = usb_get_urb (urb); + + /* completion triggered by the spanning dummy, not the + * original sitd */ + dummy->index = index; + sitd->index = -1; + + /* find the proper insertion point */ + d_here = periodic_translate_budget (ehci, frame+1, b, &d_hw_p); + + /* set the sitd fields */ + sitd_patch (ehci,stream, b, dummy, &sched->packet [index], + sitd->sitd_dma); + dummy->frame = (frame+1)%ehci->periodic_size; + + + dummy->sitd_next = *d_here; + dummy->hw_next = *d_hw_p; + d_here->sitd = dummy; + wmb (); + *d_hw_p = cpu_to_le32 (dummy->sitd_dma) | Q_TYPE_SITD; + + } + + /* make sitd reacahable from periodic schedule */ + *hw_p = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD; + } -/* fit urb's sitds into the selected schedule slot; activate as needed */ +/* sitd_link_urb - link urb's sITDs into the hardware schedule + * acording to shadow budget + * + * @ehci: pointer to ehci host controller device structure + * @urb: current USB request + * @mod: size of periodic hardware schedule [in uFrames] + * @stream: stream into which to queue request + */ static int sitd_link_urb ( struct ehci_hcd *ehci, struct urb *urb, unsigned mod, struct ehci_iso_stream *stream -) + ) { int packet; unsigned next_uframe; struct ehci_iso_sched *sched = urb->hcpriv; - struct ehci_sitd *sitd; next_uframe = stream->next_uframe; @@ -1663,62 +3069,45 @@ /* usbfs ignores TT bandwidth */ ehci_to_hcd(ehci)->self.bandwidth_allocated += stream->bandwidth; - ehci_vdbg (ehci, - "sched devp %s ep%d%s-iso [%d] %dms/%04x\n", - urb->dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", - (next_uframe >> 3) % ehci->periodic_size, - stream->interval, le32_to_cpu (stream->splits)); - stream->start = jiffies; } ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; /* fill sITDs frame by frame */ - for (packet = 0, sitd = NULL; - packet < urb->number_of_packets; - packet++) { - - /* ASSERT: we have all necessary sitds */ - BUG_ON (list_empty (&sched->td_list)); - - /* ASSERT: no itds for this endpoint in this frame */ - - sitd = list_entry (sched->td_list.next, - struct ehci_sitd, sitd_list); - list_move_tail (&sitd->sitd_list, &stream->td_list); - sitd->stream = iso_stream_get (stream); - sitd->urb = usb_get_urb (urb); - - sitd_patch (stream, sitd, sched, packet); - sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size, - sitd); - - next_uframe += stream->interval << 3; - stream->depth += stream->interval << 3; + for (packet = 0; packet < urb->number_of_packets; packet++) { + sitd_link (ehci, urb, next_uframe >> 3, stream, sched, packet); + next_uframe += stream->interval; + stream->depth += stream->interval; } + stream->next_uframe = next_uframe % mod; /* don't need that schedule data any more */ + /* also releases unused sITDs that weren't needed for spanning */ iso_sched_free (stream, sched); urb->hcpriv = NULL; timer_action (ehci, TIMER_IO_WATCHDOG); - if (!ehci->periodic_sched++) - return enable_periodic (ehci); return 0; } -/*-------------------------------------------------------------------------*/ - #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ | SITD_STS_XACT | SITD_STS_MMF) +/* sitd_complete - process a single complete sITD that has been removed + * from the periodic schedule and handed to us by scan_periodic. If + * this is the last sITD that needed to be completed for a URB, hand + * the URB back to the upper level driver. + * + * @ehci: pointer to ehci host controller device structure + * @sitd: completed sITD + * @regs: ptrace registers + */ static unsigned sitd_complete ( struct ehci_hcd *ehci, struct ehci_sitd *sitd, struct pt_regs *regs -) { + ) { struct urb *urb = sitd->urb; struct usb_iso_packet_descriptor *desc; u32 t; @@ -1750,12 +3139,15 @@ sitd->urb = NULL; sitd->stream = NULL; list_move (&sitd->sitd_list, &stream->free_list); - stream->depth -= stream->interval << 3; + if(urb_index >= 0) /* if a sitd has a spanning dummy, + * decrement at the dummy */ + stream->depth -= stream->interval; iso_stream_put (ehci, stream); /* handle completion now? */ - if ((urb_index + 1) != urb->number_of_packets) + if ((urb_index + 1) != urb->number_of_packets){ return 0; + } /* ASSERT: it's really the last sitd for this urb list_for_each_entry (sitd, &stream->td_list, sitd_list) @@ -1768,57 +3160,60 @@ urb = NULL; /* defer stopping schedule; completion can submit */ - ehci->periodic_sched--; - if (!ehci->periodic_sched) - (void) disable_periodic (ehci); ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; + /* XXX Hm... now that we have the budget, this should be moved + to where the budget is released */ if (list_empty (&stream->td_list)) { ehci_to_hcd(ehci)->self.bandwidth_allocated -= stream->bandwidth; ehci_vdbg (ehci, "deschedule devp %s ep%d%s-iso\n", dev->devpath, stream->bEndpointAddress & 0x0f, - (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); + (stream->bEndpointAddress & USB_DIR_IN) ? + "in" : "out"); } iso_stream_put (ehci, stream); return 1; } - +/* sitd_submit - top level submission point for a FS split isochronous + * endpoint request. + * + * @ehci: pointer to ehci host controller device structure + * @urb: new USB request + * @mem_flags: flags for memory allocation + */ static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags) { int status = -EINVAL; unsigned long flags; struct ehci_iso_stream *stream; + int period,period_8; /* Get iso_stream head */ stream = iso_stream_find (ehci, urb); if (stream == NULL) { - ehci_dbg (ehci, "can't get iso stream\n"); + ehci_dbg(ehci,"can't get iso stream\n"); return -ENOMEM; } - if (urb->interval != stream->interval) { - ehci_dbg (ehci, "can't change iso interval %d --> %d\n", - stream->interval, urb->interval); + + period = stream->interval; + period_8 = (period>8?period>>3:1); + + + if (urb->interval != period_8) { + ehci_dbg(ehci,"can't change iso %p interval %d/%d --> %d\n", + stream, period_8, period,urb->interval); goto done; } -#ifdef EHCI_URB_TRACE - ehci_dbg (ehci, - "submit %p dev%s ep%d%s-iso len %d\n", - urb, urb->dev->devpath, - usb_pipeendpoint (urb->pipe), - usb_pipein (urb->pipe) ? "in" : "out", - urb->transfer_buffer_length); -#endif - /* allocate SITDs */ status = sitd_urb_transaction (stream, ehci, urb, mem_flags); if (status < 0) { - ehci_dbg (ehci, "can't init sitds\n"); + ehci_dbg(ehci, "can't init sitds\n"); goto done; } @@ -1831,37 +3226,69 @@ status = iso_stream_schedule (ehci, urb, stream); if (status == 0) sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); + spin_unlock_irqrestore (&ehci->lock, flags); done: if (status < 0) iso_stream_put (ehci, stream); + return status; } -#else +/*-------------------------------------------------------------------------*/ -static inline int -sitd_submit (struct ehci_hcd *ehci, struct urb *urb, gfp_t mem_flags) -{ - ehci_dbg (ehci, "split iso support is disabled\n"); - return -ENOSYS; -} +/* scan_recovery - follow FSTN 'recvery path' to handle QH completions + * that may have just completed this frame. + * + * @ehci: pointer to ehci host controller device structure + * @regs: ptrace registers + * @qh: initial QH in recovery path. + */ +static int scan_recovery (struct ehci_hcd *ehci, + struct pt_regs *regs, + struct ehci_qh *qh){ + union ehci_shadow temp; + __le32 type = Q_TYPE_QH; + int modified = 0; -static inline unsigned -sitd_complete ( - struct ehci_hcd *ehci, - struct ehci_sitd *sitd, - struct pt_regs *regs -) { - ehci_err (ehci, "sitd_complete %p?\n", sitd); - return 0; -} + temp.qh = qh; -#endif /* USB_EHCI_SPLIT_ISO */ + while(temp.ptr){ + union ehci_shadow next = *periodic_next_shadow(&temp, type); + int next_type = Q_NEXT_TYPE (*temp.hw_next); -/*-------------------------------------------------------------------------*/ + /* stop at restore fstn */ + if(type == Q_TYPE_FSTN) break; + /* otherwise, process only QHs + (although we should only see + QHs) */ + if(type == Q_TYPE_QH){ + qh_get (temp.qh); + modified = qh_completions (ehci, temp.qh, regs); + if (unlikely (list_empty (&temp.qh->qtd_list))) + intr_deschedule (ehci, temp.qh); + qh_put (temp.qh); + } + + /* advance */ + temp = next; + type = next_type; + } + + return modified; +} + +/* scan_periodic - completion worker; called out of interrupt (or + * poll) handler to finish processing of transactions that have been + * completed by the host controller. Also now has the responsibility + * of halting the hardware periodic schedule after a complete pass + * through the schedule with nothing to do. + * + * @ehci: pointer to ehci host controller device structure + * @regs: ptrace registers + */ static void scan_periodic (struct ehci_hcd *ehci, struct pt_regs *regs) { @@ -1897,7 +3324,7 @@ uframes = 8; } -restart: + restart: /* scan each element in frame's queue for completions */ q_p = &ehci->pshadow [frame]; hw_p = &ehci->periodic [frame]; @@ -1917,21 +3344,36 @@ temp.qh = qh_get (q.qh); type = Q_NEXT_TYPE (q.qh->hw_next); q = q.qh->qh_next; - modified = qh_completions (ehci, temp.qh, regs); + modified = qh_completions (ehci, temp.qh, + regs); if (unlikely (list_empty (&temp.qh->qtd_list))) intr_deschedule (ehci, temp.qh); qh_put (temp.qh); break; case Q_TYPE_FSTN: - /* for "save place" FSTNs, look at QH entries - * in the previous frame for completions. - */ + /* for save-place FSTNs, in uFrame 0 + * and 1, inspect all QH entries in + * the previous frame for completions. + * FSTNs are not recursive, so do it + * the straightforward way here */ + if (q.fstn->hw_prev != EHCI_LIST_END) { - dbg ("ignoring completions from FSTNs"); + + /* an FSTN can only point back to a + * QH; if it's pointing to any other + * type, that's a bug in the scheduling + * code. */ + + modified = scan_recovery ( + ehci, regs, q.fstn->fstn_prev); + } + type = Q_NEXT_TYPE (q.fstn->hw_next); q = q.fstn->fstn_next; + break; + case Q_TYPE_ITD: /* skip itds for later in the frame */ rmb (); @@ -1959,6 +3401,10 @@ q = *q_p; break; case Q_TYPE_SITD: + /* unlike in the shadow budget, dummy + spanning sITDs and normal sITDs are not + handled differently in the hardware + schedule */ if ((q.sitd->hw_results & SITD_ACTIVE) && live) { q_p = &q.sitd->sitd_next; @@ -2001,6 +3447,7 @@ if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)) break; + ehci->next_uframe = now_uframe; now = readl (&ehci->regs->frame_index) % mod; if (now_uframe == now) @@ -2011,6 +3458,25 @@ } else { now_uframe++; now_uframe %= mod; + + if(now_uframe == 0){ + /* if the periodic schedule is still marked + to be disabled after a complete pass, + disable it now */ + if(ehci->periodic_sched < 0) + disable_periodic(ehci); + + /* if the periodic schedule is running but + currently empty, do not disable it + immediately; that would utterly screw + streaming isoch (as Linux is not using a + true periodic schedule). Mark the + refcount so that it will be disabled + after this pass if no new transactions + are enabled */ + if(ehci->periodic_sched == 0) + ehci->periodic_sched = -1; + } } } } diff -rub --minimal a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig --- a/drivers/usb/host/Kconfig 2006-04-26 22:19:25.000000000 -0400 +++ b/drivers/usb/host/Kconfig 2006-08-08 21:12:55.000000000 -0400 @@ -29,14 +29,16 @@ To compile this driver as a module, choose M here: the module will be called ehci-hcd. -config USB_EHCI_SPLIT_ISO - bool "Full speed ISO transactions (EXPERIMENTAL)" - depends on USB_EHCI_HCD && EXPERIMENTAL - default n +config USB_EHCI_BUDGET_DEBUG + bool "Verbose budget debugging information" + depends on USB_EHCI_HCD + default N ---help--- - This code is new and hasn't been used with many different - EHCI or USB 2.0 transaction translator implementations. - It should work for ISO-OUT transfers, like audio. + This option causes the new EHCI periodic budgeter to dump volumes of + debugging information detailing budgeting decisions and + state in addition to any normally logged messages. This + information is only useful for debugging purposes and must not be + enabled on a production machine. If unsure, say N. config USB_EHCI_ROOT_HUB_TT bool "Root Hub Transaction Translators (EXPERIMENTAL)"