diff options
Diffstat (limited to 'drivers/usb/host/xhci-mem.c')
| -rw-r--r-- | drivers/usb/host/xhci-mem.c | 2382 |
1 files changed, 1199 insertions, 1183 deletions
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index df6978abd7e6..c708bdd69f16 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * xHCI host controller driver * @@ -5,27 +6,18 @@ * * Author: Sarah Sharp * Some code borrowed from the Linux EHCI driver. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include <linux/usb.h> +#include <linux/overflow.h> #include <linux/pci.h> #include <linux/slab.h> #include <linux/dmapool.h> +#include <linux/dma-mapping.h> #include "xhci.h" +#include "xhci-trace.h" +#include "xhci-debugfs.h" /* * Allocates a generic ring segment from the ring pool, sets the dma address, @@ -35,28 +27,34 @@ * "All components of all Command and Transfer TRBs shall be initialized to '0'" */ static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, - unsigned int cycle_state, gfp_t flags) + unsigned int max_packet, + unsigned int num, + gfp_t flags) { struct xhci_segment *seg; dma_addr_t dma; - int i; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - seg = kzalloc(sizeof *seg, flags); + seg = kzalloc_node(sizeof(*seg), flags, dev_to_node(dev)); if (!seg) return NULL; - seg->trbs = dma_pool_alloc(xhci->segment_pool, flags, &dma); + seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma); if (!seg->trbs) { kfree(seg); return NULL; } - memset(seg->trbs, 0, TRB_SEGMENT_SIZE); - /* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */ - if (cycle_state == 0) { - for (i = 0; i < TRBS_PER_SEGMENT; i++) - seg->trbs[i].link.control |= TRB_CYCLE; + if (max_packet) { + seg->bounce_buf = kzalloc_node(max_packet, flags, + dev_to_node(dev)); + if (!seg->bounce_buf) { + dma_pool_free(xhci->segment_pool, seg->trbs, dma); + kfree(seg); + return NULL; + } } + seg->num = num; seg->dma = dma; seg->next = NULL; @@ -69,82 +67,222 @@ static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); seg->trbs = NULL; } + kfree(seg->bounce_buf); kfree(seg); } -static void xhci_free_segments_for_ring(struct xhci_hcd *xhci, - struct xhci_segment *first) +static void xhci_ring_segments_free(struct xhci_hcd *xhci, struct xhci_ring *ring) { - struct xhci_segment *seg; + struct xhci_segment *seg, *next; + + ring->last_seg->next = NULL; + seg = ring->first_seg; - seg = first->next; - while (seg != first) { - struct xhci_segment *next = seg->next; + while (seg) { + next = seg->next; xhci_segment_free(xhci, seg); seg = next; } - xhci_segment_free(xhci, first); } /* - * Make the prev segment point to the next segment. + * Only for transfer and command rings where driver is the producer, not for + * event rings. * - * Change the last TRB in the prev segment to be a Link TRB which points to the + * Change the last TRB in the segment to be a Link TRB which points to the * DMA address of the next segment. The caller needs to set any Link TRB * related flags, such as End TRB, Toggle Cycle, and no snoop. */ -static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev, - struct xhci_segment *next, enum xhci_ring_type type) +static void xhci_set_link_trb(struct xhci_segment *seg, bool chain_links) { + union xhci_trb *trb; u32 val; - if (!prev || !next) + if (!seg || !seg->next) return; - prev->next = next; - if (type != TYPE_EVENT) { - prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr = - cpu_to_le64(next->dma); - /* Set the last TRB in the segment to have a TRB type ID of Link TRB */ - val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control); - val &= ~TRB_TYPE_BITMASK; - val |= TRB_TYPE(TRB_LINK); - /* Always set the chain bit with 0.95 hardware */ - /* Set chain bit for isoc rings on AMD 0.96 host */ - if (xhci_link_trb_quirk(xhci) || - (type == TYPE_ISOC && - (xhci->quirks & XHCI_AMD_0x96_HOST))) - val |= TRB_CHAIN; - prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val); - } + trb = &seg->trbs[TRBS_PER_SEGMENT - 1]; + + /* Set the last TRB in the segment to have a TRB type ID of Link TRB */ + val = le32_to_cpu(trb->link.control); + val &= ~TRB_TYPE_BITMASK; + val |= TRB_TYPE(TRB_LINK); + if (chain_links) + val |= TRB_CHAIN; + trb->link.control = cpu_to_le32(val); + trb->link.segment_ptr = cpu_to_le64(seg->next->dma); +} + +static void xhci_initialize_ring_segments(struct xhci_hcd *xhci, struct xhci_ring *ring) +{ + struct xhci_segment *seg; + bool chain_links; + + if (ring->type == TYPE_EVENT) + return; + + chain_links = xhci_link_chain_quirk(xhci, ring->type); + xhci_for_each_ring_seg(ring->first_seg, seg) + xhci_set_link_trb(seg, chain_links); + + /* See section 4.9.2.1 and 6.4.4.1 */ + ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= cpu_to_le32(LINK_TOGGLE); } /* - * Link the ring to the new segments. + * Link the src ring segments to the dst ring. * Set Toggle Cycle for the new ring if needed. */ -static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring, - struct xhci_segment *first, struct xhci_segment *last, - unsigned int num_segs) +static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *src, struct xhci_ring *dst) { - struct xhci_segment *next; + struct xhci_segment *seg; + bool chain_links; - if (!ring || !first || !last) + if (!src || !dst) return; - next = ring->enq_seg->next; - xhci_link_segments(xhci, ring->enq_seg, first, ring->type); - xhci_link_segments(xhci, last, next, ring->type); - ring->num_segs += num_segs; - ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs; + /* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */ + if (dst->cycle_state == 0) { + xhci_for_each_ring_seg(src->first_seg, seg) { + for (int i = 0; i < TRBS_PER_SEGMENT; i++) + seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE); + } + } + + src->last_seg->next = dst->enq_seg->next; + dst->enq_seg->next = src->first_seg; + if (dst->type != TYPE_EVENT) { + chain_links = xhci_link_chain_quirk(xhci, dst->type); + xhci_set_link_trb(dst->enq_seg, chain_links); + xhci_set_link_trb(src->last_seg, chain_links); + } + dst->num_segs += src->num_segs; + + if (dst->enq_seg == dst->last_seg) { + if (dst->type != TYPE_EVENT) + dst->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control + &= ~cpu_to_le32(LINK_TOGGLE); + + dst->last_seg = src->last_seg; + } else if (dst->type != TYPE_EVENT) { + src->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control &= ~cpu_to_le32(LINK_TOGGLE); + } + + for (seg = dst->enq_seg; seg != dst->last_seg; seg = seg->next) + seg->next->num = seg->num + 1; +} + +/* + * We need a radix tree for mapping physical addresses of TRBs to which stream + * ID they belong to. We need to do this because the host controller won't tell + * us which stream ring the TRB came from. We could store the stream ID in an + * event data TRB, but that doesn't help us for the cancellation case, since the + * endpoint may stop before it reaches that event data TRB. + * + * The radix tree maps the upper portion of the TRB DMA address to a ring + * segment that has the same upper portion of DMA addresses. For example, say I + * have segments of size 1KB, that are always 1KB aligned. A segment may + * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the + * key to the stream ID is 0x43244. I can use the DMA address of the TRB to + * pass the radix tree a key to get the right stream ID: + * + * 0x10c90fff >> 10 = 0x43243 + * 0x10c912c0 >> 10 = 0x43244 + * 0x10c91400 >> 10 = 0x43245 + * + * Obviously, only those TRBs with DMA addresses that are within the segment + * will make the radix tree return the stream ID for that ring. + * + * Caveats for the radix tree: + * + * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an + * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be + * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the + * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit + * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit + * extended systems (where the DMA address can be bigger than 32-bits), + * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that. + */ +static int xhci_insert_segment_mapping(struct radix_tree_root *trb_address_map, + struct xhci_ring *ring, + struct xhci_segment *seg, + gfp_t mem_flags) +{ + unsigned long key; + int ret; + + key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT); + /* Skip any segments that were already added. */ + if (radix_tree_lookup(trb_address_map, key)) + return 0; + + ret = radix_tree_maybe_preload(mem_flags); + if (ret) + return ret; + ret = radix_tree_insert(trb_address_map, + key, ring); + radix_tree_preload_end(); + return ret; +} + +static void xhci_remove_segment_mapping(struct radix_tree_root *trb_address_map, + struct xhci_segment *seg) +{ + unsigned long key; + + key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT); + if (radix_tree_lookup(trb_address_map, key)) + radix_tree_delete(trb_address_map, key); +} + +static int xhci_update_stream_segment_mapping( + struct radix_tree_root *trb_address_map, + struct xhci_ring *ring, + struct xhci_segment *first_seg, + gfp_t mem_flags) +{ + struct xhci_segment *seg; + struct xhci_segment *failed_seg; + int ret; + + if (WARN_ON_ONCE(trb_address_map == NULL)) + return 0; + + xhci_for_each_ring_seg(first_seg, seg) { + ret = xhci_insert_segment_mapping(trb_address_map, + ring, seg, mem_flags); + if (ret) + goto remove_streams; + } - if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) { - ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control - &= ~cpu_to_le32(LINK_TOGGLE); - last->trbs[TRBS_PER_SEGMENT-1].link.control - |= cpu_to_le32(LINK_TOGGLE); - ring->last_seg = last; + return 0; + +remove_streams: + failed_seg = seg; + xhci_for_each_ring_seg(first_seg, seg) { + xhci_remove_segment_mapping(trb_address_map, seg); + if (seg == failed_seg) + return ret; } + + return ret; +} + +static void xhci_remove_stream_mapping(struct xhci_ring *ring) +{ + struct xhci_segment *seg; + + if (WARN_ON_ONCE(ring->trb_address_map == NULL)) + return; + + xhci_for_each_ring_seg(ring->first_seg, seg) + xhci_remove_segment_mapping(ring->trb_address_map, seg); +} + +static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags) +{ + return xhci_update_stream_segment_mapping(ring->trb_address_map, ring, + ring->first_seg, mem_flags); } /* XXX: Do we need the hcd structure in all these functions? */ @@ -153,14 +291,18 @@ void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) if (!ring) return; - if (ring->first_seg) - xhci_free_segments_for_ring(xhci, ring->first_seg); + trace_xhci_ring_free(ring); + + if (ring->first_seg) { + if (ring->type == TYPE_STREAM) + xhci_remove_stream_mapping(ring); + xhci_ring_segments_free(xhci, ring); + } kfree(ring); } -static void xhci_initialize_ring_info(struct xhci_ring *ring, - unsigned int cycle_state) +void xhci_initialize_ring_info(struct xhci_ring *ring) { /* The ring is empty, so the enqueue pointer == dequeue pointer */ ring->enqueue = ring->first_seg->trbs; @@ -174,10 +316,7 @@ static void xhci_initialize_ring_info(struct xhci_ring *ring, * New rings are initialized with cycle state equal to 1; if we are * handling ring expansion, set the cycle state equal to the old ring. */ - ring->cycle_state = cycle_state; - /* Not necessary for new rings, but needed for re-initialized rings */ - ring->enq_updates = 0; - ring->deq_updates = 0; + ring->cycle_state = 1; /* * Each segment has a link TRB, and leave an extra TRB for SW @@ -185,81 +324,74 @@ static void xhci_initialize_ring_info(struct xhci_ring *ring, */ ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1; } +EXPORT_SYMBOL_GPL(xhci_initialize_ring_info); /* Allocate segments and link them for a ring */ -static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, - struct xhci_segment **first, struct xhci_segment **last, - unsigned int num_segs, unsigned int cycle_state, - enum xhci_ring_type type, gfp_t flags) +static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, struct xhci_ring *ring, gfp_t flags) { struct xhci_segment *prev; + unsigned int num = 0; - prev = xhci_segment_alloc(xhci, cycle_state, flags); + prev = xhci_segment_alloc(xhci, ring->bounce_buf_len, num, flags); if (!prev) return -ENOMEM; - num_segs--; + num++; - *first = prev; - while (num_segs > 0) { + ring->first_seg = prev; + while (num < ring->num_segs) { struct xhci_segment *next; - next = xhci_segment_alloc(xhci, cycle_state, flags); - if (!next) { - prev = *first; - while (prev) { - next = prev->next; - xhci_segment_free(xhci, prev); - prev = next; - } - return -ENOMEM; - } - xhci_link_segments(xhci, prev, next, type); + next = xhci_segment_alloc(xhci, ring->bounce_buf_len, num, flags); + if (!next) + goto free_segments; + prev->next = next; prev = next; - num_segs--; + num++; } - xhci_link_segments(xhci, prev, *first, type); - *last = prev; + ring->last_seg = prev; + ring->last_seg->next = ring->first_seg; return 0; + +free_segments: + ring->last_seg = prev; + xhci_ring_segments_free(xhci, ring); + return -ENOMEM; } -/** +/* * Create a new ring with zero or more segments. * * Link each segment together into a ring. * Set the end flag and the cycle toggle bit on the last segment. * See section 4.9.1 and figures 15 and 16. */ -static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, - unsigned int num_segs, unsigned int cycle_state, - enum xhci_ring_type type, gfp_t flags) +struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, unsigned int num_segs, + enum xhci_ring_type type, unsigned int max_packet, gfp_t flags) { struct xhci_ring *ring; int ret; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - ring = kzalloc(sizeof *(ring), flags); + ring = kzalloc_node(sizeof(*ring), flags, dev_to_node(dev)); if (!ring) return NULL; ring->num_segs = num_segs; + ring->bounce_buf_len = max_packet; INIT_LIST_HEAD(&ring->td_list); ring->type = type; if (num_segs == 0) return ring; - ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg, - &ring->last_seg, num_segs, cycle_state, type, flags); + ret = xhci_alloc_segments_for_ring(xhci, ring, flags); if (ret) goto fail; - /* Only event ring does not use link TRB */ - if (type != TYPE_EVENT) { - /* See section 4.9.2.1 and 6.4.4.1 */ - ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= - cpu_to_le32(LINK_TOGGLE); - } - xhci_initialize_ring_info(ring, cycle_state); + xhci_initialize_ring_segments(xhci, ring); + xhci_initialize_ring_info(ring); + trace_xhci_ring_alloc(ring); return ring; fail: @@ -267,121 +399,83 @@ fail: return NULL; } -void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci, +void xhci_free_endpoint_ring(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev, unsigned int ep_index) { - int rings_cached; - - rings_cached = virt_dev->num_rings_cached; - if (rings_cached < XHCI_MAX_RINGS_CACHED) { - virt_dev->ring_cache[rings_cached] = - virt_dev->eps[ep_index].ring; - virt_dev->num_rings_cached++; - xhci_dbg(xhci, "Cached old ring, " - "%d ring%s cached\n", - virt_dev->num_rings_cached, - (virt_dev->num_rings_cached > 1) ? "s" : ""); - } else { - xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); - xhci_dbg(xhci, "Ring cache full (%d rings), " - "freeing ring\n", - virt_dev->num_rings_cached); - } + xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); virt_dev->eps[ep_index].ring = NULL; } -/* Zero an endpoint ring (except for link TRBs) and move the enqueue and dequeue - * pointers to the beginning of the ring. - */ -static void xhci_reinit_cached_ring(struct xhci_hcd *xhci, - struct xhci_ring *ring, unsigned int cycle_state, - enum xhci_ring_type type) -{ - struct xhci_segment *seg = ring->first_seg; - int i; - - do { - memset(seg->trbs, 0, - sizeof(union xhci_trb)*TRBS_PER_SEGMENT); - if (cycle_state == 0) { - for (i = 0; i < TRBS_PER_SEGMENT; i++) - seg->trbs[i].link.control |= TRB_CYCLE; - } - /* All endpoint rings have link TRBs */ - xhci_link_segments(xhci, seg, seg->next, type); - seg = seg->next; - } while (seg != ring->first_seg); - ring->type = type; - xhci_initialize_ring_info(ring, cycle_state); - /* td list should be empty since all URBs have been cancelled, - * but just in case... - */ - INIT_LIST_HEAD(&ring->td_list); -} - /* * Expand an existing ring. - * Look for a cached ring or allocate a new ring which has same segment numbers - * and link the two rings. + * Allocate a new ring which has same segment numbers and link the two rings. */ int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, - unsigned int num_trbs, gfp_t flags) + unsigned int num_new_segs, gfp_t flags) { - struct xhci_segment *first; - struct xhci_segment *last; - unsigned int num_segs; - unsigned int num_segs_needed; - int ret; - - num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) / - (TRBS_PER_SEGMENT - 1); + struct xhci_ring new_ring; + int ret; - /* Allocate number of segments we needed, or double the ring size */ - num_segs = ring->num_segs > num_segs_needed ? - ring->num_segs : num_segs_needed; + if (num_new_segs == 0) + return 0; - ret = xhci_alloc_segments_for_ring(xhci, &first, &last, - num_segs, ring->cycle_state, ring->type, flags); + new_ring.num_segs = num_new_segs; + new_ring.bounce_buf_len = ring->bounce_buf_len; + new_ring.type = ring->type; + ret = xhci_alloc_segments_for_ring(xhci, &new_ring, flags); if (ret) return -ENOMEM; - xhci_link_rings(xhci, ring, first, last, num_segs); - xhci_dbg(xhci, "ring expansion succeed, now has %d segments\n", + xhci_initialize_ring_segments(xhci, &new_ring); + + if (ring->type == TYPE_STREAM) { + ret = xhci_update_stream_segment_mapping(ring->trb_address_map, ring, + new_ring.first_seg, flags); + if (ret) + goto free_segments; + } + + xhci_link_rings(xhci, &new_ring, ring); + trace_xhci_ring_expansion(ring); + xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, + "ring expansion succeed, now has %d segments", ring->num_segs); return 0; -} -#define CTX_SIZE(_hcc) (HCC_64BYTE_CONTEXT(_hcc) ? 64 : 32) +free_segments: + xhci_ring_segments_free(xhci, &new_ring); + return ret; +} -static struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, +struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, int type, gfp_t flags) { struct xhci_container_ctx *ctx; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; if ((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT)) return NULL; - ctx = kzalloc(sizeof(*ctx), flags); + ctx = kzalloc_node(sizeof(*ctx), flags, dev_to_node(dev)); if (!ctx) return NULL; ctx->type = type; - ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024; + ctx->size = xhci->hcc_params & HCC_64BYTE_CONTEXT ? 2048 : 1024; if (type == XHCI_CTX_TYPE_INPUT) ctx->size += CTX_SIZE(xhci->hcc_params); - ctx->bytes = dma_pool_alloc(xhci->device_pool, flags, &ctx->dma); + ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma); if (!ctx->bytes) { kfree(ctx); return NULL; } - memset(ctx->bytes, 0, ctx->size); return ctx; } -static void xhci_free_container_ctx(struct xhci_hcd *xhci, +void xhci_free_container_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx) { if (!ctx) @@ -390,7 +484,36 @@ static void xhci_free_container_ctx(struct xhci_hcd *xhci, kfree(ctx); } -struct xhci_input_control_ctx *xhci_get_input_control_ctx(struct xhci_hcd *xhci, +struct xhci_container_ctx *xhci_alloc_port_bw_ctx(struct xhci_hcd *xhci, + gfp_t flags) +{ + struct xhci_container_ctx *ctx; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + + ctx = kzalloc_node(sizeof(*ctx), flags, dev_to_node(dev)); + if (!ctx) + return NULL; + + ctx->size = GET_PORT_BW_ARRAY_SIZE; + + ctx->bytes = dma_pool_zalloc(xhci->port_bw_pool, flags, &ctx->dma); + if (!ctx->bytes) { + kfree(ctx); + return NULL; + } + return ctx; +} + +void xhci_free_port_bw_ctx(struct xhci_hcd *xhci, + struct xhci_container_ctx *ctx) +{ + if (!ctx) + return; + dma_pool_free(xhci->port_bw_pool, ctx->bytes, ctx->dma); + kfree(ctx); +} + +struct xhci_input_control_ctx *xhci_get_input_control_ctx( struct xhci_container_ctx *ctx) { if (ctx->type != XHCI_CTX_TYPE_INPUT) @@ -421,7 +544,7 @@ struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, return (struct xhci_ep_ctx *) (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params))); } - +EXPORT_SYMBOL_GPL(xhci_get_ep_ctx); /***************** Streams structures manipulation *************************/ @@ -429,18 +552,15 @@ static void xhci_free_stream_ctx(struct xhci_hcd *xhci, unsigned int num_stream_ctxs, struct xhci_stream_ctx *stream_ctx, dma_addr_t dma) { - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + size_t size = array_size(sizeof(struct xhci_stream_ctx), num_stream_ctxs); - if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE) - dma_free_coherent(&pdev->dev, - sizeof(struct xhci_stream_ctx)*num_stream_ctxs, - stream_ctx, dma); - else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE) - return dma_pool_free(xhci->small_streams_pool, - stream_ctx, dma); + if (size > MEDIUM_STREAM_ARRAY_SIZE) + dma_free_coherent(dev, size, stream_ctx, dma); + else if (size > SMALL_STREAM_ARRAY_SIZE) + dma_pool_free(xhci->medium_streams_pool, stream_ctx, dma); else - return dma_pool_free(xhci->medium_streams_pool, - stream_ctx, dma); + dma_pool_free(xhci->small_streams_pool, stream_ctx, dma); } /* @@ -457,18 +577,15 @@ static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, unsigned int num_stream_ctxs, dma_addr_t *dma, gfp_t mem_flags) { - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + size_t size = array_size(sizeof(struct xhci_stream_ctx), num_stream_ctxs); - if (num_stream_ctxs > MEDIUM_STREAM_ARRAY_SIZE) - return dma_alloc_coherent(&pdev->dev, - sizeof(struct xhci_stream_ctx)*num_stream_ctxs, - dma, mem_flags); - else if (num_stream_ctxs <= SMALL_STREAM_ARRAY_SIZE) - return dma_pool_alloc(xhci->small_streams_pool, - mem_flags, dma); + if (size > MEDIUM_STREAM_ARRAY_SIZE) + return dma_alloc_coherent(dev, size, dma, mem_flags); + if (size > SMALL_STREAM_ARRAY_SIZE) + return dma_pool_zalloc(xhci->medium_streams_pool, mem_flags, dma); else - return dma_pool_alloc(xhci->medium_streams_pool, - mem_flags, dma); + return dma_pool_zalloc(xhci->small_streams_pool, mem_flags, dma); } struct xhci_ring *xhci_dma_to_transfer_ring( @@ -481,86 +598,6 @@ struct xhci_ring *xhci_dma_to_transfer_ring( return ep->ring; } -/* Only use this when you know stream_info is valid */ -#ifdef CONFIG_USB_XHCI_HCD_DEBUGGING -static struct xhci_ring *dma_to_stream_ring( - struct xhci_stream_info *stream_info, - u64 address) -{ - return radix_tree_lookup(&stream_info->trb_address_map, - address >> TRB_SEGMENT_SHIFT); -} -#endif /* CONFIG_USB_XHCI_HCD_DEBUGGING */ - -struct xhci_ring *xhci_stream_id_to_ring( - struct xhci_virt_device *dev, - unsigned int ep_index, - unsigned int stream_id) -{ - struct xhci_virt_ep *ep = &dev->eps[ep_index]; - - if (stream_id == 0) - return ep->ring; - if (!ep->stream_info) - return NULL; - - if (stream_id > ep->stream_info->num_streams) - return NULL; - return ep->stream_info->stream_rings[stream_id]; -} - -#ifdef CONFIG_USB_XHCI_HCD_DEBUGGING -static int xhci_test_radix_tree(struct xhci_hcd *xhci, - unsigned int num_streams, - struct xhci_stream_info *stream_info) -{ - u32 cur_stream; - struct xhci_ring *cur_ring; - u64 addr; - - for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { - struct xhci_ring *mapped_ring; - int trb_size = sizeof(union xhci_trb); - - cur_ring = stream_info->stream_rings[cur_stream]; - for (addr = cur_ring->first_seg->dma; - addr < cur_ring->first_seg->dma + TRB_SEGMENT_SIZE; - addr += trb_size) { - mapped_ring = dma_to_stream_ring(stream_info, addr); - if (cur_ring != mapped_ring) { - xhci_warn(xhci, "WARN: DMA address 0x%08llx " - "didn't map to stream ID %u; " - "mapped to ring %p\n", - (unsigned long long) addr, - cur_stream, - mapped_ring); - return -EINVAL; - } - } - /* One TRB after the end of the ring segment shouldn't return a - * pointer to the current ring (although it may be a part of a - * different ring). - */ - mapped_ring = dma_to_stream_ring(stream_info, addr); - if (mapped_ring != cur_ring) { - /* One TRB before should also fail */ - addr = cur_ring->first_seg->dma - trb_size; - mapped_ring = dma_to_stream_ring(stream_info, addr); - } - if (mapped_ring == cur_ring) { - xhci_warn(xhci, "WARN: Bad DMA address 0x%08llx " - "mapped to valid stream ID %u; " - "mapped ring = %p\n", - (unsigned long long) addr, - cur_stream, - mapped_ring); - return -EINVAL; - } - } - return 0; -} -#endif /* CONFIG_USB_XHCI_HCD_DEBUGGING */ - /* * Change an endpoint's internal structure so it supports stream IDs. The * number of requested streams includes stream 0, which cannot be used by device @@ -569,50 +606,20 @@ static int xhci_test_radix_tree(struct xhci_hcd *xhci, * The number of stream contexts in the stream context array may be bigger than * the number of streams the driver wants to use. This is because the number of * stream context array entries must be a power of two. - * - * We need a radix tree for mapping physical addresses of TRBs to which stream - * ID they belong to. We need to do this because the host controller won't tell - * us which stream ring the TRB came from. We could store the stream ID in an - * event data TRB, but that doesn't help us for the cancellation case, since the - * endpoint may stop before it reaches that event data TRB. - * - * The radix tree maps the upper portion of the TRB DMA address to a ring - * segment that has the same upper portion of DMA addresses. For example, say I - * have segments of size 1KB, that are always 64-byte aligned. A segment may - * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the - * key to the stream ID is 0x43244. I can use the DMA address of the TRB to - * pass the radix tree a key to get the right stream ID: - * - * 0x10c90fff >> 10 = 0x43243 - * 0x10c912c0 >> 10 = 0x43244 - * 0x10c91400 >> 10 = 0x43245 - * - * Obviously, only those TRBs with DMA addresses that are within the segment - * will make the radix tree return the stream ID for that ring. - * - * Caveats for the radix tree: - * - * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an - * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be - * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the - * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit - * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit - * extended systems (where the DMA address can be bigger than 32-bits), - * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that. */ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, unsigned int num_stream_ctxs, - unsigned int num_streams, gfp_t mem_flags) + unsigned int num_streams, + unsigned int max_packet, gfp_t mem_flags) { struct xhci_stream_info *stream_info; u32 cur_stream; struct xhci_ring *cur_ring; - unsigned long key; u64 addr; int ret; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - xhci_dbg(xhci, "Allocating %u streams and %u " - "stream context array entries.\n", + xhci_dbg(xhci, "Allocating %u streams and %u stream context array entries.\n", num_streams, num_stream_ctxs); if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) { xhci_dbg(xhci, "Command ring has no reserved TRBs available\n"); @@ -620,7 +627,8 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, } xhci->cmd_ring_reserved_trbs++; - stream_info = kzalloc(sizeof(struct xhci_stream_info), mem_flags); + stream_info = kzalloc_node(sizeof(*stream_info), mem_flags, + dev_to_node(dev)); if (!stream_info) goto cleanup_trbs; @@ -628,9 +636,9 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, stream_info->num_stream_ctxs = num_stream_ctxs; /* Initialize the array of virtual pointers to stream rings. */ - stream_info->stream_rings = kzalloc( - sizeof(struct xhci_ring *)*num_streams, - mem_flags); + stream_info->stream_rings = kcalloc_node( + num_streams, sizeof(struct xhci_ring *), mem_flags, + dev_to_node(dev)); if (!stream_info->stream_rings) goto cleanup_info; @@ -639,13 +647,11 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, num_stream_ctxs, &stream_info->ctx_array_dma, mem_flags); if (!stream_info->stream_ctx_array) - goto cleanup_ctx; - memset(stream_info->stream_ctx_array, 0, - sizeof(struct xhci_stream_ctx)*num_stream_ctxs); + goto cleanup_ring_array; /* Allocate everything needed to free the stream rings later */ stream_info->free_streams_command = - xhci_alloc_command(xhci, true, true, mem_flags); + xhci_alloc_command_with_ctx(xhci, true, mem_flags); if (!stream_info->free_streams_command) goto cleanup_ctx; @@ -655,26 +661,26 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, * and add their segment DMA addresses to the radix tree. * Stream 0 is reserved. */ + for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { stream_info->stream_rings[cur_stream] = - xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, mem_flags); + xhci_ring_alloc(xhci, 2, TYPE_STREAM, max_packet, mem_flags); cur_ring = stream_info->stream_rings[cur_stream]; if (!cur_ring) goto cleanup_rings; cur_ring->stream_id = cur_stream; + cur_ring->trb_address_map = &stream_info->trb_address_map; /* Set deq ptr, cycle bit, and stream context type */ addr = cur_ring->first_seg->dma | SCT_FOR_CTX(SCT_PRI_TR) | cur_ring->cycle_state; stream_info->stream_ctx_array[cur_stream].stream_ring = cpu_to_le64(addr); - xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", - cur_stream, (unsigned long long) addr); + xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", cur_stream, addr); - key = (unsigned long) - (cur_ring->first_seg->dma >> TRB_SEGMENT_SHIFT); - ret = radix_tree_insert(&stream_info->trb_address_map, - key, cur_ring); + ret = xhci_update_stream_mapping(cur_ring, mem_flags); + + trace_xhci_alloc_stream_info_ctx(stream_info, cur_stream); if (ret) { xhci_ring_free(xhci, cur_ring); stream_info->stream_rings[cur_stream] = NULL; @@ -687,13 +693,6 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, * was any other way, the host controller would assume the ring is * "empty" and wait forever for data to be queued to that stream ID). */ -#if XHCI_DEBUG - /* Do a little test on the radix tree to make sure it returns the - * correct values. - */ - if (xhci_test_radix_tree(xhci, num_streams, stream_info)) - goto cleanup_rings; -#endif return stream_info; @@ -701,15 +700,17 @@ cleanup_rings: for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { cur_ring = stream_info->stream_rings[cur_stream]; if (cur_ring) { - addr = cur_ring->first_seg->dma; - radix_tree_delete(&stream_info->trb_address_map, - addr >> TRB_SEGMENT_SHIFT); xhci_ring_free(xhci, cur_ring); stream_info->stream_rings[cur_stream] = NULL; } } xhci_free_command(xhci, stream_info->free_streams_command); cleanup_ctx: + xhci_free_stream_ctx(xhci, + stream_info->num_stream_ctxs, + stream_info->stream_ctx_array, + stream_info->ctx_array_dma); +cleanup_ring_array: kfree(stream_info->stream_rings); cleanup_info: kfree(stream_info); @@ -731,7 +732,8 @@ void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc. */ max_primary_streams = fls(stream_info->num_stream_ctxs) - 2; - xhci_dbg(xhci, "Setting number of stream ctx array entries to %u\n", + xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, + "Setting number of stream ctx array entries to %u", 1 << (max_primary_streams + 1)); ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK); ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams) @@ -744,8 +746,7 @@ void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, * Reinstalls the "normal" endpoint ring (at its previous dequeue mark, * not at the beginning of the ring). */ -void xhci_setup_no_streams_ep_input_ctx(struct xhci_hcd *xhci, - struct xhci_ep_ctx *ep_ctx, +void xhci_setup_no_streams_ep_input_ctx(struct xhci_ep_ctx *ep_ctx, struct xhci_virt_ep *ep) { dma_addr_t addr; @@ -763,7 +764,6 @@ void xhci_free_stream_info(struct xhci_hcd *xhci, { int cur_stream; struct xhci_ring *cur_ring; - dma_addr_t addr; if (!stream_info) return; @@ -772,9 +772,6 @@ void xhci_free_stream_info(struct xhci_hcd *xhci, cur_stream++) { cur_ring = stream_info->stream_rings[cur_stream]; if (cur_ring) { - addr = cur_ring->first_seg->dma; - radix_tree_delete(&stream_info->trb_address_map, - addr >> TRB_SEGMENT_SHIFT); xhci_ring_free(xhci, cur_ring); stream_info->stream_rings[cur_stream] = NULL; } @@ -787,23 +784,13 @@ void xhci_free_stream_info(struct xhci_hcd *xhci, stream_info->stream_ctx_array, stream_info->ctx_array_dma); - if (stream_info) - kfree(stream_info->stream_rings); + kfree(stream_info->stream_rings); kfree(stream_info); } /***************** Device context manipulation *************************/ -static void xhci_init_endpoint_timer(struct xhci_hcd *xhci, - struct xhci_virt_ep *ep) -{ - init_timer(&ep->stop_cmd_timer); - ep->stop_cmd_timer.data = (unsigned long) ep; - ep->stop_cmd_timer.function = xhci_stop_endpoint_command_watchdog; - ep->xhci = xhci; -} - static void xhci_free_tt_info(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev, int slot_id) @@ -813,15 +800,14 @@ static void xhci_free_tt_info(struct xhci_hcd *xhci, bool slot_found = false; /* If the device never made it past the Set Address stage, - * it may not have the real_port set correctly. + * it may not have the root hub port pointer set correctly. */ - if (virt_dev->real_port == 0 || - virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { - xhci_dbg(xhci, "Bad real port.\n"); + if (!virt_dev->rhub_port) { + xhci_dbg(xhci, "Bad rhub port.\n"); return; } - tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts); + tt_list_head = &(xhci->rh_bw[virt_dev->rhub_port->hw_portnum].tts); list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) { /* Multi-TT hubs will have more than one entry */ if (tt_info->slot_id == slot_id) { @@ -842,6 +828,7 @@ int xhci_alloc_tt_info(struct xhci_hcd *xhci, struct xhci_tt_bw_info *tt_info; unsigned int num_ports; int i, j; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; if (!tt->multi) num_ports = 1; @@ -851,12 +838,13 @@ int xhci_alloc_tt_info(struct xhci_hcd *xhci, for (i = 0; i < num_ports; i++, tt_info++) { struct xhci_interval_bw_table *bw_table; - tt_info = kzalloc(sizeof(*tt_info), mem_flags); + tt_info = kzalloc_node(sizeof(*tt_info), mem_flags, + dev_to_node(dev)); if (!tt_info) goto free_tts; INIT_LIST_HEAD(&tt_info->tt_list); list_add(&tt_info->tt_list, - &xhci->rh_bw[virt_dev->real_port - 1].tts); + &xhci->rh_bw[virt_dev->rhub_port->hw_portnum].tts); tt_info->slot_id = virt_dev->udev->slot_id; if (tt->multi) tt_info->ttport = i+1; @@ -877,58 +865,104 @@ free_tts: * will be manipulated by the configure endpoint, allocate device, or update * hub functions while this function is removing the TT entries from the list. */ -void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) +void xhci_free_virt_device(struct xhci_hcd *xhci, struct xhci_virt_device *dev, + int slot_id) { - struct xhci_virt_device *dev; int i; int old_active_eps = 0; /* Slot ID 0 is reserved */ - if (slot_id == 0 || !xhci->devs[slot_id]) + if (slot_id == 0 || !dev) return; - dev = xhci->devs[slot_id]; - xhci->dcbaa->dev_context_ptrs[slot_id] = 0; - if (!dev) - return; + /* If device ctx array still points to _this_ device, clear it */ + if (dev->out_ctx && + xhci->dcbaa->dev_context_ptrs[slot_id] == cpu_to_le64(dev->out_ctx->dma)) + xhci->dcbaa->dev_context_ptrs[slot_id] = 0; + + trace_xhci_free_virt_device(dev); if (dev->tt_info) old_active_eps = dev->tt_info->active_eps; - for (i = 0; i < 31; ++i) { + for (i = 0; i < 31; i++) { if (dev->eps[i].ring) xhci_ring_free(xhci, dev->eps[i].ring); if (dev->eps[i].stream_info) xhci_free_stream_info(xhci, dev->eps[i].stream_info); - /* Endpoints on the TT/root port lists should have been removed - * when usb_disable_device() was called for the device. - * We can't drop them anyway, because the udev might have gone - * away by this point, and we can't tell what speed it was. + /* + * Endpoints are normally deleted from the bandwidth list when + * endpoints are dropped, before device is freed. + * If host is dying or being removed then endpoints aren't + * dropped cleanly, so delete the endpoint from list here. + * Only applicable for hosts with software bandwidth checking. */ - if (!list_empty(&dev->eps[i].bw_endpoint_list)) - xhci_warn(xhci, "Slot %u endpoint %u " - "not removed from BW list!\n", - slot_id, i); + + if (!list_empty(&dev->eps[i].bw_endpoint_list)) { + list_del_init(&dev->eps[i].bw_endpoint_list); + xhci_dbg(xhci, "Slot %u endpoint %u not removed from BW list!\n", + slot_id, i); + } } /* If this is a hub, free the TT(s) from the TT list */ xhci_free_tt_info(xhci, dev, slot_id); /* If necessary, update the number of active TTs on this root port */ xhci_update_tt_active_eps(xhci, dev, old_active_eps); - if (dev->ring_cache) { - for (i = 0; i < dev->num_rings_cached; i++) - xhci_ring_free(xhci, dev->ring_cache[i]); - kfree(dev->ring_cache); - } - if (dev->in_ctx) xhci_free_container_ctx(xhci, dev->in_ctx); if (dev->out_ctx) xhci_free_container_ctx(xhci, dev->out_ctx); - kfree(xhci->devs[slot_id]); - xhci->devs[slot_id] = NULL; + if (dev->udev && dev->udev->slot_id) + dev->udev->slot_id = 0; + if (dev->rhub_port && dev->rhub_port->slot_id == slot_id) + dev->rhub_port->slot_id = 0; + if (xhci->devs[slot_id] == dev) + xhci->devs[slot_id] = NULL; + kfree(dev); +} + +/* + * Free a virt_device structure. + * If the virt_device added a tt_info (a hub) and has children pointing to + * that tt_info, then free the child first. Recursive. + * We can't rely on udev at this point to find child-parent relationships. + */ +static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) +{ + struct xhci_virt_device *vdev; + struct list_head *tt_list_head; + struct xhci_tt_bw_info *tt_info, *next; + int i; + + vdev = xhci->devs[slot_id]; + if (!vdev) + return; + + if (!vdev->rhub_port) { + xhci_dbg(xhci, "Bad rhub port.\n"); + goto out; + } + + tt_list_head = &(xhci->rh_bw[vdev->rhub_port->hw_portnum].tts); + list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) { + /* is this a hub device that added a tt_info to the tts list */ + if (tt_info->slot_id == slot_id) { + /* are any devices using this tt_info? */ + for (i = 1; i < xhci->max_slots; i++) { + vdev = xhci->devs[i]; + if (vdev && (vdev->tt_info == tt_info)) + xhci_free_virt_devices_depth_first( + xhci, i); + } + } + } +out: + /* we are now at a leaf device */ + xhci_debugfs_remove_slot(xhci, slot_id); + xhci_free_virt_device(xhci, xhci->devs[slot_id], slot_id); } int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, @@ -943,49 +977,40 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, return 0; } - xhci->devs[slot_id] = kzalloc(sizeof(*xhci->devs[slot_id]), flags); - if (!xhci->devs[slot_id]) + dev = kzalloc(sizeof(*dev), flags); + if (!dev) return 0; - dev = xhci->devs[slot_id]; + + dev->slot_id = slot_id; /* Allocate the (output) device context that will be used in the HC. */ dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags); if (!dev->out_ctx) goto fail; - xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id, - (unsigned long long)dev->out_ctx->dma); + xhci_dbg(xhci, "Slot %d output ctx = 0x%pad (dma)\n", slot_id, &dev->out_ctx->dma); /* Allocate the (input) device context for address device command */ dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags); if (!dev->in_ctx) goto fail; - xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id, - (unsigned long long)dev->in_ctx->dma); + xhci_dbg(xhci, "Slot %d input ctx = 0x%pad (dma)\n", slot_id, &dev->in_ctx->dma); - /* Initialize the cancellation list and watchdog timers for each ep */ + /* Initialize the cancellation and bandwidth list for each ep */ for (i = 0; i < 31; i++) { - xhci_init_endpoint_timer(xhci, &dev->eps[i]); + dev->eps[i].ep_index = i; + dev->eps[i].vdev = dev; + dev->eps[i].xhci = xhci; INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list); INIT_LIST_HEAD(&dev->eps[i].bw_endpoint_list); } /* Allocate endpoint 0 ring */ - dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, flags); + dev->eps[0].ring = xhci_ring_alloc(xhci, 2, TYPE_CTRL, 0, flags); if (!dev->eps[0].ring) goto fail; - /* Allocate pointers to the ring cache */ - dev->ring_cache = kzalloc( - sizeof(struct xhci_ring *)*XHCI_MAX_RINGS_CACHED, - flags); - if (!dev->ring_cache) - goto fail; - dev->num_rings_cached = 0; - - init_completion(&dev->cmd_completion); - INIT_LIST_HEAD(&dev->cmd_list); dev->udev = udev; /* Point to output device context in dcbaa. */ @@ -995,9 +1020,19 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, &xhci->dcbaa->dev_context_ptrs[slot_id], le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); + trace_xhci_alloc_virt_device(dev); + + xhci->devs[slot_id] = dev; + return 1; fail: - xhci_free_virt_device(xhci, slot_id); + + if (dev->in_ctx) + xhci_free_container_ctx(xhci, dev->in_ctx); + if (dev->out_ctx) + xhci_free_container_ctx(xhci, dev->out_ctx); + kfree(dev); + return 0; } @@ -1025,23 +1060,22 @@ void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, /* * The xHCI roothub may have ports of differing speeds in any order in the port - * status registers. xhci->port_array provides an array of the port speed for - * each offset into the port status registers. + * status registers. * - * The xHCI hardware wants to know the roothub port number that the USB device + * The xHCI hardware wants to know the roothub port that the USB device * is attached to (or the roothub port its ancestor hub is attached to). All we * know is the index of that port under either the USB 2.0 or the USB 3.0 * roothub, but that doesn't give us the real index into the HW port status - * registers. Call xhci_find_raw_port_number() to get real index. + * registers. */ -static u32 xhci_find_real_port_number(struct xhci_hcd *xhci, - struct usb_device *udev) +static struct xhci_port *xhci_find_rhub_port(struct xhci_hcd *xhci, struct usb_device *udev) { struct usb_device *top_dev; + struct xhci_hub *rhub; struct usb_hcd *hcd; - if (udev->speed == USB_SPEED_SUPER) - hcd = xhci->shared_hcd; + if (udev->speed >= USB_SPEED_SUPER) + hcd = xhci_get_usb3_hcd(xhci); else hcd = xhci->main_hcd; @@ -1049,7 +1083,8 @@ static u32 xhci_find_real_port_number(struct xhci_hcd *xhci, top_dev = top_dev->parent) /* Found device below root hub */; - return xhci_find_raw_port_number(hcd, top_dev->portnum); + rhub = xhci_get_rhub(hcd); + return rhub->ports[top_dev->portnum - 1]; } /* Setup an xHCI virtual device for a Set Address command */ @@ -1058,9 +1093,7 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud struct xhci_virt_device *dev; struct xhci_ep_ctx *ep0_ctx; struct xhci_slot_ctx *slot_ctx; - u32 port_num; u32 max_packets; - struct usb_device *top_dev; dev = xhci->devs[udev->slot_id]; /* Slot ID 0 is reserved */ @@ -1075,6 +1108,10 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud /* 3) Only the control endpoint is valid - one endpoint context */ slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route); switch (udev->speed) { + case USB_SPEED_SUPER_PLUS: + slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP); + max_packets = MAX_PACKET(512); + break; case USB_SPEED_SUPER: slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS); max_packets = MAX_PACKET(512); @@ -1092,27 +1129,20 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS); max_packets = MAX_PACKET(8); break; - case USB_SPEED_WIRELESS: - xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n"); - return -EINVAL; - break; default: /* Speed was set earlier, this shouldn't happen. */ return -EINVAL; } /* Find the root hub port this device is under */ - port_num = xhci_find_real_port_number(xhci, udev); - if (!port_num) + dev->rhub_port = xhci_find_rhub_port(xhci, udev); + if (!dev->rhub_port) return -EINVAL; - slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num)); - /* Set the port number in the virtual_device to the faked port number */ - for (top_dev = udev; top_dev->parent && top_dev->parent->parent; - top_dev = top_dev->parent) - /* Found device below root hub */; - dev->fake_port = top_dev->portnum; - dev->real_port = port_num; - xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num); - xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port); + /* Slot ID is set to the device directly below the root hub */ + if (!udev->parent->parent) + dev->rhub_port->slot_id = udev->slot_id; + slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(dev->rhub_port->hw_portnum + 1)); + xhci_dbg(xhci, "Slot ID %d: HW portnum %d, hcd portnum %d\n", + udev->slot_id, dev->rhub_port->hw_portnum, dev->rhub_port->hcd_portnum); /* Find the right bandwidth table that this device will be a part of. * If this is a full speed device attached directly to a root port (or a @@ -1121,12 +1151,12 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud * will never be created for the HS root hub. */ if (!udev->tt || !udev->tt->hub->parent) { - dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table; + dev->bw_table = &xhci->rh_bw[dev->rhub_port->hw_portnum].bw_table; } else { struct xhci_root_port_bw_info *rh_bw; struct xhci_tt_bw_info *tt_bw; - rh_bw = &xhci->rh_bw[port_num - 1]; + rh_bw = &xhci->rh_bw[dev->rhub_port->hw_portnum]; /* Find the right TT. */ list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) { if (tt_bw->slot_id != udev->tt->hub->slot_id) @@ -1165,6 +1195,10 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma | dev->eps[0].ring->cycle_state); + ep0_ctx->tx_info = cpu_to_le32(EP_AVG_TRB_LENGTH(8)); + + trace_xhci_setup_addressable_virt_device(dev); + /* Steps 7 and 8 were done in xhci_alloc_virt_device() */ return 0; @@ -1213,7 +1247,7 @@ static unsigned int xhci_microframes_to_exponent(struct usb_device *udev, interval = fls(desc_interval) - 1; interval = clamp_val(interval, min_exponent, max_exponent); if ((1 << interval) != desc_interval) - dev_warn(&udev->dev, + dev_dbg(&udev->dev, "ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n", ep->desc.bEndpointAddress, 1 << interval, @@ -1260,8 +1294,9 @@ static unsigned int xhci_get_endpoint_interval(struct usb_device *udev, interval = xhci_parse_microframe_interval(udev, ep); break; } - /* Fall through - SS and HS isoc/int have same decoding */ + fallthrough; /* SS and HS isoc/int have same decoding */ + case USB_SPEED_SUPER_PLUS: case USB_SPEED_SUPER: if (usb_endpoint_xfer_int(&ep->desc) || usb_endpoint_xfer_isoc(&ep->desc)) { @@ -1279,6 +1314,7 @@ static unsigned int xhci_get_endpoint_interval(struct usb_device *udev, * since it uses the same rules as low speed interrupt * endpoints. */ + fallthrough; case USB_SPEED_LOW: if (usb_endpoint_xfer_int(&ep->desc) || @@ -1291,76 +1327,78 @@ static unsigned int xhci_get_endpoint_interval(struct usb_device *udev, default: BUG(); } - return EP_INTERVAL(interval); + return interval; } -/* The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps. +/* + * xHCs without LEC use the "Mult" field in the endpoint context for SuperSpeed + * isoc eps, and High speed isoc eps that support bandwidth doubling. Standard * High speed endpoint descriptors can define "the number of additional * transaction opportunities per microframe", but that goes in the Max Burst * endpoint context field. */ -static u32 xhci_get_endpoint_mult(struct usb_device *udev, - struct usb_host_endpoint *ep) +static u32 xhci_get_endpoint_mult(struct xhci_hcd *xhci, + struct usb_device *udev, + struct usb_host_endpoint *ep) { - if (udev->speed != USB_SPEED_SUPER || - !usb_endpoint_xfer_isoc(&ep->desc)) - return 0; - return ep->ss_ep_comp.bmAttributes; + bool lec; + + /* xHCI 1.1 with LEC set does not use mult field, except intel eUSB2 */ + lec = xhci->hci_version > 0x100 && (xhci->hcc_params2 & HCC2_LEC); + + /* eUSB2 double isoc bw devices are the only USB2 devices using mult */ + if (usb_endpoint_is_hs_isoc_double(udev, ep) && + (!lec || xhci->quirks & XHCI_INTEL_HOST)) + return 1; + + /* SuperSpeed isoc transfers on hosts without LEC uses mult field */ + if (udev->speed >= USB_SPEED_SUPER && + usb_endpoint_xfer_isoc(&ep->desc) && !lec) + return ep->ss_ep_comp.bmAttributes; + + return 0; } -static u32 xhci_get_endpoint_type(struct usb_device *udev, - struct usb_host_endpoint *ep) +static u32 xhci_get_endpoint_max_burst(struct usb_device *udev, + struct usb_host_endpoint *ep) { - int in; - u32 type; + /* Super speed and Plus have max burst in ep companion desc */ + if (udev->speed >= USB_SPEED_SUPER) + return ep->ss_ep_comp.bMaxBurst; - in = usb_endpoint_dir_in(&ep->desc); - if (usb_endpoint_xfer_control(&ep->desc)) { - type = EP_TYPE(CTRL_EP); - } else if (usb_endpoint_xfer_bulk(&ep->desc)) { - if (in) - type = EP_TYPE(BULK_IN_EP); - else - type = EP_TYPE(BULK_OUT_EP); - } else if (usb_endpoint_xfer_isoc(&ep->desc)) { - if (in) - type = EP_TYPE(ISOC_IN_EP); - else - type = EP_TYPE(ISOC_OUT_EP); - } else if (usb_endpoint_xfer_int(&ep->desc)) { - if (in) - type = EP_TYPE(INT_IN_EP); - else - type = EP_TYPE(INT_OUT_EP); - } else { - type = 0; + if (udev->speed == USB_SPEED_HIGH && + (usb_endpoint_xfer_isoc(&ep->desc) || + usb_endpoint_xfer_int(&ep->desc))) { + /* + * USB 2 Isochronous Double IN Bandwidth ECN uses fixed burst + * size and max packets bits 12:11 are invalid. + */ + if (usb_endpoint_is_hs_isoc_double(udev, ep)) + return 2; + + return usb_endpoint_maxp_mult(&ep->desc) - 1; } - return type; + + return 0; } -/* Return the maximum endpoint service interval time (ESIT) payload. - * Basically, this is the maxpacket size, multiplied by the burst size - * and mult size. - */ -static u32 xhci_get_max_esit_payload(struct xhci_hcd *xhci, - struct usb_device *udev, - struct usb_host_endpoint *ep) +static u32 xhci_get_endpoint_type(struct usb_host_endpoint *ep) { - int max_burst; - int max_packet; - - /* Only applies for interrupt or isochronous endpoints */ - if (usb_endpoint_xfer_control(&ep->desc) || - usb_endpoint_xfer_bulk(&ep->desc)) - return 0; + int in; - if (udev->speed == USB_SPEED_SUPER) - return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval); + in = usb_endpoint_dir_in(&ep->desc); - max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc)); - max_burst = (usb_endpoint_maxp(&ep->desc) & 0x1800) >> 11; - /* A 0 in max burst means 1 transfer per ESIT */ - return max_packet * (max_burst + 1); + switch (usb_endpoint_type(&ep->desc)) { + case USB_ENDPOINT_XFER_CONTROL: + return CTRL_EP; + case USB_ENDPOINT_XFER_BULK: + return in ? BULK_IN_EP : BULK_OUT_EP; + case USB_ENDPOINT_XFER_ISOC: + return in ? ISOC_IN_EP : ISOC_OUT_EP; + case USB_ENDPOINT_XFER_INT: + return in ? INT_IN_EP : INT_OUT_EP; + } + return 0; } /* Set up an endpoint with one ring segment. Do not allocate stream rings. @@ -1376,108 +1414,99 @@ int xhci_endpoint_init(struct xhci_hcd *xhci, struct xhci_ep_ctx *ep_ctx; struct xhci_ring *ep_ring; unsigned int max_packet; - unsigned int max_burst; - enum xhci_ring_type type; + enum xhci_ring_type ring_type; u32 max_esit_payload; u32 endpoint_type; + unsigned int max_burst; + unsigned int interval; + unsigned int mult; + unsigned int avg_trb_len; + unsigned int err_count = 0; ep_index = xhci_get_endpoint_index(&ep->desc); ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); - endpoint_type = xhci_get_endpoint_type(udev, ep); + endpoint_type = xhci_get_endpoint_type(ep); if (!endpoint_type) return -EINVAL; - ep_ctx->ep_info2 = cpu_to_le32(endpoint_type); - type = usb_endpoint_type(&ep->desc); - /* Set up the endpoint ring */ - virt_dev->eps[ep_index].new_ring = - xhci_ring_alloc(xhci, 2, 1, type, mem_flags); - if (!virt_dev->eps[ep_index].new_ring) { - /* Attempt to use the ring cache */ - if (virt_dev->num_rings_cached == 0) - return -ENOMEM; - virt_dev->eps[ep_index].new_ring = - virt_dev->ring_cache[virt_dev->num_rings_cached]; - virt_dev->ring_cache[virt_dev->num_rings_cached] = NULL; - virt_dev->num_rings_cached--; - xhci_reinit_cached_ring(xhci, virt_dev->eps[ep_index].new_ring, - 1, type); + ring_type = usb_endpoint_type(&ep->desc); + + /* Ensure host supports double isoc bandwidth for eUSB2 devices */ + if (usb_endpoint_is_hs_isoc_double(udev, ep) && !(xhci->hcc_params2 & HCC2_EUSB2_DIC)) { + dev_dbg(&udev->dev, "Double Isoc Bandwidth not supported by xhci\n"); + return -EINVAL; + } + + /* + * Get values to fill the endpoint context, mostly from ep descriptor. + * The average TRB buffer lengt for bulk endpoints is unclear as we + * have no clue on scatter gather list entry size. For Isoc and Int, + * set it to max available. See xHCI 1.1 spec 4.14.1.1 for details. + */ + max_esit_payload = usb_endpoint_max_periodic_payload(udev, ep); + interval = xhci_get_endpoint_interval(udev, ep); + + /* Periodic endpoint bInterval limit quirk */ + if (usb_endpoint_xfer_int(&ep->desc) || + usb_endpoint_xfer_isoc(&ep->desc)) { + if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_9) && + interval >= 9) { + interval = 8; + } + if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && + udev->speed >= USB_SPEED_HIGH && + interval >= 7) { + interval = 6; + } } - virt_dev->eps[ep_index].skip = false; - ep_ring = virt_dev->eps[ep_index].new_ring; - ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma | ep_ring->cycle_state); - ep_ctx->ep_info = cpu_to_le32(xhci_get_endpoint_interval(udev, ep) - | EP_MULT(xhci_get_endpoint_mult(udev, ep))); + mult = xhci_get_endpoint_mult(xhci, udev, ep); + max_packet = xhci_usb_endpoint_maxp(udev, ep); + max_burst = xhci_get_endpoint_max_burst(udev, ep); + avg_trb_len = max_esit_payload; /* FIXME dig Mult and streams info out of ep companion desc */ - /* Allow 3 retries for everything but isoc; - * CErr shall be set to 0 for Isoch endpoints. - */ + /* Allow 3 retries for everything but isoc, set CErr = 3 */ if (!usb_endpoint_xfer_isoc(&ep->desc)) - ep_ctx->ep_info2 |= cpu_to_le32(ERROR_COUNT(3)); - else - ep_ctx->ep_info2 |= cpu_to_le32(ERROR_COUNT(0)); - - /* Set the max packet size and max burst */ - max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc)); - max_burst = 0; - switch (udev->speed) { - case USB_SPEED_SUPER: - /* dig out max burst from ep companion desc */ - max_burst = ep->ss_ep_comp.bMaxBurst; - break; - case USB_SPEED_HIGH: - /* Some devices get this wrong */ - if (usb_endpoint_xfer_bulk(&ep->desc)) + err_count = 3; + /* HS bulk max packet should be 512, FS bulk supports 8, 16, 32 or 64 */ + if (usb_endpoint_xfer_bulk(&ep->desc)) { + if (udev->speed == USB_SPEED_HIGH) max_packet = 512; - /* bits 11:12 specify the number of additional transaction - * opportunities per microframe (USB 2.0, section 9.6.6) - */ - if (usb_endpoint_xfer_isoc(&ep->desc) || - usb_endpoint_xfer_int(&ep->desc)) { - max_burst = (usb_endpoint_maxp(&ep->desc) - & 0x1800) >> 11; + if (udev->speed == USB_SPEED_FULL) { + max_packet = rounddown_pow_of_two(max_packet); + max_packet = clamp_val(max_packet, 8, 64); } - break; - case USB_SPEED_FULL: - case USB_SPEED_LOW: - break; - default: - BUG(); } - ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet) | - MAX_BURST(max_burst)); - max_esit_payload = xhci_get_max_esit_payload(xhci, udev, ep); - ep_ctx->tx_info = cpu_to_le32(MAX_ESIT_PAYLOAD_FOR_EP(max_esit_payload)); + /* xHCI 1.0 and 1.1 indicates that ctrl ep avg TRB Length should be 8 */ + if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100) + avg_trb_len = 8; - /* - * XXX no idea how to calculate the average TRB buffer length for bulk - * endpoints, as the driver gives us no clue how big each scatter gather - * list entry (or buffer) is going to be. - * - * For isochronous and interrupt endpoints, we set it to the max - * available, until we have new API in the USB core to allow drivers to - * declare how much bandwidth they actually need. - * - * Normally, it would be calculated by taking the total of the buffer - * lengths in the TD and then dividing by the number of TRBs in a TD, - * including link TRBs, No-op TRBs, and Event data TRBs. Since we don't - * use Event Data TRBs, and we don't chain in a link TRB on short - * transfers, we're basically dividing by 1. - * - * xHCI 1.0 specification indicates that the Average TRB Length should - * be set to 8 for control endpoints. - */ - if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version == 0x100) - ep_ctx->tx_info |= cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(8)); - else - ep_ctx->tx_info |= - cpu_to_le32(AVG_TRB_LENGTH_FOR_EP(max_esit_payload)); + /* Set up the endpoint ring */ + virt_dev->eps[ep_index].new_ring = + xhci_ring_alloc(xhci, 2, ring_type, max_packet, mem_flags); + if (!virt_dev->eps[ep_index].new_ring) + return -ENOMEM; + + virt_dev->eps[ep_index].skip = false; + ep_ring = virt_dev->eps[ep_index].new_ring; + + /* Fill the endpoint context */ + ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) | + EP_INTERVAL(interval) | + EP_MULT(mult)); + ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) | + MAX_PACKET(max_packet) | + MAX_BURST(max_burst) | + ERROR_COUNT(err_count)); + ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma | + ep_ring->cycle_state); + + ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) | + EP_AVG_TRB_LENGTH(avg_trb_len)); - /* FIXME Debug endpoint context */ return 0; } @@ -1520,7 +1549,7 @@ void xhci_update_bw_info(struct xhci_hcd *xhci, unsigned int ep_type; int i; - for (i = 1; i < 31; ++i) { + for (i = 1; i < 31; i++) { bw_info = &virt_dev->eps[i].bw_info; /* We can't tell what endpoint type is being dropped, but @@ -1583,6 +1612,10 @@ void xhci_endpoint_copy(struct xhci_hcd *xhci, in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2; in_ep_ctx->deq = out_ep_ctx->deq; in_ep_ctx->tx_info = out_ep_ctx->tx_info; + if (xhci->quirks & XHCI_MTK_HOST) { + in_ep_ctx->reserved[0] = out_ep_ctx->reserved[0]; + in_ep_ctx->reserved[1] = out_ep_ctx->reserved[1]; + } } /* Copy output xhci_slot_ctx to the input xhci_slot_ctx. @@ -1610,62 +1643,55 @@ void xhci_slot_copy(struct xhci_hcd *xhci, static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) { int i; - struct device *dev = xhci_to_hcd(xhci)->self.controller; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); - xhci_dbg(xhci, "Allocating %d scratchpad buffers\n", num_sp); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Allocating %d scratchpad buffers", num_sp); if (!num_sp) return 0; - xhci->scratchpad = kzalloc(sizeof(*xhci->scratchpad), flags); + xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags, + dev_to_node(dev)); if (!xhci->scratchpad) goto fail_sp; xhci->scratchpad->sp_array = dma_alloc_coherent(dev, - num_sp * sizeof(u64), + array_size(sizeof(u64), num_sp), &xhci->scratchpad->sp_dma, flags); if (!xhci->scratchpad->sp_array) goto fail_sp2; - xhci->scratchpad->sp_buffers = kzalloc(sizeof(void *) * num_sp, flags); + xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *), + flags, dev_to_node(dev)); if (!xhci->scratchpad->sp_buffers) goto fail_sp3; - xhci->scratchpad->sp_dma_buffers = - kzalloc(sizeof(dma_addr_t) * num_sp, flags); - - if (!xhci->scratchpad->sp_dma_buffers) - goto fail_sp4; - xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); for (i = 0; i < num_sp; i++) { dma_addr_t dma; void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma, - flags); + flags); if (!buf) - goto fail_sp5; + goto fail_sp4; xhci->scratchpad->sp_array[i] = dma; xhci->scratchpad->sp_buffers[i] = buf; - xhci->scratchpad->sp_dma_buffers[i] = dma; } return 0; - fail_sp5: - for (i = i - 1; i >= 0; i--) { + fail_sp4: + while (i--) dma_free_coherent(dev, xhci->page_size, xhci->scratchpad->sp_buffers[i], - xhci->scratchpad->sp_dma_buffers[i]); - } - kfree(xhci->scratchpad->sp_dma_buffers); + xhci->scratchpad->sp_array[i]); - fail_sp4: kfree(xhci->scratchpad->sp_buffers); fail_sp3: - dma_free_coherent(dev, num_sp * sizeof(u64), + dma_free_coherent(dev, array_size(sizeof(u64), num_sp), xhci->scratchpad->sp_array, xhci->scratchpad->sp_dma); @@ -1681,7 +1707,7 @@ static void scratchpad_free(struct xhci_hcd *xhci) { int num_sp; int i; - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; if (!xhci->scratchpad) return; @@ -1689,13 +1715,12 @@ static void scratchpad_free(struct xhci_hcd *xhci) num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); for (i = 0; i < num_sp; i++) { - dma_free_coherent(&pdev->dev, xhci->page_size, + dma_free_coherent(dev, xhci->page_size, xhci->scratchpad->sp_buffers[i], - xhci->scratchpad->sp_dma_buffers[i]); + xhci->scratchpad->sp_array[i]); } - kfree(xhci->scratchpad->sp_dma_buffers); kfree(xhci->scratchpad->sp_buffers); - dma_free_coherent(&pdev->dev, num_sp * sizeof(u64), + dma_free_coherent(dev, array_size(sizeof(u64), num_sp), xhci->scratchpad->sp_array, xhci->scratchpad->sp_dma); kfree(xhci->scratchpad); @@ -1703,30 +1728,20 @@ static void scratchpad_free(struct xhci_hcd *xhci) } struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, - bool allocate_in_ctx, bool allocate_completion, - gfp_t mem_flags) + bool allocate_completion, gfp_t mem_flags) { struct xhci_command *command; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - command = kzalloc(sizeof(*command), mem_flags); + command = kzalloc_node(sizeof(*command), mem_flags, dev_to_node(dev)); if (!command) return NULL; - if (allocate_in_ctx) { - command->in_ctx = - xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, - mem_flags); - if (!command->in_ctx) { - kfree(command); - return NULL; - } - } - if (allocate_completion) { command->completion = - kzalloc(sizeof(struct completion), mem_flags); + kzalloc_node(sizeof(struct completion), mem_flags, + dev_to_node(dev)); if (!command->completion) { - xhci_free_container_ctx(xhci, command->in_ctx); kfree(command); return NULL; } @@ -1734,16 +1749,34 @@ struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, } command->status = 0; + /* set default timeout to 5000 ms */ + command->timeout_ms = XHCI_CMD_DEFAULT_TIMEOUT; INIT_LIST_HEAD(&command->cmd_list); return command; } -void xhci_urb_free_priv(struct xhci_hcd *xhci, struct urb_priv *urb_priv) +struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci, + bool allocate_completion, gfp_t mem_flags) { - if (urb_priv) { - kfree(urb_priv->td[0]); - kfree(urb_priv); + struct xhci_command *command; + + command = xhci_alloc_command(xhci, allocate_completion, mem_flags); + if (!command) + return NULL; + + command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, + mem_flags); + if (!command->in_ctx) { + kfree(command->completion); + kfree(command); + return NULL; } + return command; +} + +void xhci_urb_free_priv(struct urb_priv *urb_priv) +{ + kfree(urb_priv); } void xhci_free_command(struct xhci_hcd *xhci, @@ -1755,91 +1788,181 @@ void xhci_free_command(struct xhci_hcd *xhci, kfree(command); } +static int xhci_alloc_erst(struct xhci_hcd *xhci, + struct xhci_ring *evt_ring, + struct xhci_erst *erst, + gfp_t flags) +{ + size_t size; + unsigned int val; + struct xhci_segment *seg; + struct xhci_erst_entry *entry; + + size = array_size(sizeof(struct xhci_erst_entry), evt_ring->num_segs); + erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev, + size, &erst->erst_dma_addr, flags); + if (!erst->entries) + return -ENOMEM; + + erst->num_entries = evt_ring->num_segs; + + seg = evt_ring->first_seg; + for (val = 0; val < evt_ring->num_segs; val++) { + entry = &erst->entries[val]; + entry->seg_addr = cpu_to_le64(seg->dma); + entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT); + entry->rsvd = 0; + seg = seg->next; + } + + return 0; +} + +static void +xhci_remove_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir) +{ + u32 tmp; + + if (!ir) + return; + + /* + * Clean out interrupter registers except ERSTBA. Clearing either the + * low or high 32 bits of ERSTBA immediately causes the controller to + * dereference the partially cleared 64 bit address, causing IOMMU error. + */ + if (ir->ir_set) { + tmp = readl(&ir->ir_set->erst_size); + tmp &= ~ERST_SIZE_MASK; + writel(tmp, &ir->ir_set->erst_size); + + xhci_update_erst_dequeue(xhci, ir, true); + } +} + +static void +xhci_free_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir) +{ + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + size_t erst_size; + + if (!ir) + return; + + erst_size = array_size(sizeof(struct xhci_erst_entry), ir->erst.num_entries); + if (ir->erst.entries) + dma_free_coherent(dev, erst_size, + ir->erst.entries, + ir->erst.erst_dma_addr); + ir->erst.entries = NULL; + + /* free interrupter event ring */ + if (ir->event_ring) + xhci_ring_free(xhci, ir->event_ring); + + ir->event_ring = NULL; + + kfree(ir); +} + +void xhci_remove_secondary_interrupter(struct usb_hcd *hcd, struct xhci_interrupter *ir) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + unsigned int intr_num; + + spin_lock_irq(&xhci->lock); + + /* interrupter 0 is primary interrupter, don't touch it */ + if (!ir || !ir->intr_num || ir->intr_num >= xhci->max_interrupters) { + xhci_dbg(xhci, "Invalid secondary interrupter, can't remove\n"); + spin_unlock_irq(&xhci->lock); + return; + } + + /* + * Cleanup secondary interrupter to ensure there are no pending events. + * This also updates event ring dequeue pointer back to the start. + */ + xhci_skip_sec_intr_events(xhci, ir->event_ring, ir); + intr_num = ir->intr_num; + + xhci_remove_interrupter(xhci, ir); + xhci->interrupters[intr_num] = NULL; + + spin_unlock_irq(&xhci->lock); + + xhci_free_interrupter(xhci, ir); +} +EXPORT_SYMBOL_GPL(xhci_remove_secondary_interrupter); + void xhci_mem_cleanup(struct xhci_hcd *xhci) { - struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller); - struct dev_info *dev_info, *next; - struct xhci_cd *cur_cd, *next_cd; - unsigned long flags; - int size; - int i, j, num_ports; - - /* Free the Event Ring Segment Table and the actual Event Ring */ - size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries); - if (xhci->erst.entries) - dma_free_coherent(&pdev->dev, size, - xhci->erst.entries, xhci->erst.erst_dma_addr); - xhci->erst.entries = NULL; - xhci_dbg(xhci, "Freed ERST\n"); - if (xhci->event_ring) - xhci_ring_free(xhci, xhci->event_ring); - xhci->event_ring = NULL; - xhci_dbg(xhci, "Freed event ring\n"); - - if (xhci->lpm_command) - xhci_free_command(xhci, xhci->lpm_command); - xhci->cmd_ring_reserved_trbs = 0; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + int i, j; + + cancel_delayed_work_sync(&xhci->cmd_timer); + + for (i = 0; xhci->interrupters && i < xhci->max_interrupters; i++) { + if (xhci->interrupters[i]) { + xhci_remove_interrupter(xhci, xhci->interrupters[i]); + xhci_free_interrupter(xhci, xhci->interrupters[i]); + xhci->interrupters[i] = NULL; + } + } + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed interrupters"); + if (xhci->cmd_ring) xhci_ring_free(xhci, xhci->cmd_ring); xhci->cmd_ring = NULL; - xhci_dbg(xhci, "Freed command ring\n"); - list_for_each_entry_safe(cur_cd, next_cd, - &xhci->cancel_cmd_list, cancel_cmd_list) { - list_del(&cur_cd->cancel_cmd_list); - kfree(cur_cd); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); + xhci_cleanup_command_queue(xhci); + + for (i = 0; i < xhci->max_ports && xhci->rh_bw; i++) { + struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; + for (j = 0; j < XHCI_MAX_INTERVAL; j++) { + struct list_head *ep = &bwt->interval_bw[j].endpoints; + while (!list_empty(ep)) + list_del_init(ep->next); + } } - for (i = 1; i < MAX_HC_SLOTS; ++i) - xhci_free_virt_device(xhci, i); + for (i = xhci->max_slots; i > 0; i--) + xhci_free_virt_devices_depth_first(xhci, i); - if (xhci->segment_pool) - dma_pool_destroy(xhci->segment_pool); + dma_pool_destroy(xhci->segment_pool); xhci->segment_pool = NULL; - xhci_dbg(xhci, "Freed segment pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); - if (xhci->device_pool) - dma_pool_destroy(xhci->device_pool); + dma_pool_destroy(xhci->device_pool); xhci->device_pool = NULL; - xhci_dbg(xhci, "Freed device context pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); - if (xhci->small_streams_pool) - dma_pool_destroy(xhci->small_streams_pool); + dma_pool_destroy(xhci->small_streams_pool); xhci->small_streams_pool = NULL; - xhci_dbg(xhci, "Freed small stream array pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Freed small stream array pool"); + + dma_pool_destroy(xhci->port_bw_pool); + xhci->port_bw_pool = NULL; + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Freed xhci port bw array pool"); - if (xhci->medium_streams_pool) - dma_pool_destroy(xhci->medium_streams_pool); + dma_pool_destroy(xhci->medium_streams_pool); xhci->medium_streams_pool = NULL; - xhci_dbg(xhci, "Freed medium stream array pool\n"); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Freed medium stream array pool"); if (xhci->dcbaa) - dma_free_coherent(&pdev->dev, sizeof(*xhci->dcbaa), + dma_free_coherent(dev, sizeof(*xhci->dcbaa), xhci->dcbaa, xhci->dcbaa->dma); xhci->dcbaa = NULL; scratchpad_free(xhci); - spin_lock_irqsave(&xhci->lock, flags); - list_for_each_entry_safe(dev_info, next, &xhci->lpm_failed_devs, list) { - list_del(&dev_info->list); - kfree(dev_info); - } - spin_unlock_irqrestore(&xhci->lock, flags); - if (!xhci->rh_bw) goto no_bw; - num_ports = HCS_MAX_PORTS(xhci->hcs_params1); - for (i = 0; i < num_ports; i++) { - struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; - for (j = 0; j < XHCI_MAX_INTERVAL; j++) { - struct list_head *ep = &bwt->interval_bw[j].endpoints; - while (!list_empty(ep)) - list_del_init(ep->next); - } - } - - for (i = 0; i < num_ports; i++) { + for (i = 0; i < xhci->max_ports; i++) { struct xhci_tt_bw_info *tt, *n; list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { list_del(&tt->tt_list); @@ -1848,278 +1971,210 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) } no_bw: - xhci->num_usb2_ports = 0; - xhci->num_usb3_ports = 0; + xhci->cmd_ring_reserved_trbs = 0; + xhci->usb2_rhub.num_ports = 0; + xhci->usb3_rhub.num_ports = 0; xhci->num_active_eps = 0; - kfree(xhci->usb2_ports); - kfree(xhci->usb3_ports); - kfree(xhci->port_array); + kfree(xhci->usb2_rhub.ports); + kfree(xhci->usb3_rhub.ports); + kfree(xhci->hw_ports); kfree(xhci->rh_bw); - kfree(xhci->ext_caps); + for (i = 0; i < xhci->num_port_caps; i++) + kfree(xhci->port_caps[i].psi); + kfree(xhci->port_caps); + kfree(xhci->interrupters); + xhci->num_port_caps = 0; + + xhci->usb2_rhub.ports = NULL; + xhci->usb3_rhub.ports = NULL; + xhci->hw_ports = NULL; + xhci->rh_bw = NULL; + xhci->port_caps = NULL; + xhci->interrupters = NULL; xhci->page_size = 0; - xhci->page_shift = 0; - xhci->bus_state[0].bus_suspended = 0; - xhci->bus_state[1].bus_suspended = 0; + xhci->usb2_rhub.bus_state.bus_suspended = 0; + xhci->usb3_rhub.bus_state.bus_suspended = 0; } -static int xhci_test_trb_in_td(struct xhci_hcd *xhci, - struct xhci_segment *input_seg, - union xhci_trb *start_trb, - union xhci_trb *end_trb, - dma_addr_t input_dma, - struct xhci_segment *result_seg, - char *test_name, int test_number) +static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_interrupter *ir) { - unsigned long long start_dma; - unsigned long long end_dma; - struct xhci_segment *seg; - - start_dma = xhci_trb_virt_to_dma(input_seg, start_trb); - end_dma = xhci_trb_virt_to_dma(input_seg, end_trb); - - seg = trb_in_td(input_seg, start_trb, end_trb, input_dma); - if (seg != result_seg) { - xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n", - test_name, test_number); - xhci_warn(xhci, "Tested TRB math w/ seg %p and " - "input DMA 0x%llx\n", - input_seg, - (unsigned long long) input_dma); - xhci_warn(xhci, "starting TRB %p (0x%llx DMA), " - "ending TRB %p (0x%llx DMA)\n", - start_trb, start_dma, - end_trb, end_dma); - xhci_warn(xhci, "Expected seg %p, got seg %p\n", - result_seg, seg); - return -1; - } - return 0; -} - -/* TRB math checks for xhci_trb_in_td(), using the command and event rings. */ -static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci, gfp_t mem_flags) -{ - struct { - dma_addr_t input_dma; - struct xhci_segment *result_seg; - } simple_test_vector [] = { - /* A zeroed DMA field should fail */ - { 0, NULL }, - /* One TRB before the ring start should fail */ - { xhci->event_ring->first_seg->dma - 16, NULL }, - /* One byte before the ring start should fail */ - { xhci->event_ring->first_seg->dma - 1, NULL }, - /* Starting TRB should succeed */ - { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg }, - /* Ending TRB should succeed */ - { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16, - xhci->event_ring->first_seg }, - /* One byte after the ring end should fail */ - { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL }, - /* One TRB after the ring end should fail */ - { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL }, - /* An address of all ones should fail */ - { (dma_addr_t) (~0), NULL }, - }; - struct { - struct xhci_segment *input_seg; - union xhci_trb *start_trb; - union xhci_trb *end_trb; - dma_addr_t input_dma; - struct xhci_segment *result_seg; - } complex_test_vector [] = { - /* Test feeding a valid DMA address from a different ring */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = xhci->event_ring->first_seg->trbs, - .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], - .input_dma = xhci->cmd_ring->first_seg->dma, - .result_seg = NULL, - }, - /* Test feeding a valid end TRB from a different ring */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = xhci->event_ring->first_seg->trbs, - .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], - .input_dma = xhci->cmd_ring->first_seg->dma, - .result_seg = NULL, - }, - /* Test feeding a valid start and end TRB from a different ring */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = xhci->cmd_ring->first_seg->trbs, - .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], - .input_dma = xhci->cmd_ring->first_seg->dma, - .result_seg = NULL, - }, - /* TRB in this ring, but after this TD */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = &xhci->event_ring->first_seg->trbs[0], - .end_trb = &xhci->event_ring->first_seg->trbs[3], - .input_dma = xhci->event_ring->first_seg->dma + 4*16, - .result_seg = NULL, - }, - /* TRB in this ring, but before this TD */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = &xhci->event_ring->first_seg->trbs[3], - .end_trb = &xhci->event_ring->first_seg->trbs[6], - .input_dma = xhci->event_ring->first_seg->dma + 2*16, - .result_seg = NULL, - }, - /* TRB in this ring, but after this wrapped TD */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], - .end_trb = &xhci->event_ring->first_seg->trbs[1], - .input_dma = xhci->event_ring->first_seg->dma + 2*16, - .result_seg = NULL, - }, - /* TRB in this ring, but before this wrapped TD */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], - .end_trb = &xhci->event_ring->first_seg->trbs[1], - .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16, - .result_seg = NULL, - }, - /* TRB not in this ring, and we have a wrapped TD */ - { .input_seg = xhci->event_ring->first_seg, - .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], - .end_trb = &xhci->event_ring->first_seg->trbs[1], - .input_dma = xhci->cmd_ring->first_seg->dma + 2*16, - .result_seg = NULL, - }, - }; - - unsigned int num_tests; - int i, ret; - - num_tests = ARRAY_SIZE(simple_test_vector); - for (i = 0; i < num_tests; i++) { - ret = xhci_test_trb_in_td(xhci, - xhci->event_ring->first_seg, - xhci->event_ring->first_seg->trbs, - &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], - simple_test_vector[i].input_dma, - simple_test_vector[i].result_seg, - "Simple", i); - if (ret < 0) - return ret; - } - - num_tests = ARRAY_SIZE(complex_test_vector); - for (i = 0; i < num_tests; i++) { - ret = xhci_test_trb_in_td(xhci, - complex_test_vector[i].input_seg, - complex_test_vector[i].start_trb, - complex_test_vector[i].end_trb, - complex_test_vector[i].input_dma, - complex_test_vector[i].result_seg, - "Complex", i); - if (ret < 0) - return ret; - } - xhci_dbg(xhci, "TRB math tests passed.\n"); - return 0; -} - -static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) -{ - u64 temp; dma_addr_t deq; - deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, - xhci->event_ring->dequeue); - if (deq == 0 && !in_interrupt()) - xhci_warn(xhci, "WARN something wrong with SW event ring " - "dequeue ptr.\n"); + deq = xhci_trb_virt_to_dma(ir->event_ring->deq_seg, + ir->event_ring->dequeue); + if (!deq) + xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr.\n"); /* Update HC event ring dequeue pointer */ - temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); - temp &= ERST_PTR_MASK; /* Don't clear the EHB bit (which is RW1C) because * there might be more events to service. */ - temp &= ~ERST_EHB; - xhci_dbg(xhci, "// Write event ring dequeue pointer, " - "preserving EHB bit\n"); - xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, - &xhci->ir_set->erst_dequeue); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "// Write event ring dequeue pointer, preserving EHB bit"); + xhci_write_64(xhci, deq & ERST_PTR_MASK, &ir->ir_set->erst_dequeue); } static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, - __le32 __iomem *addr, u8 major_revision, int max_caps) + __le32 __iomem *addr, int max_caps) { u32 temp, port_offset, port_count; int i; + u8 major_revision, minor_revision, tmp_minor_revision; + struct xhci_hub *rhub; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + struct xhci_port_cap *port_cap; + + temp = readl(addr); + major_revision = XHCI_EXT_PORT_MAJOR(temp); + minor_revision = XHCI_EXT_PORT_MINOR(temp); - if (major_revision > 0x03) { - xhci_warn(xhci, "Ignoring unknown port speed, " - "Ext Cap %p, revision = 0x%x\n", + if (major_revision == 0x03) { + rhub = &xhci->usb3_rhub; + /* + * Some hosts incorrectly use sub-minor version for minor + * version (i.e. 0x02 instead of 0x20 for bcdUSB 0x320 and 0x01 + * for bcdUSB 0x310). Since there is no USB release with sub + * minor version 0x301 to 0x309, we can assume that they are + * incorrect and fix it here. + */ + if (minor_revision > 0x00 && minor_revision < 0x10) + minor_revision <<= 4; + /* + * Some zhaoxin's xHCI controller that follow usb3.1 spec + * but only support Gen1. + */ + if (xhci->quirks & XHCI_ZHAOXIN_HOST) { + tmp_minor_revision = minor_revision; + minor_revision = 0; + } + + } else if (major_revision <= 0x02) { + rhub = &xhci->usb2_rhub; + } else { + xhci_warn(xhci, "Ignoring unknown port speed, Ext Cap %p, revision = 0x%x\n", addr, major_revision); /* Ignoring port protocol we can't understand. FIXME */ return; } /* Port offset and count in the third dword, see section 7.2 */ - temp = xhci_readl(xhci, addr + 2); + temp = readl(addr + 2); port_offset = XHCI_EXT_PORT_OFF(temp); port_count = XHCI_EXT_PORT_COUNT(temp); - xhci_dbg(xhci, "Ext Cap %p, port offset = %u, " - "count = %u, revision = 0x%x\n", - addr, port_offset, port_count, major_revision); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Ext Cap %p, port offset = %u, count = %u, revision = 0x%x", + addr, port_offset, port_count, major_revision); /* Port count includes the current port offset */ if (port_offset == 0 || (port_offset + port_count - 1) > num_ports) /* WTF? "Valid values are ‘1’ to MaxPorts" */ return; - /* cache usb2 port capabilities */ - if (major_revision < 0x03 && xhci->num_ext_caps < max_caps) - xhci->ext_caps[xhci->num_ext_caps++] = temp; + port_cap = &xhci->port_caps[xhci->num_port_caps++]; + if (xhci->num_port_caps > max_caps) + return; - /* Check the host's USB2 LPM capability */ - if ((xhci->hci_version == 0x96) && (major_revision != 0x03) && - (temp & XHCI_L1C)) { - xhci_dbg(xhci, "xHCI 0.96: support USB2 software lpm\n"); - xhci->sw_lpm_support = 1; - } + port_cap->psi_count = XHCI_EXT_PORT_PSIC(temp); + + if (port_cap->psi_count) { + port_cap->psi = kcalloc_node(port_cap->psi_count, + sizeof(*port_cap->psi), + GFP_KERNEL, dev_to_node(dev)); + if (!port_cap->psi) + port_cap->psi_count = 0; - if ((xhci->hci_version >= 0x100) && (major_revision != 0x03)) { - xhci_dbg(xhci, "xHCI 1.0: support USB2 software lpm\n"); - xhci->sw_lpm_support = 1; - if (temp & XHCI_HLC) { - xhci_dbg(xhci, "xHCI 1.0: support USB2 hardware lpm\n"); - xhci->hw_lpm_support = 1; + port_cap->psi_uid_count++; + for (i = 0; i < port_cap->psi_count; i++) { + port_cap->psi[i] = readl(addr + 4 + i); + + /* count unique ID values, two consecutive entries can + * have the same ID if link is assymetric + */ + if (i && (XHCI_EXT_PORT_PSIV(port_cap->psi[i]) != + XHCI_EXT_PORT_PSIV(port_cap->psi[i - 1]))) + port_cap->psi_uid_count++; + + if (xhci->quirks & XHCI_ZHAOXIN_HOST && + major_revision == 0x03 && + XHCI_EXT_PORT_PSIV(port_cap->psi[i]) >= 5) + minor_revision = tmp_minor_revision; + + xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n", + XHCI_EXT_PORT_PSIV(port_cap->psi[i]), + XHCI_EXT_PORT_PSIE(port_cap->psi[i]), + XHCI_EXT_PORT_PLT(port_cap->psi[i]), + XHCI_EXT_PORT_PFD(port_cap->psi[i]), + XHCI_EXT_PORT_LP(port_cap->psi[i]), + XHCI_EXT_PORT_PSIM(port_cap->psi[i])); } } + rhub->maj_rev = major_revision; + + if (rhub->min_rev < minor_revision) + rhub->min_rev = minor_revision; + + port_cap->maj_rev = major_revision; + port_cap->min_rev = minor_revision; + port_cap->protocol_caps = temp; + + if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) && + (temp & XHCI_HLC)) { + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "xHCI 1.0: support USB2 hardware lpm"); + xhci->hw_lpm_support = 1; + } + port_offset--; for (i = port_offset; i < (port_offset + port_count); i++) { + struct xhci_port *hw_port = &xhci->hw_ports[i]; /* Duplicate entry. Ignore the port if the revisions differ. */ - if (xhci->port_array[i] != 0) { - xhci_warn(xhci, "Duplicate port entry, Ext Cap %p," - " port %u\n", addr, i); - xhci_warn(xhci, "Port was marked as USB %u, " - "duplicated as USB %u\n", - xhci->port_array[i], major_revision); + if (hw_port->rhub) { + xhci_warn(xhci, "Duplicate port entry, Ext Cap %p, port %u\n", addr, i); + xhci_warn(xhci, "Port was marked as USB %u, duplicated as USB %u\n", + hw_port->rhub->maj_rev, major_revision); /* Only adjust the roothub port counts if we haven't * found a similar duplicate. */ - if (xhci->port_array[i] != major_revision && - xhci->port_array[i] != DUPLICATE_ENTRY) { - if (xhci->port_array[i] == 0x03) - xhci->num_usb3_ports--; - else - xhci->num_usb2_ports--; - xhci->port_array[i] = DUPLICATE_ENTRY; + if (hw_port->rhub != rhub && + hw_port->hcd_portnum != DUPLICATE_ENTRY) { + hw_port->rhub->num_ports--; + hw_port->hcd_portnum = DUPLICATE_ENTRY; } - /* FIXME: Should we disable the port? */ continue; } - xhci->port_array[i] = major_revision; - if (major_revision == 0x03) - xhci->num_usb3_ports++; - else - xhci->num_usb2_ports++; + hw_port->rhub = rhub; + hw_port->port_cap = port_cap; + rhub->num_ports++; } /* FIXME: Should we disable ports not in the Extended Capabilities? */ } +static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, + struct xhci_hub *rhub, gfp_t flags) +{ + int port_index = 0; + int i; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + + if (!rhub->num_ports) + return; + rhub->ports = kcalloc_node(rhub->num_ports, sizeof(*rhub->ports), + flags, dev_to_node(dev)); + if (!rhub->ports) + return; + + for (i = 0; i < xhci->max_ports; i++) { + if (xhci->hw_ports[i].rhub != rhub || + xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) + continue; + xhci->hw_ports[i].hcd_portnum = port_index; + rhub->ports[port_index] = &xhci->hw_ports[i]; + port_index++; + if (port_index == rhub->num_ports) + break; + } +} + /* * Scan the Extended Capabilities for the "Supported Protocol Capabilities" that * specify what speeds each port is supposed to be. We can't count on the port @@ -2129,29 +2184,30 @@ static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, */ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) { - __le32 __iomem *addr, *tmp_addr; - u32 offset, tmp_offset; - unsigned int num_ports; - int i, j, port_index; + void __iomem *base; + u32 offset; + int i, j; int cap_count = 0; + u32 cap_start; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - addr = &xhci->cap_regs->hcc_params; - offset = XHCI_HCC_EXT_CAPS(xhci_readl(xhci, addr)); - if (offset == 0) { - xhci_err(xhci, "No Extended Capability registers, " - "unable to set up roothub.\n"); - return -ENODEV; - } - - num_ports = HCS_MAX_PORTS(xhci->hcs_params1); - xhci->port_array = kzalloc(sizeof(*xhci->port_array)*num_ports, flags); - if (!xhci->port_array) + xhci->hw_ports = kcalloc_node(xhci->max_ports, sizeof(*xhci->hw_ports), + flags, dev_to_node(dev)); + if (!xhci->hw_ports) return -ENOMEM; - xhci->rh_bw = kzalloc(sizeof(*xhci->rh_bw)*num_ports, flags); + for (i = 0; i < xhci->max_ports; i++) { + xhci->hw_ports[i].port_reg = &xhci->op_regs->port_regs[i]; + xhci->hw_ports[i].hw_portnum = i; + + init_completion(&xhci->hw_ports[i].rexit_done); + init_completion(&xhci->hw_ports[i].u3exit_done); + } + + xhci->rh_bw = kcalloc_node(xhci->max_ports, sizeof(*xhci->rh_bw), flags, dev_to_node(dev)); if (!xhci->rh_bw) return -ENOMEM; - for (i = 0; i < num_ports; i++) { + for (i = 0; i < xhci->max_ports; i++) { struct xhci_interval_bw_table *bw_table; INIT_LIST_HEAD(&xhci->rh_bw[i].tts); @@ -2159,337 +2215,297 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) for (j = 0; j < XHCI_MAX_INTERVAL; j++) INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints); } + base = &xhci->cap_regs->hc_capbase; - /* - * For whatever reason, the first capability offset is from the - * capability register base, not from the HCCPARAMS register. - * See section 5.3.6 for offset calculation. - */ - addr = &xhci->cap_regs->hc_capbase + offset; - - tmp_addr = addr; - tmp_offset = offset; + cap_start = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_PROTOCOL); + if (!cap_start) { + xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n"); + return -ENODEV; + } + offset = cap_start; /* count extended protocol capability entries for later caching */ - do { - u32 cap_id; - cap_id = xhci_readl(xhci, tmp_addr); - if (XHCI_EXT_CAPS_ID(cap_id) == XHCI_EXT_CAPS_PROTOCOL) - cap_count++; - tmp_offset = XHCI_EXT_CAPS_NEXT(cap_id); - tmp_addr += tmp_offset; - } while (tmp_offset); - - xhci->ext_caps = kzalloc(sizeof(*xhci->ext_caps) * cap_count, flags); - if (!xhci->ext_caps) + while (offset) { + cap_count++; + offset = xhci_find_next_ext_cap(base, offset, + XHCI_EXT_CAPS_PROTOCOL); + } + + xhci->port_caps = kcalloc_node(cap_count, sizeof(*xhci->port_caps), + flags, dev_to_node(dev)); + if (!xhci->port_caps) return -ENOMEM; - while (1) { - u32 cap_id; - - cap_id = xhci_readl(xhci, addr); - if (XHCI_EXT_CAPS_ID(cap_id) == XHCI_EXT_CAPS_PROTOCOL) - xhci_add_in_port(xhci, num_ports, addr, - (u8) XHCI_EXT_PORT_MAJOR(cap_id), - cap_count); - offset = XHCI_EXT_CAPS_NEXT(cap_id); - if (!offset || (xhci->num_usb2_ports + xhci->num_usb3_ports) - == num_ports) + offset = cap_start; + + while (offset) { + xhci_add_in_port(xhci, xhci->max_ports, base + offset, cap_count); + if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == xhci->max_ports) break; - /* - * Once you're into the Extended Capabilities, the offset is - * always relative to the register holding the offset. - */ - addr += offset; + offset = xhci_find_next_ext_cap(base, offset, + XHCI_EXT_CAPS_PROTOCOL); } - - if (xhci->num_usb2_ports == 0 && xhci->num_usb3_ports == 0) { + if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) { xhci_warn(xhci, "No ports on the roothubs?\n"); return -ENODEV; } - xhci_dbg(xhci, "Found %u USB 2.0 ports and %u USB 3.0 ports.\n", - xhci->num_usb2_ports, xhci->num_usb3_ports); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Found %u USB 2.0 ports and %u USB 3.0 ports.", + xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports); /* Place limits on the number of roothub ports so that the hub * descriptors aren't longer than the USB core will allocate. */ - if (xhci->num_usb3_ports > 15) { - xhci_dbg(xhci, "Limiting USB 3.0 roothub ports to 15.\n"); - xhci->num_usb3_ports = 15; + if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) { + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Limiting USB 3.0 roothub ports to %u.", + USB_SS_MAXPORTS); + xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS; } - if (xhci->num_usb2_ports > USB_MAXCHILDREN) { - xhci_dbg(xhci, "Limiting USB 2.0 roothub ports to %u.\n", + if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) { + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Limiting USB 2.0 roothub ports to %u.", USB_MAXCHILDREN); - xhci->num_usb2_ports = USB_MAXCHILDREN; + xhci->usb2_rhub.num_ports = USB_MAXCHILDREN; } - /* - * Note we could have all USB 3.0 ports, or all USB 2.0 ports. - * Not sure how the USB core will handle a hub with no ports... - */ - if (xhci->num_usb2_ports) { - xhci->usb2_ports = kmalloc(sizeof(*xhci->usb2_ports)* - xhci->num_usb2_ports, flags); - if (!xhci->usb2_ports) - return -ENOMEM; - - port_index = 0; - for (i = 0; i < num_ports; i++) { - if (xhci->port_array[i] == 0x03 || - xhci->port_array[i] == 0 || - xhci->port_array[i] == DUPLICATE_ENTRY) - continue; + if (!xhci->usb2_rhub.num_ports) + xhci_info(xhci, "USB2 root hub has no ports\n"); + + if (!xhci->usb3_rhub.num_ports) + xhci_info(xhci, "USB3 root hub has no ports\n"); + + xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags); + xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags); - xhci->usb2_ports[port_index] = - &xhci->op_regs->port_status_base + - NUM_PORT_REGS*i; - xhci_dbg(xhci, "USB 2.0 port at index %u, " - "addr = %p\n", i, - xhci->usb2_ports[port_index]); - port_index++; - if (port_index == xhci->num_usb2_ports) - break; - } - } - if (xhci->num_usb3_ports) { - xhci->usb3_ports = kmalloc(sizeof(*xhci->usb3_ports)* - xhci->num_usb3_ports, flags); - if (!xhci->usb3_ports) - return -ENOMEM; - - port_index = 0; - for (i = 0; i < num_ports; i++) - if (xhci->port_array[i] == 0x03) { - xhci->usb3_ports[port_index] = - &xhci->op_regs->port_status_base + - NUM_PORT_REGS*i; - xhci_dbg(xhci, "USB 3.0 port at index %u, " - "addr = %p\n", i, - xhci->usb3_ports[port_index]); - port_index++; - if (port_index == xhci->num_usb3_ports) - break; - } - } return 0; } -int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) +static struct xhci_interrupter * +xhci_alloc_interrupter(struct xhci_hcd *xhci, unsigned int segs, gfp_t flags) { - dma_addr_t dma; - struct device *dev = xhci_to_hcd(xhci)->self.controller; - unsigned int val, val2; - u64 val_64; - struct xhci_segment *seg; - u32 page_size, temp; - int i; + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + struct xhci_interrupter *ir; + unsigned int max_segs; + int ret; - INIT_LIST_HEAD(&xhci->lpm_failed_devs); - INIT_LIST_HEAD(&xhci->cancel_cmd_list); + if (!segs) + segs = ERST_DEFAULT_SEGS; - page_size = xhci_readl(xhci, &xhci->op_regs->page_size); - xhci_dbg(xhci, "Supported page size register = 0x%x\n", page_size); - for (i = 0; i < 16; i++) { - if ((0x1 & page_size) != 0) - break; - page_size = page_size >> 1; + max_segs = BIT(HCS_ERST_MAX(xhci->hcs_params2)); + segs = min(segs, max_segs); + + ir = kzalloc_node(sizeof(*ir), flags, dev_to_node(dev)); + if (!ir) + return NULL; + + ir->event_ring = xhci_ring_alloc(xhci, segs, TYPE_EVENT, 0, flags); + if (!ir->event_ring) { + xhci_warn(xhci, "Failed to allocate interrupter event ring\n"); + kfree(ir); + return NULL; } - if (i < 16) - xhci_dbg(xhci, "Supported page size of %iK\n", (1 << (i+12)) / 1024); + + ret = xhci_alloc_erst(xhci, ir->event_ring, &ir->erst, flags); + if (ret) { + xhci_warn(xhci, "Failed to allocate interrupter erst\n"); + xhci_ring_free(xhci, ir->event_ring); + kfree(ir); + return NULL; + } + + return ir; +} + +void xhci_add_interrupter(struct xhci_hcd *xhci, unsigned int intr_num) +{ + struct xhci_interrupter *ir; + u64 erst_base; + u32 erst_size; + + ir = xhci->interrupters[intr_num]; + ir->intr_num = intr_num; + ir->ir_set = &xhci->run_regs->ir_set[intr_num]; + + /* set ERST count with the number of entries in the segment table */ + erst_size = readl(&ir->ir_set->erst_size); + erst_size &= ~ERST_SIZE_MASK; + erst_size |= ir->event_ring->num_segs; + writel(erst_size, &ir->ir_set->erst_size); + + erst_base = xhci_read_64(xhci, &ir->ir_set->erst_base); + erst_base &= ~ERST_BASE_ADDRESS_MASK; + erst_base |= ir->erst.erst_dma_addr & ERST_BASE_ADDRESS_MASK; + if (xhci->quirks & XHCI_WRITE_64_HI_LO) + hi_lo_writeq(erst_base, &ir->ir_set->erst_base); else - xhci_warn(xhci, "WARN: no supported page size\n"); - /* Use 4K pages, since that's common and the minimum the HC supports */ - xhci->page_shift = 12; - xhci->page_size = 1 << xhci->page_shift; - xhci_dbg(xhci, "HCD page size set to %iK\n", xhci->page_size / 1024); + xhci_write_64(xhci, erst_base, &ir->ir_set->erst_base); - /* - * Program the Number of Device Slots Enabled field in the CONFIG - * register with the max value of slots the HC can handle. - */ - val = HCS_MAX_SLOTS(xhci_readl(xhci, &xhci->cap_regs->hcs_params1)); - xhci_dbg(xhci, "// xHC can handle at most %d device slots.\n", - (unsigned int) val); - val2 = xhci_readl(xhci, &xhci->op_regs->config_reg); - val |= (val2 & ~HCS_SLOTS_MASK); - xhci_dbg(xhci, "// Setting Max device slots reg = 0x%x.\n", - (unsigned int) val); - xhci_writel(xhci, val, &xhci->op_regs->config_reg); + /* Set the event ring dequeue address of this interrupter */ + xhci_set_hc_event_deq(xhci, ir); +} + +struct xhci_interrupter * +xhci_create_secondary_interrupter(struct usb_hcd *hcd, unsigned int segs, + u32 imod_interval, unsigned int intr_num) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + struct xhci_interrupter *ir; + unsigned int i; + int err = -ENOSPC; + + if (!xhci->interrupters || xhci->max_interrupters <= 1 || + intr_num >= xhci->max_interrupters) + return NULL; + + ir = xhci_alloc_interrupter(xhci, segs, GFP_KERNEL); + if (!ir) + return NULL; + + spin_lock_irq(&xhci->lock); + if (!intr_num) { + /* Find available secondary interrupter, interrupter 0 is reserved for primary */ + for (i = 1; i < xhci->max_interrupters; i++) { + if (!xhci->interrupters[i]) { + xhci->interrupters[i] = ir; + xhci_add_interrupter(xhci, i); + err = 0; + break; + } + } + } else { + if (!xhci->interrupters[intr_num]) { + xhci->interrupters[intr_num] = ir; + xhci_add_interrupter(xhci, intr_num); + err = 0; + } + } + spin_unlock_irq(&xhci->lock); + + if (err) { + xhci_warn(xhci, "Failed to add secondary interrupter, max interrupters %d\n", + xhci->max_interrupters); + xhci_free_interrupter(xhci, ir); + return NULL; + } + + xhci_set_interrupter_moderation(ir, imod_interval); + + xhci_dbg(xhci, "Add secondary interrupter %d, max interrupters %d\n", + ir->intr_num, xhci->max_interrupters); + + return ir; +} +EXPORT_SYMBOL_GPL(xhci_create_secondary_interrupter); + +int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) +{ + struct device *dev = xhci_to_hcd(xhci)->self.sysdev; + dma_addr_t dma; /* - * Section 5.4.8 - doorbell array must be + * xHCI section 5.4.6 - Device Context array must be * "physically contiguous and 64-byte (cache line) aligned". */ - xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, - GFP_KERNEL); + xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, flags); if (!xhci->dcbaa) goto fail; - memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa)); + xhci->dcbaa->dma = dma; - xhci_dbg(xhci, "// Device context base array address = 0x%llx (DMA), %p (virt)\n", - (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa); - xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, + "Device context base array address = 0x%pad (DMA), %p (virt)", + &xhci->dcbaa->dma, xhci->dcbaa); /* * Initialize the ring segment pool. The ring must be a contiguous * structure comprised of TRBs. The TRBs must be 16 byte aligned, - * however, the command ring segment needs 64-byte aligned segments, - * so we pick the greater alignment need. + * however, the command ring segment needs 64-byte aligned segments + * and our use of dma addresses in the trb_address_map radix tree needs + * TRB_SEGMENT_SIZE alignment, so we pick the greater alignment need. */ - xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, - TRB_SEGMENT_SIZE, 64, xhci->page_size); + if (xhci->quirks & XHCI_TRB_OVERFETCH) + /* Buggy HC prefetches beyond segment bounds - allocate dummy space at the end */ + xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, + TRB_SEGMENT_SIZE * 2, TRB_SEGMENT_SIZE * 2, xhci->page_size * 2); + else + xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, + TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); + if (!xhci->segment_pool) + goto fail; /* See Table 46 and Note on Figure 55 */ - xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, - 2112, 64, xhci->page_size); - if (!xhci->segment_pool || !xhci->device_pool) + xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, 2112, 64, + xhci->page_size); + if (!xhci->device_pool) goto fail; - /* Linear stream context arrays don't have any boundary restrictions, + /* + * Linear stream context arrays don't have any boundary restrictions, * and only need to be 16-byte aligned. */ - xhci->small_streams_pool = - dma_pool_create("xHCI 256 byte stream ctx arrays", - dev, SMALL_STREAM_ARRAY_SIZE, 16, 0); - xhci->medium_streams_pool = - dma_pool_create("xHCI 1KB stream ctx arrays", - dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0); - /* Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE - * will be allocated with dma_alloc_coherent() + xhci->small_streams_pool = dma_pool_create("xHCI 256 byte stream ctx arrays", + dev, SMALL_STREAM_ARRAY_SIZE, 16, 0); + if (!xhci->small_streams_pool) + goto fail; + + /* + * Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE will be + * allocated with dma_alloc_coherent(). */ - if (!xhci->small_streams_pool || !xhci->medium_streams_pool) + xhci->medium_streams_pool = dma_pool_create("xHCI 1KB stream ctx arrays", + dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0); + if (!xhci->medium_streams_pool) + goto fail; + + /* + * refer to xhci rev1_2 protocol 5.3.3 max ports is 255. + * refer to xhci rev1_2 protocol 6.4.3.14 port bandwidth buffer need + * to be 16-byte aligned. + */ + xhci->port_bw_pool = dma_pool_create("xHCI 256 port bw ctx arrays", + dev, GET_PORT_BW_ARRAY_SIZE, 16, 0); + if (!xhci->port_bw_pool) goto fail; /* Set up the command ring to have one segments for now. */ - xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, flags); + xhci->cmd_ring = xhci_ring_alloc(xhci, 1, TYPE_COMMAND, 0, flags); if (!xhci->cmd_ring) goto fail; - xhci_dbg(xhci, "Allocated command ring at %p\n", xhci->cmd_ring); - xhci_dbg(xhci, "First segment DMA is 0x%llx\n", - (unsigned long long)xhci->cmd_ring->first_seg->dma); - - /* Set the address in the Command Ring Control register */ - val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); - val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) | - (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) | - xhci->cmd_ring->cycle_state; - xhci_dbg(xhci, "// Setting command ring address to 0x%x\n", val); - xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); - xhci_dbg_cmd_ptrs(xhci); - - xhci->lpm_command = xhci_alloc_command(xhci, true, true, flags); - if (!xhci->lpm_command) - goto fail; - /* Reserve one command ring TRB for disabling LPM. + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Allocated command ring at %p", xhci->cmd_ring); + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%pad", + &xhci->cmd_ring->first_seg->dma); + + /* + * Reserve one command ring TRB for disabling LPM. * Since the USB core grabs the shared usb_bus bandwidth mutex before * disabling LPM, we only need to reserve one TRB for all devices. */ xhci->cmd_ring_reserved_trbs++; - val = xhci_readl(xhci, &xhci->cap_regs->db_off); - val &= DBOFF_MASK; - xhci_dbg(xhci, "// Doorbell array is located at offset 0x%x" - " from cap regs base addr\n", val); - xhci->dba = (void __iomem *) xhci->cap_regs + val; - xhci_dbg_regs(xhci); - xhci_print_run_regs(xhci); - /* Set ir_set to interrupt register set 0 */ - xhci->ir_set = &xhci->run_regs->ir_set[0]; - - /* - * Event ring setup: Allocate a normal ring, but also setup - * the event ring segment table (ERST). Section 4.9.3. - */ - xhci_dbg(xhci, "// Allocating event ring\n"); - xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, - flags); - if (!xhci->event_ring) - goto fail; - if (xhci_check_trb_in_td_math(xhci, flags) < 0) + /* Allocate and set up primary interrupter 0 with an event ring. */ + xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Allocating primary event ring"); + xhci->interrupters = kcalloc_node(xhci->max_interrupters, sizeof(*xhci->interrupters), + flags, dev_to_node(dev)); + if (!xhci->interrupters) goto fail; - xhci->erst.entries = dma_alloc_coherent(dev, - sizeof(struct xhci_erst_entry) * ERST_NUM_SEGS, &dma, - GFP_KERNEL); - if (!xhci->erst.entries) + xhci->interrupters[0] = xhci_alloc_interrupter(xhci, 0, flags); + if (!xhci->interrupters[0]) goto fail; - xhci_dbg(xhci, "// Allocated event ring segment table at 0x%llx\n", - (unsigned long long)dma); - - memset(xhci->erst.entries, 0, sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS); - xhci->erst.num_entries = ERST_NUM_SEGS; - xhci->erst.erst_dma_addr = dma; - xhci_dbg(xhci, "Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx\n", - xhci->erst.num_entries, - xhci->erst.entries, - (unsigned long long)xhci->erst.erst_dma_addr); - - /* set ring base address and size for each segment table entry */ - for (val = 0, seg = xhci->event_ring->first_seg; val < ERST_NUM_SEGS; val++) { - struct xhci_erst_entry *entry = &xhci->erst.entries[val]; - entry->seg_addr = cpu_to_le64(seg->dma); - entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT); - entry->rsvd = 0; - seg = seg->next; - } - - /* set ERST count with the number of entries in the segment table */ - val = xhci_readl(xhci, &xhci->ir_set->erst_size); - val &= ERST_SIZE_MASK; - val |= ERST_NUM_SEGS; - xhci_dbg(xhci, "// Write ERST size = %i to ir_set 0 (some bits preserved)\n", - val); - xhci_writel(xhci, val, &xhci->ir_set->erst_size); - - xhci_dbg(xhci, "// Set ERST entries to point to event ring.\n"); - /* set the segment table base address */ - xhci_dbg(xhci, "// Set ERST base address for ir_set 0 = 0x%llx\n", - (unsigned long long)xhci->erst.erst_dma_addr); - val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); - val_64 &= ERST_PTR_MASK; - val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); - xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); - - /* Set the event ring dequeue address */ - xhci_set_hc_event_deq(xhci); - xhci_dbg(xhci, "Wrote ERST address to ir_set 0.\n"); - xhci_print_ir_set(xhci, 0); - - /* - * XXX: Might need to set the Interrupter Moderation Register to - * something other than the default (~1ms minimum between interrupts). - * See section 5.5.1.2. - */ - init_completion(&xhci->addr_dev); - for (i = 0; i < MAX_HC_SLOTS; ++i) - xhci->devs[i] = NULL; - for (i = 0; i < USB_MAXCHILDREN; ++i) { - xhci->bus_state[0].resume_done[i] = 0; - xhci->bus_state[1].resume_done[i] = 0; - } if (scratchpad_alloc(xhci, flags)) goto fail; + if (xhci_setup_port_arrays(xhci, flags)) goto fail; - /* Enable USB 3.0 device notifications for function remote wake, which - * is necessary for allowing USB 3.0 devices to do remote wakeup from - * U3 (device suspend). - */ - temp = xhci_readl(xhci, &xhci->op_regs->dev_notification); - temp &= ~DEV_NOTE_MASK; - temp |= DEV_NOTE_FWAKE; - xhci_writel(xhci, temp, &xhci->op_regs->dev_notification); - return 0; fail: - xhci_warn(xhci, "Couldn't initialize memory\n"); xhci_halt(xhci); - xhci_reset(xhci); + xhci_reset(xhci, XHCI_RESET_SHORT_USEC); xhci_mem_cleanup(xhci); return -ENOMEM; } |