diff --git a/Documentation/ABI/testing/configfs-thunderbolt_stream b/Documentation/ABI/testing/configfs-thunderbolt_stream index 7abc6b73a1e4..cbecb3d8db50 100644 --- a/Documentation/ABI/testing/configfs-thunderbolt_stream +++ b/Documentation/ABI/testing/configfs-thunderbolt_stream @@ -27,6 +27,21 @@ Description: default values. If there is an advertised remote stream with the same name, uses its values as the default. +What: /sys/kernel/config/thunderbolt/stream/./$name/busy_poll +Date: Nov 2026 +KernelVersion: v7.3 +Contact: Mika Westerberg +Description: + Instead of using interrupts for completing the frames in + the TX/RX rings, busy poll them directly from the + read(2) and write(2) calls. This burns more CPU cycles + but provides lower latency for applications that need it. + + This also makes poll(2) return EPOLLERR because + interrupts do not provide wakeup anymore. Likewise a + blocking read(2) without available data busy-spins until + data arrives or a signal is received. + What: /sys/kernel/config/thunderbolt/stream/./$name/index Date: Sep 2026 KernelVersion: v7.2 diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 98893732bc6e..ff42e28c307c 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -1350,7 +1350,7 @@ static void tbnet_generate_mac(struct net_device *dev) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; } -static int tbnet_probe(struct tb_service *svc, const struct tb_service_id *id) +static int tbnet_probe(struct tb_service *svc) { struct tb_xdomain *xd = tb_service_parent(svc); struct net_device *dev; @@ -1470,7 +1470,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(tbnet_pm_ops, tbnet_suspend, tbnet_resume); static const struct tb_service_id tbnet_ids[] = { { TB_SERVICE("network", 1) }, - { }, + { } }; MODULE_DEVICE_TABLE(tbsvc, tbnet_ids); diff --git a/drivers/thunderbolt/debugfs.c b/drivers/thunderbolt/debugfs.c index f5cf0e177f40..6e9080e7bcec 100644 --- a/drivers/thunderbolt/debugfs.c +++ b/drivers/thunderbolt/debugfs.c @@ -136,13 +136,13 @@ static void *validate_and_copy_from_user(const void __user *user_buf, if (!access_ok(user_buf, *count)) return ERR_PTR(-EFAULT); - buf = (void *)get_zeroed_page(GFP_KERNEL); + buf = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!buf) return ERR_PTR(-ENOMEM); nbytes = min_t(size_t, *count, PAGE_SIZE); if (copy_from_user(buf, user_buf, nbytes)) { - free_page((unsigned long)buf); + kfree(buf); return ERR_PTR(-EFAULT); } @@ -265,7 +265,7 @@ static ssize_t regs_write(struct tb_switch *sw, struct tb_port *port, out: pm_runtime_mark_last_busy(&sw->dev); pm_runtime_put_autosuspend(&sw->dev); - free_page((unsigned long)buf); + kfree(buf); return ret < 0 ? ret : count; } @@ -406,7 +406,7 @@ static ssize_t port_sb_regs_write(struct file *file, const char __user *user_buf out: pm_runtime_mark_last_busy(&sw->dev); pm_runtime_put_autosuspend(&sw->dev); - free_page((unsigned long)buf); + kfree(buf); return ret < 0 ? ret : count; } @@ -439,7 +439,7 @@ static ssize_t retimer_sb_regs_write(struct file *file, out: pm_runtime_mark_last_busy(&rt->dev); pm_runtime_put_autosuspend(&rt->dev); - free_page((unsigned long)buf); + kfree(buf); return ret < 0 ? ret : count; } @@ -652,7 +652,7 @@ margining_ber_level_write(struct file *file, const char __user *user_buf, margining->ber_level = val; out_free: - free_page((unsigned long)buf); + kfree(buf); out_unlock: mutex_unlock(&tb->lock); @@ -829,7 +829,7 @@ margining_lanes_write(struct file *file, const char __user *user_buf, } } - free_page((unsigned long)buf); + kfree(buf); if (lane == -1) return -EINVAL; @@ -958,7 +958,7 @@ margining_error_counter_write(struct file *file, const char __user *user_buf, else goto err_free; - free_page((unsigned long)buf); + kfree(buf); scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &tb->lock) { if (!margining->software) @@ -970,7 +970,7 @@ margining_error_counter_write(struct file *file, const char __user *user_buf, return count; err_free: - free_page((unsigned long)buf); + kfree(buf); return -EINVAL; } @@ -1116,7 +1116,7 @@ static ssize_t margining_mode_write(struct file *file, mutex_unlock(&tb->lock); out_free: - free_page((unsigned long)buf); + kfree(buf); return ret ? ret : count; } @@ -1503,7 +1503,7 @@ static ssize_t margining_test_write(struct file *file, mutex_unlock(&tb->lock); out_free: - free_page((unsigned long)buf); + kfree(buf); return ret ? ret : count; } @@ -1569,7 +1569,7 @@ static ssize_t margining_margin_write(struct file *file, mutex_unlock(&tb->lock); out_free: - free_page((unsigned long)buf); + kfree(buf); return ret ? ret : count; } @@ -1624,7 +1624,7 @@ static ssize_t margining_eye_write(struct file *file, ret = -EINVAL; } - free_page((unsigned long)buf); + kfree(buf); return ret ? ret : count; } @@ -1934,7 +1934,7 @@ static ssize_t counters_write(struct file *file, const char __user *user_buf, out: pm_runtime_mark_last_busy(&sw->dev); pm_runtime_put_autosuspend(&sw->dev); - free_page((unsigned long)buf); + kfree(buf); return ret < 0 ? ret : count; } diff --git a/drivers/thunderbolt/dma_test.c b/drivers/thunderbolt/dma_test.c index 7877319b1b03..519c67678b08 100644 --- a/drivers/thunderbolt/dma_test.c +++ b/drivers/thunderbolt/dma_test.c @@ -636,7 +636,7 @@ static void dma_test_debugfs_init(struct tb_service *svc) debugfs_create_file("test", 0200, debugfs_dir, svc, &test_fops); } -static int dma_test_probe(struct tb_service *svc, const struct tb_service_id *id) +static int dma_test_probe(struct tb_service *svc) { struct tb_xdomain *xd = tb_service_parent(svc); struct dma_test *dt; @@ -689,7 +689,7 @@ static const struct dev_pm_ops dma_test_pm_ops = { static const struct tb_service_id dma_test_ids[] = { { TB_SERVICE("dma_test", 1) }, - { }, + { } }; MODULE_DEVICE_TABLE(tbsvc, dma_test_ids); diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 479fa4d265c2..12c88509a54f 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -77,12 +77,10 @@ static int tb_service_probe(struct device *dev) { struct tb_service *svc = tb_to_service(dev); struct tb_service_driver *driver; - const struct tb_service_id *id; driver = container_of(dev->driver, struct tb_service_driver, driver); - id = __tb_service_match(dev, &driver->driver); - return driver->probe(svc, id); + return driver->probe(svc); } static void tb_service_remove(struct device *dev) @@ -790,6 +788,21 @@ int tb_domain_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, transmit_ring, receive_path, receive_ring); } +static void tb_domain_reset_interface(struct tb *tb) +{ + struct tb_nhi *nhi = tb->nhi; + + if (!nhi->ops->reset_interface) + return; + + guard(mutex)(&tb->lock); + + /* The reset clears the ring state so stop the control channel */ + tb_ctl_stop(tb->ctl); + nhi->ops->reset_interface(nhi); + tb_ctl_start(tb->ctl); +} + /** * tb_domain_disconnect_xdomain_paths() - Disable DMA paths for XDomain * @tb: Domain disabling the DMA paths @@ -812,11 +825,20 @@ int tb_domain_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, int transmit_path, int transmit_ring, int receive_path, int receive_ring) { + int ret; + if (!tb->cm_ops->disconnect_xdomain_paths) return -ENOTSUPP; - return tb->cm_ops->disconnect_xdomain_paths(tb, xd, transmit_path, + ret = tb->cm_ops->disconnect_xdomain_paths(tb, xd, transmit_path, transmit_ring, receive_path, receive_ring); + if (ret) + return ret; + + if (tb->nhi->quirks & QUIRK_RESET_DMA_ON_TEARDOWN) + tb_domain_reset_interface(tb); + + return 0; } static int disconnect_xdomain(struct device *dev, void *data) diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index 35e3c119d5ee..5809809f64d4 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -235,6 +235,12 @@ static void ring_write_descriptors(struct tb_ring *ring) { struct ring_frame *frame, *n; struct ring_desc *descriptor; + u32 flags; + + flags = RING_DESC_POSTED; + if (!(ring->flags & RING_FLAG_NO_INTERRUPT)) + flags |= RING_DESC_INTERRUPT; + list_for_each_entry_safe(frame, n, &ring->queue, list) { if (ring_full(ring)) break; @@ -242,7 +248,7 @@ static void ring_write_descriptors(struct tb_ring *ring) descriptor = &ring->descriptors[ring->head]; descriptor->phys = frame->buffer_phy; descriptor->time = 0; - descriptor->flags = RING_DESC_POSTED | RING_DESC_INTERRUPT; + descriptor->flags = flags; if (ring->is_tx) { descriptor->length = frame->size; descriptor->eof = frame->eof; @@ -339,8 +345,9 @@ EXPORT_SYMBOL_GPL(__tb_ring_enqueue); * @ring: Ring to poll * * This function can be called when @start_poll callback of the @ring - * has been called. It will read one completed frame from the ring and - * return it to the caller. + * has been called or the ring is created with %RING_FLAG_NO_INTERRUPT. + * It will read one completed frame from the ring and return it to the + * caller. * * Return: Pointer to &struct ring_frame, %NULL if there is no more * completed frames. @@ -538,6 +545,12 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size, dev_dbg(nhi->dev, "allocating %s ring %d of size %d\n", transmit ? "TX" : "RX", hop, size); + if ((flags & RING_FLAG_NO_INTERRUPT) && start_poll) { + dev_WARN(nhi->dev, + "start_poll() and NO_INTERRUPT cannot be used at the same time\n"); + return NULL; + } + ring = kzalloc_obj(*ring); if (!ring) return NULL; @@ -568,7 +581,7 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size, if (!ring->descriptors) goto err_free_ring; - if (nhi->ops->request_ring_irq) { + if (!(flags & RING_FLAG_NO_INTERRUPT) && nhi->ops->request_ring_irq) { if (nhi->ops->request_ring_irq(ring, flags & RING_FLAG_NO_SUSPEND)) goto err_free_descs; } @@ -701,7 +714,8 @@ void tb_ring_start(struct tb_ring *ring) ring_iowrite32options(ring, flags, 0); } - ring_interrupt_active(ring, true); + if (!(ring->flags & RING_FLAG_NO_INTERRUPT)) + ring_interrupt_active(ring, true); ring->running = true; err: spin_unlock(&ring->lock); @@ -761,7 +775,8 @@ void tb_ring_stop(struct tb_ring *ring) RING_TYPE(ring), ring->hop); goto err; } - ring_interrupt_active(ring, false); + if (!(ring->flags & RING_FLAG_NO_INTERRUPT)) + ring_interrupt_active(ring, false); ring_iowrite32options(ring, 0, 0); ring_iowrite64desc(ring, 0, 0); @@ -1160,6 +1175,32 @@ static void nhi_reset(struct tb_nhi *nhi) dev_warn(nhi->dev, "timeout resetting host router\n"); } +/** + * nhi_reset_interface() - Reset the host interface + * @nhi: Host interface to reset + * + * Brings the registers in the memory BAR back to their default state and + * clears the End-to-End Flow Control state. The caller is responsible for + * stopping the control channel over the reset because it clears the ring + * state as well. + */ +void nhi_reset_interface(struct tb_nhi *nhi) +{ + u32 val; + + val = ioread32(nhi->iobase + REG_CAPS); + /* Only v1 host interfaces implement the reset */ + if (FIELD_GET(REG_CAPS_VERSION_MASK, val) >= REG_CAPS_VERSION_2) + return; + + dev_dbg(nhi->dev, "issuing host interface reset\n"); + + iowrite32(REG_HOST_INTERFACE_RESET_RST, + nhi->iobase + REG_HOST_INTERFACE_RESET); + /* Wait for tHIReset (10 ms) to complete */ + usleep_range(10000, 20000); +} + static struct tb *nhi_select_cm(struct tb_nhi *nhi) { struct tb *tb; @@ -1235,6 +1276,8 @@ int nhi_probe(struct tb_nhi *nhi) dev_dbg(dev, "NHI initialized, starting thunderbolt\n"); + nhi->host_reset = host_reset; + res = tb_domain_add(tb, host_reset); if (res) { /* diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index d488eadadfce..f72d6b274501 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -36,6 +36,8 @@ irqreturn_t nhi_msi(int irq, void *data); irqreturn_t ring_msix(int irq, void *data); int nhi_probe(struct tb_nhi *nhi); void nhi_shutdown(struct tb_nhi *nhi); +void nhi_reset_interface(struct tb_nhi *nhi); + extern const struct dev_pm_ops nhi_pm_ops; /** @@ -52,6 +54,7 @@ extern const struct dev_pm_ops nhi_pm_ops; * @release_ring_irq: NHI specific interrupt release hook * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook + * @reset_interface: Resets the host interface */ struct tb_nhi_ops { int (*init)(struct tb_nhi *nhi); @@ -66,6 +69,7 @@ struct tb_nhi_ops { void (*release_ring_irq)(struct tb_ring *ring); bool (*is_present)(struct tb_nhi *nhi); int (*init_interrupts)(struct tb_nhi *nhi); + void (*reset_interface)(struct tb_nhi *nhi); }; /* @@ -116,11 +120,24 @@ struct tb_nhi_ops { #define PCI_DEVICE_ID_INTEL_PTL_P_NHI0 0xe433 #define PCI_DEVICE_ID_INTEL_PTL_P_NHI1 0xe434 +#define PCI_DEVICE_ID_AMD_1AH_M60H_NHI0 0x1120 +#define PCI_DEVICE_ID_AMD_1AH_M60H_NHI1 0x1121 +#define PCI_DEVICE_ID_AMD_1AH_M68H_NHI0 0x113b +#define PCI_DEVICE_ID_AMD_1AH_M68H_NHI1 0x113c +#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI0 0x1155 +#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI1 0x1158 +#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI2 0x1159 +#define PCI_DEVICE_ID_AMD_1AH_M24H_NHI0 0x151c +#define PCI_DEVICE_ID_AMD_1AH_M24H_NHI1 0x151d +#define PCI_DEVICE_ID_AMD_1AH_M70H_NHI0 0x158d +#define PCI_DEVICE_ID_AMD_1AH_M70H_NHI1 0x158e + #define PCI_CLASS_SERIAL_USB_USB4 0x0c0340 /* Host interface quirks */ -#define QUIRK_AUTO_CLEAR_INT BIT(0) -#define QUIRK_E2E BIT(1) +#define QUIRK_AUTO_CLEAR_INT BIT(0) +#define QUIRK_E2E BIT(1) +#define QUIRK_RESET_DMA_ON_TEARDOWN BIT(2) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/nhi_regs.h b/drivers/thunderbolt/nhi_regs.h index d6a197fabc74..99df60b6db36 100644 --- a/drivers/thunderbolt/nhi_regs.h +++ b/drivers/thunderbolt/nhi_regs.h @@ -115,6 +115,10 @@ struct ring_desc { #define REG_CAPS_VERSION_MASK GENMASK(23, 16) #define REG_CAPS_VERSION_2 0x40 +/* Host Interface Reset - resets TX/RX rings and E2E flow control counters */ +#define REG_HOST_INTERFACE_RESET 0x39858 +#define REG_HOST_INTERFACE_RESET_RST BIT(0) + #define REG_DMA_MISC 0x39864 #define REG_DMA_MISC_INT_AUTO_CLEAR BIT(2) #define REG_DMA_MISC_DISABLE_AUTO_CLEAR BIT(17) diff --git a/drivers/thunderbolt/pci.c b/drivers/thunderbolt/pci.c index bbd186c29ef7..99333729f3c2 100644 --- a/drivers/thunderbolt/pci.c +++ b/drivers/thunderbolt/pci.c @@ -62,6 +62,27 @@ static void nhi_pci_check_quirks(struct tb_nhi_pci *nhi_pci) nhi->quirks |= QUIRK_E2E; break; } + } else if (pdev->vendor == PCI_VENDOR_ID_AMD) { + switch (pdev->device) { + case PCI_DEVICE_ID_AMD_1AH_M60H_NHI0: + case PCI_DEVICE_ID_AMD_1AH_M60H_NHI1: + case PCI_DEVICE_ID_AMD_1AH_M68H_NHI0: + case PCI_DEVICE_ID_AMD_1AH_M68H_NHI1: + case PCI_DEVICE_ID_AMD_1AH_M80H_NHI0: + case PCI_DEVICE_ID_AMD_1AH_M80H_NHI1: + case PCI_DEVICE_ID_AMD_1AH_M80H_NHI2: + case PCI_DEVICE_ID_AMD_1AH_M24H_NHI0: + case PCI_DEVICE_ID_AMD_1AH_M24H_NHI1: + case PCI_DEVICE_ID_AMD_1AH_M70H_NHI0: + case PCI_DEVICE_ID_AMD_1AH_M70H_NHI1: + /* + * These AMD hosts may hang the Tx ring when the + * DMA paths are torn down so they need the host + * interface reset after each teardown. + */ + nhi->quirks |= QUIRK_RESET_DMA_ON_TEARDOWN; + break; + } } } @@ -112,7 +133,6 @@ static int nhi_pci_init_msi(struct tb_nhi *nhi) { struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi); struct pci_dev *pdev = to_pci_dev(nhi->dev); - struct device *dev = &pdev->dev; int res, irq, nvec; ida_init(&nhi_pci->msix_ida); @@ -139,7 +159,7 @@ static int nhi_pci_init_msi(struct tb_nhi *nhi) res = devm_request_irq(&pdev->dev, irq, nhi_msi, IRQF_NO_SUSPEND, "thunderbolt", nhi); if (res) - return dev_err_probe(dev, res, "request_irq failed, aborting\n"); + return res; } return 0; @@ -230,7 +250,7 @@ static void nhi_pci_ring_release_msix(struct tb_ring *ring) ring->irq = 0; } -static void nhi_pci_shutdown(struct tb_nhi *nhi) +static void nhi_pci_release_irq(struct tb_nhi *nhi) { struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi); struct pci_dev *pdev = to_pci_dev(nhi->dev); @@ -256,9 +276,10 @@ static const struct tb_nhi_ops pci_nhi_default_ops = { .post_nvm_auth = nhi_pci_complete_dma_port, .request_ring_irq = nhi_pci_ring_request_msix, .release_ring_irq = nhi_pci_ring_release_msix, - .shutdown = nhi_pci_shutdown, + .shutdown = nhi_pci_release_irq, .is_present = nhi_pci_is_present, .init_interrupts = nhi_pci_init_msi, + .reset_interface = nhi_reset_interface, }; /* Ice Lake specific NHI operations */ @@ -424,7 +445,7 @@ static int icl_nhi_resume(struct tb_nhi *nhi) static void icl_nhi_shutdown(struct tb_nhi *nhi) { - nhi_pci_shutdown(nhi); + nhi_pci_release_irq(nhi); icl_nhi_force_power(nhi, false); } @@ -442,6 +463,7 @@ static const struct tb_nhi_ops icl_nhi_ops = { .release_ring_irq = nhi_pci_ring_release_msix, .is_present = nhi_pci_is_present, .init_interrupts = nhi_pci_init_msi, + .reset_interface = nhi_reset_interface, }; static int nhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -479,11 +501,19 @@ static int nhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return nhi_probe(&nhi_pci->nhi); } -static void nhi_pci_remove(struct pci_dev *pdev) +static void nhi_pci_do_remove(struct pci_dev *pdev, bool reset) { struct tb *tb = pci_get_drvdata(pdev); struct tb_nhi *nhi = tb->nhi; + /* + * On system shutdown/reboot force a host router reset so the + * connection manager asserts DPR on connected Thunderbolt 3 devices + * before the router tree is removed (see tb_stop()). + */ + if (reset) + nhi->host_reset = true; + pm_runtime_get_sync(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_forbid(&pdev->dev); @@ -493,6 +523,16 @@ static void nhi_pci_remove(struct pci_dev *pdev) nhi_shutdown(nhi); } +static void nhi_pci_remove(struct pci_dev *pdev) +{ + nhi_pci_do_remove(pdev, false); +} + +static void nhi_pci_shutdown(struct pci_dev *pdev) +{ + nhi_pci_do_remove(pdev, true); +} + static struct pci_device_id nhi_ids[] = { /* * We have to specify class, the TB bridges use the same device and @@ -593,7 +633,7 @@ static struct pci_driver nhi_driver = { .id_table = nhi_ids, .probe = nhi_pci_probe, .remove = nhi_pci_remove, - .shutdown = nhi_pci_remove, + .shutdown = nhi_pci_shutdown, .driver.pm = &nhi_pm_ops, }; diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c index 4cc86d8d6491..c737dd0ca6e7 100644 --- a/drivers/thunderbolt/stream.c +++ b/drivers/thunderbolt/stream.c @@ -9,10 +9,12 @@ #define pr_fmt(fmt) "tbstream: " fmt +#include #include #include #include #include +#include #include #include #include @@ -128,6 +130,7 @@ struct tbstream_ring { * @out_hopid: Out HopID * @ring_size: Size of the rings * @throttling: Interrupt throttling rate in ns + * @busy_poll: Instead of interrupts, busy poll the rings * @users: Number of times @cdev has been opened * @closed: CLOSE packet was received * @removed: Userspace removed the ConfigFS group underneath. @@ -147,6 +150,7 @@ struct tbstream_dev { int out_hopid; unsigned int ring_size; unsigned int throttling; + bool busy_poll; int users; bool closed; bool removed; @@ -512,8 +516,10 @@ tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf, dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, size, DMA_TO_DEVICE); if (pdf == TBSTREAM_DATA) { - if (copy_page_from_iter(sf->page, 0, size, from) != size) + if (copy_page_from_iter(sf->page, 0, size, from) != size) { + sdev->tx_ring.cons--; return ERR_PTR(-EFAULT); + } } else { memset(page_address(sf->page), 0, size); } @@ -534,10 +540,39 @@ tbstream_dev_send_data(struct tbstream_dev *sdev, struct iov_iter *from, return tb_ring_tx(sdev->tx_ring.ring, &sf->frame); } +static void +tbstream_dev_poll_ring(struct tbstream_dev *sdev, struct tbstream_ring *ring) +{ + struct ring_frame *frame; + + if (!sdev->busy_poll) + return; + + while ((frame = tb_ring_poll(ring->ring))) + frame->callback(ring->ring, frame, false); +} + static int tbstream_dev_send_close(struct tbstream_dev *sdev) { struct tbstream_frame *sf; + if (sdev->busy_poll) { + /* + * When busy polling it's the write(2) path that + * advances the completions so it is possible that the + * ring is full at this point. Advance the ring here so + * that there is room for the CLOSE packet to be sent. + */ + ktime_t timeout = ktime_add_ms(ktime_get(), 500); + + do { + if (tbstream_ring_available(&sdev->tx_ring)) + break; + tbstream_dev_poll_ring(sdev, &sdev->tx_ring); + fsleep(15); + } while (ktime_before(ktime_get(), timeout)); + } + sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256); if (IS_ERR(sf)) return PTR_ERR(sf); @@ -547,12 +582,15 @@ static int tbstream_dev_send_close(struct tbstream_dev *sdev) static int tbstream_dev_start(struct tbstream_dev *sdev) { struct tb_xdomain *xd = tbstream_dev_xdomain(sdev); + unsigned int flags = RING_FLAG_FRAME | RING_FLAG_E2E; u16 sof_mask, eof_mask; struct tb_ring *ring; int ret, e2e_tx_hop; - ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size, - RING_FLAG_FRAME | RING_FLAG_E2E); + if (sdev->busy_poll) + flags |= RING_FLAG_NO_INTERRUPT; + + ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size, flags); if (!ring) return -ENOMEM; sdev->tx_ring.ring = ring; @@ -565,9 +603,8 @@ static int tbstream_dev_start(struct tbstream_dev *sdev) sof_mask = BIT(TBSTREAM_FRAME_START); eof_mask = BIT(TBSTREAM_DATA) | BIT(TBSTREAM_CLOSE); - ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size, - RING_FLAG_FRAME | RING_FLAG_E2E, e2e_tx_hop, - sof_mask, eof_mask, NULL, NULL); + ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size, flags, + e2e_tx_hop, sof_mask, eof_mask, NULL, NULL); if (!ring) { ret = -ENOMEM; goto err_free_tx_buffers; @@ -605,15 +642,43 @@ static int tbstream_dev_start(struct tbstream_dev *sdev) return ret; } +static bool tbstream_dev_tx_drained(const struct tbstream_dev *sdev) +{ + const struct tbstream_ring *ring = &sdev->tx_ring; + + /* + * Everything is completed when number of free TX slots is back + * to the maximum. + */ + return ring->prod - ring->cons == tb_ring_size(ring->ring) - 1; +} + static void tbstream_dev_stop(struct tbstream_dev *sdev) { struct tb_xdomain *xd; - /* Wait for the ring to complete any outstanding frames */ - tb_ring_flush(sdev->tx_ring.ring, 500); - tb_ring_stop(sdev->tx_ring.ring); - tb_ring_flush(sdev->rx_ring.ring, 500); - tb_ring_stop(sdev->rx_ring.ring); + if (sdev->busy_poll) { + /* + * When busy polling we must advance the ring ourselves + * to push all outstanding frames on the wire. + */ + ktime_t timeout = ktime_add_ms(ktime_get(), 500); + + do { + if (tbstream_dev_tx_drained(sdev)) + break; + tbstream_dev_poll_ring(sdev, &sdev->tx_ring); + fsleep(15); + } while (ktime_before(ktime_get(), timeout)); + + tb_ring_stop(sdev->tx_ring.ring); + tb_ring_stop(sdev->rx_ring.ring); + } else { + tb_ring_flush(sdev->tx_ring.ring, 500); + tb_ring_stop(sdev->tx_ring.ring); + tb_ring_flush(sdev->rx_ring.ring, 500); + tb_ring_stop(sdev->rx_ring.ring); + } xd = tbstream_dev_xdomain(sdev); if (xd) { @@ -631,10 +696,24 @@ static void tbstream_dev_stop(struct tbstream_dev *sdev) sdev->tx_ring.ring = NULL; } +/* Use only with read_iter/write_iter() to handle nowait */ +static int tbstream_dev_lock(struct tbstream_dev *sdev, bool nowait) +{ + if (nowait) { + if (!mutex_trylock(&sdev->lock)) + return -EAGAIN; + } else { + if (mutex_lock_interruptible(&sdev->lock)) + return -ERESTARTSYS; + } + return 0; +} + static ssize_t tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) { struct file *file = kiocb->ki_filp; + bool nowait = file->f_flags & O_NONBLOCK || kiocb->ki_flags & IOCB_NOWAIT; struct tbstream_dev *sdev = to_tbstream_dev(file->private_data); size_t nbytes; int ret; @@ -643,35 +722,54 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) if (ret) return ret; - if (mutex_lock_interruptible(&sdev->lock)) - return -ERESTARTSYS; + ret = tbstream_dev_lock(sdev, nowait); + if (ret) + return ret; - while (!tbstream_ring_available(&sdev->rx_ring)) { - mutex_unlock(&sdev->lock); - - if (file->f_flags & O_NONBLOCK) - return -EAGAIN; - ret = wait_event_interruptible(sdev->wait, - tbstream_ring_available(&sdev->rx_ring) || - tbstream_dev_valid(sdev) != 0 || - tbstream_dev_closed(sdev) || - tbstream_dev_removed(sdev)); - if (ret) - return ret; + for (;;) { + /* When busy polling, advance any completions manually */ + tbstream_dev_poll_ring(sdev, &sdev->rx_ring); ret = tbstream_dev_valid(sdev); + if (ret) { + mutex_unlock(&sdev->lock); + return ret; + } + + if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) { + mutex_unlock(&sdev->lock); + return 0; + } + + if (tbstream_ring_available(&sdev->rx_ring)) + break; + + mutex_unlock(&sdev->lock); + + if (nowait) + return -EAGAIN; + + if (sdev->busy_poll) { + if (signal_pending(current)) + return -ERESTARTSYS; + cond_resched(); + } else { + ret = wait_event_interruptible(sdev->wait, + tbstream_ring_available(&sdev->rx_ring) || + tbstream_dev_valid(sdev) != 0 || + tbstream_dev_closed(sdev) || + tbstream_dev_removed(sdev)); + if (ret) + return ret; + } + + ret = tbstream_dev_lock(sdev, nowait); if (ret) return ret; - - if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) - return 0; - - if (mutex_lock_interruptible(&sdev->lock)) - return -ERESTARTSYS; } nbytes = 0; - while (nbytes < iov_iter_count(to)) { + while (iov_iter_count(to)) { struct tbstream_frame *sf; size_t size, sf_size; @@ -693,7 +791,7 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to) } sf_size = tb_ring_frame_size(&sf->frame); - size = min(iov_iter_count(to) - nbytes, sf_size); + size = min(iov_iter_count(to), sf_size); if (copy_page_to_iter(sf->page, sf->offset, size, to) != size) { ret = -EFAULT; @@ -727,6 +825,7 @@ static ssize_t tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from) { struct file *file = kiocb->ki_filp; + bool nowait = file->f_flags & O_NONBLOCK || kiocb->ki_flags & IOCB_NOWAIT; struct tbstream_dev *sdev = to_tbstream_dev(file->private_data); size_t nbytes; int ret; @@ -735,38 +834,56 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from) if (ret) return ret; - if (mutex_lock_interruptible(&sdev->lock)) - return -ERESTARTSYS; + ret = tbstream_dev_lock(sdev, nowait); + if (ret) + return ret; - while (!tbstream_ring_available(&sdev->tx_ring)) { - mutex_unlock(&sdev->lock); - - if (file->f_flags & O_NONBLOCK) - return -EAGAIN; - ret = wait_event_interruptible(sdev->wait, - tbstream_ring_available(&sdev->tx_ring) || - tbstream_dev_valid(sdev) != 0 || - tbstream_dev_closed(sdev) || - tbstream_dev_removed(sdev)); - if (ret) - return ret; + for (;;) { + tbstream_dev_poll_ring(sdev, &sdev->tx_ring); ret = tbstream_dev_valid(sdev); + if (ret) { + mutex_unlock(&sdev->lock); + return ret; + } + + if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) { + mutex_unlock(&sdev->lock); + return -ENXIO; + } + + if (tbstream_ring_available(&sdev->tx_ring)) + break; + + mutex_unlock(&sdev->lock); + + if (nowait) + return -EAGAIN; + + if (sdev->busy_poll) { + if (signal_pending(current)) + return -ERESTARTSYS; + cond_resched(); + } else { + ret = wait_event_interruptible(sdev->wait, + tbstream_ring_available(&sdev->tx_ring) || + tbstream_dev_valid(sdev) != 0 || + tbstream_dev_closed(sdev) || + tbstream_dev_removed(sdev)); + if (ret) + return ret; + } + + ret = tbstream_dev_lock(sdev, nowait); if (ret) return ret; - - if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) - return -ENXIO; - - if (mutex_lock_interruptible(&sdev->lock)) - return -ERESTARTSYS; } nbytes = 0; - while (nbytes < iov_iter_count(from)) { + while (iov_iter_count(from)) { size_t size; - size = min(iov_iter_count(from) - nbytes, TB_MAX_FRAME_SIZE); + size = min(iov_iter_count(from), TB_MAX_FRAME_SIZE); ret = tbstream_dev_send_data(sdev, from, size); if (ret) { /* @@ -793,6 +910,13 @@ tbstream_dev_fops_poll(struct file *file, struct poll_table_struct *wait) struct tbstream_dev *sdev = to_tbstream_dev(file->private_data); __poll_t mask = 0; + /* + * Without interrupts there is nothing that can wake us up so + * return failure instead. + */ + if (sdev->busy_poll) + return EPOLLERR; + poll_wait(file, &sdev->wait, wait); guard(mutex)(&sdev->lock); if (tbstream_dev_valid(sdev) != 0) { @@ -902,6 +1026,35 @@ tbstream_dev_from_group(struct config_group *group) return container_of(group, struct tbstream_dev, group); } +static ssize_t tbstream_dev_busy_poll_show(struct config_item *item, char *buf) +{ + struct config_group *group = to_config_group(item); + struct tbstream_dev *sdev = tbstream_dev_from_group(group); + + return sysfs_emit(buf, "%u\n", sdev->busy_poll); +} + +static ssize_t +tbstream_dev_busy_poll_store(struct config_item *item, const char *buf, + size_t count) +{ + struct config_group *group = to_config_group(item); + struct tbstream_dev *sdev = tbstream_dev_from_group(group); + bool busy_poll; + int ret; + + ret = kstrtobool(buf, &busy_poll); + if (ret) + return ret; + + guard(mutex)(&sdev->lock); + if (sdev->users) + return -EBUSY; + sdev->busy_poll = busy_poll; + return count; +} +CONFIGFS_ATTR(tbstream_dev_, busy_poll); + static ssize_t tbstream_dev_index_show(struct config_item *item, char *buf) { struct config_group *group = to_config_group(item); @@ -1208,6 +1361,7 @@ tbstream_dev_throttling_store(struct config_item *item, const char *buf, CONFIGFS_ATTR(tbstream_dev_, throttling); static struct configfs_attribute *tbstream_dev_attrs[] = { + &tbstream_dev_attr_busy_poll, &tbstream_dev_attr_index, &tbstream_dev_attr_in_hopid, &tbstream_dev_attr_out_hopid, @@ -1540,7 +1694,7 @@ static void tbstream_group_detach_stream(struct tbstream *stream) config_group_put(&sg->group); } -static int tbstream_probe(struct tb_service *svc, const struct tb_service_id *id) +static int tbstream_probe(struct tb_service *svc) { struct tbstream *stream; @@ -1630,7 +1784,7 @@ static const struct dev_pm_ops tbstream_pm_ops = { static const struct tb_service_id tbstream_ids[] = { { TB_SERVICE("stream", 1) }, - { }, + { } }; MODULE_DEVICE_TABLE(tbsvc, tbstream_ids); diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index a830c82bb905..404c0693df50 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -682,7 +682,16 @@ int tb_port_disable(struct tb_port *port) return __tb_port_enable(port, false); } -static int tb_port_reset(struct tb_port *port) +/** + * tb_port_reset() - Reset the port + * @port: Port to reset + * + * Resets @port. For USB4 ports this issues a USB4 port reset and for + * legacy ports the link controller port is reset. + * + * Return: %0 on success, negative errno otherwise. + */ +int tb_port_reset(struct tb_port *port) { if (tb_switch_is_usb4(port->sw)) return port->cap_usb4 ? usb4_port_reset(port) : 0; diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index f43f2d952372..47753a5c0f2e 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -2941,7 +2941,9 @@ static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type, static void tb_stop(struct tb *tb) { struct tb_cm *tcm = tb_priv(tb); + struct tb_nhi *nhi = tb->nhi; struct tb_tunnel *tunnel; + struct tb_port *port; struct tb_tunnel *n; cancel_delayed_work(&tcm->remove_work); @@ -2956,6 +2958,25 @@ static void tb_stop(struct tb *tb) tb_tunnel_deactivate(tunnel); tb_tunnel_put(tunnel); } + /* + * Signal disconnect to connected devices before the router tree is + * removed below. A Thunderbolt 3 device directly connected to a USB4 + * host otherwise never receives a disconnect indication, leaving + * firmware to poll the dead link for up to ~60 s which on some + * platforms turns the shutdown into a warm reset. Asserting + * PORT_CS_19.DPR drives SBTX low (USB4 spec section 6.9) so the device + * detects SBRX low and goes to Uninitialized Unplugged immediately. + */ + if (nhi->host_reset) { + tb_switch_for_each_port(tb->root_switch, port) { + if (!tb_port_is_null(port) || !tb_port_has_remote(port)) + continue; + if (tb_switch_is_usb4(port->remote->sw)) + continue; + if (tb_port_reset(port)) + tb_port_dbg(port, "downstream port reset failed, continuing\n"); + } + } tb_switch_remove(tb->root_switch); tb->root_switch = NULL; tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index ec9192b61bc0..4373336d9425 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1103,6 +1103,7 @@ int tb_port_clear_counter(struct tb_port *port, int counter); int tb_port_unlock(struct tb_port *port); int tb_port_enable(struct tb_port *port); int tb_port_disable(struct tb_port *port); +int tb_port_reset(struct tb_port *port); int tb_port_alloc_in_hopid(struct tb_port *port, int hopid, int max_hopid); void tb_port_release_in_hopid(struct tb_port *port, int hopid); int tb_port_alloc_out_hopid(struct tb_port *port, int hopid, int max_hopid); diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index b7f32305f14a..7e8284575dff 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -48,6 +48,9 @@ #define TB_DP_AUX_PRIORITY 2 #define TB_DP_AUX_WEIGHT 1 +/* struct tb_regs_hop::initial_credits is 7 bits wide */ +#define TB_MAX_CREDITS 127 + /* Minimum number of credits needed for PCIe path */ #define TB_MIN_PCIE_CREDITS 6U /* @@ -1778,8 +1781,7 @@ static int tb_dma_reserve_credits(struct tb_path_hop *hop, unsigned int credits) if (available < TB_MIN_DMA_CREDITS) return -ENOSPC; - while (credits > available) - credits--; + credits = min(credits, available); tb_port_dbg(port, "reserving %u credits for DMA path\n", credits); @@ -1908,7 +1910,7 @@ struct tb_tunnel *tb_tunnel_alloc_dma(struct tb *tb, struct tb_port *nhi, struct tb_tunnel *tunnel; size_t npaths = 0, i = 0; struct tb_path *path; - int credits; + unsigned int credits; /* Ring 0 is reserved for control channel */ if (WARN_ON(!receive_ring || !transmit_ring)) @@ -1931,6 +1933,11 @@ struct tb_tunnel *tb_tunnel_alloc_dma(struct tb *tb, struct tb_port *nhi, tunnel->destroy = tb_dma_destroy; credits = min_not_zero(dma_credits, nhi->sw->max_dma_credits); + if (credits > TB_MAX_CREDITS) { + tb_tunnel_dbg(tunnel, "%u credits do not fit a hop, using %u\n", + credits, TB_MAX_CREDITS); + credits = TB_MAX_CREDITS; + } if (receive_ring > 0) { path = tb_path_alloc(tb, dst, receive_path, nhi, receive_ring, 0, diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c index 86b2f7474670..c179bd751fe4 100644 --- a/drivers/thunderbolt/xdomain.c +++ b/drivers/thunderbolt/xdomain.c @@ -968,6 +968,9 @@ tb_xdp_schedule_request(struct tb *tb, const struct tb_xdp_header *hdr, */ int tb_register_service_driver(struct tb_service_driver *drv) { + if (!drv->probe) + return -EINVAL; + drv->driver.bus = &tb_bus_type; return driver_register(&drv->driver); } @@ -1811,6 +1814,7 @@ static void tb_xdomain_state_work(struct work_struct *work) tb_xdomain_failed(xd); } else { xd->state = XDOMAIN_STATE_ENUMERATED; + tb_xdomain_queue_properties_changed(xd); } break; diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 557288c0274b..d48623fda79b 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -465,7 +465,7 @@ static inline struct tb_service *tb_to_service(struct device *dev) */ struct tb_service_driver { struct device_driver driver; - int (*probe)(struct tb_service *svc, const struct tb_service_id *id); + int (*probe)(struct tb_service *svc); void (*remove)(struct tb_service *svc); void (*shutdown)(struct tb_service *svc); const struct tb_service_id *id_table; @@ -514,6 +514,11 @@ void tb_service_properties_changed(struct tb_service *svc); * @hop_count: Number of rings (end point hops) supported by NHI. * @quirks: NHI specific quirks if any * @domain_released: Completed when domain has been fully released + * @host_reset: Host router was reset on driver load, or forced on system + * shutdown/reboot. When set, tb_stop() asserts DPR on connected + * downstream ports to signal disconnect before tearing down the + * router tree. Only Thunderbolt 3 devices are reset; USB4 + * routers are skipped. */ struct tb_nhi { spinlock_t lock; @@ -528,6 +533,7 @@ struct tb_nhi { u32 hop_count; unsigned long quirks; struct completion domain_released; + bool host_reset; }; /** @@ -592,6 +598,8 @@ struct tb_ring { #define RING_FLAG_FRAME BIT(1) /* Enable end-to-end flow control */ #define RING_FLAG_E2E BIT(2) +/* Do not enable interrupt for the ring */ +#define RING_FLAG_NO_INTERRUPT BIT(3) struct ring_frame; typedef void (*ring_cb)(struct tb_ring *, struct ring_frame *, bool canceled);