From 456d8aa37d0f56fc9e985e812496e861dcd6f2f2 Mon Sep 17 00:00:00 2001 From: Ding Hui Date: Sun, 7 May 2023 11:40:57 +0800 Subject: [PATCH 1/9] PCI/ASPM: Disable ASPM on MFD function removal to avoid use-after-free Struct pcie_link_state->downstream is a pointer to the pci_dev of function 0. Previously we retained that pointer when removing function 0, and subsequent ASPM policy changes dereferenced it, resulting in a use-after-free warning from KASAN, e.g.: # echo 1 > /sys/bus/pci/devices/0000:03:00.0/remove # echo powersave > /sys/module/pcie_aspm/parameters/policy BUG: KASAN: slab-use-after-free in pcie_config_aspm_link+0x42d/0x500 Call Trace: kasan_report+0xae/0xe0 pcie_config_aspm_link+0x42d/0x500 pcie_aspm_set_policy+0x8e/0x1a0 param_attr_store+0x162/0x2c0 module_attr_store+0x3e/0x80 PCIe spec r6.0, sec 7.5.3.7, recommends that software program the same ASPM Control value in all functions of multi-function devices. Disable ASPM and free the pcie_link_state when any child function is removed so we can discard the dangling pcie_link_state->downstream pointer and maintain the same ASPM Control configuration for all functions. [bhelgaas: commit log and comment] Debugged-by: Zongquan Qin Suggested-by: Bjorn Helgaas Fixes: b5a0a9b59c81 ("PCI/ASPM: Read and set up L1 substate capabilities") Link: https://lore.kernel.org/r/20230507034057.20970-1-dinghui@sangfor.com.cn Signed-off-by: Ding Hui Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 66d7514ca111..db32335039d6 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -1010,21 +1010,24 @@ void pcie_aspm_exit_link_state(struct pci_dev *pdev) down_read(&pci_bus_sem); mutex_lock(&aspm_lock); - /* - * All PCIe functions are in one slot, remove one function will remove - * the whole slot, so just wait until we are the last function left. - */ - if (!list_empty(&parent->subordinate->devices)) - goto out; link = parent->link_state; root = link->root; parent_link = link->parent; - /* All functions are removed, so just disable ASPM for the link */ + /* + * link->downstream is a pointer to the pci_dev of function 0. If + * we remove that function, the pci_dev is about to be deallocated, + * so we can't use link->downstream again. Free the link state to + * avoid this. + * + * If we're removing a non-0 function, it's possible we could + * retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends + * programming the same ASPM Control value for all functions of + * multi-function devices, so disable ASPM for all of them. + */ pcie_config_aspm_link(link, 0); list_del(&link->sibling); - /* Clock PM is for endpoint device */ free_link_state(link); /* Recheck latencies and configure upstream links */ @@ -1032,7 +1035,7 @@ void pcie_aspm_exit_link_state(struct pci_dev *pdev) pcie_update_aspm_capable(root); pcie_config_aspm_path(parent_link); } -out: + mutex_unlock(&aspm_lock); up_read(&pci_bus_sem); } From fb097dcd5a28c0a2325632405c76a66777a6bed9 Mon Sep 17 00:00:00 2001 From: Ajay Agarwal Date: Thu, 4 May 2023 16:42:57 +0530 Subject: [PATCH 2/9] PCI/ASPM: Disable only ASPM_STATE_L1 when driver disables L1 Previously pci_disable_link_state(PCIE_LINK_STATE_L1) disabled L1SS as well as L1. This is unnecessary since pcie_config_aspm_link() takes care that L1SS is not enabled if L1 is disabled. Disable only ASPM_STATE_L1 when the caller disables L1. No functional changes intended. This is consistent with aspm_attr_store_common(), which disables only L1, not L1SS, when L1 is disabled via the sysfs "l1_aspm" file. [bhelgaas: commit log] Link: https://lore.kernel.org/r/20230504111301.229358-2-ajayagarwal@google.com Signed-off-by: Ajay Agarwal Signed-off-by: Bjorn Helgaas Reviewed-by: Kuppuswamy Sathyanarayanan --- drivers/pci/pcie/aspm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index db32335039d6..8c8352eeee52 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -1098,8 +1098,7 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) if (state & PCIE_LINK_STATE_L0S) link->aspm_disable |= ASPM_STATE_L0S; if (state & PCIE_LINK_STATE_L1) - /* L1 PM substates require L1 */ - link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS; + link->aspm_disable |= ASPM_STATE_L1; if (state & PCIE_LINK_STATE_L1_1) link->aspm_disable |= ASPM_STATE_L1_1; if (state & PCIE_LINK_STATE_L1_2) From 25edb25d7972414022c1fa098e2d85876bd7fab2 Mon Sep 17 00:00:00 2001 From: Ajay Agarwal Date: Thu, 4 May 2023 16:42:58 +0530 Subject: [PATCH 3/9] PCI/ASPM: Set only ASPM_STATE_L1 when driver enables L1 Previously pci_enable_link_state(PCIE_LINK_STATE_L1) enabled L1SS as well as L1. Enable only ASPM_STATE_L1 when the caller enables L1. The only current caller is vmd_pm_enable_quirk(), which enables *all* ASPM states, so this should have no functional effect. [bhelgaas: commit log] Link: https://lore.kernel.org/r/20230504111301.229358-3-ajayagarwal@google.com Signed-off-by: Ajay Agarwal Signed-off-by: Bjorn Helgaas Reviewed-by: Kuppuswamy Sathyanarayanan --- drivers/pci/pcie/aspm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 8c8352eeee52..a341019f9d9b 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -1173,8 +1173,7 @@ int pci_enable_link_state(struct pci_dev *pdev, int state) if (state & PCIE_LINK_STATE_L0S) link->aspm_default |= ASPM_STATE_L0S; if (state & PCIE_LINK_STATE_L1) - /* L1 PM substates require L1 */ - link->aspm_default |= ASPM_STATE_L1 | ASPM_STATE_L1SS; + link->aspm_default |= ASPM_STATE_L1; if (state & PCIE_LINK_STATE_L1_1) link->aspm_default |= ASPM_STATE_L1_1; if (state & PCIE_LINK_STATE_L1_2) From 80950a546089690ff6f02ca15930b9be695a668a Mon Sep 17 00:00:00 2001 From: Ajay Agarwal Date: Thu, 4 May 2023 16:42:59 +0530 Subject: [PATCH 4/9] PCI/ASPM: Set ASPM_STATE_L1 when driver enables L1.1 or L1.2 Previously pci_enable_link_state(PCIE_LINK_STATE_L1_1) enabled only ASPM_STATE_L1_1 and did not enable ASPM_STATE_L1. The L1.1 state only works when L1 is enabled, so enable ASPM_STATE_L1 in addition, and do the same for L1.2. The only current caller is vmd_pm_enable_quirk(), which enables *all* ASPM states, so this should have no functional effect. [bhelgaas: commit log] Link: https://lore.kernel.org/r/20230504111301.229358-4-ajayagarwal@google.com Signed-off-by: Ajay Agarwal Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index a341019f9d9b..338eedef12f1 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -1174,14 +1174,15 @@ int pci_enable_link_state(struct pci_dev *pdev, int state) link->aspm_default |= ASPM_STATE_L0S; if (state & PCIE_LINK_STATE_L1) link->aspm_default |= ASPM_STATE_L1; + /* L1 PM substates require L1 */ if (state & PCIE_LINK_STATE_L1_1) - link->aspm_default |= ASPM_STATE_L1_1; + link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1; if (state & PCIE_LINK_STATE_L1_2) - link->aspm_default |= ASPM_STATE_L1_2; + link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1; if (state & PCIE_LINK_STATE_L1_1_PCIPM) - link->aspm_default |= ASPM_STATE_L1_1_PCIPM; + link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1; if (state & PCIE_LINK_STATE_L1_2_PCIPM) - link->aspm_default |= ASPM_STATE_L1_2_PCIPM; + link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1; pcie_config_aspm_link(link, policy_to_aspm_state(link)); link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0; From 05a55d9ca1457295db73f127e39ec3b18f0b32b2 Mon Sep 17 00:00:00 2001 From: Ajay Agarwal Date: Thu, 4 May 2023 16:43:00 +0530 Subject: [PATCH 5/9] PCI/ASPM: Rename L1.2-specific functions from 'l1ss' to 'l12' The functions aspm_calc_l1ss_info() and calc_l1ss_pwron() perform calculations and register programming specific to L1.2 state. Rename them to aspm_calc_l12_info() and calc_l12_pwron() respectively. Link: https://lore.kernel.org/r/20230504111301.229358-5-ajayagarwal@google.com Signed-off-by: Ajay Agarwal Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 338eedef12f1..d3d8dfe1fb73 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -337,7 +337,7 @@ static u32 calc_l1_acceptable(u32 encoding) } /* Convert L1SS T_pwr encoding to usec */ -static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) +static u32 calc_l12_pwron(struct pci_dev *pdev, u32 scale, u32 val) { switch (scale) { case 0: @@ -471,7 +471,7 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, } /* Calculate L1.2 PM substate timing parameters */ -static void aspm_calc_l1ss_info(struct pcie_link_state *link, +static void aspm_calc_l12_info(struct pcie_link_state *link, u32 parent_l1ss_cap, u32 child_l1ss_cap) { struct pci_dev *child = link->downstream, *parent = link->pdev; @@ -495,13 +495,13 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, val2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; - if (calc_l1ss_pwron(parent, scale1, val1) > - calc_l1ss_pwron(child, scale2, val2)) { + if (calc_l12_pwron(parent, scale1, val1) > + calc_l12_pwron(child, scale2, val2)) { ctl2 |= scale1 | (val1 << 3); - t_power_on = calc_l1ss_pwron(parent, scale1, val1); + t_power_on = calc_l12_pwron(parent, scale1, val1); } else { ctl2 |= scale2 | (val2 << 3); - t_power_on = calc_l1ss_pwron(child, scale2, val2); + t_power_on = calc_l12_pwron(child, scale2, val2); } /* @@ -617,7 +617,7 @@ static void aspm_l1ss_init(struct pcie_link_state *link) link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; if (link->aspm_support & ASPM_STATE_L1SS) - aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap); + aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap); } static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) From 911afb9f9516a8ea2db1d15b18436c19a591dc5c Mon Sep 17 00:00:00 2001 From: Ajay Agarwal Date: Thu, 4 May 2023 16:43:01 +0530 Subject: [PATCH 6/9] PCI/ASPM: Remove unnecessary ASPM_STATE_L1SS check Previously aspm_l1ss_init() checked if ASPM_STATE_L1SS is supported before calling aspm_calc_l12_info(), only for that function to return if ASPM_STATE_L1_2_MASK is not supported. Simplify the logic by directly checking for ASPM_STATE_L1_2_MASK. Link: https://lore.kernel.org/r/20230504111301.229358-6-ajayagarwal@google.com Signed-off-by: Ajay Agarwal Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index d3d8dfe1fb73..72cdb30a924a 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -481,9 +481,6 @@ static void aspm_calc_l12_info(struct pcie_link_state *link, u32 pctl1, pctl2, cctl1, cctl2; u32 pl1_2_enables, cl1_2_enables; - if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) - return; - /* Choose the greater of the two Port Common_Mode_Restore_Times */ val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; @@ -616,7 +613,7 @@ static void aspm_l1ss_init(struct pcie_link_state *link) if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; - if (link->aspm_support & ASPM_STATE_L1SS) + if (link->aspm_support & ASPM_STATE_L1_2_MASK) aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap); } From f5297a01ee805d7fa569d288ed65fc0f9ac9b03d Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 20 Jun 2023 14:44:55 -0500 Subject: [PATCH 7/9] PCI/ASPM: Return 0 or -ETIMEDOUT from pcie_retrain_link() "pcie_retrain_link" is not a question with a true/false answer, so "bool" isn't quite the right return type. Return 0 for success or -ETIMEDOUT if the retrain failed. No functional change intended. [bhelgaas: based on Ilpo's patch below] Link: https://lore.kernel.org/r/20230502083923.34562-1-ilpo.jarvinen@linux.intel.com Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 72cdb30a924a..ee6323ded1c8 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -193,7 +193,7 @@ static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) link->clkpm_disable = blacklist ? 1 : 0; } -static bool pcie_retrain_link(struct pcie_link_state *link) +static int pcie_retrain_link(struct pcie_link_state *link) { struct pci_dev *parent = link->pdev; unsigned long end_jiffies; @@ -220,7 +220,9 @@ static bool pcie_retrain_link(struct pcie_link_state *link) break; msleep(1); } while (time_before(jiffies, end_jiffies)); - return !(reg16 & PCI_EXP_LNKSTA_LT); + if (reg16 & PCI_EXP_LNKSTA_LT) + return -ETIMEDOUT; + return 0; } /* @@ -289,15 +291,15 @@ static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) reg16 &= ~PCI_EXP_LNKCTL_CCC; pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); - if (pcie_retrain_link(link)) - return; + if (pcie_retrain_link(link)) { - /* Training failed. Restore common clock configurations */ - pci_err(parent, "ASPM: Could not configure common clock\n"); - list_for_each_entry(child, &linkbus->devices, bus_list) - pcie_capability_write_word(child, PCI_EXP_LNKCTL, + /* Training failed. Restore common clock configurations */ + pci_err(parent, "ASPM: Could not configure common clock\n"); + list_for_each_entry(child, &linkbus->devices, bus_list) + pcie_capability_write_word(child, PCI_EXP_LNKCTL, child_reg[PCI_FUNC(child->devfn)]); - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); + } } /* Convert L0s latency encoding to ns */ From 9c7f136433d26592cb4d9cd00b4e15c33d9797c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Tue, 20 Jun 2023 14:49:33 -0500 Subject: [PATCH 8/9] PCI/ASPM: Factor out pcie_wait_for_retrain() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Factor pcie_wait_for_retrain() out from pcie_retrain_link(). No functional change intended. [bhelgaas: split out from https://lore.kernel.org/r/20230502083923.34562-1-ilpo.jarvinen@linux.intel.com] Signed-off-by: Ilpo Järvinen Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index ee6323ded1c8..954717d7033f 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -193,10 +193,26 @@ static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) link->clkpm_disable = blacklist ? 1 : 0; } +static int pcie_wait_for_retrain(struct pci_dev *pdev) +{ + unsigned long end_jiffies; + u16 reg16; + + /* Wait for Link Training to be cleared by hardware */ + end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT; + do { + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, ®16); + if (!(reg16 & PCI_EXP_LNKSTA_LT)) + return 0; + msleep(1); + } while (time_before(jiffies, end_jiffies)); + + return -ETIMEDOUT; +} + static int pcie_retrain_link(struct pcie_link_state *link) { struct pci_dev *parent = link->pdev; - unsigned long end_jiffies; u16 reg16; pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); @@ -212,17 +228,7 @@ static int pcie_retrain_link(struct pcie_link_state *link) pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); } - /* Wait for link training end. Break out after waiting for timeout */ - end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT; - do { - pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); - if (!(reg16 & PCI_EXP_LNKSTA_LT)) - break; - msleep(1); - } while (time_before(jiffies, end_jiffies)); - if (reg16 & PCI_EXP_LNKSTA_LT) - return -ETIMEDOUT; - return 0; + return pcie_wait_for_retrain(parent); } /* From e7e39756363ad5bd83ddeae1063193d0f13870fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Tue, 2 May 2023 11:39:23 +0300 Subject: [PATCH 9/9] PCI/ASPM: Avoid link retraining race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PCIe r6.0.1, sec 7.5.3.7, recommends setting the link control parameters, then waiting for the Link Training bit to be clear before setting the Retrain Link bit. This avoids a race where the LTSSM may not use the updated parameters if it is already in the midst of link training because of other normal link activity. Wait for the Link Training bit to be clear before toggling the Retrain Link bit to ensure that the LTSSM uses the updated link control parameters. [bhelgaas: commit log, return 0 (success)/-ETIMEDOUT instead of bool for both pcie_wait_for_retrain() and the existing pcie_retrain_link()] Suggested-by: Lukas Wunner Fixes: 7d715a6c1ae5 ("PCI: add PCI Express ASPM support") Link: https://lore.kernel.org/r/20230502083923.34562-1-ilpo.jarvinen@linux.intel.com Signed-off-by: Ilpo Järvinen Signed-off-by: Bjorn Helgaas Reviewed-by: Lukas Wunner Cc: stable@vger.kernel.org --- drivers/pci/pcie/aspm.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 954717d7033f..3aa73ecdf86f 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -213,8 +213,19 @@ static int pcie_wait_for_retrain(struct pci_dev *pdev) static int pcie_retrain_link(struct pcie_link_state *link) { struct pci_dev *parent = link->pdev; + int rc; u16 reg16; + /* + * Ensure the updated LNKCTL parameters are used during link + * training by checking that there is no ongoing link training to + * avoid LTSSM race as recommended in Implementation Note at the + * end of PCIe r6.0.1 sec 7.5.3.7. + */ + rc = pcie_wait_for_retrain(parent); + if (rc) + return rc; + pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); reg16 |= PCI_EXP_LNKCTL_RL; pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);