Skip to content

Commit c1c2d89

Browse files
bjorn-helgaasbjorn-helgaas
committed
Merge branch 'pci/aspm'
- Save L1 PM Substates Capability across suspend/resume so L1SS keeps working after resume (Vidya Sagar) - If device lacks L1 PM Substates Capability, don't read junk and treat it as such a Capability (Bjorn Helgaas) - Fix the LTR_L1.2_THRESHOLD computation, which previously configured the threshold for entering L1.2 to be lower than intended, so L1.2 could be used when it shouldn't be (Bjorn Helgaas) * pci/aspm: PCI/ASPM: Correct LTR_L1.2_THRESHOLD computation PCI/ASPM: Ignore L1 PM Substates if device lacks capability PCI/ASPM: Factor out L1 PM Substates configuration PCI/ASPM: Save L1 PM Substates Capability for suspend/resume PCI/ASPM: Refactor L1 PM Substates Control Register programming
2 parents 568035b + 7afeb84 commit c1c2d89

File tree

3 files changed

+174
-95
lines changed

3 files changed

+174
-95
lines changed

‎drivers/pci/pci.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1663,6 +1663,7 @@ int pci_save_state(struct pci_dev *dev)
16631663
return i;
16641664

16651665
pci_save_ltr_state(dev);
1666+
pci_save_aspm_l1ss_state(dev);
16661667
pci_save_dpc_state(dev);
16671668
pci_save_aer_state(dev);
16681669
pci_save_ptm_state(dev);
@@ -1769,6 +1770,7 @@ void pci_restore_state(struct pci_dev *dev)
17691770
* LTR itself (in the PCIe capability).
17701771
*/
17711772
pci_restore_ltr_state(dev);
1773+
pci_restore_aspm_l1ss_state(dev);
17721774

17731775
pci_restore_pcie_state(dev);
17741776
pci_restore_pasid_state(dev);
@@ -3485,6 +3487,11 @@ void pci_allocate_cap_save_buffers(struct pci_dev *dev)
34853487
if (error)
34863488
pci_err(dev, "unable to allocate suspend buffer for LTR\n");
34873489

3490+
error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_L1SS,
3491+
2 * sizeof(u32));
3492+
if (error)
3493+
pci_err(dev, "unable to allocate suspend buffer for ASPM-L1SS\n");
3494+
34883495
pci_allocate_vc_save_buffers(dev);
34893496
}
34903497

‎drivers/pci/pci.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,10 +561,14 @@ bool pcie_wait_for_link(struct pci_dev *pdev, bool active);
561561
void pcie_aspm_init_link_state(struct pci_dev *pdev);
562562
void pcie_aspm_exit_link_state(struct pci_dev *pdev);
563563
void pcie_aspm_powersave_config_link(struct pci_dev *pdev);
564+
void pci_save_aspm_l1ss_state(struct pci_dev *dev);
565+
void pci_restore_aspm_l1ss_state(struct pci_dev *dev);
564566
#else
565567
static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { }
566568
static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev) { }
567569
static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { }
570+
static inline void pci_save_aspm_l1ss_state(struct pci_dev *dev) { }
571+
static inline void pci_restore_aspm_l1ss_state(struct pci_dev *dev) { }
568572
#endif
569573

570574
#ifdef CONFIG_PCIE_ECRC

‎drivers/pci/pcie/aspm.c

Lines changed: 163 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
*/
99

1010
#include <linux/kernel.h>
11+
#include <linux/math.h>
1112
#include <linux/module.h>
1213
#include <linux/moduleparam.h>
1314
#include <linux/pci.h>
@@ -350,29 +351,43 @@ static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
350351
return 0;
351352
}
352353

354+
/*
355+
* Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1
356+
* register. Ports enter L1.2 when the most recent LTR value is greater
357+
* than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we
358+
* don't enter L1.2 too aggressively.
359+
*
360+
* See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.
361+
*/
353362
static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
354363
{
355-
u32 threshold_ns = threshold_us * 1000;
364+
u64 threshold_ns = (u64) threshold_us * 1000;
356365

357-
/* See PCIe r3.1, sec 7.33.3 and sec 6.18 */
358-
if (threshold_ns < 32) {
359-
*scale = 0;
366+
/*
367+
* LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max
368+
* value of 0x3ff.
369+
*/
370+
if (threshold_ns <= 0x3ff * 1) {
371+
*scale = 0; /* Value times 1ns */
360372
*value = threshold_ns;
361-
} else if (threshold_ns < 1024) {
362-
*scale = 1;
363-
*value = threshold_ns >> 5;
364-
} else if (threshold_ns < 32768) {
365-
*scale = 2;
366-
*value = threshold_ns >> 10;
367-
} else if (threshold_ns < 1048576) {
368-
*scale = 3;
369-
*value = threshold_ns >> 15;
370-
} else if (threshold_ns < 33554432) {
371-
*scale = 4;
372-
*value = threshold_ns >> 20;
373+
} else if (threshold_ns <= 0x3ff * 32) {
374+
*scale = 1; /* Value times 32ns */
375+
*value = roundup(threshold_ns, 32) / 32;
376+
} else if (threshold_ns <= 0x3ff * 1024) {
377+
*scale = 2; /* Value times 1024ns */
378+
*value = roundup(threshold_ns, 1024) / 1024;
379+
} else if (threshold_ns <= 0x3ff * 32768) {
380+
*scale = 3; /* Value times 32768ns */
381+
*value = roundup(threshold_ns, 32768) / 32768;
382+
} else if (threshold_ns <= 0x3ff * 1048576) {
383+
*scale = 4; /* Value times 1048576ns */
384+
*value = roundup(threshold_ns, 1048576) / 1048576;
385+
} else if (threshold_ns <= 0x3ff * (u64) 33554432) {
386+
*scale = 5; /* Value times 33554432ns */
387+
*value = roundup(threshold_ns, 33554432) / 33554432;
373388
} else {
374389
*scale = 5;
375-
*value = threshold_ns >> 25;
390+
*value = 0x3ff; /* Max representable value */
376391
}
377392
}
378393

@@ -455,6 +470,31 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
455470
pci_write_config_dword(pdev, pos, val);
456471
}
457472

473+
static void aspm_program_l1ss(struct pci_dev *dev, u32 ctl1, u32 ctl2)
474+
{
475+
u16 l1ss = dev->l1ss;
476+
u32 l1_2_enable;
477+
478+
/*
479+
* Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be
480+
* programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1.
481+
*/
482+
pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL2, ctl2);
483+
484+
/*
485+
* In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in
486+
* PCI_L1SS_CTL1 must be programmed *before* setting the L1.2
487+
* enable bits, even though they're all in PCI_L1SS_CTL1.
488+
*/
489+
l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK;
490+
ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;
491+
492+
pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, ctl1);
493+
if (l1_2_enable)
494+
pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1,
495+
ctl1 | l1_2_enable);
496+
}
497+
458498
/* Calculate L1.2 PM substate timing parameters */
459499
static void aspm_calc_l1ss_info(struct pcie_link_state *link,
460500
u32 parent_l1ss_cap, u32 child_l1ss_cap)
@@ -464,7 +504,6 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
464504
u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
465505
u32 ctl1 = 0, ctl2 = 0;
466506
u32 pctl1, pctl2, cctl1, cctl2;
467-
u32 pl1_2_enables, cl1_2_enables;
468507

469508
if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
470509
return;
@@ -513,48 +552,85 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
513552
ctl2 == pctl2 && ctl2 == cctl2)
514553
return;
515554

516-
/* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
517-
pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
518-
cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
555+
pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
556+
PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
557+
PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
558+
pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
559+
PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
560+
PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
561+
aspm_program_l1ss(parent, pctl1, ctl2);
562+
563+
cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
564+
PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
565+
PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
566+
cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
567+
PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
568+
PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
569+
aspm_program_l1ss(child, cctl1, ctl2);
570+
}
519571

520-
if (pl1_2_enables || cl1_2_enables) {
521-
pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
522-
PCI_L1SS_CTL1_L1_2_MASK, 0);
523-
pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
524-
PCI_L1SS_CTL1_L1_2_MASK, 0);
525-
}
572+
static void aspm_l1ss_init(struct pcie_link_state *link)
573+
{
574+
struct pci_dev *child = link->downstream, *parent = link->pdev;
575+
u32 parent_l1ss_cap, child_l1ss_cap;
576+
u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
526577

527-
/* Program T_POWER_ON times in both ports */
528-
pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
529-
pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
578+
if (!parent->l1ss || !child->l1ss)
579+
return;
530580

531-
/* Program Common_Mode_Restore_Time in upstream device */
532-
pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
533-
PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
581+
/* Setup L1 substate */
582+
pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
583+
&parent_l1ss_cap);
584+
pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
585+
&child_l1ss_cap);
534586

535-
/* Program LTR_L1.2_THRESHOLD time in both ports */
536-
pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
537-
PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
538-
PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
539-
pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
540-
PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
541-
PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
542-
543-
if (pl1_2_enables || cl1_2_enables) {
544-
pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
545-
pl1_2_enables);
546-
pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
547-
cl1_2_enables);
548-
}
587+
if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
588+
parent_l1ss_cap = 0;
589+
if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
590+
child_l1ss_cap = 0;
591+
592+
/*
593+
* If we don't have LTR for the entire path from the Root Complex
594+
* to this device, we can't use ASPM L1.2 because it relies on the
595+
* LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
596+
*/
597+
if (!child->ltr_path)
598+
child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
599+
600+
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
601+
link->aspm_support |= ASPM_STATE_L1_1;
602+
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
603+
link->aspm_support |= ASPM_STATE_L1_2;
604+
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
605+
link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
606+
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
607+
link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
608+
609+
if (parent_l1ss_cap)
610+
pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
611+
&parent_l1ss_ctl1);
612+
if (child_l1ss_cap)
613+
pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
614+
&child_l1ss_ctl1);
615+
616+
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
617+
link->aspm_enabled |= ASPM_STATE_L1_1;
618+
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
619+
link->aspm_enabled |= ASPM_STATE_L1_2;
620+
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
621+
link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
622+
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
623+
link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
624+
625+
if (link->aspm_support & ASPM_STATE_L1SS)
626+
aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
549627
}
550628

551629
static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
552630
{
553631
struct pci_dev *child = link->downstream, *parent = link->pdev;
554632
u32 parent_lnkcap, child_lnkcap;
555633
u16 parent_lnkctl, child_lnkctl;
556-
u32 parent_l1ss_cap, child_l1ss_cap;
557-
u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
558634
struct pci_bus *linkbus = parent->subordinate;
559635

560636
if (blacklist) {
@@ -609,52 +685,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
609685
if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
610686
link->aspm_enabled |= ASPM_STATE_L1;
611687

612-
/* Setup L1 substate */
613-
pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
614-
&parent_l1ss_cap);
615-
pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
616-
&child_l1ss_cap);
617-
618-
if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
619-
parent_l1ss_cap = 0;
620-
if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
621-
child_l1ss_cap = 0;
622-
623-
/*
624-
* If we don't have LTR for the entire path from the Root Complex
625-
* to this device, we can't use ASPM L1.2 because it relies on the
626-
* LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
627-
*/
628-
if (!child->ltr_path)
629-
child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
630-
631-
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
632-
link->aspm_support |= ASPM_STATE_L1_1;
633-
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
634-
link->aspm_support |= ASPM_STATE_L1_2;
635-
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
636-
link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
637-
if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
638-
link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
639-
640-
if (parent_l1ss_cap)
641-
pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
642-
&parent_l1ss_ctl1);
643-
if (child_l1ss_cap)
644-
pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
645-
&child_l1ss_ctl1);
646-
647-
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
648-
link->aspm_enabled |= ASPM_STATE_L1_1;
649-
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
650-
link->aspm_enabled |= ASPM_STATE_L1_2;
651-
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
652-
link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
653-
if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
654-
link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
655-
656-
if (link->aspm_support & ASPM_STATE_L1SS)
657-
aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
688+
aspm_l1ss_init(link);
658689

659690
/* Save default state */
660691
link->aspm_default = link->aspm_enabled;
@@ -726,6 +757,43 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
726757
PCI_L1SS_CTL1_L1SS_MASK, val);
727758
}
728759

760+
void pci_save_aspm_l1ss_state(struct pci_dev *dev)
761+
{
762+
struct pci_cap_saved_state *save_state;
763+
u16 l1ss = dev->l1ss;
764+
u32 *cap;
765+
766+
if (!l1ss)
767+
return;
768+
769+
save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS);
770+
if (!save_state)
771+
return;
772+
773+
cap = (u32 *)&save_state->cap.data[0];
774+
pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL2, cap++);
775+
pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL1, cap++);
776+
}
777+
778+
void pci_restore_aspm_l1ss_state(struct pci_dev *dev)
779+
{
780+
struct pci_cap_saved_state *save_state;
781+
u32 *cap, ctl1, ctl2;
782+
u16 l1ss = dev->l1ss;
783+
784+
if (!l1ss)
785+
return;
786+
787+
save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS);
788+
if (!save_state)
789+
return;
790+
791+
cap = (u32 *)&save_state->cap.data[0];
792+
ctl2 = *cap++;
793+
ctl1 = *cap;
794+
aspm_program_l1ss(dev, ctl1, ctl2);
795+
}
796+
729797
static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
730798
{
731799
pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,

0 commit comments

Comments
 (0)