8
8
*/
9
9
10
10
#include <linux/kernel.h>
11
+ #include <linux/math.h>
11
12
#include <linux/module.h>
12
13
#include <linux/moduleparam.h>
13
14
#include <linux/pci.h>
@@ -350,29 +351,43 @@ static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
350
351
return 0 ;
351
352
}
352
353
354
+ /*
355
+ * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1
356
+ * register. Ports enter L1.2 when the most recent LTR value is greater
357
+ * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we
358
+ * don't enter L1.2 too aggressively.
359
+ *
360
+ * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.
361
+ */
353
362
static void encode_l12_threshold (u32 threshold_us , u32 * scale , u32 * value )
354
363
{
355
- u32 threshold_ns = threshold_us * 1000 ;
364
+ u64 threshold_ns = ( u64 ) threshold_us * 1000 ;
356
365
357
- /* See PCIe r3.1, sec 7.33.3 and sec 6.18 */
358
- if (threshold_ns < 32 ) {
359
- * scale = 0 ;
366
+ /*
367
+ * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max
368
+ * value of 0x3ff.
369
+ */
370
+ if (threshold_ns <= 0x3ff * 1 ) {
371
+ * scale = 0 ; /* Value times 1ns */
360
372
* value = threshold_ns ;
361
- } else if (threshold_ns < 1024 ) {
362
- * scale = 1 ;
363
- * value = threshold_ns >> 5 ;
364
- } else if (threshold_ns < 32768 ) {
365
- * scale = 2 ;
366
- * value = threshold_ns >> 10 ;
367
- } else if (threshold_ns < 1048576 ) {
368
- * scale = 3 ;
369
- * value = threshold_ns >> 15 ;
370
- } else if (threshold_ns < 33554432 ) {
371
- * scale = 4 ;
372
- * value = threshold_ns >> 20 ;
373
+ } else if (threshold_ns <= 0x3ff * 32 ) {
374
+ * scale = 1 ; /* Value times 32ns */
375
+ * value = roundup (threshold_ns , 32 ) / 32 ;
376
+ } else if (threshold_ns <= 0x3ff * 1024 ) {
377
+ * scale = 2 ; /* Value times 1024ns */
378
+ * value = roundup (threshold_ns , 1024 ) / 1024 ;
379
+ } else if (threshold_ns <= 0x3ff * 32768 ) {
380
+ * scale = 3 ; /* Value times 32768ns */
381
+ * value = roundup (threshold_ns , 32768 ) / 32768 ;
382
+ } else if (threshold_ns <= 0x3ff * 1048576 ) {
383
+ * scale = 4 ; /* Value times 1048576ns */
384
+ * value = roundup (threshold_ns , 1048576 ) / 1048576 ;
385
+ } else if (threshold_ns <= 0x3ff * (u64 ) 33554432 ) {
386
+ * scale = 5 ; /* Value times 33554432ns */
387
+ * value = roundup (threshold_ns , 33554432 ) / 33554432 ;
373
388
} else {
374
389
* scale = 5 ;
375
- * value = threshold_ns >> 25 ;
390
+ * value = 0x3ff ; /* Max representable value */
376
391
}
377
392
}
378
393
@@ -455,6 +470,31 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
455
470
pci_write_config_dword (pdev , pos , val );
456
471
}
457
472
473
+ static void aspm_program_l1ss (struct pci_dev * dev , u32 ctl1 , u32 ctl2 )
474
+ {
475
+ u16 l1ss = dev -> l1ss ;
476
+ u32 l1_2_enable ;
477
+
478
+ /*
479
+ * Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be
480
+ * programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1.
481
+ */
482
+ pci_write_config_dword (dev , l1ss + PCI_L1SS_CTL2 , ctl2 );
483
+
484
+ /*
485
+ * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in
486
+ * PCI_L1SS_CTL1 must be programmed *before* setting the L1.2
487
+ * enable bits, even though they're all in PCI_L1SS_CTL1.
488
+ */
489
+ l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK ;
490
+ ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK ;
491
+
492
+ pci_write_config_dword (dev , l1ss + PCI_L1SS_CTL1 , ctl1 );
493
+ if (l1_2_enable )
494
+ pci_write_config_dword (dev , l1ss + PCI_L1SS_CTL1 ,
495
+ ctl1 | l1_2_enable );
496
+ }
497
+
458
498
/* Calculate L1.2 PM substate timing parameters */
459
499
static void aspm_calc_l1ss_info (struct pcie_link_state * link ,
460
500
u32 parent_l1ss_cap , u32 child_l1ss_cap )
@@ -464,7 +504,6 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
464
504
u32 t_common_mode , t_power_on , l1_2_threshold , scale , value ;
465
505
u32 ctl1 = 0 , ctl2 = 0 ;
466
506
u32 pctl1 , pctl2 , cctl1 , cctl2 ;
467
- u32 pl1_2_enables , cl1_2_enables ;
468
507
469
508
if (!(link -> aspm_support & ASPM_STATE_L1_2_MASK ))
470
509
return ;
@@ -513,48 +552,85 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link,
513
552
ctl2 == pctl2 && ctl2 == cctl2 )
514
553
return ;
515
554
516
- /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
517
- pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK ;
518
- cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK ;
555
+ pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
556
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
557
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE );
558
+ pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
559
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
560
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE ));
561
+ aspm_program_l1ss (parent , pctl1 , ctl2 );
562
+
563
+ cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
564
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
565
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE );
566
+ cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
567
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
568
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE ));
569
+ aspm_program_l1ss (child , cctl1 , ctl2 );
570
+ }
519
571
520
- if (pl1_2_enables || cl1_2_enables ) {
521
- pci_clear_and_set_dword (child , child -> l1ss + PCI_L1SS_CTL1 ,
522
- PCI_L1SS_CTL1_L1_2_MASK , 0 );
523
- pci_clear_and_set_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 ,
524
- PCI_L1SS_CTL1_L1_2_MASK , 0 );
525
- }
572
+ static void aspm_l1ss_init (struct pcie_link_state * link )
573
+ {
574
+ struct pci_dev * child = link -> downstream , * parent = link -> pdev ;
575
+ u32 parent_l1ss_cap , child_l1ss_cap ;
576
+ u32 parent_l1ss_ctl1 = 0 , child_l1ss_ctl1 = 0 ;
526
577
527
- /* Program T_POWER_ON times in both ports */
528
- pci_write_config_dword (parent , parent -> l1ss + PCI_L1SS_CTL2 , ctl2 );
529
- pci_write_config_dword (child , child -> l1ss + PCI_L1SS_CTL2 , ctl2 );
578
+ if (!parent -> l1ss || !child -> l1ss )
579
+ return ;
530
580
531
- /* Program Common_Mode_Restore_Time in upstream device */
532
- pci_clear_and_set_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 ,
533
- PCI_L1SS_CTL1_CM_RESTORE_TIME , ctl1 );
581
+ /* Setup L1 substate */
582
+ pci_read_config_dword (parent , parent -> l1ss + PCI_L1SS_CAP ,
583
+ & parent_l1ss_cap );
584
+ pci_read_config_dword (child , child -> l1ss + PCI_L1SS_CAP ,
585
+ & child_l1ss_cap );
534
586
535
- /* Program LTR_L1.2_THRESHOLD time in both ports */
536
- pci_clear_and_set_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 ,
537
- PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
538
- PCI_L1SS_CTL1_LTR_L12_TH_SCALE , ctl1 );
539
- pci_clear_and_set_dword (child , child -> l1ss + PCI_L1SS_CTL1 ,
540
- PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
541
- PCI_L1SS_CTL1_LTR_L12_TH_SCALE , ctl1 );
542
-
543
- if (pl1_2_enables || cl1_2_enables ) {
544
- pci_clear_and_set_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 , 0 ,
545
- pl1_2_enables );
546
- pci_clear_and_set_dword (child , child -> l1ss + PCI_L1SS_CTL1 , 0 ,
547
- cl1_2_enables );
548
- }
587
+ if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS ))
588
+ parent_l1ss_cap = 0 ;
589
+ if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS ))
590
+ child_l1ss_cap = 0 ;
591
+
592
+ /*
593
+ * If we don't have LTR for the entire path from the Root Complex
594
+ * to this device, we can't use ASPM L1.2 because it relies on the
595
+ * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
596
+ */
597
+ if (!child -> ltr_path )
598
+ child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2 ;
599
+
600
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1 )
601
+ link -> aspm_support |= ASPM_STATE_L1_1 ;
602
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2 )
603
+ link -> aspm_support |= ASPM_STATE_L1_2 ;
604
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1 )
605
+ link -> aspm_support |= ASPM_STATE_L1_1_PCIPM ;
606
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2 )
607
+ link -> aspm_support |= ASPM_STATE_L1_2_PCIPM ;
608
+
609
+ if (parent_l1ss_cap )
610
+ pci_read_config_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 ,
611
+ & parent_l1ss_ctl1 );
612
+ if (child_l1ss_cap )
613
+ pci_read_config_dword (child , child -> l1ss + PCI_L1SS_CTL1 ,
614
+ & child_l1ss_ctl1 );
615
+
616
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1 )
617
+ link -> aspm_enabled |= ASPM_STATE_L1_1 ;
618
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2 )
619
+ link -> aspm_enabled |= ASPM_STATE_L1_2 ;
620
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1 )
621
+ link -> aspm_enabled |= ASPM_STATE_L1_1_PCIPM ;
622
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2 )
623
+ link -> aspm_enabled |= ASPM_STATE_L1_2_PCIPM ;
624
+
625
+ if (link -> aspm_support & ASPM_STATE_L1SS )
626
+ aspm_calc_l1ss_info (link , parent_l1ss_cap , child_l1ss_cap );
549
627
}
550
628
551
629
static void pcie_aspm_cap_init (struct pcie_link_state * link , int blacklist )
552
630
{
553
631
struct pci_dev * child = link -> downstream , * parent = link -> pdev ;
554
632
u32 parent_lnkcap , child_lnkcap ;
555
633
u16 parent_lnkctl , child_lnkctl ;
556
- u32 parent_l1ss_cap , child_l1ss_cap ;
557
- u32 parent_l1ss_ctl1 = 0 , child_l1ss_ctl1 = 0 ;
558
634
struct pci_bus * linkbus = parent -> subordinate ;
559
635
560
636
if (blacklist ) {
@@ -609,52 +685,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
609
685
if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1 )
610
686
link -> aspm_enabled |= ASPM_STATE_L1 ;
611
687
612
- /* Setup L1 substate */
613
- pci_read_config_dword (parent , parent -> l1ss + PCI_L1SS_CAP ,
614
- & parent_l1ss_cap );
615
- pci_read_config_dword (child , child -> l1ss + PCI_L1SS_CAP ,
616
- & child_l1ss_cap );
617
-
618
- if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS ))
619
- parent_l1ss_cap = 0 ;
620
- if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS ))
621
- child_l1ss_cap = 0 ;
622
-
623
- /*
624
- * If we don't have LTR for the entire path from the Root Complex
625
- * to this device, we can't use ASPM L1.2 because it relies on the
626
- * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
627
- */
628
- if (!child -> ltr_path )
629
- child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2 ;
630
-
631
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1 )
632
- link -> aspm_support |= ASPM_STATE_L1_1 ;
633
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2 )
634
- link -> aspm_support |= ASPM_STATE_L1_2 ;
635
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1 )
636
- link -> aspm_support |= ASPM_STATE_L1_1_PCIPM ;
637
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2 )
638
- link -> aspm_support |= ASPM_STATE_L1_2_PCIPM ;
639
-
640
- if (parent_l1ss_cap )
641
- pci_read_config_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 ,
642
- & parent_l1ss_ctl1 );
643
- if (child_l1ss_cap )
644
- pci_read_config_dword (child , child -> l1ss + PCI_L1SS_CTL1 ,
645
- & child_l1ss_ctl1 );
646
-
647
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1 )
648
- link -> aspm_enabled |= ASPM_STATE_L1_1 ;
649
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2 )
650
- link -> aspm_enabled |= ASPM_STATE_L1_2 ;
651
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1 )
652
- link -> aspm_enabled |= ASPM_STATE_L1_1_PCIPM ;
653
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2 )
654
- link -> aspm_enabled |= ASPM_STATE_L1_2_PCIPM ;
655
-
656
- if (link -> aspm_support & ASPM_STATE_L1SS )
657
- aspm_calc_l1ss_info (link , parent_l1ss_cap , child_l1ss_cap );
688
+ aspm_l1ss_init (link );
658
689
659
690
/* Save default state */
660
691
link -> aspm_default = link -> aspm_enabled ;
@@ -726,6 +757,43 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
726
757
PCI_L1SS_CTL1_L1SS_MASK , val );
727
758
}
728
759
760
+ void pci_save_aspm_l1ss_state (struct pci_dev * dev )
761
+ {
762
+ struct pci_cap_saved_state * save_state ;
763
+ u16 l1ss = dev -> l1ss ;
764
+ u32 * cap ;
765
+
766
+ if (!l1ss )
767
+ return ;
768
+
769
+ save_state = pci_find_saved_ext_cap (dev , PCI_EXT_CAP_ID_L1SS );
770
+ if (!save_state )
771
+ return ;
772
+
773
+ cap = (u32 * )& save_state -> cap .data [0 ];
774
+ pci_read_config_dword (dev , l1ss + PCI_L1SS_CTL2 , cap ++ );
775
+ pci_read_config_dword (dev , l1ss + PCI_L1SS_CTL1 , cap ++ );
776
+ }
777
+
778
+ void pci_restore_aspm_l1ss_state (struct pci_dev * dev )
779
+ {
780
+ struct pci_cap_saved_state * save_state ;
781
+ u32 * cap , ctl1 , ctl2 ;
782
+ u16 l1ss = dev -> l1ss ;
783
+
784
+ if (!l1ss )
785
+ return ;
786
+
787
+ save_state = pci_find_saved_ext_cap (dev , PCI_EXT_CAP_ID_L1SS );
788
+ if (!save_state )
789
+ return ;
790
+
791
+ cap = (u32 * )& save_state -> cap .data [0 ];
792
+ ctl2 = * cap ++ ;
793
+ ctl1 = * cap ;
794
+ aspm_program_l1ss (dev , ctl1 , ctl2 );
795
+ }
796
+
729
797
static void pcie_config_aspm_dev (struct pci_dev * pdev , u32 val )
730
798
{
731
799
pcie_capability_clear_and_set_word (pdev , PCI_EXP_LNKCTL ,
0 commit comments