1/* 2 * File: drivers/pci/pcie/aspm.c 3 * Enabling PCIe link L0s/L1 state and Clock Power Management 4 * 5 * Copyright (C) 2007 Intel 6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 7 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 8 */ 9 10#include <linux/kernel.h> 11#include <linux/module.h> 12#include <linux/moduleparam.h> 13#include <linux/pci.h> 14#include <linux/pci_regs.h> 15#include <linux/errno.h> 16#include <linux/pm.h> 17#include <linux/init.h> 18#include <linux/slab.h> 19#include <linux/jiffies.h> 20#include <linux/delay.h> 21#include <linux/pci-aspm.h> 22#include "../pci.h" 23 24#ifdef MODULE_PARAM_PREFIX 25#undef MODULE_PARAM_PREFIX 26#endif 27#define MODULE_PARAM_PREFIX "pcie_aspm." 28 29/* Note: those are not register definitions */ 30#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 31#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 32#define ASPM_STATE_L1 (4) /* L1 state */ 33#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 34#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1) 35 36struct aspm_latency { 37 u32 l0s; /* L0s latency (nsec) */ 38 u32 l1; /* L1 latency (nsec) */ 39}; 40 41struct pcie_link_state { 42 struct pci_dev *pdev; /* Upstream component of the Link */ 43 struct pcie_link_state *root; /* pointer to the root port link */ 44 struct pcie_link_state *parent; /* pointer to the parent Link state */ 45 struct list_head sibling; /* node in link_list */ 46 struct list_head children; /* list of child link states */ 47 struct list_head link; /* node in parent's children list */ 48 49 /* ASPM state */ 50 u32 aspm_support:3; /* Supported ASPM state */ 51 u32 aspm_enabled:3; /* Enabled ASPM state */ 52 u32 aspm_capable:3; /* Capable ASPM state with latency */ 53 u32 aspm_default:3; /* Default ASPM state by BIOS */ 54 u32 aspm_disable:3; /* Disabled ASPM state */ 55 56 /* Clock PM state */ 57 u32 clkpm_capable:1; /* Clock PM capable? */ 58 u32 clkpm_enabled:1; /* Current Clock PM state */ 59 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 60 61 /* Exit latencies */ 62 struct aspm_latency latency_up; /* Upstream direction exit latency */ 63 struct aspm_latency latency_dw; /* Downstream direction exit latency */ 64 /* 65 * Endpoint acceptable latencies. A pcie downstream port only 66 * has one slot under it, so at most there are 8 functions. 67 */ 68 struct aspm_latency acceptable[8]; 69}; 70 71static int aspm_disabled, aspm_force; 72static bool aspm_support_enabled = true; 73static DEFINE_MUTEX(aspm_lock); 74static LIST_HEAD(link_list); 75 76#define POLICY_DEFAULT 0 /* BIOS default setting */ 77#define POLICY_PERFORMANCE 1 /* high performance */ 78#define POLICY_POWERSAVE 2 /* high power saving */ 79 80#ifdef CONFIG_PCIEASPM_PERFORMANCE 81static int aspm_policy = POLICY_PERFORMANCE; 82#elif defined CONFIG_PCIEASPM_POWERSAVE 83static int aspm_policy = POLICY_POWERSAVE; 84#else 85static int aspm_policy; 86#endif 87 88static const char *policy_str[] = { 89 [POLICY_DEFAULT] = "default", 90 [POLICY_PERFORMANCE] = "performance", 91 [POLICY_POWERSAVE] = "powersave" 92}; 93 94#define LINK_RETRAIN_TIMEOUT HZ 95 96static int policy_to_aspm_state(struct pcie_link_state *link) 97{ 98 switch (aspm_policy) { 99 case POLICY_PERFORMANCE: 100 /* Disable ASPM and Clock PM */ 101 return 0; 102 case POLICY_POWERSAVE: 103 /* Enable ASPM L0s/L1 */ 104 return ASPM_STATE_ALL; 105 case POLICY_DEFAULT: 106 return link->aspm_default; 107 } 108 return 0; 109} 110 111static int policy_to_clkpm_state(struct pcie_link_state *link) 112{ 113 switch (aspm_policy) { 114 case POLICY_PERFORMANCE: 115 /* Disable ASPM and Clock PM */ 116 return 0; 117 case POLICY_POWERSAVE: 118 /* Disable Clock PM */ 119 return 1; 120 case POLICY_DEFAULT: 121 return link->clkpm_default; 122 } 123 return 0; 124} 125 126static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 127{ 128 struct pci_dev *child; 129 struct pci_bus *linkbus = link->pdev->subordinate; 130 131 list_for_each_entry(child, &linkbus->devices, bus_list) { 132 if (enable) 133 pcie_capability_set_word(child, PCI_EXP_LNKCTL, 134 PCI_EXP_LNKCTL_CLKREQ_EN); 135 else 136 pcie_capability_clear_word(child, PCI_EXP_LNKCTL, 137 PCI_EXP_LNKCTL_CLKREQ_EN); 138 } 139 link->clkpm_enabled = !!enable; 140} 141 142static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 143{ 144 /* Don't enable Clock PM if the link is not Clock PM capable */ 145 if (!link->clkpm_capable && enable) 146 enable = 0; 147 /* Need nothing if the specified equals to current state */ 148 if (link->clkpm_enabled == enable) 149 return; 150 pcie_set_clkpm_nocheck(link, enable); 151} 152 153static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 154{ 155 int capable = 1, enabled = 1; 156 u32 reg32; 157 u16 reg16; 158 struct pci_dev *child; 159 struct pci_bus *linkbus = link->pdev->subordinate; 160 161 /* All functions should have the same cap and state, take the worst */ 162 list_for_each_entry(child, &linkbus->devices, bus_list) { 163 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); 164 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 165 capable = 0; 166 enabled = 0; 167 break; 168 } 169 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 170 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 171 enabled = 0; 172 } 173 link->clkpm_enabled = enabled; 174 link->clkpm_default = enabled; 175 link->clkpm_capable = (blacklist) ? 0 : capable; 176} 177 178/* 179 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 180 * could use common clock. If they are, configure them to use the 181 * common clock. That will reduce the ASPM state exit latency. 182 */ 183static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 184{ 185 int same_clock = 1; 186 u16 reg16, parent_reg, child_reg[8]; 187 unsigned long start_jiffies; 188 struct pci_dev *child, *parent = link->pdev; 189 struct pci_bus *linkbus = parent->subordinate; 190 /* 191 * All functions of a slot should have the same Slot Clock 192 * Configuration, so just check one function 193 */ 194 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 195 BUG_ON(!pci_is_pcie(child)); 196 197 /* Check downstream component if bit Slot Clock Configuration is 1 */ 198 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); 199 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 200 same_clock = 0; 201 202 /* Check upstream component if bit Slot Clock Configuration is 1 */ 203 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 204 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 205 same_clock = 0; 206 207 /* Configure downstream component, all functions */ 208 list_for_each_entry(child, &linkbus->devices, bus_list) { 209 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 210 child_reg[PCI_FUNC(child->devfn)] = reg16; 211 if (same_clock) 212 reg16 |= PCI_EXP_LNKCTL_CCC; 213 else 214 reg16 &= ~PCI_EXP_LNKCTL_CCC; 215 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); 216 } 217 218 /* Configure upstream component */ 219 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); 220 parent_reg = reg16; 221 if (same_clock) 222 reg16 |= PCI_EXP_LNKCTL_CCC; 223 else 224 reg16 &= ~PCI_EXP_LNKCTL_CCC; 225 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 226 227 /* Retrain link */ 228 reg16 |= PCI_EXP_LNKCTL_RL; 229 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 230 231 /* Wait for link training end. Break out after waiting for timeout */ 232 start_jiffies = jiffies; 233 for (;;) { 234 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 235 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 236 break; 237 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 238 break; 239 msleep(1); 240 } 241 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 242 return; 243 244 /* Training failed. Restore common clock configurations */ 245 dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); 246 list_for_each_entry(child, &linkbus->devices, bus_list) 247 pcie_capability_write_word(child, PCI_EXP_LNKCTL, 248 child_reg[PCI_FUNC(child->devfn)]); 249 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); 250} 251 252/* Convert L0s latency encoding to ns */ 253static u32 calc_l0s_latency(u32 encoding) 254{ 255 if (encoding == 0x7) 256 return (5 * 1000); /* > 4us */ 257 return (64 << encoding); 258} 259 260/* Convert L0s acceptable latency encoding to ns */ 261static u32 calc_l0s_acceptable(u32 encoding) 262{ 263 if (encoding == 0x7) 264 return -1U; 265 return (64 << encoding); 266} 267 268/* Convert L1 latency encoding to ns */ 269static u32 calc_l1_latency(u32 encoding) 270{ 271 if (encoding == 0x7) 272 return (65 * 1000); /* > 64us */ 273 return (1000 << encoding); 274} 275 276/* Convert L1 acceptable latency encoding to ns */ 277static u32 calc_l1_acceptable(u32 encoding) 278{ 279 if (encoding == 0x7) 280 return -1U; 281 return (1000 << encoding); 282} 283 284struct aspm_register_info { 285 u32 support:2; 286 u32 enabled:2; 287 u32 latency_encoding_l0s; 288 u32 latency_encoding_l1; 289}; 290 291static void pcie_get_aspm_reg(struct pci_dev *pdev, 292 struct aspm_register_info *info) 293{ 294 u16 reg16; 295 u32 reg32; 296 297 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); 298 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 299 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 300 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 301 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); 302 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 303} 304 305static void pcie_aspm_check_latency(struct pci_dev *endpoint) 306{ 307 u32 latency, l1_switch_latency = 0; 308 struct aspm_latency *acceptable; 309 struct pcie_link_state *link; 310 311 /* Device not in D0 doesn't need latency check */ 312 if ((endpoint->current_state != PCI_D0) && 313 (endpoint->current_state != PCI_UNKNOWN)) 314 return; 315 316 link = endpoint->bus->self->link_state; 317 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 318 319 while (link) { 320 /* Check upstream direction L0s latency */ 321 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 322 (link->latency_up.l0s > acceptable->l0s)) 323 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 324 325 /* Check downstream direction L0s latency */ 326 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 327 (link->latency_dw.l0s > acceptable->l0s)) 328 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 329 /* 330 * Check L1 latency. 331 * Every switch on the path to root complex need 1 332 * more microsecond for L1. Spec doesn't mention L0s. 333 */ 334 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 335 if ((link->aspm_capable & ASPM_STATE_L1) && 336 (latency + l1_switch_latency > acceptable->l1)) 337 link->aspm_capable &= ~ASPM_STATE_L1; 338 l1_switch_latency += 1000; 339 340 link = link->parent; 341 } 342} 343 344static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 345{ 346 struct pci_dev *child, *parent = link->pdev; 347 struct pci_bus *linkbus = parent->subordinate; 348 struct aspm_register_info upreg, dwreg; 349 350 if (blacklist) { 351 /* Set enabled/disable so that we will disable ASPM later */ 352 link->aspm_enabled = ASPM_STATE_ALL; 353 link->aspm_disable = ASPM_STATE_ALL; 354 return; 355 } 356 357 /* Configure common clock before checking latencies */ 358 pcie_aspm_configure_common_clock(link); 359 360 /* Get upstream/downstream components' register state */ 361 pcie_get_aspm_reg(parent, &upreg); 362 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 363 pcie_get_aspm_reg(child, &dwreg); 364 365 /* 366 * Setup L0s state 367 * 368 * Note that we must not enable L0s in either direction on a 369 * given link unless components on both sides of the link each 370 * support L0s. 371 */ 372 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 373 link->aspm_support |= ASPM_STATE_L0S; 374 if (dwreg.enabled & PCIE_LINK_STATE_L0S) 375 link->aspm_enabled |= ASPM_STATE_L0S_UP; 376 if (upreg.enabled & PCIE_LINK_STATE_L0S) 377 link->aspm_enabled |= ASPM_STATE_L0S_DW; 378 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 379 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 380 381 /* Setup L1 state */ 382 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 383 link->aspm_support |= ASPM_STATE_L1; 384 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 385 link->aspm_enabled |= ASPM_STATE_L1; 386 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 387 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 388 389 /* Save default state */ 390 link->aspm_default = link->aspm_enabled; 391 392 /* Setup initial capable state. Will be updated later */ 393 link->aspm_capable = link->aspm_support; 394 /* 395 * If the downstream component has pci bridge function, don't 396 * do ASPM for now. 397 */ 398 list_for_each_entry(child, &linkbus->devices, bus_list) { 399 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { 400 link->aspm_disable = ASPM_STATE_ALL; 401 break; 402 } 403 } 404 405 /* Get and check endpoint acceptable latencies */ 406 list_for_each_entry(child, &linkbus->devices, bus_list) { 407 u32 reg32, encoding; 408 struct aspm_latency *acceptable = 409 &link->acceptable[PCI_FUNC(child->devfn)]; 410 411 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && 412 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) 413 continue; 414 415 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 416 /* Calculate endpoint L0s acceptable latency */ 417 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 418 acceptable->l0s = calc_l0s_acceptable(encoding); 419 /* Calculate endpoint L1 acceptable latency */ 420 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 421 acceptable->l1 = calc_l1_acceptable(encoding); 422 423 pcie_aspm_check_latency(child); 424 } 425} 426 427static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 428{ 429 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, 430 PCI_EXP_LNKCTL_ASPMC, val); 431} 432 433static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 434{ 435 u32 upstream = 0, dwstream = 0; 436 struct pci_dev *child, *parent = link->pdev; 437 struct pci_bus *linkbus = parent->subordinate; 438 439 /* Nothing to do if the link is already in the requested state */ 440 state &= (link->aspm_capable & ~link->aspm_disable); 441 if (link->aspm_enabled == state) 442 return; 443 /* Convert ASPM state to upstream/downstream ASPM register state */ 444 if (state & ASPM_STATE_L0S_UP) 445 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; 446 if (state & ASPM_STATE_L0S_DW) 447 upstream |= PCI_EXP_LNKCTL_ASPM_L0S; 448 if (state & ASPM_STATE_L1) { 449 upstream |= PCI_EXP_LNKCTL_ASPM_L1; 450 dwstream |= PCI_EXP_LNKCTL_ASPM_L1; 451 } 452 /* 453 * Spec 2.0 suggests all functions should be configured the 454 * same setting for ASPM. Enabling ASPM L1 should be done in 455 * upstream component first and then downstream, and vice 456 * versa for disabling ASPM L1. Spec doesn't mention L0S. 457 */ 458 if (state & ASPM_STATE_L1) 459 pcie_config_aspm_dev(parent, upstream); 460 list_for_each_entry(child, &linkbus->devices, bus_list) 461 pcie_config_aspm_dev(child, dwstream); 462 if (!(state & ASPM_STATE_L1)) 463 pcie_config_aspm_dev(parent, upstream); 464 465 link->aspm_enabled = state; 466} 467 468static void pcie_config_aspm_path(struct pcie_link_state *link) 469{ 470 while (link) { 471 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 472 link = link->parent; 473 } 474} 475 476static void free_link_state(struct pcie_link_state *link) 477{ 478 link->pdev->link_state = NULL; 479 kfree(link); 480} 481 482static int pcie_aspm_sanity_check(struct pci_dev *pdev) 483{ 484 struct pci_dev *child; 485 u32 reg32; 486 487 /* 488 * Some functions in a slot might not all be PCIe functions, 489 * very strange. Disable ASPM for the whole slot 490 */ 491 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 492 if (!pci_is_pcie(child)) 493 return -EINVAL; 494 495 /* 496 * If ASPM is disabled then we're not going to change 497 * the BIOS state. It's safe to continue even if it's a 498 * pre-1.1 device 499 */ 500 501 if (aspm_disabled) 502 continue; 503 504 /* 505 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 506 * RBER bit to determine if a function is 1.1 version device 507 */ 508 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 509 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 510 dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); 511 return -EINVAL; 512 } 513 } 514 return 0; 515} 516 517static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 518{ 519 struct pcie_link_state *link; 520 521 link = kzalloc(sizeof(*link), GFP_KERNEL); 522 if (!link) 523 return NULL; 524 INIT_LIST_HEAD(&link->sibling); 525 INIT_LIST_HEAD(&link->children); 526 INIT_LIST_HEAD(&link->link); 527 link->pdev = pdev; 528 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) { 529 struct pcie_link_state *parent; 530 parent = pdev->bus->parent->self->link_state; 531 if (!parent) { 532 kfree(link); 533 return NULL; 534 } 535 link->parent = parent; 536 list_add(&link->link, &parent->children); 537 } 538 /* Setup a pointer to the root port link */ 539 if (!link->parent) 540 link->root = link; 541 else 542 link->root = link->parent->root; 543 544 list_add(&link->sibling, &link_list); 545 pdev->link_state = link; 546 return link; 547} 548 549/* 550 * pcie_aspm_init_link_state: Initiate PCI express link state. 551 * It is called after the pcie and its children devices are scanned. 552 * @pdev: the root port or switch downstream port 553 */ 554void pcie_aspm_init_link_state(struct pci_dev *pdev) 555{ 556 struct pcie_link_state *link; 557 int blacklist = !!pcie_aspm_sanity_check(pdev); 558 559 if (!aspm_support_enabled) 560 return; 561 562 if (!pci_is_pcie(pdev) || pdev->link_state) 563 return; 564 if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && 565 pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) 566 return; 567 568 /* VIA has a strange chipset, root port is under a bridge */ 569 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && 570 pdev->bus->self) 571 return; 572 573 down_read(&pci_bus_sem); 574 if (list_empty(&pdev->subordinate->devices)) 575 goto out; 576 577 mutex_lock(&aspm_lock); 578 link = alloc_pcie_link_state(pdev); 579 if (!link) 580 goto unlock; 581 /* 582 * Setup initial ASPM state. Note that we need to configure 583 * upstream links also because capable state of them can be 584 * update through pcie_aspm_cap_init(). 585 */ 586 pcie_aspm_cap_init(link, blacklist); 587 588 /* Setup initial Clock PM state */ 589 pcie_clkpm_cap_init(link, blacklist); 590 591 /* 592 * At this stage drivers haven't had an opportunity to change the 593 * link policy setting. Enabling ASPM on broken hardware can cripple 594 * it even before the driver has had a chance to disable ASPM, so 595 * default to a safe level right now. If we're enabling ASPM beyond 596 * the BIOS's expectation, we'll do so once pci_enable_device() is 597 * called. 598 */ 599 if (aspm_policy != POLICY_POWERSAVE) { 600 pcie_config_aspm_path(link); 601 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 602 } 603 604unlock: 605 mutex_unlock(&aspm_lock); 606out: 607 up_read(&pci_bus_sem); 608} 609 610/* Recheck latencies and update aspm_capable for links under the root */ 611static void pcie_update_aspm_capable(struct pcie_link_state *root) 612{ 613 struct pcie_link_state *link; 614 BUG_ON(root->parent); 615 list_for_each_entry(link, &link_list, sibling) { 616 if (link->root != root) 617 continue; 618 link->aspm_capable = link->aspm_support; 619 } 620 list_for_each_entry(link, &link_list, sibling) { 621 struct pci_dev *child; 622 struct pci_bus *linkbus = link->pdev->subordinate; 623 if (link->root != root) 624 continue; 625 list_for_each_entry(child, &linkbus->devices, bus_list) { 626 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && 627 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) 628 continue; 629 pcie_aspm_check_latency(child); 630 } 631 } 632} 633 634/* @pdev: the endpoint device */ 635void pcie_aspm_exit_link_state(struct pci_dev *pdev) 636{ 637 struct pci_dev *parent = pdev->bus->self; 638 struct pcie_link_state *link, *root, *parent_link; 639 640 if (!parent || !parent->link_state) 641 return; 642 643 down_read(&pci_bus_sem); 644 mutex_lock(&aspm_lock); 645 /* 646 * All PCIe functions are in one slot, remove one function will remove 647 * the whole slot, so just wait until we are the last function left. 648 */ 649 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 650 goto out; 651 652 link = parent->link_state; 653 root = link->root; 654 parent_link = link->parent; 655 656 /* All functions are removed, so just disable ASPM for the link */ 657 pcie_config_aspm_link(link, 0); 658 list_del(&link->sibling); 659 list_del(&link->link); 660 /* Clock PM is for endpoint device */ 661 free_link_state(link); 662 663 /* Recheck latencies and configure upstream links */ 664 if (parent_link) { 665 pcie_update_aspm_capable(root); 666 pcie_config_aspm_path(parent_link); 667 } 668out: 669 mutex_unlock(&aspm_lock); 670 up_read(&pci_bus_sem); 671} 672 673/* @pdev: the root port or switch downstream port */ 674void pcie_aspm_pm_state_change(struct pci_dev *pdev) 675{ 676 struct pcie_link_state *link = pdev->link_state; 677 678 if (aspm_disabled || !pci_is_pcie(pdev) || !link) 679 return; 680 if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 681 (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)) 682 return; 683 /* 684 * Devices changed PM state, we should recheck if latency 685 * meets all functions' requirement 686 */ 687 down_read(&pci_bus_sem); 688 mutex_lock(&aspm_lock); 689 pcie_update_aspm_capable(link->root); 690 pcie_config_aspm_path(link); 691 mutex_unlock(&aspm_lock); 692 up_read(&pci_bus_sem); 693} 694 695void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 696{ 697 struct pcie_link_state *link = pdev->link_state; 698 699 if (aspm_disabled || !pci_is_pcie(pdev) || !link) 700 return; 701 702 if (aspm_policy != POLICY_POWERSAVE) 703 return; 704 705 if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 706 (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)) 707 return; 708 709 down_read(&pci_bus_sem); 710 mutex_lock(&aspm_lock); 711 pcie_config_aspm_path(link); 712 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 713 mutex_unlock(&aspm_lock); 714 up_read(&pci_bus_sem); 715} 716 717static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem, 718 bool force) 719{ 720 struct pci_dev *parent = pdev->bus->self; 721 struct pcie_link_state *link; 722 723 if (!pci_is_pcie(pdev)) 724 return; 725 726 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || 727 pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) 728 parent = pdev; 729 if (!parent || !parent->link_state) 730 return; 731 732 /* 733 * A driver requested that ASPM be disabled on this device, but 734 * if we don't have permission to manage ASPM (e.g., on ACPI 735 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 736 * the _OSC method), we can't honor that request. Windows has 737 * a similar mechanism using "PciASPMOptOut", which is also 738 * ignored in this situation. 739 */ 740 if (aspm_disabled && !force) { 741 dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); 742 return; 743 } 744 745 if (sem) 746 down_read(&pci_bus_sem); 747 mutex_lock(&aspm_lock); 748 link = parent->link_state; 749 if (state & PCIE_LINK_STATE_L0S) 750 link->aspm_disable |= ASPM_STATE_L0S; 751 if (state & PCIE_LINK_STATE_L1) 752 link->aspm_disable |= ASPM_STATE_L1; 753 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 754 755 if (state & PCIE_LINK_STATE_CLKPM) { 756 link->clkpm_capable = 0; 757 pcie_set_clkpm(link, 0); 758 } 759 mutex_unlock(&aspm_lock); 760 if (sem) 761 up_read(&pci_bus_sem); 762} 763 764void pci_disable_link_state_locked(struct pci_dev *pdev, int state) 765{ 766 __pci_disable_link_state(pdev, state, false, false); 767} 768EXPORT_SYMBOL(pci_disable_link_state_locked); 769 770/** 771 * pci_disable_link_state - Disable device's link state, so the link will 772 * never enter specific states. Note that if the BIOS didn't grant ASPM 773 * control to the OS, this does nothing because we can't touch the LNKCTL 774 * register. 775 * 776 * @pdev: PCI device 777 * @state: ASPM link state to disable 778 */ 779void pci_disable_link_state(struct pci_dev *pdev, int state) 780{ 781 __pci_disable_link_state(pdev, state, true, false); 782} 783EXPORT_SYMBOL(pci_disable_link_state); 784 785void pcie_clear_aspm(struct pci_bus *bus) 786{ 787 struct pci_dev *child; 788 789 if (aspm_force) 790 return; 791 792 /* 793 * Clear any ASPM setup that the firmware has carried out on this bus 794 */ 795 list_for_each_entry(child, &bus->devices, bus_list) { 796 __pci_disable_link_state(child, PCIE_LINK_STATE_L0S | 797 PCIE_LINK_STATE_L1 | 798 PCIE_LINK_STATE_CLKPM, 799 false, true); 800 } 801} 802 803static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 804{ 805 int i; 806 struct pcie_link_state *link; 807 808 if (aspm_disabled) 809 return -EPERM; 810 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 811 if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 812 break; 813 if (i >= ARRAY_SIZE(policy_str)) 814 return -EINVAL; 815 if (i == aspm_policy) 816 return 0; 817 818 down_read(&pci_bus_sem); 819 mutex_lock(&aspm_lock); 820 aspm_policy = i; 821 list_for_each_entry(link, &link_list, sibling) { 822 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 823 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 824 } 825 mutex_unlock(&aspm_lock); 826 up_read(&pci_bus_sem); 827 return 0; 828} 829 830static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) 831{ 832 int i, cnt = 0; 833 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 834 if (i == aspm_policy) 835 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 836 else 837 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 838 return cnt; 839} 840 841module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 842 NULL, 0644); 843 844#ifdef CONFIG_PCIEASPM_DEBUG 845static ssize_t link_state_show(struct device *dev, 846 struct device_attribute *attr, 847 char *buf) 848{ 849 struct pci_dev *pci_device = to_pci_dev(dev); 850 struct pcie_link_state *link_state = pci_device->link_state; 851 852 return sprintf(buf, "%d\n", link_state->aspm_enabled); 853} 854 855static ssize_t link_state_store(struct device *dev, 856 struct device_attribute *attr, 857 const char *buf, 858 size_t n) 859{ 860 struct pci_dev *pdev = to_pci_dev(dev); 861 struct pcie_link_state *link, *root = pdev->link_state->root; 862 u32 val = buf[0] - '0', state = 0; 863 864 if (aspm_disabled) 865 return -EPERM; 866 if (n < 1 || val > 3) 867 return -EINVAL; 868 869 /* Convert requested state to ASPM state */ 870 if (val & PCIE_LINK_STATE_L0S) 871 state |= ASPM_STATE_L0S; 872 if (val & PCIE_LINK_STATE_L1) 873 state |= ASPM_STATE_L1; 874 875 down_read(&pci_bus_sem); 876 mutex_lock(&aspm_lock); 877 list_for_each_entry(link, &link_list, sibling) { 878 if (link->root != root) 879 continue; 880 pcie_config_aspm_link(link, state); 881 } 882 mutex_unlock(&aspm_lock); 883 up_read(&pci_bus_sem); 884 return n; 885} 886 887static ssize_t clk_ctl_show(struct device *dev, 888 struct device_attribute *attr, 889 char *buf) 890{ 891 struct pci_dev *pci_device = to_pci_dev(dev); 892 struct pcie_link_state *link_state = pci_device->link_state; 893 894 return sprintf(buf, "%d\n", link_state->clkpm_enabled); 895} 896 897static ssize_t clk_ctl_store(struct device *dev, 898 struct device_attribute *attr, 899 const char *buf, 900 size_t n) 901{ 902 struct pci_dev *pdev = to_pci_dev(dev); 903 int state; 904 905 if (n < 1) 906 return -EINVAL; 907 state = buf[0]-'0'; 908 909 down_read(&pci_bus_sem); 910 mutex_lock(&aspm_lock); 911 pcie_set_clkpm_nocheck(pdev->link_state, !!state); 912 mutex_unlock(&aspm_lock); 913 up_read(&pci_bus_sem); 914 915 return n; 916} 917 918static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store); 919static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store); 920 921static char power_group[] = "power"; 922void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 923{ 924 struct pcie_link_state *link_state = pdev->link_state; 925 926 if (!pci_is_pcie(pdev) || 927 (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && 928 pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 929 return; 930 931 if (link_state->aspm_support) 932 sysfs_add_file_to_group(&pdev->dev.kobj, 933 &dev_attr_link_state.attr, power_group); 934 if (link_state->clkpm_capable) 935 sysfs_add_file_to_group(&pdev->dev.kobj, 936 &dev_attr_clk_ctl.attr, power_group); 937} 938 939void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 940{ 941 struct pcie_link_state *link_state = pdev->link_state; 942 943 if (!pci_is_pcie(pdev) || 944 (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && 945 pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 946 return; 947 948 if (link_state->aspm_support) 949 sysfs_remove_file_from_group(&pdev->dev.kobj, 950 &dev_attr_link_state.attr, power_group); 951 if (link_state->clkpm_capable) 952 sysfs_remove_file_from_group(&pdev->dev.kobj, 953 &dev_attr_clk_ctl.attr, power_group); 954} 955#endif 956 957static int __init pcie_aspm_disable(char *str) 958{ 959 if (!strcmp(str, "off")) { 960 aspm_policy = POLICY_DEFAULT; 961 aspm_disabled = 1; 962 aspm_support_enabled = false; 963 printk(KERN_INFO "PCIe ASPM is disabled\n"); 964 } else if (!strcmp(str, "force")) { 965 aspm_force = 1; 966 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 967 } 968 return 1; 969} 970 971__setup("pcie_aspm=", pcie_aspm_disable); 972 973void pcie_no_aspm(void) 974{ 975 /* 976 * Disabling ASPM is intended to prevent the kernel from modifying 977 * existing hardware state, not to clear existing state. To that end: 978 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 979 * (b) prevent userspace from changing policy 980 */ 981 if (!aspm_force) { 982 aspm_policy = POLICY_DEFAULT; 983 aspm_disabled = 1; 984 } 985} 986 987bool pcie_aspm_support_enabled(void) 988{ 989 return aspm_support_enabled; 990} 991EXPORT_SYMBOL(pcie_aspm_support_enabled); 992