1/* 2 * PCI Virtual Channel support 3 * 4 * Copyright (C) 2013 Red Hat, Inc. All rights reserved. 5 * Author: Alex Williamson <alex.williamson@redhat.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12#include <linux/device.h> 13#include <linux/kernel.h> 14#include <linux/module.h> 15#include <linux/pci.h> 16#include <linux/pci_regs.h> 17#include <linux/types.h> 18 19/** 20 * pci_vc_save_restore_dwords - Save or restore a series of dwords 21 * @dev: device 22 * @pos: starting config space position 23 * @buf: buffer to save to or restore from 24 * @dwords: number of dwords to save/restore 25 * @save: whether to save or restore 26 */ 27static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, 28 u32 *buf, int dwords, bool save) 29{ 30 int i; 31 32 for (i = 0; i < dwords; i++, buf++) { 33 if (save) 34 pci_read_config_dword(dev, pos + (i * 4), buf); 35 else 36 pci_write_config_dword(dev, pos + (i * 4), *buf); 37 } 38} 39 40/** 41 * pci_vc_load_arb_table - load and wait for VC arbitration table 42 * @dev: device 43 * @pos: starting position of VC capability (VC/VC9/MFVC) 44 * 45 * Set Load VC Arbitration Table bit requesting hardware to apply the VC 46 * Arbitration Table (previously loaded). When the VC Arbitration Table 47 * Status clears, hardware has latched the table into VC arbitration logic. 48 */ 49static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) 50{ 51 u16 ctrl; 52 53 pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl); 54 pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, 55 ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); 56 if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS, 57 PCI_VC_PORT_STATUS_TABLE)) 58 return; 59 60 dev_err(&dev->dev, "VC arbitration table failed to load\n"); 61} 62 63/** 64 * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table 65 * @dev: device 66 * @pos: starting position of VC capability (VC/VC9/MFVC) 67 * @res: VC resource number, ie. VCn (0-7) 68 * 69 * Set Load Port Arbitration Table bit requesting hardware to apply the Port 70 * Arbitration Table (previously loaded). When the Port Arbitration Table 71 * Status clears, hardware has latched the table into port arbitration logic. 72 */ 73static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) 74{ 75 int ctrl_pos, status_pos; 76 u32 ctrl; 77 78 ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); 79 status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); 80 81 pci_read_config_dword(dev, ctrl_pos, &ctrl); 82 pci_write_config_dword(dev, ctrl_pos, 83 ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); 84 85 if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE)) 86 return; 87 88 dev_err(&dev->dev, "VC%d port arbitration table failed to load\n", res); 89} 90 91/** 92 * pci_vc_enable - Enable virtual channel 93 * @dev: device 94 * @pos: starting position of VC capability (VC/VC9/MFVC) 95 * @res: VC res number, ie. VCn (0-7) 96 * 97 * A VC is enabled by setting the enable bit in matching resource control 98 * registers on both sides of a link. We therefore need to find the opposite 99 * end of the link. To keep this simple we enable from the downstream device. 100 * RC devices do not have an upstream device, nor does it seem that VC9 do 101 * (spec is unclear). Once we find the upstream device, match the VC ID to 102 * get the correct resource, disable and enable on both ends. 103 */ 104static void pci_vc_enable(struct pci_dev *dev, int pos, int res) 105{ 106 int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; 107 u32 ctrl, header, cap1, ctrl2; 108 struct pci_dev *link = NULL; 109 110 /* Enable VCs from the downstream device */ 111 if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || 112 pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) 113 return; 114 115 ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); 116 status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); 117 118 pci_read_config_dword(dev, ctrl_pos, &ctrl); 119 id = ctrl & PCI_VC_RES_CTRL_ID; 120 121 pci_read_config_dword(dev, pos, &header); 122 123 /* If there is no opposite end of the link, skip to enable */ 124 if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || 125 pci_is_root_bus(dev->bus)) 126 goto enable; 127 128 pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC); 129 if (!pos2) 130 goto enable; 131 132 pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1); 133 evcc = cap1 & PCI_VC_CAP1_EVCC; 134 135 /* VC0 is hardwired enabled, so we can start with 1 */ 136 for (i = 1; i < evcc + 1; i++) { 137 ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + 138 (i * PCI_CAP_VC_PER_VC_SIZEOF); 139 status_pos2 = pos2 + PCI_VC_RES_STATUS + 140 (i * PCI_CAP_VC_PER_VC_SIZEOF); 141 pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2); 142 if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { 143 link = dev->bus->self; 144 break; 145 } 146 } 147 148 if (!link) 149 goto enable; 150 151 /* Disable if enabled */ 152 if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { 153 ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; 154 pci_write_config_dword(link, ctrl_pos2, ctrl2); 155 } 156 157 /* Enable on both ends */ 158 ctrl2 |= PCI_VC_RES_CTRL_ENABLE; 159 pci_write_config_dword(link, ctrl_pos2, ctrl2); 160enable: 161 ctrl |= PCI_VC_RES_CTRL_ENABLE; 162 pci_write_config_dword(dev, ctrl_pos, ctrl); 163 164 if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO)) 165 dev_err(&dev->dev, "VC%d negotiation stuck pending\n", id); 166 167 if (link && !pci_wait_for_pending(link, status_pos2, 168 PCI_VC_RES_STATUS_NEGO)) 169 dev_err(&link->dev, "VC%d negotiation stuck pending\n", id); 170} 171 172/** 173 * pci_vc_do_save_buffer - Size, save, or restore VC state 174 * @dev: device 175 * @pos: starting position of VC capability (VC/VC9/MFVC) 176 * @save_state: buffer for save/restore 177 * @name: for error message 178 * @save: if provided a buffer, this indicates what to do with it 179 * 180 * Walking Virtual Channel config space to size, save, or restore it 181 * is complicated, so we do it all from one function to reduce code and 182 * guarantee ordering matches in the buffer. When called with NULL 183 * @save_state, return the size of the necessary save buffer. When called 184 * with a non-NULL @save_state, @save determines whether we save to the 185 * buffer or restore from it. 186 */ 187static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, 188 struct pci_cap_saved_state *save_state, 189 bool save) 190{ 191 u32 cap1; 192 char evcc, lpevcc, parb_size; 193 int i, len = 0; 194 u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; 195 196 /* Sanity check buffer size for save/restore */ 197 if (buf && save_state->cap.size != 198 pci_vc_do_save_buffer(dev, pos, NULL, save)) { 199 dev_err(&dev->dev, 200 "VC save buffer size does not match @0x%x\n", pos); 201 return -ENOMEM; 202 } 203 204 pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1); 205 /* Extended VC Count (not counting VC0) */ 206 evcc = cap1 & PCI_VC_CAP1_EVCC; 207 /* Low Priority Extended VC Count (not counting VC0) */ 208 lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4; 209 /* Port Arbitration Table Entry Size (bits) */ 210 parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10); 211 212 /* 213 * Port VC Control Register contains VC Arbitration Select, which 214 * cannot be modified when more than one LPVC is in operation. We 215 * therefore save/restore it first, as only VC0 should be enabled 216 * after device reset. 217 */ 218 if (buf) { 219 if (save) 220 pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, 221 (u16 *)buf); 222 else 223 pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL, 224 *(u16 *)buf); 225 buf += 2; 226 } 227 len += 2; 228 229 /* 230 * If we have any Low Priority VCs and a VC Arbitration Table Offset 231 * in Port VC Capability Register 2 then save/restore it next. 232 */ 233 if (lpevcc) { 234 u32 cap2; 235 int vcarb_offset; 236 237 pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2); 238 vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16; 239 240 if (vcarb_offset) { 241 int size, vcarb_phases = 0; 242 243 if (cap2 & PCI_VC_CAP2_128_PHASE) 244 vcarb_phases = 128; 245 else if (cap2 & PCI_VC_CAP2_64_PHASE) 246 vcarb_phases = 64; 247 else if (cap2 & PCI_VC_CAP2_32_PHASE) 248 vcarb_phases = 32; 249 250 /* Fixed 4 bits per phase per lpevcc (plus VC0) */ 251 size = ((lpevcc + 1) * vcarb_phases * 4) / 8; 252 253 if (size && buf) { 254 pci_vc_save_restore_dwords(dev, 255 pos + vcarb_offset, 256 (u32 *)buf, 257 size / 4, save); 258 /* 259 * On restore, we need to signal hardware to 260 * re-load the VC Arbitration Table. 261 */ 262 if (!save) 263 pci_vc_load_arb_table(dev, pos); 264 265 buf += size; 266 } 267 len += size; 268 } 269 } 270 271 /* 272 * In addition to each VC Resource Control Register, we may have a 273 * Port Arbitration Table attached to each VC. The Port Arbitration 274 * Table Offset in each VC Resource Capability Register tells us if 275 * it exists. The entry size is global from the Port VC Capability 276 * Register1 above. The number of phases is determined per VC. 277 */ 278 for (i = 0; i < evcc + 1; i++) { 279 u32 cap; 280 int parb_offset; 281 282 pci_read_config_dword(dev, pos + PCI_VC_RES_CAP + 283 (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap); 284 parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16; 285 if (parb_offset) { 286 int size, parb_phases = 0; 287 288 if (cap & PCI_VC_RES_CAP_256_PHASE) 289 parb_phases = 256; 290 else if (cap & (PCI_VC_RES_CAP_128_PHASE | 291 PCI_VC_RES_CAP_128_PHASE_TB)) 292 parb_phases = 128; 293 else if (cap & PCI_VC_RES_CAP_64_PHASE) 294 parb_phases = 64; 295 else if (cap & PCI_VC_RES_CAP_32_PHASE) 296 parb_phases = 32; 297 298 size = (parb_size * parb_phases) / 8; 299 300 if (size && buf) { 301 pci_vc_save_restore_dwords(dev, 302 pos + parb_offset, 303 (u32 *)buf, 304 size / 4, save); 305 buf += size; 306 } 307 len += size; 308 } 309 310 /* VC Resource Control Register */ 311 if (buf) { 312 int ctrl_pos = pos + PCI_VC_RES_CTRL + 313 (i * PCI_CAP_VC_PER_VC_SIZEOF); 314 if (save) 315 pci_read_config_dword(dev, ctrl_pos, 316 (u32 *)buf); 317 else { 318 u32 tmp, ctrl = *(u32 *)buf; 319 /* 320 * For an FLR case, the VC config may remain. 321 * Preserve enable bit, restore the rest. 322 */ 323 pci_read_config_dword(dev, ctrl_pos, &tmp); 324 tmp &= PCI_VC_RES_CTRL_ENABLE; 325 tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; 326 pci_write_config_dword(dev, ctrl_pos, tmp); 327 /* Load port arbitration table if used */ 328 if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) 329 pci_vc_load_port_arb_table(dev, pos, i); 330 /* Re-enable if needed */ 331 if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) 332 pci_vc_enable(dev, pos, i); 333 } 334 buf += 4; 335 } 336 len += 4; 337 } 338 339 return buf ? 0 : len; 340} 341 342static struct { 343 u16 id; 344 const char *name; 345} vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, 346 { PCI_EXT_CAP_ID_VC, "VC" }, 347 { PCI_EXT_CAP_ID_VC9, "VC9" } }; 348 349/** 350 * pci_save_vc_state - Save VC state to pre-allocate save buffer 351 * @dev: device 352 * 353 * For each type of VC capability, VC/VC9/MFVC, find the capability and 354 * save it to the pre-allocated save buffer. 355 */ 356int pci_save_vc_state(struct pci_dev *dev) 357{ 358 int i; 359 360 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 361 int pos, ret; 362 struct pci_cap_saved_state *save_state; 363 364 pos = pci_find_ext_capability(dev, vc_caps[i].id); 365 if (!pos) 366 continue; 367 368 save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); 369 if (!save_state) { 370 dev_err(&dev->dev, "%s buffer not found in %s\n", 371 vc_caps[i].name, __func__); 372 return -ENOMEM; 373 } 374 375 ret = pci_vc_do_save_buffer(dev, pos, save_state, true); 376 if (ret) { 377 dev_err(&dev->dev, "%s save unsuccessful %s\n", 378 vc_caps[i].name, __func__); 379 return ret; 380 } 381 } 382 383 return 0; 384} 385 386/** 387 * pci_restore_vc_state - Restore VC state from save buffer 388 * @dev: device 389 * 390 * For each type of VC capability, VC/VC9/MFVC, find the capability and 391 * restore it from the previously saved buffer. 392 */ 393void pci_restore_vc_state(struct pci_dev *dev) 394{ 395 int i; 396 397 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 398 int pos; 399 struct pci_cap_saved_state *save_state; 400 401 pos = pci_find_ext_capability(dev, vc_caps[i].id); 402 save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id); 403 if (!save_state || !pos) 404 continue; 405 406 pci_vc_do_save_buffer(dev, pos, save_state, false); 407 } 408} 409 410/** 411 * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps 412 * @dev: device 413 * 414 * For each type of VC capability, VC/VC9/MFVC, find the capability, size 415 * it, and allocate a buffer for save/restore. 416 */ 417 418void pci_allocate_vc_save_buffers(struct pci_dev *dev) 419{ 420 int i; 421 422 for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { 423 int len, pos = pci_find_ext_capability(dev, vc_caps[i].id); 424 425 if (!pos) 426 continue; 427 428 len = pci_vc_do_save_buffer(dev, pos, NULL, false); 429 if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len)) 430 dev_err(&dev->dev, 431 "unable to preallocate %s save buffer\n", 432 vc_caps[i].name); 433 } 434} 435