dma.c revision 4dcaf47258d59010802bd0eda933f69ee7d98cc7
1/* 2* Filename: dma.c 3* 4* 5* Authors: Joshua Morris <josh.h.morris@us.ibm.com> 6* Philip Kelleher <pjk1939@linux.vnet.ibm.com> 7* 8* (C) Copyright 2013 IBM Corporation 9* 10* This program is free software; you can redistribute it and/or 11* modify it under the terms of the GNU General Public License as 12* published by the Free Software Foundation; either version 2 of the 13* License, or (at your option) any later version. 14* 15* This program is distributed in the hope that it will be useful, but 16* WITHOUT ANY WARRANTY; without even the implied warranty of 17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18* General Public License for more details. 19* 20* You should have received a copy of the GNU General Public License 21* along with this program; if not, write to the Free Software Foundation, 22* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 23*/ 24 25#include <linux/slab.h> 26#include "rsxx_priv.h" 27 28struct rsxx_dma { 29 struct list_head list; 30 u8 cmd; 31 unsigned int laddr; /* Logical address */ 32 struct { 33 u32 off; 34 u32 cnt; 35 } sub_page; 36 dma_addr_t dma_addr; 37 struct page *page; 38 unsigned int pg_off; /* Page Offset */ 39 rsxx_dma_cb cb; 40 void *cb_data; 41}; 42 43/* This timeout is used to detect a stalled DMA channel */ 44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000) 45 46struct hw_status { 47 u8 status; 48 u8 tag; 49 __le16 count; 50 __le32 _rsvd2; 51 __le64 _rsvd3; 52} __packed; 53 54enum rsxx_dma_status { 55 DMA_SW_ERR = 0x1, 56 DMA_HW_FAULT = 0x2, 57 DMA_CANCELLED = 0x4, 58}; 59 60struct hw_cmd { 61 u8 command; 62 u8 tag; 63 u8 _rsvd; 64 u8 sub_page; /* Bit[0:2]: 512byte offset */ 65 /* Bit[4:6]: 512byte count */ 66 __le32 device_addr; 67 __le64 host_addr; 68} __packed; 69 70enum rsxx_hw_cmd { 71 HW_CMD_BLK_DISCARD = 0x70, 72 HW_CMD_BLK_WRITE = 0x80, 73 HW_CMD_BLK_READ = 0xC0, 74 HW_CMD_BLK_RECON_READ = 0xE0, 75}; 76 77enum rsxx_hw_status { 78 HW_STATUS_CRC = 0x01, 79 HW_STATUS_HARD_ERR = 0x02, 80 HW_STATUS_SOFT_ERR = 0x04, 81 HW_STATUS_FAULT = 0x08, 82}; 83 84static struct kmem_cache *rsxx_dma_pool; 85 86struct dma_tracker { 87 int next_tag; 88 struct rsxx_dma *dma; 89}; 90 91#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \ 92 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS)) 93 94struct dma_tracker_list { 95 spinlock_t lock; 96 int head; 97 struct dma_tracker list[0]; 98}; 99 100 101/*----------------- Misc Utility Functions -------------------*/ 102static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card) 103{ 104 unsigned long long tgt_addr8; 105 106 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) & 107 card->_stripe.upper_mask) | 108 ((addr8) & card->_stripe.lower_mask); 109 do_div(tgt_addr8, RSXX_HW_BLK_SIZE); 110 return tgt_addr8; 111} 112 113static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8) 114{ 115 unsigned int tgt; 116 117 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask; 118 119 return tgt; 120} 121 122void rsxx_dma_queue_reset(struct rsxx_cardinfo *card) 123{ 124 /* Reset all DMA Command/Status Queues */ 125 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET); 126} 127 128static unsigned int get_dma_size(struct rsxx_dma *dma) 129{ 130 if (dma->sub_page.cnt) 131 return dma->sub_page.cnt << 9; 132 else 133 return RSXX_HW_BLK_SIZE; 134} 135 136 137/*----------------- DMA Tracker -------------------*/ 138static void set_tracker_dma(struct dma_tracker_list *trackers, 139 int tag, 140 struct rsxx_dma *dma) 141{ 142 trackers->list[tag].dma = dma; 143} 144 145static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers, 146 int tag) 147{ 148 return trackers->list[tag].dma; 149} 150 151static int pop_tracker(struct dma_tracker_list *trackers) 152{ 153 int tag; 154 155 spin_lock(&trackers->lock); 156 tag = trackers->head; 157 if (tag != -1) { 158 trackers->head = trackers->list[tag].next_tag; 159 trackers->list[tag].next_tag = -1; 160 } 161 spin_unlock(&trackers->lock); 162 163 return tag; 164} 165 166static void push_tracker(struct dma_tracker_list *trackers, int tag) 167{ 168 spin_lock(&trackers->lock); 169 trackers->list[tag].next_tag = trackers->head; 170 trackers->head = tag; 171 trackers->list[tag].dma = NULL; 172 spin_unlock(&trackers->lock); 173} 174 175 176/*----------------- Interrupt Coalescing -------------*/ 177/* 178 * Interrupt Coalescing Register Format: 179 * Interrupt Timer (64ns units) [15:0] 180 * Interrupt Count [24:16] 181 * Reserved [31:25] 182*/ 183#define INTR_COAL_LATENCY_MASK (0x0000ffff) 184 185#define INTR_COAL_COUNT_SHIFT 16 186#define INTR_COAL_COUNT_BITS 9 187#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \ 188 INTR_COAL_COUNT_SHIFT) 189#define INTR_COAL_LATENCY_UNITS_NS 64 190 191 192static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency) 193{ 194 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS; 195 196 if (mode == RSXX_INTR_COAL_DISABLED) 197 return 0; 198 199 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) | 200 (latency_units & INTR_COAL_LATENCY_MASK); 201 202} 203 204static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card) 205{ 206 int i; 207 u32 q_depth = 0; 208 u32 intr_coal; 209 210 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE || 211 unlikely(card->eeh_state)) 212 return; 213 214 for (i = 0; i < card->n_targets; i++) 215 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth); 216 217 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 218 q_depth / 2, 219 card->config.data.intr_coal.latency); 220 iowrite32(intr_coal, card->regmap + INTR_COAL); 221} 222 223/*----------------- RSXX DMA Handling -------------------*/ 224static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl, 225 struct rsxx_dma *dma, 226 unsigned int status) 227{ 228 if (status & DMA_SW_ERR) 229 ctrl->stats.dma_sw_err++; 230 if (status & DMA_HW_FAULT) 231 ctrl->stats.dma_hw_fault++; 232 if (status & DMA_CANCELLED) 233 ctrl->stats.dma_cancelled++; 234 235 if (dma->dma_addr) 236 pci_unmap_page(ctrl->card->dev, dma->dma_addr, 237 get_dma_size(dma), 238 dma->cmd == HW_CMD_BLK_WRITE ? 239 PCI_DMA_TODEVICE : 240 PCI_DMA_FROMDEVICE); 241 242 if (dma->cb) 243 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0); 244 245 kmem_cache_free(rsxx_dma_pool, dma); 246} 247 248static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl, 249 struct rsxx_dma *dma) 250{ 251 /* 252 * Requeued DMAs go to the front of the queue so they are issued 253 * first. 254 */ 255 spin_lock(&ctrl->queue_lock); 256 list_add(&dma->list, &ctrl->queue); 257 spin_unlock(&ctrl->queue_lock); 258} 259 260static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl, 261 struct rsxx_dma *dma, 262 u8 hw_st) 263{ 264 unsigned int status = 0; 265 int requeue_cmd = 0; 266 267 dev_dbg(CARD_TO_DEV(ctrl->card), 268 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n", 269 dma->cmd, dma->laddr, hw_st); 270 271 if (hw_st & HW_STATUS_CRC) 272 ctrl->stats.crc_errors++; 273 if (hw_st & HW_STATUS_HARD_ERR) 274 ctrl->stats.hard_errors++; 275 if (hw_st & HW_STATUS_SOFT_ERR) 276 ctrl->stats.soft_errors++; 277 278 switch (dma->cmd) { 279 case HW_CMD_BLK_READ: 280 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 281 if (ctrl->card->scrub_hard) { 282 dma->cmd = HW_CMD_BLK_RECON_READ; 283 requeue_cmd = 1; 284 ctrl->stats.reads_retried++; 285 } else { 286 status |= DMA_HW_FAULT; 287 ctrl->stats.reads_failed++; 288 } 289 } else if (hw_st & HW_STATUS_FAULT) { 290 status |= DMA_HW_FAULT; 291 ctrl->stats.reads_failed++; 292 } 293 294 break; 295 case HW_CMD_BLK_RECON_READ: 296 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 297 /* Data could not be reconstructed. */ 298 status |= DMA_HW_FAULT; 299 ctrl->stats.reads_failed++; 300 } 301 302 break; 303 case HW_CMD_BLK_WRITE: 304 status |= DMA_HW_FAULT; 305 ctrl->stats.writes_failed++; 306 307 break; 308 case HW_CMD_BLK_DISCARD: 309 status |= DMA_HW_FAULT; 310 ctrl->stats.discards_failed++; 311 312 break; 313 default: 314 dev_err(CARD_TO_DEV(ctrl->card), 315 "Unknown command in DMA!(cmd: x%02x " 316 "laddr x%08x st: x%02x\n", 317 dma->cmd, dma->laddr, hw_st); 318 status |= DMA_SW_ERR; 319 320 break; 321 } 322 323 if (requeue_cmd) 324 rsxx_requeue_dma(ctrl, dma); 325 else 326 rsxx_complete_dma(ctrl, dma, status); 327} 328 329static void dma_engine_stalled(unsigned long data) 330{ 331 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data; 332 333 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 || 334 unlikely(ctrl->card->eeh_state)) 335 return; 336 337 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) { 338 /* 339 * The dma engine was stalled because the SW_CMD_IDX write 340 * was lost. Issue it again to recover. 341 */ 342 dev_warn(CARD_TO_DEV(ctrl->card), 343 "SW_CMD_IDX write was lost, re-writing...\n"); 344 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 345 mod_timer(&ctrl->activity_timer, 346 jiffies + DMA_ACTIVITY_TIMEOUT); 347 } else { 348 dev_warn(CARD_TO_DEV(ctrl->card), 349 "DMA channel %d has stalled, faulting interface.\n", 350 ctrl->id); 351 ctrl->card->dma_fault = 1; 352 } 353} 354 355static void rsxx_issue_dmas(struct work_struct *work) 356{ 357 struct rsxx_dma_ctrl *ctrl; 358 struct rsxx_dma *dma; 359 int tag; 360 int cmds_pending = 0; 361 struct hw_cmd *hw_cmd_buf; 362 363 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work); 364 hw_cmd_buf = ctrl->cmd.buf; 365 366 if (unlikely(ctrl->card->halt) || 367 unlikely(ctrl->card->eeh_state)) 368 return; 369 370 while (1) { 371 spin_lock(&ctrl->queue_lock); 372 if (list_empty(&ctrl->queue)) { 373 spin_unlock(&ctrl->queue_lock); 374 break; 375 } 376 spin_unlock(&ctrl->queue_lock); 377 378 tag = pop_tracker(ctrl->trackers); 379 if (tag == -1) 380 break; 381 382 spin_lock(&ctrl->queue_lock); 383 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list); 384 list_del(&dma->list); 385 ctrl->stats.sw_q_depth--; 386 spin_unlock(&ctrl->queue_lock); 387 388 /* 389 * This will catch any DMAs that slipped in right before the 390 * fault, but was queued after all the other DMAs were 391 * cancelled. 392 */ 393 if (unlikely(ctrl->card->dma_fault)) { 394 push_tracker(ctrl->trackers, tag); 395 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 396 continue; 397 } 398 399 set_tracker_dma(ctrl->trackers, tag, dma); 400 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd; 401 hw_cmd_buf[ctrl->cmd.idx].tag = tag; 402 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0; 403 hw_cmd_buf[ctrl->cmd.idx].sub_page = 404 ((dma->sub_page.cnt & 0x7) << 4) | 405 (dma->sub_page.off & 0x7); 406 407 hw_cmd_buf[ctrl->cmd.idx].device_addr = 408 cpu_to_le32(dma->laddr); 409 410 hw_cmd_buf[ctrl->cmd.idx].host_addr = 411 cpu_to_le64(dma->dma_addr); 412 413 dev_dbg(CARD_TO_DEV(ctrl->card), 414 "Issue DMA%d(laddr %d tag %d) to idx %d\n", 415 ctrl->id, dma->laddr, tag, ctrl->cmd.idx); 416 417 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK; 418 cmds_pending++; 419 420 if (dma->cmd == HW_CMD_BLK_WRITE) 421 ctrl->stats.writes_issued++; 422 else if (dma->cmd == HW_CMD_BLK_DISCARD) 423 ctrl->stats.discards_issued++; 424 else 425 ctrl->stats.reads_issued++; 426 } 427 428 /* Let HW know we've queued commands. */ 429 if (cmds_pending) { 430 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth); 431 mod_timer(&ctrl->activity_timer, 432 jiffies + DMA_ACTIVITY_TIMEOUT); 433 434 if (unlikely(ctrl->card->eeh_state)) { 435 del_timer_sync(&ctrl->activity_timer); 436 return; 437 } 438 439 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 440 } 441} 442 443static void rsxx_dma_done(struct work_struct *work) 444{ 445 struct rsxx_dma_ctrl *ctrl; 446 struct rsxx_dma *dma; 447 unsigned long flags; 448 u16 count; 449 u8 status; 450 u8 tag; 451 struct hw_status *hw_st_buf; 452 453 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work); 454 hw_st_buf = ctrl->status.buf; 455 456 if (unlikely(ctrl->card->halt) || 457 unlikely(ctrl->card->dma_fault) || 458 unlikely(ctrl->card->eeh_state)) 459 return; 460 461 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 462 463 while (count == ctrl->e_cnt) { 464 /* 465 * The read memory-barrier is necessary to keep aggressive 466 * processors/optimizers (such as the PPC Apple G5) from 467 * reordering the following status-buffer tag & status read 468 * *before* the count read on subsequent iterations of the 469 * loop! 470 */ 471 rmb(); 472 473 status = hw_st_buf[ctrl->status.idx].status; 474 tag = hw_st_buf[ctrl->status.idx].tag; 475 476 dma = get_tracker_dma(ctrl->trackers, tag); 477 if (dma == NULL) { 478 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 479 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL); 480 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 481 482 dev_err(CARD_TO_DEV(ctrl->card), 483 "No tracker for tag %d " 484 "(idx %d id %d)\n", 485 tag, ctrl->status.idx, ctrl->id); 486 return; 487 } 488 489 dev_dbg(CARD_TO_DEV(ctrl->card), 490 "Completing DMA%d" 491 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n", 492 ctrl->id, dma->laddr, tag, status, count, 493 ctrl->status.idx); 494 495 atomic_dec(&ctrl->stats.hw_q_depth); 496 497 mod_timer(&ctrl->activity_timer, 498 jiffies + DMA_ACTIVITY_TIMEOUT); 499 500 if (status) 501 rsxx_handle_dma_error(ctrl, dma, status); 502 else 503 rsxx_complete_dma(ctrl, dma, 0); 504 505 push_tracker(ctrl->trackers, tag); 506 507 ctrl->status.idx = (ctrl->status.idx + 1) & 508 RSXX_CS_IDX_MASK; 509 ctrl->e_cnt++; 510 511 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 512 } 513 514 dma_intr_coal_auto_tune(ctrl->card); 515 516 if (atomic_read(&ctrl->stats.hw_q_depth) == 0) 517 del_timer_sync(&ctrl->activity_timer); 518 519 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 520 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id)); 521 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 522 523 spin_lock(&ctrl->queue_lock); 524 if (ctrl->stats.sw_q_depth) 525 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work); 526 spin_unlock(&ctrl->queue_lock); 527} 528 529static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card, 530 struct list_head *q) 531{ 532 struct rsxx_dma *dma; 533 struct rsxx_dma *tmp; 534 int cnt = 0; 535 536 list_for_each_entry_safe(dma, tmp, q, list) { 537 list_del(&dma->list); 538 539 if (dma->dma_addr) 540 pci_unmap_page(card->dev, dma->dma_addr, 541 get_dma_size(dma), 542 (dma->cmd == HW_CMD_BLK_WRITE) ? 543 PCI_DMA_TODEVICE : 544 PCI_DMA_FROMDEVICE); 545 kmem_cache_free(rsxx_dma_pool, dma); 546 cnt++; 547 } 548 549 return cnt; 550} 551 552static int rsxx_queue_discard(struct rsxx_cardinfo *card, 553 struct list_head *q, 554 unsigned int laddr, 555 rsxx_dma_cb cb, 556 void *cb_data) 557{ 558 struct rsxx_dma *dma; 559 560 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 561 if (!dma) 562 return -ENOMEM; 563 564 dma->cmd = HW_CMD_BLK_DISCARD; 565 dma->laddr = laddr; 566 dma->dma_addr = 0; 567 dma->sub_page.off = 0; 568 dma->sub_page.cnt = 0; 569 dma->page = NULL; 570 dma->pg_off = 0; 571 dma->cb = cb; 572 dma->cb_data = cb_data; 573 574 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr); 575 576 list_add_tail(&dma->list, q); 577 578 return 0; 579} 580 581static int rsxx_queue_dma(struct rsxx_cardinfo *card, 582 struct list_head *q, 583 int dir, 584 unsigned int dma_off, 585 unsigned int dma_len, 586 unsigned int laddr, 587 struct page *page, 588 unsigned int pg_off, 589 rsxx_dma_cb cb, 590 void *cb_data) 591{ 592 struct rsxx_dma *dma; 593 594 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 595 if (!dma) 596 return -ENOMEM; 597 598 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len, 599 dir ? PCI_DMA_TODEVICE : 600 PCI_DMA_FROMDEVICE); 601 if (!dma->dma_addr) { 602 kmem_cache_free(rsxx_dma_pool, dma); 603 return -ENOMEM; 604 } 605 606 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ; 607 dma->laddr = laddr; 608 dma->sub_page.off = (dma_off >> 9); 609 dma->sub_page.cnt = (dma_len >> 9); 610 dma->page = page; 611 dma->pg_off = pg_off; 612 dma->cb = cb; 613 dma->cb_data = cb_data; 614 615 dev_dbg(CARD_TO_DEV(card), 616 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n", 617 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off, 618 dma->sub_page.cnt, dma->page, dma->pg_off); 619 620 /* Queue the DMA */ 621 list_add_tail(&dma->list, q); 622 623 return 0; 624} 625 626int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, 627 struct bio *bio, 628 atomic_t *n_dmas, 629 rsxx_dma_cb cb, 630 void *cb_data) 631{ 632 struct list_head dma_list[RSXX_MAX_TARGETS]; 633 struct bio_vec *bvec; 634 unsigned long long addr8; 635 unsigned int laddr; 636 unsigned int bv_len; 637 unsigned int bv_off; 638 unsigned int dma_off; 639 unsigned int dma_len; 640 int dma_cnt[RSXX_MAX_TARGETS]; 641 int tgt; 642 int st; 643 int i; 644 645 addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */ 646 atomic_set(n_dmas, 0); 647 648 for (i = 0; i < card->n_targets; i++) { 649 INIT_LIST_HEAD(&dma_list[i]); 650 dma_cnt[i] = 0; 651 } 652 653 if (bio->bi_rw & REQ_DISCARD) { 654 bv_len = bio->bi_size; 655 656 while (bv_len > 0) { 657 tgt = rsxx_get_dma_tgt(card, addr8); 658 laddr = rsxx_addr8_to_laddr(addr8, card); 659 660 st = rsxx_queue_discard(card, &dma_list[tgt], laddr, 661 cb, cb_data); 662 if (st) 663 goto bvec_err; 664 665 dma_cnt[tgt]++; 666 atomic_inc(n_dmas); 667 addr8 += RSXX_HW_BLK_SIZE; 668 bv_len -= RSXX_HW_BLK_SIZE; 669 } 670 } else { 671 bio_for_each_segment(bvec, bio, i) { 672 bv_len = bvec->bv_len; 673 bv_off = bvec->bv_offset; 674 675 while (bv_len > 0) { 676 tgt = rsxx_get_dma_tgt(card, addr8); 677 laddr = rsxx_addr8_to_laddr(addr8, card); 678 dma_off = addr8 & RSXX_HW_BLK_MASK; 679 dma_len = min(bv_len, 680 RSXX_HW_BLK_SIZE - dma_off); 681 682 st = rsxx_queue_dma(card, &dma_list[tgt], 683 bio_data_dir(bio), 684 dma_off, dma_len, 685 laddr, bvec->bv_page, 686 bv_off, cb, cb_data); 687 if (st) 688 goto bvec_err; 689 690 dma_cnt[tgt]++; 691 atomic_inc(n_dmas); 692 addr8 += dma_len; 693 bv_off += dma_len; 694 bv_len -= dma_len; 695 } 696 } 697 } 698 699 for (i = 0; i < card->n_targets; i++) { 700 if (!list_empty(&dma_list[i])) { 701 spin_lock(&card->ctrl[i].queue_lock); 702 card->ctrl[i].stats.sw_q_depth += dma_cnt[i]; 703 list_splice_tail(&dma_list[i], &card->ctrl[i].queue); 704 spin_unlock(&card->ctrl[i].queue_lock); 705 706 queue_work(card->ctrl[i].issue_wq, 707 &card->ctrl[i].issue_dma_work); 708 } 709 } 710 711 return 0; 712 713bvec_err: 714 for (i = 0; i < card->n_targets; i++) 715 rsxx_cleanup_dma_queue(card, &dma_list[i]); 716 717 return st; 718} 719 720 721/*----------------- DMA Engine Initialization & Setup -------------------*/ 722int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl) 723{ 724 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8, 725 &ctrl->status.dma_addr); 726 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8, 727 &ctrl->cmd.dma_addr); 728 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL) 729 return -ENOMEM; 730 731 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8); 732 iowrite32(lower_32_bits(ctrl->status.dma_addr), 733 ctrl->regmap + SB_ADD_LO); 734 iowrite32(upper_32_bits(ctrl->status.dma_addr), 735 ctrl->regmap + SB_ADD_HI); 736 737 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8); 738 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO); 739 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI); 740 741 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT); 742 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) { 743 dev_crit(&dev->dev, "Failed reading status cnt x%x\n", 744 ctrl->status.idx); 745 return -EINVAL; 746 } 747 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT); 748 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT); 749 750 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX); 751 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) { 752 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n", 753 ctrl->status.idx); 754 return -EINVAL; 755 } 756 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX); 757 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 758 759 return 0; 760} 761 762static int rsxx_dma_ctrl_init(struct pci_dev *dev, 763 struct rsxx_dma_ctrl *ctrl) 764{ 765 int i; 766 int st; 767 768 memset(&ctrl->stats, 0, sizeof(ctrl->stats)); 769 770 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8); 771 if (!ctrl->trackers) 772 return -ENOMEM; 773 774 ctrl->trackers->head = 0; 775 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { 776 ctrl->trackers->list[i].next_tag = i + 1; 777 ctrl->trackers->list[i].dma = NULL; 778 } 779 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1; 780 spin_lock_init(&ctrl->trackers->lock); 781 782 spin_lock_init(&ctrl->queue_lock); 783 INIT_LIST_HEAD(&ctrl->queue); 784 785 setup_timer(&ctrl->activity_timer, dma_engine_stalled, 786 (unsigned long)ctrl); 787 788 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0); 789 if (!ctrl->issue_wq) 790 return -ENOMEM; 791 792 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0); 793 if (!ctrl->done_wq) 794 return -ENOMEM; 795 796 INIT_WORK(&ctrl->issue_dma_work, rsxx_issue_dmas); 797 INIT_WORK(&ctrl->dma_done_work, rsxx_dma_done); 798 799 st = rsxx_hw_buffers_init(dev, ctrl); 800 if (st) 801 return st; 802 803 return 0; 804} 805 806static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card, 807 unsigned int stripe_size8) 808{ 809 if (!is_power_of_2(stripe_size8)) { 810 dev_err(CARD_TO_DEV(card), 811 "stripe_size is NOT a power of 2!\n"); 812 return -EINVAL; 813 } 814 815 card->_stripe.lower_mask = stripe_size8 - 1; 816 817 card->_stripe.upper_mask = ~(card->_stripe.lower_mask); 818 card->_stripe.upper_shift = ffs(card->n_targets) - 1; 819 820 card->_stripe.target_mask = card->n_targets - 1; 821 card->_stripe.target_shift = ffs(stripe_size8) - 1; 822 823 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n", 824 card->_stripe.lower_mask); 825 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n", 826 card->_stripe.upper_shift); 827 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n", 828 card->_stripe.upper_mask); 829 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n", 830 card->_stripe.target_mask); 831 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n", 832 card->_stripe.target_shift); 833 834 return 0; 835} 836 837int rsxx_dma_configure(struct rsxx_cardinfo *card) 838{ 839 u32 intr_coal; 840 841 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 842 card->config.data.intr_coal.count, 843 card->config.data.intr_coal.latency); 844 iowrite32(intr_coal, card->regmap + INTR_COAL); 845 846 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size); 847} 848 849int rsxx_dma_setup(struct rsxx_cardinfo *card) 850{ 851 unsigned long flags; 852 int st; 853 int i; 854 855 dev_info(CARD_TO_DEV(card), 856 "Initializing %d DMA targets\n", 857 card->n_targets); 858 859 /* Regmap is divided up into 4K chunks. One for each DMA channel */ 860 for (i = 0; i < card->n_targets; i++) 861 card->ctrl[i].regmap = card->regmap + (i * 4096); 862 863 card->dma_fault = 0; 864 865 /* Reset the DMA queues */ 866 rsxx_dma_queue_reset(card); 867 868 /************* Setup DMA Control *************/ 869 for (i = 0; i < card->n_targets; i++) { 870 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]); 871 if (st) 872 goto failed_dma_setup; 873 874 card->ctrl[i].card = card; 875 card->ctrl[i].id = i; 876 } 877 878 card->scrub_hard = 1; 879 880 if (card->config_valid) 881 rsxx_dma_configure(card); 882 883 /* Enable the interrupts after all setup has completed. */ 884 for (i = 0; i < card->n_targets; i++) { 885 spin_lock_irqsave(&card->irq_lock, flags); 886 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i)); 887 spin_unlock_irqrestore(&card->irq_lock, flags); 888 } 889 890 return 0; 891 892failed_dma_setup: 893 for (i = 0; i < card->n_targets; i++) { 894 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i]; 895 896 if (ctrl->issue_wq) { 897 destroy_workqueue(ctrl->issue_wq); 898 ctrl->issue_wq = NULL; 899 } 900 901 if (ctrl->done_wq) { 902 destroy_workqueue(ctrl->done_wq); 903 ctrl->done_wq = NULL; 904 } 905 906 if (ctrl->trackers) 907 vfree(ctrl->trackers); 908 909 if (ctrl->status.buf) 910 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 911 ctrl->status.buf, 912 ctrl->status.dma_addr); 913 if (ctrl->cmd.buf) 914 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 915 ctrl->cmd.buf, ctrl->cmd.dma_addr); 916 } 917 918 return st; 919} 920 921 922void rsxx_dma_destroy(struct rsxx_cardinfo *card) 923{ 924 struct rsxx_dma_ctrl *ctrl; 925 struct rsxx_dma *dma; 926 int i, j; 927 int cnt = 0; 928 929 for (i = 0; i < card->n_targets; i++) { 930 ctrl = &card->ctrl[i]; 931 932 if (ctrl->issue_wq) { 933 destroy_workqueue(ctrl->issue_wq); 934 ctrl->issue_wq = NULL; 935 } 936 937 if (ctrl->done_wq) { 938 destroy_workqueue(ctrl->done_wq); 939 ctrl->done_wq = NULL; 940 } 941 942 if (timer_pending(&ctrl->activity_timer)) 943 del_timer_sync(&ctrl->activity_timer); 944 945 /* Clean up the DMA queue */ 946 spin_lock(&ctrl->queue_lock); 947 cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue); 948 spin_unlock(&ctrl->queue_lock); 949 950 if (cnt) 951 dev_info(CARD_TO_DEV(card), 952 "Freed %d queued DMAs on channel %d\n", 953 cnt, i); 954 955 /* Clean up issued DMAs */ 956 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) { 957 dma = get_tracker_dma(ctrl->trackers, j); 958 if (dma) { 959 pci_unmap_page(card->dev, dma->dma_addr, 960 get_dma_size(dma), 961 (dma->cmd == HW_CMD_BLK_WRITE) ? 962 PCI_DMA_TODEVICE : 963 PCI_DMA_FROMDEVICE); 964 kmem_cache_free(rsxx_dma_pool, dma); 965 cnt++; 966 } 967 } 968 969 if (cnt) 970 dev_info(CARD_TO_DEV(card), 971 "Freed %d pending DMAs on channel %d\n", 972 cnt, i); 973 974 vfree(ctrl->trackers); 975 976 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 977 ctrl->status.buf, ctrl->status.dma_addr); 978 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 979 ctrl->cmd.buf, ctrl->cmd.dma_addr); 980 } 981} 982 983int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card) 984{ 985 int i; 986 int j; 987 int cnt; 988 struct rsxx_dma *dma; 989 struct list_head *issued_dmas; 990 991 issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets, 992 GFP_KERNEL); 993 if (!issued_dmas) 994 return -ENOMEM; 995 996 for (i = 0; i < card->n_targets; i++) { 997 INIT_LIST_HEAD(&issued_dmas[i]); 998 cnt = 0; 999 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) { 1000 dma = get_tracker_dma(card->ctrl[i].trackers, j); 1001 if (dma == NULL) 1002 continue; 1003 1004 if (dma->cmd == HW_CMD_BLK_WRITE) 1005 card->ctrl[i].stats.writes_issued--; 1006 else if (dma->cmd == HW_CMD_BLK_DISCARD) 1007 card->ctrl[i].stats.discards_issued--; 1008 else 1009 card->ctrl[i].stats.reads_issued--; 1010 1011 list_add_tail(&dma->list, &issued_dmas[i]); 1012 push_tracker(card->ctrl[i].trackers, j); 1013 cnt++; 1014 } 1015 1016 spin_lock(&card->ctrl[i].queue_lock); 1017 list_splice(&issued_dmas[i], &card->ctrl[i].queue); 1018 1019 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth); 1020 card->ctrl[i].stats.sw_q_depth += cnt; 1021 card->ctrl[i].e_cnt = 0; 1022 1023 list_for_each_entry(dma, &card->ctrl[i].queue, list) { 1024 if (dma->dma_addr) 1025 pci_unmap_page(card->dev, dma->dma_addr, 1026 get_dma_size(dma), 1027 dma->cmd == HW_CMD_BLK_WRITE ? 1028 PCI_DMA_TODEVICE : 1029 PCI_DMA_FROMDEVICE); 1030 } 1031 spin_unlock(&card->ctrl[i].queue_lock); 1032 } 1033 1034 kfree(issued_dmas); 1035 1036 return 0; 1037} 1038 1039void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card) 1040{ 1041 struct rsxx_dma *dma; 1042 struct rsxx_dma *tmp; 1043 int i; 1044 1045 for (i = 0; i < card->n_targets; i++) { 1046 spin_lock(&card->ctrl[i].queue_lock); 1047 list_for_each_entry_safe(dma, tmp, &card->ctrl[i].queue, list) { 1048 list_del(&dma->list); 1049 1050 rsxx_complete_dma(&card->ctrl[i], dma, DMA_CANCELLED); 1051 } 1052 spin_unlock(&card->ctrl[i].queue_lock); 1053 } 1054} 1055 1056int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card) 1057{ 1058 struct rsxx_dma *dma; 1059 struct rsxx_dma *tmp; 1060 int i; 1061 1062 for (i = 0; i < card->n_targets; i++) { 1063 spin_lock(&card->ctrl[i].queue_lock); 1064 list_for_each_entry(dma, &card->ctrl[i].queue, list) { 1065 dma->dma_addr = pci_map_page(card->dev, dma->page, 1066 dma->pg_off, get_dma_size(dma), 1067 dma->cmd == HW_CMD_BLK_WRITE ? 1068 PCI_DMA_TODEVICE : 1069 PCI_DMA_FROMDEVICE); 1070 if (!dma->dma_addr) { 1071 spin_unlock(&card->ctrl[i].queue_lock); 1072 kmem_cache_free(rsxx_dma_pool, dma); 1073 return -ENOMEM; 1074 } 1075 } 1076 spin_unlock(&card->ctrl[i].queue_lock); 1077 } 1078 1079 return 0; 1080} 1081 1082int rsxx_dma_init(void) 1083{ 1084 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN); 1085 if (!rsxx_dma_pool) 1086 return -ENOMEM; 1087 1088 return 0; 1089} 1090 1091 1092void rsxx_dma_cleanup(void) 1093{ 1094 kmem_cache_destroy(rsxx_dma_pool); 1095} 1096 1097