dma.c revision 31a70bb4440c963e69ce210389d8119c70b5c39d
1/* 2* Filename: dma.c 3* 4* 5* Authors: Joshua Morris <josh.h.morris@us.ibm.com> 6* Philip Kelleher <pjk1939@linux.vnet.ibm.com> 7* 8* (C) Copyright 2013 IBM Corporation 9* 10* This program is free software; you can redistribute it and/or 11* modify it under the terms of the GNU General Public License as 12* published by the Free Software Foundation; either version 2 of the 13* License, or (at your option) any later version. 14* 15* This program is distributed in the hope that it will be useful, but 16* WITHOUT ANY WARRANTY; without even the implied warranty of 17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18* General Public License for more details. 19* 20* You should have received a copy of the GNU General Public License 21* along with this program; if not, write to the Free Software Foundation, 22* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 23*/ 24 25#include <linux/slab.h> 26#include "rsxx_priv.h" 27 28struct rsxx_dma { 29 struct list_head list; 30 u8 cmd; 31 unsigned int laddr; /* Logical address */ 32 struct { 33 u32 off; 34 u32 cnt; 35 } sub_page; 36 dma_addr_t dma_addr; 37 struct page *page; 38 unsigned int pg_off; /* Page Offset */ 39 rsxx_dma_cb cb; 40 void *cb_data; 41}; 42 43/* This timeout is used to detect a stalled DMA channel */ 44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000) 45 46struct hw_status { 47 u8 status; 48 u8 tag; 49 __le16 count; 50 __le32 _rsvd2; 51 __le64 _rsvd3; 52} __packed; 53 54enum rsxx_dma_status { 55 DMA_SW_ERR = 0x1, 56 DMA_HW_FAULT = 0x2, 57 DMA_CANCELLED = 0x4, 58}; 59 60struct hw_cmd { 61 u8 command; 62 u8 tag; 63 u8 _rsvd; 64 u8 sub_page; /* Bit[0:2]: 512byte offset */ 65 /* Bit[4:6]: 512byte count */ 66 __le32 device_addr; 67 __le64 host_addr; 68} __packed; 69 70enum rsxx_hw_cmd { 71 HW_CMD_BLK_DISCARD = 0x70, 72 HW_CMD_BLK_WRITE = 0x80, 73 HW_CMD_BLK_READ = 0xC0, 74 HW_CMD_BLK_RECON_READ = 0xE0, 75}; 76 77enum rsxx_hw_status { 78 HW_STATUS_CRC = 0x01, 79 HW_STATUS_HARD_ERR = 0x02, 80 HW_STATUS_SOFT_ERR = 0x04, 81 HW_STATUS_FAULT = 0x08, 82}; 83 84static struct kmem_cache *rsxx_dma_pool; 85 86struct dma_tracker { 87 int next_tag; 88 struct rsxx_dma *dma; 89}; 90 91#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \ 92 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS)) 93 94struct dma_tracker_list { 95 spinlock_t lock; 96 int head; 97 struct dma_tracker list[0]; 98}; 99 100 101/*----------------- Misc Utility Functions -------------------*/ 102static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card) 103{ 104 unsigned long long tgt_addr8; 105 106 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) & 107 card->_stripe.upper_mask) | 108 ((addr8) & card->_stripe.lower_mask); 109 do_div(tgt_addr8, RSXX_HW_BLK_SIZE); 110 return tgt_addr8; 111} 112 113static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8) 114{ 115 unsigned int tgt; 116 117 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask; 118 119 return tgt; 120} 121 122void rsxx_dma_queue_reset(struct rsxx_cardinfo *card) 123{ 124 /* Reset all DMA Command/Status Queues */ 125 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET); 126} 127 128static unsigned int get_dma_size(struct rsxx_dma *dma) 129{ 130 if (dma->sub_page.cnt) 131 return dma->sub_page.cnt << 9; 132 else 133 return RSXX_HW_BLK_SIZE; 134} 135 136 137/*----------------- DMA Tracker -------------------*/ 138static void set_tracker_dma(struct dma_tracker_list *trackers, 139 int tag, 140 struct rsxx_dma *dma) 141{ 142 trackers->list[tag].dma = dma; 143} 144 145static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers, 146 int tag) 147{ 148 return trackers->list[tag].dma; 149} 150 151static int pop_tracker(struct dma_tracker_list *trackers) 152{ 153 int tag; 154 155 spin_lock(&trackers->lock); 156 tag = trackers->head; 157 if (tag != -1) { 158 trackers->head = trackers->list[tag].next_tag; 159 trackers->list[tag].next_tag = -1; 160 } 161 spin_unlock(&trackers->lock); 162 163 return tag; 164} 165 166static void push_tracker(struct dma_tracker_list *trackers, int tag) 167{ 168 spin_lock(&trackers->lock); 169 trackers->list[tag].next_tag = trackers->head; 170 trackers->head = tag; 171 trackers->list[tag].dma = NULL; 172 spin_unlock(&trackers->lock); 173} 174 175 176/*----------------- Interrupt Coalescing -------------*/ 177/* 178 * Interrupt Coalescing Register Format: 179 * Interrupt Timer (64ns units) [15:0] 180 * Interrupt Count [24:16] 181 * Reserved [31:25] 182*/ 183#define INTR_COAL_LATENCY_MASK (0x0000ffff) 184 185#define INTR_COAL_COUNT_SHIFT 16 186#define INTR_COAL_COUNT_BITS 9 187#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \ 188 INTR_COAL_COUNT_SHIFT) 189#define INTR_COAL_LATENCY_UNITS_NS 64 190 191 192static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency) 193{ 194 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS; 195 196 if (mode == RSXX_INTR_COAL_DISABLED) 197 return 0; 198 199 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) | 200 (latency_units & INTR_COAL_LATENCY_MASK); 201 202} 203 204static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card) 205{ 206 int i; 207 u32 q_depth = 0; 208 u32 intr_coal; 209 210 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE || 211 unlikely(card->eeh_state)) 212 return; 213 214 for (i = 0; i < card->n_targets; i++) 215 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth); 216 217 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 218 q_depth / 2, 219 card->config.data.intr_coal.latency); 220 iowrite32(intr_coal, card->regmap + INTR_COAL); 221} 222 223/*----------------- RSXX DMA Handling -------------------*/ 224static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl, 225 struct rsxx_dma *dma, 226 unsigned int status) 227{ 228 if (status & DMA_SW_ERR) 229 ctrl->stats.dma_sw_err++; 230 if (status & DMA_HW_FAULT) 231 ctrl->stats.dma_hw_fault++; 232 if (status & DMA_CANCELLED) 233 ctrl->stats.dma_cancelled++; 234 235 if (dma->dma_addr) 236 pci_unmap_page(ctrl->card->dev, dma->dma_addr, 237 get_dma_size(dma), 238 dma->cmd == HW_CMD_BLK_WRITE ? 239 PCI_DMA_TODEVICE : 240 PCI_DMA_FROMDEVICE); 241 242 if (dma->cb) 243 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0); 244 245 kmem_cache_free(rsxx_dma_pool, dma); 246} 247 248int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl, 249 struct list_head *q) 250{ 251 struct rsxx_dma *dma; 252 struct rsxx_dma *tmp; 253 int cnt = 0; 254 255 list_for_each_entry_safe(dma, tmp, q, list) { 256 list_del(&dma->list); 257 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 258 cnt++; 259 } 260 261 return cnt; 262} 263 264static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl, 265 struct rsxx_dma *dma) 266{ 267 /* 268 * Requeued DMAs go to the front of the queue so they are issued 269 * first. 270 */ 271 spin_lock_bh(&ctrl->queue_lock); 272 list_add(&dma->list, &ctrl->queue); 273 spin_unlock_bh(&ctrl->queue_lock); 274} 275 276static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl, 277 struct rsxx_dma *dma, 278 u8 hw_st) 279{ 280 unsigned int status = 0; 281 int requeue_cmd = 0; 282 283 dev_dbg(CARD_TO_DEV(ctrl->card), 284 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n", 285 dma->cmd, dma->laddr, hw_st); 286 287 if (hw_st & HW_STATUS_CRC) 288 ctrl->stats.crc_errors++; 289 if (hw_st & HW_STATUS_HARD_ERR) 290 ctrl->stats.hard_errors++; 291 if (hw_st & HW_STATUS_SOFT_ERR) 292 ctrl->stats.soft_errors++; 293 294 switch (dma->cmd) { 295 case HW_CMD_BLK_READ: 296 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 297 if (ctrl->card->scrub_hard) { 298 dma->cmd = HW_CMD_BLK_RECON_READ; 299 requeue_cmd = 1; 300 ctrl->stats.reads_retried++; 301 } else { 302 status |= DMA_HW_FAULT; 303 ctrl->stats.reads_failed++; 304 } 305 } else if (hw_st & HW_STATUS_FAULT) { 306 status |= DMA_HW_FAULT; 307 ctrl->stats.reads_failed++; 308 } 309 310 break; 311 case HW_CMD_BLK_RECON_READ: 312 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 313 /* Data could not be reconstructed. */ 314 status |= DMA_HW_FAULT; 315 ctrl->stats.reads_failed++; 316 } 317 318 break; 319 case HW_CMD_BLK_WRITE: 320 status |= DMA_HW_FAULT; 321 ctrl->stats.writes_failed++; 322 323 break; 324 case HW_CMD_BLK_DISCARD: 325 status |= DMA_HW_FAULT; 326 ctrl->stats.discards_failed++; 327 328 break; 329 default: 330 dev_err(CARD_TO_DEV(ctrl->card), 331 "Unknown command in DMA!(cmd: x%02x " 332 "laddr x%08x st: x%02x\n", 333 dma->cmd, dma->laddr, hw_st); 334 status |= DMA_SW_ERR; 335 336 break; 337 } 338 339 if (requeue_cmd) 340 rsxx_requeue_dma(ctrl, dma); 341 else 342 rsxx_complete_dma(ctrl, dma, status); 343} 344 345static void dma_engine_stalled(unsigned long data) 346{ 347 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data; 348 int cnt; 349 350 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 || 351 unlikely(ctrl->card->eeh_state)) 352 return; 353 354 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) { 355 /* 356 * The dma engine was stalled because the SW_CMD_IDX write 357 * was lost. Issue it again to recover. 358 */ 359 dev_warn(CARD_TO_DEV(ctrl->card), 360 "SW_CMD_IDX write was lost, re-writing...\n"); 361 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 362 mod_timer(&ctrl->activity_timer, 363 jiffies + DMA_ACTIVITY_TIMEOUT); 364 } else { 365 dev_warn(CARD_TO_DEV(ctrl->card), 366 "DMA channel %d has stalled, faulting interface.\n", 367 ctrl->id); 368 ctrl->card->dma_fault = 1; 369 370 /* Clean up the DMA queue */ 371 spin_lock(&ctrl->queue_lock); 372 cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue); 373 spin_unlock(&ctrl->queue_lock); 374 375 cnt += rsxx_dma_cancel(ctrl); 376 377 if (cnt) 378 dev_info(CARD_TO_DEV(ctrl->card), 379 "Freed %d queued DMAs on channel %d\n", 380 cnt, ctrl->id); 381 } 382} 383 384static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl) 385{ 386 struct rsxx_dma *dma; 387 int tag; 388 int cmds_pending = 0; 389 struct hw_cmd *hw_cmd_buf; 390 391 hw_cmd_buf = ctrl->cmd.buf; 392 393 if (unlikely(ctrl->card->halt) || 394 unlikely(ctrl->card->eeh_state)) 395 return; 396 397 while (1) { 398 spin_lock_bh(&ctrl->queue_lock); 399 if (list_empty(&ctrl->queue)) { 400 spin_unlock_bh(&ctrl->queue_lock); 401 break; 402 } 403 spin_unlock_bh(&ctrl->queue_lock); 404 405 tag = pop_tracker(ctrl->trackers); 406 if (tag == -1) 407 break; 408 409 spin_lock_bh(&ctrl->queue_lock); 410 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list); 411 list_del(&dma->list); 412 ctrl->stats.sw_q_depth--; 413 spin_unlock_bh(&ctrl->queue_lock); 414 415 /* 416 * This will catch any DMAs that slipped in right before the 417 * fault, but was queued after all the other DMAs were 418 * cancelled. 419 */ 420 if (unlikely(ctrl->card->dma_fault)) { 421 push_tracker(ctrl->trackers, tag); 422 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 423 continue; 424 } 425 426 set_tracker_dma(ctrl->trackers, tag, dma); 427 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd; 428 hw_cmd_buf[ctrl->cmd.idx].tag = tag; 429 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0; 430 hw_cmd_buf[ctrl->cmd.idx].sub_page = 431 ((dma->sub_page.cnt & 0x7) << 4) | 432 (dma->sub_page.off & 0x7); 433 434 hw_cmd_buf[ctrl->cmd.idx].device_addr = 435 cpu_to_le32(dma->laddr); 436 437 hw_cmd_buf[ctrl->cmd.idx].host_addr = 438 cpu_to_le64(dma->dma_addr); 439 440 dev_dbg(CARD_TO_DEV(ctrl->card), 441 "Issue DMA%d(laddr %d tag %d) to idx %d\n", 442 ctrl->id, dma->laddr, tag, ctrl->cmd.idx); 443 444 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK; 445 cmds_pending++; 446 447 if (dma->cmd == HW_CMD_BLK_WRITE) 448 ctrl->stats.writes_issued++; 449 else if (dma->cmd == HW_CMD_BLK_DISCARD) 450 ctrl->stats.discards_issued++; 451 else 452 ctrl->stats.reads_issued++; 453 } 454 455 /* Let HW know we've queued commands. */ 456 if (cmds_pending) { 457 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth); 458 mod_timer(&ctrl->activity_timer, 459 jiffies + DMA_ACTIVITY_TIMEOUT); 460 461 if (unlikely(ctrl->card->eeh_state)) { 462 del_timer_sync(&ctrl->activity_timer); 463 return; 464 } 465 466 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 467 } 468} 469 470static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl) 471{ 472 struct rsxx_dma *dma; 473 unsigned long flags; 474 u16 count; 475 u8 status; 476 u8 tag; 477 struct hw_status *hw_st_buf; 478 479 hw_st_buf = ctrl->status.buf; 480 481 if (unlikely(ctrl->card->halt) || 482 unlikely(ctrl->card->dma_fault) || 483 unlikely(ctrl->card->eeh_state)) 484 return; 485 486 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 487 488 while (count == ctrl->e_cnt) { 489 /* 490 * The read memory-barrier is necessary to keep aggressive 491 * processors/optimizers (such as the PPC Apple G5) from 492 * reordering the following status-buffer tag & status read 493 * *before* the count read on subsequent iterations of the 494 * loop! 495 */ 496 rmb(); 497 498 status = hw_st_buf[ctrl->status.idx].status; 499 tag = hw_st_buf[ctrl->status.idx].tag; 500 501 dma = get_tracker_dma(ctrl->trackers, tag); 502 if (dma == NULL) { 503 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 504 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL); 505 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 506 507 dev_err(CARD_TO_DEV(ctrl->card), 508 "No tracker for tag %d " 509 "(idx %d id %d)\n", 510 tag, ctrl->status.idx, ctrl->id); 511 return; 512 } 513 514 dev_dbg(CARD_TO_DEV(ctrl->card), 515 "Completing DMA%d" 516 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n", 517 ctrl->id, dma->laddr, tag, status, count, 518 ctrl->status.idx); 519 520 atomic_dec(&ctrl->stats.hw_q_depth); 521 522 mod_timer(&ctrl->activity_timer, 523 jiffies + DMA_ACTIVITY_TIMEOUT); 524 525 if (status) 526 rsxx_handle_dma_error(ctrl, dma, status); 527 else 528 rsxx_complete_dma(ctrl, dma, 0); 529 530 push_tracker(ctrl->trackers, tag); 531 532 ctrl->status.idx = (ctrl->status.idx + 1) & 533 RSXX_CS_IDX_MASK; 534 ctrl->e_cnt++; 535 536 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 537 } 538 539 dma_intr_coal_auto_tune(ctrl->card); 540 541 if (atomic_read(&ctrl->stats.hw_q_depth) == 0) 542 del_timer_sync(&ctrl->activity_timer); 543 544 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 545 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id)); 546 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 547 548 spin_lock_bh(&ctrl->queue_lock); 549 if (ctrl->stats.sw_q_depth) 550 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work); 551 spin_unlock_bh(&ctrl->queue_lock); 552} 553 554static void rsxx_schedule_issue(struct work_struct *work) 555{ 556 struct rsxx_dma_ctrl *ctrl; 557 558 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work); 559 560 mutex_lock(&ctrl->work_lock); 561 rsxx_issue_dmas(ctrl); 562 mutex_unlock(&ctrl->work_lock); 563} 564 565static void rsxx_schedule_done(struct work_struct *work) 566{ 567 struct rsxx_dma_ctrl *ctrl; 568 569 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work); 570 571 mutex_lock(&ctrl->work_lock); 572 rsxx_dma_done(ctrl); 573 mutex_unlock(&ctrl->work_lock); 574} 575 576static int rsxx_queue_discard(struct rsxx_cardinfo *card, 577 struct list_head *q, 578 unsigned int laddr, 579 rsxx_dma_cb cb, 580 void *cb_data) 581{ 582 struct rsxx_dma *dma; 583 584 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 585 if (!dma) 586 return -ENOMEM; 587 588 dma->cmd = HW_CMD_BLK_DISCARD; 589 dma->laddr = laddr; 590 dma->dma_addr = 0; 591 dma->sub_page.off = 0; 592 dma->sub_page.cnt = 0; 593 dma->page = NULL; 594 dma->pg_off = 0; 595 dma->cb = cb; 596 dma->cb_data = cb_data; 597 598 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr); 599 600 list_add_tail(&dma->list, q); 601 602 return 0; 603} 604 605static int rsxx_queue_dma(struct rsxx_cardinfo *card, 606 struct list_head *q, 607 int dir, 608 unsigned int dma_off, 609 unsigned int dma_len, 610 unsigned int laddr, 611 struct page *page, 612 unsigned int pg_off, 613 rsxx_dma_cb cb, 614 void *cb_data) 615{ 616 struct rsxx_dma *dma; 617 618 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 619 if (!dma) 620 return -ENOMEM; 621 622 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len, 623 dir ? PCI_DMA_TODEVICE : 624 PCI_DMA_FROMDEVICE); 625 if (!dma->dma_addr) { 626 kmem_cache_free(rsxx_dma_pool, dma); 627 return -ENOMEM; 628 } 629 630 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ; 631 dma->laddr = laddr; 632 dma->sub_page.off = (dma_off >> 9); 633 dma->sub_page.cnt = (dma_len >> 9); 634 dma->page = page; 635 dma->pg_off = pg_off; 636 dma->cb = cb; 637 dma->cb_data = cb_data; 638 639 dev_dbg(CARD_TO_DEV(card), 640 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n", 641 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off, 642 dma->sub_page.cnt, dma->page, dma->pg_off); 643 644 /* Queue the DMA */ 645 list_add_tail(&dma->list, q); 646 647 return 0; 648} 649 650int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, 651 struct bio *bio, 652 atomic_t *n_dmas, 653 rsxx_dma_cb cb, 654 void *cb_data) 655{ 656 struct list_head dma_list[RSXX_MAX_TARGETS]; 657 struct bio_vec *bvec; 658 unsigned long long addr8; 659 unsigned int laddr; 660 unsigned int bv_len; 661 unsigned int bv_off; 662 unsigned int dma_off; 663 unsigned int dma_len; 664 int dma_cnt[RSXX_MAX_TARGETS]; 665 int tgt; 666 int st; 667 int i; 668 669 addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */ 670 atomic_set(n_dmas, 0); 671 672 for (i = 0; i < card->n_targets; i++) { 673 INIT_LIST_HEAD(&dma_list[i]); 674 dma_cnt[i] = 0; 675 } 676 677 if (bio->bi_rw & REQ_DISCARD) { 678 bv_len = bio->bi_size; 679 680 while (bv_len > 0) { 681 tgt = rsxx_get_dma_tgt(card, addr8); 682 laddr = rsxx_addr8_to_laddr(addr8, card); 683 684 st = rsxx_queue_discard(card, &dma_list[tgt], laddr, 685 cb, cb_data); 686 if (st) 687 goto bvec_err; 688 689 dma_cnt[tgt]++; 690 atomic_inc(n_dmas); 691 addr8 += RSXX_HW_BLK_SIZE; 692 bv_len -= RSXX_HW_BLK_SIZE; 693 } 694 } else { 695 bio_for_each_segment(bvec, bio, i) { 696 bv_len = bvec->bv_len; 697 bv_off = bvec->bv_offset; 698 699 while (bv_len > 0) { 700 tgt = rsxx_get_dma_tgt(card, addr8); 701 laddr = rsxx_addr8_to_laddr(addr8, card); 702 dma_off = addr8 & RSXX_HW_BLK_MASK; 703 dma_len = min(bv_len, 704 RSXX_HW_BLK_SIZE - dma_off); 705 706 st = rsxx_queue_dma(card, &dma_list[tgt], 707 bio_data_dir(bio), 708 dma_off, dma_len, 709 laddr, bvec->bv_page, 710 bv_off, cb, cb_data); 711 if (st) 712 goto bvec_err; 713 714 dma_cnt[tgt]++; 715 atomic_inc(n_dmas); 716 addr8 += dma_len; 717 bv_off += dma_len; 718 bv_len -= dma_len; 719 } 720 } 721 } 722 723 for (i = 0; i < card->n_targets; i++) { 724 if (!list_empty(&dma_list[i])) { 725 spin_lock_bh(&card->ctrl[i].queue_lock); 726 card->ctrl[i].stats.sw_q_depth += dma_cnt[i]; 727 list_splice_tail(&dma_list[i], &card->ctrl[i].queue); 728 spin_unlock_bh(&card->ctrl[i].queue_lock); 729 730 queue_work(card->ctrl[i].issue_wq, 731 &card->ctrl[i].issue_dma_work); 732 } 733 } 734 735 return 0; 736 737bvec_err: 738 for (i = 0; i < card->n_targets; i++) { 739 spin_lock_bh(&card->ctrl[i].queue_lock); 740 rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]); 741 spin_unlock_bh(&card->ctrl[i].queue_lock); 742 } 743 744 return st; 745} 746 747 748/*----------------- DMA Engine Initialization & Setup -------------------*/ 749int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl) 750{ 751 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8, 752 &ctrl->status.dma_addr); 753 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8, 754 &ctrl->cmd.dma_addr); 755 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL) 756 return -ENOMEM; 757 758 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8); 759 iowrite32(lower_32_bits(ctrl->status.dma_addr), 760 ctrl->regmap + SB_ADD_LO); 761 iowrite32(upper_32_bits(ctrl->status.dma_addr), 762 ctrl->regmap + SB_ADD_HI); 763 764 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8); 765 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO); 766 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI); 767 768 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT); 769 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) { 770 dev_crit(&dev->dev, "Failed reading status cnt x%x\n", 771 ctrl->status.idx); 772 return -EINVAL; 773 } 774 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT); 775 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT); 776 777 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX); 778 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) { 779 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n", 780 ctrl->status.idx); 781 return -EINVAL; 782 } 783 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX); 784 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 785 786 return 0; 787} 788 789static int rsxx_dma_ctrl_init(struct pci_dev *dev, 790 struct rsxx_dma_ctrl *ctrl) 791{ 792 int i; 793 int st; 794 795 memset(&ctrl->stats, 0, sizeof(ctrl->stats)); 796 797 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8); 798 if (!ctrl->trackers) 799 return -ENOMEM; 800 801 ctrl->trackers->head = 0; 802 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { 803 ctrl->trackers->list[i].next_tag = i + 1; 804 ctrl->trackers->list[i].dma = NULL; 805 } 806 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1; 807 spin_lock_init(&ctrl->trackers->lock); 808 809 spin_lock_init(&ctrl->queue_lock); 810 mutex_init(&ctrl->work_lock); 811 INIT_LIST_HEAD(&ctrl->queue); 812 813 setup_timer(&ctrl->activity_timer, dma_engine_stalled, 814 (unsigned long)ctrl); 815 816 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0); 817 if (!ctrl->issue_wq) 818 return -ENOMEM; 819 820 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0); 821 if (!ctrl->done_wq) 822 return -ENOMEM; 823 824 INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue); 825 INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done); 826 827 st = rsxx_hw_buffers_init(dev, ctrl); 828 if (st) 829 return st; 830 831 return 0; 832} 833 834static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card, 835 unsigned int stripe_size8) 836{ 837 if (!is_power_of_2(stripe_size8)) { 838 dev_err(CARD_TO_DEV(card), 839 "stripe_size is NOT a power of 2!\n"); 840 return -EINVAL; 841 } 842 843 card->_stripe.lower_mask = stripe_size8 - 1; 844 845 card->_stripe.upper_mask = ~(card->_stripe.lower_mask); 846 card->_stripe.upper_shift = ffs(card->n_targets) - 1; 847 848 card->_stripe.target_mask = card->n_targets - 1; 849 card->_stripe.target_shift = ffs(stripe_size8) - 1; 850 851 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n", 852 card->_stripe.lower_mask); 853 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n", 854 card->_stripe.upper_shift); 855 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n", 856 card->_stripe.upper_mask); 857 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n", 858 card->_stripe.target_mask); 859 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n", 860 card->_stripe.target_shift); 861 862 return 0; 863} 864 865int rsxx_dma_configure(struct rsxx_cardinfo *card) 866{ 867 u32 intr_coal; 868 869 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 870 card->config.data.intr_coal.count, 871 card->config.data.intr_coal.latency); 872 iowrite32(intr_coal, card->regmap + INTR_COAL); 873 874 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size); 875} 876 877int rsxx_dma_setup(struct rsxx_cardinfo *card) 878{ 879 unsigned long flags; 880 int st; 881 int i; 882 883 dev_info(CARD_TO_DEV(card), 884 "Initializing %d DMA targets\n", 885 card->n_targets); 886 887 /* Regmap is divided up into 4K chunks. One for each DMA channel */ 888 for (i = 0; i < card->n_targets; i++) 889 card->ctrl[i].regmap = card->regmap + (i * 4096); 890 891 card->dma_fault = 0; 892 893 /* Reset the DMA queues */ 894 rsxx_dma_queue_reset(card); 895 896 /************* Setup DMA Control *************/ 897 for (i = 0; i < card->n_targets; i++) { 898 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]); 899 if (st) 900 goto failed_dma_setup; 901 902 card->ctrl[i].card = card; 903 card->ctrl[i].id = i; 904 } 905 906 card->scrub_hard = 1; 907 908 if (card->config_valid) 909 rsxx_dma_configure(card); 910 911 /* Enable the interrupts after all setup has completed. */ 912 for (i = 0; i < card->n_targets; i++) { 913 spin_lock_irqsave(&card->irq_lock, flags); 914 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i)); 915 spin_unlock_irqrestore(&card->irq_lock, flags); 916 } 917 918 return 0; 919 920failed_dma_setup: 921 for (i = 0; i < card->n_targets; i++) { 922 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i]; 923 924 if (ctrl->issue_wq) { 925 destroy_workqueue(ctrl->issue_wq); 926 ctrl->issue_wq = NULL; 927 } 928 929 if (ctrl->done_wq) { 930 destroy_workqueue(ctrl->done_wq); 931 ctrl->done_wq = NULL; 932 } 933 934 if (ctrl->trackers) 935 vfree(ctrl->trackers); 936 937 if (ctrl->status.buf) 938 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 939 ctrl->status.buf, 940 ctrl->status.dma_addr); 941 if (ctrl->cmd.buf) 942 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 943 ctrl->cmd.buf, ctrl->cmd.dma_addr); 944 } 945 946 return st; 947} 948 949int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl) 950{ 951 struct rsxx_dma *dma; 952 int i; 953 int cnt = 0; 954 955 /* Clean up issued DMAs */ 956 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { 957 dma = get_tracker_dma(ctrl->trackers, i); 958 if (dma) { 959 atomic_dec(&ctrl->stats.hw_q_depth); 960 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 961 push_tracker(ctrl->trackers, i); 962 cnt++; 963 } 964 } 965 966 return cnt; 967} 968 969void rsxx_dma_destroy(struct rsxx_cardinfo *card) 970{ 971 struct rsxx_dma_ctrl *ctrl; 972 int i; 973 974 for (i = 0; i < card->n_targets; i++) { 975 ctrl = &card->ctrl[i]; 976 977 if (ctrl->issue_wq) { 978 destroy_workqueue(ctrl->issue_wq); 979 ctrl->issue_wq = NULL; 980 } 981 982 if (ctrl->done_wq) { 983 destroy_workqueue(ctrl->done_wq); 984 ctrl->done_wq = NULL; 985 } 986 987 if (timer_pending(&ctrl->activity_timer)) 988 del_timer_sync(&ctrl->activity_timer); 989 990 /* Clean up the DMA queue */ 991 spin_lock_bh(&ctrl->queue_lock); 992 rsxx_cleanup_dma_queue(ctrl, &ctrl->queue); 993 spin_unlock_bh(&ctrl->queue_lock); 994 995 rsxx_dma_cancel(ctrl); 996 997 vfree(ctrl->trackers); 998 999 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 1000 ctrl->status.buf, ctrl->status.dma_addr); 1001 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 1002 ctrl->cmd.buf, ctrl->cmd.dma_addr); 1003 } 1004} 1005 1006int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card) 1007{ 1008 int i; 1009 int j; 1010 int cnt; 1011 struct rsxx_dma *dma; 1012 struct list_head *issued_dmas; 1013 1014 issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets, 1015 GFP_KERNEL); 1016 if (!issued_dmas) 1017 return -ENOMEM; 1018 1019 for (i = 0; i < card->n_targets; i++) { 1020 INIT_LIST_HEAD(&issued_dmas[i]); 1021 cnt = 0; 1022 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) { 1023 dma = get_tracker_dma(card->ctrl[i].trackers, j); 1024 if (dma == NULL) 1025 continue; 1026 1027 if (dma->cmd == HW_CMD_BLK_WRITE) 1028 card->ctrl[i].stats.writes_issued--; 1029 else if (dma->cmd == HW_CMD_BLK_DISCARD) 1030 card->ctrl[i].stats.discards_issued--; 1031 else 1032 card->ctrl[i].stats.reads_issued--; 1033 1034 list_add_tail(&dma->list, &issued_dmas[i]); 1035 push_tracker(card->ctrl[i].trackers, j); 1036 cnt++; 1037 } 1038 1039 spin_lock_bh(&card->ctrl[i].queue_lock); 1040 list_splice(&issued_dmas[i], &card->ctrl[i].queue); 1041 1042 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth); 1043 card->ctrl[i].stats.sw_q_depth += cnt; 1044 card->ctrl[i].e_cnt = 0; 1045 1046 list_for_each_entry(dma, &card->ctrl[i].queue, list) { 1047 if (dma->dma_addr) 1048 pci_unmap_page(card->dev, dma->dma_addr, 1049 get_dma_size(dma), 1050 dma->cmd == HW_CMD_BLK_WRITE ? 1051 PCI_DMA_TODEVICE : 1052 PCI_DMA_FROMDEVICE); 1053 } 1054 spin_unlock_bh(&card->ctrl[i].queue_lock); 1055 } 1056 1057 kfree(issued_dmas); 1058 1059 return 0; 1060} 1061 1062int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card) 1063{ 1064 struct rsxx_dma *dma; 1065 int i; 1066 1067 for (i = 0; i < card->n_targets; i++) { 1068 spin_lock_bh(&card->ctrl[i].queue_lock); 1069 list_for_each_entry(dma, &card->ctrl[i].queue, list) { 1070 dma->dma_addr = pci_map_page(card->dev, dma->page, 1071 dma->pg_off, get_dma_size(dma), 1072 dma->cmd == HW_CMD_BLK_WRITE ? 1073 PCI_DMA_TODEVICE : 1074 PCI_DMA_FROMDEVICE); 1075 if (!dma->dma_addr) { 1076 spin_unlock_bh(&card->ctrl[i].queue_lock); 1077 kmem_cache_free(rsxx_dma_pool, dma); 1078 return -ENOMEM; 1079 } 1080 } 1081 spin_unlock_bh(&card->ctrl[i].queue_lock); 1082 } 1083 1084 return 0; 1085} 1086 1087int rsxx_dma_init(void) 1088{ 1089 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN); 1090 if (!rsxx_dma_pool) 1091 return -ENOMEM; 1092 1093 return 0; 1094} 1095 1096 1097void rsxx_dma_cleanup(void) 1098{ 1099 kmem_cache_destroy(rsxx_dma_pool); 1100} 1101 1102