dma.c revision 62302508f2986720ad73494dd8037dff1c4f77d1
1/* 2* Filename: dma.c 3* 4* 5* Authors: Joshua Morris <josh.h.morris@us.ibm.com> 6* Philip Kelleher <pjk1939@linux.vnet.ibm.com> 7* 8* (C) Copyright 2013 IBM Corporation 9* 10* This program is free software; you can redistribute it and/or 11* modify it under the terms of the GNU General Public License as 12* published by the Free Software Foundation; either version 2 of the 13* License, or (at your option) any later version. 14* 15* This program is distributed in the hope that it will be useful, but 16* WITHOUT ANY WARRANTY; without even the implied warranty of 17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18* General Public License for more details. 19* 20* You should have received a copy of the GNU General Public License 21* along with this program; if not, write to the Free Software Foundation, 22* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 23*/ 24 25#include <linux/slab.h> 26#include "rsxx_priv.h" 27 28struct rsxx_dma { 29 struct list_head list; 30 u8 cmd; 31 unsigned int laddr; /* Logical address */ 32 struct { 33 u32 off; 34 u32 cnt; 35 } sub_page; 36 dma_addr_t dma_addr; 37 struct page *page; 38 unsigned int pg_off; /* Page Offset */ 39 rsxx_dma_cb cb; 40 void *cb_data; 41}; 42 43/* This timeout is used to detect a stalled DMA channel */ 44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000) 45 46struct hw_status { 47 u8 status; 48 u8 tag; 49 __le16 count; 50 __le32 _rsvd2; 51 __le64 _rsvd3; 52} __packed; 53 54enum rsxx_dma_status { 55 DMA_SW_ERR = 0x1, 56 DMA_HW_FAULT = 0x2, 57 DMA_CANCELLED = 0x4, 58}; 59 60struct hw_cmd { 61 u8 command; 62 u8 tag; 63 u8 _rsvd; 64 u8 sub_page; /* Bit[0:2]: 512byte offset */ 65 /* Bit[4:6]: 512byte count */ 66 __le32 device_addr; 67 __le64 host_addr; 68} __packed; 69 70enum rsxx_hw_cmd { 71 HW_CMD_BLK_DISCARD = 0x70, 72 HW_CMD_BLK_WRITE = 0x80, 73 HW_CMD_BLK_READ = 0xC0, 74 HW_CMD_BLK_RECON_READ = 0xE0, 75}; 76 77enum rsxx_hw_status { 78 HW_STATUS_CRC = 0x01, 79 HW_STATUS_HARD_ERR = 0x02, 80 HW_STATUS_SOFT_ERR = 0x04, 81 HW_STATUS_FAULT = 0x08, 82}; 83 84static struct kmem_cache *rsxx_dma_pool; 85 86struct dma_tracker { 87 int next_tag; 88 struct rsxx_dma *dma; 89}; 90 91#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \ 92 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS)) 93 94struct dma_tracker_list { 95 spinlock_t lock; 96 int head; 97 struct dma_tracker list[0]; 98}; 99 100 101/*----------------- Misc Utility Functions -------------------*/ 102static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card) 103{ 104 unsigned long long tgt_addr8; 105 106 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) & 107 card->_stripe.upper_mask) | 108 ((addr8) & card->_stripe.lower_mask); 109 do_div(tgt_addr8, RSXX_HW_BLK_SIZE); 110 return tgt_addr8; 111} 112 113static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8) 114{ 115 unsigned int tgt; 116 117 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask; 118 119 return tgt; 120} 121 122void rsxx_dma_queue_reset(struct rsxx_cardinfo *card) 123{ 124 /* Reset all DMA Command/Status Queues */ 125 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET); 126} 127 128static unsigned int get_dma_size(struct rsxx_dma *dma) 129{ 130 if (dma->sub_page.cnt) 131 return dma->sub_page.cnt << 9; 132 else 133 return RSXX_HW_BLK_SIZE; 134} 135 136 137/*----------------- DMA Tracker -------------------*/ 138static void set_tracker_dma(struct dma_tracker_list *trackers, 139 int tag, 140 struct rsxx_dma *dma) 141{ 142 trackers->list[tag].dma = dma; 143} 144 145static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers, 146 int tag) 147{ 148 return trackers->list[tag].dma; 149} 150 151static int pop_tracker(struct dma_tracker_list *trackers) 152{ 153 int tag; 154 155 spin_lock(&trackers->lock); 156 tag = trackers->head; 157 if (tag != -1) { 158 trackers->head = trackers->list[tag].next_tag; 159 trackers->list[tag].next_tag = -1; 160 } 161 spin_unlock(&trackers->lock); 162 163 return tag; 164} 165 166static void push_tracker(struct dma_tracker_list *trackers, int tag) 167{ 168 spin_lock(&trackers->lock); 169 trackers->list[tag].next_tag = trackers->head; 170 trackers->head = tag; 171 trackers->list[tag].dma = NULL; 172 spin_unlock(&trackers->lock); 173} 174 175 176/*----------------- Interrupt Coalescing -------------*/ 177/* 178 * Interrupt Coalescing Register Format: 179 * Interrupt Timer (64ns units) [15:0] 180 * Interrupt Count [24:16] 181 * Reserved [31:25] 182*/ 183#define INTR_COAL_LATENCY_MASK (0x0000ffff) 184 185#define INTR_COAL_COUNT_SHIFT 16 186#define INTR_COAL_COUNT_BITS 9 187#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \ 188 INTR_COAL_COUNT_SHIFT) 189#define INTR_COAL_LATENCY_UNITS_NS 64 190 191 192static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency) 193{ 194 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS; 195 196 if (mode == RSXX_INTR_COAL_DISABLED) 197 return 0; 198 199 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) | 200 (latency_units & INTR_COAL_LATENCY_MASK); 201 202} 203 204static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card) 205{ 206 int i; 207 u32 q_depth = 0; 208 u32 intr_coal; 209 210 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE || 211 unlikely(card->eeh_state)) 212 return; 213 214 for (i = 0; i < card->n_targets; i++) 215 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth); 216 217 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 218 q_depth / 2, 219 card->config.data.intr_coal.latency); 220 iowrite32(intr_coal, card->regmap + INTR_COAL); 221} 222 223/*----------------- RSXX DMA Handling -------------------*/ 224static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl, 225 struct rsxx_dma *dma, 226 unsigned int status) 227{ 228 if (status & DMA_SW_ERR) 229 ctrl->stats.dma_sw_err++; 230 if (status & DMA_HW_FAULT) 231 ctrl->stats.dma_hw_fault++; 232 if (status & DMA_CANCELLED) 233 ctrl->stats.dma_cancelled++; 234 235 if (dma->dma_addr) 236 pci_unmap_page(ctrl->card->dev, dma->dma_addr, 237 get_dma_size(dma), 238 dma->cmd == HW_CMD_BLK_WRITE ? 239 PCI_DMA_TODEVICE : 240 PCI_DMA_FROMDEVICE); 241 242 if (dma->cb) 243 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0); 244 245 kmem_cache_free(rsxx_dma_pool, dma); 246} 247 248int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl, 249 struct list_head *q) 250{ 251 struct rsxx_dma *dma; 252 struct rsxx_dma *tmp; 253 int cnt = 0; 254 255 list_for_each_entry_safe(dma, tmp, q, list) { 256 list_del(&dma->list); 257 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 258 cnt++; 259 } 260 261 return cnt; 262} 263 264static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl, 265 struct rsxx_dma *dma) 266{ 267 /* 268 * Requeued DMAs go to the front of the queue so they are issued 269 * first. 270 */ 271 spin_lock_bh(&ctrl->queue_lock); 272 ctrl->stats.sw_q_depth++; 273 list_add(&dma->list, &ctrl->queue); 274 spin_unlock_bh(&ctrl->queue_lock); 275} 276 277static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl, 278 struct rsxx_dma *dma, 279 u8 hw_st) 280{ 281 unsigned int status = 0; 282 int requeue_cmd = 0; 283 284 dev_dbg(CARD_TO_DEV(ctrl->card), 285 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n", 286 dma->cmd, dma->laddr, hw_st); 287 288 if (hw_st & HW_STATUS_CRC) 289 ctrl->stats.crc_errors++; 290 if (hw_st & HW_STATUS_HARD_ERR) 291 ctrl->stats.hard_errors++; 292 if (hw_st & HW_STATUS_SOFT_ERR) 293 ctrl->stats.soft_errors++; 294 295 switch (dma->cmd) { 296 case HW_CMD_BLK_READ: 297 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 298 if (ctrl->card->scrub_hard) { 299 dma->cmd = HW_CMD_BLK_RECON_READ; 300 requeue_cmd = 1; 301 ctrl->stats.reads_retried++; 302 } else { 303 status |= DMA_HW_FAULT; 304 ctrl->stats.reads_failed++; 305 } 306 } else if (hw_st & HW_STATUS_FAULT) { 307 status |= DMA_HW_FAULT; 308 ctrl->stats.reads_failed++; 309 } 310 311 break; 312 case HW_CMD_BLK_RECON_READ: 313 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 314 /* Data could not be reconstructed. */ 315 status |= DMA_HW_FAULT; 316 ctrl->stats.reads_failed++; 317 } 318 319 break; 320 case HW_CMD_BLK_WRITE: 321 status |= DMA_HW_FAULT; 322 ctrl->stats.writes_failed++; 323 324 break; 325 case HW_CMD_BLK_DISCARD: 326 status |= DMA_HW_FAULT; 327 ctrl->stats.discards_failed++; 328 329 break; 330 default: 331 dev_err(CARD_TO_DEV(ctrl->card), 332 "Unknown command in DMA!(cmd: x%02x " 333 "laddr x%08x st: x%02x\n", 334 dma->cmd, dma->laddr, hw_st); 335 status |= DMA_SW_ERR; 336 337 break; 338 } 339 340 if (requeue_cmd) 341 rsxx_requeue_dma(ctrl, dma); 342 else 343 rsxx_complete_dma(ctrl, dma, status); 344} 345 346static void dma_engine_stalled(unsigned long data) 347{ 348 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data; 349 int cnt; 350 351 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 || 352 unlikely(ctrl->card->eeh_state)) 353 return; 354 355 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) { 356 /* 357 * The dma engine was stalled because the SW_CMD_IDX write 358 * was lost. Issue it again to recover. 359 */ 360 dev_warn(CARD_TO_DEV(ctrl->card), 361 "SW_CMD_IDX write was lost, re-writing...\n"); 362 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 363 mod_timer(&ctrl->activity_timer, 364 jiffies + DMA_ACTIVITY_TIMEOUT); 365 } else { 366 dev_warn(CARD_TO_DEV(ctrl->card), 367 "DMA channel %d has stalled, faulting interface.\n", 368 ctrl->id); 369 ctrl->card->dma_fault = 1; 370 371 /* Clean up the DMA queue */ 372 spin_lock(&ctrl->queue_lock); 373 cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue); 374 spin_unlock(&ctrl->queue_lock); 375 376 cnt += rsxx_dma_cancel(ctrl); 377 378 if (cnt) 379 dev_info(CARD_TO_DEV(ctrl->card), 380 "Freed %d queued DMAs on channel %d\n", 381 cnt, ctrl->id); 382 } 383} 384 385static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl) 386{ 387 struct rsxx_dma *dma; 388 int tag; 389 int cmds_pending = 0; 390 struct hw_cmd *hw_cmd_buf; 391 392 hw_cmd_buf = ctrl->cmd.buf; 393 394 if (unlikely(ctrl->card->halt) || 395 unlikely(ctrl->card->eeh_state)) 396 return; 397 398 while (1) { 399 spin_lock_bh(&ctrl->queue_lock); 400 if (list_empty(&ctrl->queue)) { 401 spin_unlock_bh(&ctrl->queue_lock); 402 break; 403 } 404 spin_unlock_bh(&ctrl->queue_lock); 405 406 tag = pop_tracker(ctrl->trackers); 407 if (tag == -1) 408 break; 409 410 spin_lock_bh(&ctrl->queue_lock); 411 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list); 412 list_del(&dma->list); 413 ctrl->stats.sw_q_depth--; 414 spin_unlock_bh(&ctrl->queue_lock); 415 416 /* 417 * This will catch any DMAs that slipped in right before the 418 * fault, but was queued after all the other DMAs were 419 * cancelled. 420 */ 421 if (unlikely(ctrl->card->dma_fault)) { 422 push_tracker(ctrl->trackers, tag); 423 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 424 continue; 425 } 426 427 set_tracker_dma(ctrl->trackers, tag, dma); 428 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd; 429 hw_cmd_buf[ctrl->cmd.idx].tag = tag; 430 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0; 431 hw_cmd_buf[ctrl->cmd.idx].sub_page = 432 ((dma->sub_page.cnt & 0x7) << 4) | 433 (dma->sub_page.off & 0x7); 434 435 hw_cmd_buf[ctrl->cmd.idx].device_addr = 436 cpu_to_le32(dma->laddr); 437 438 hw_cmd_buf[ctrl->cmd.idx].host_addr = 439 cpu_to_le64(dma->dma_addr); 440 441 dev_dbg(CARD_TO_DEV(ctrl->card), 442 "Issue DMA%d(laddr %d tag %d) to idx %d\n", 443 ctrl->id, dma->laddr, tag, ctrl->cmd.idx); 444 445 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK; 446 cmds_pending++; 447 448 if (dma->cmd == HW_CMD_BLK_WRITE) 449 ctrl->stats.writes_issued++; 450 else if (dma->cmd == HW_CMD_BLK_DISCARD) 451 ctrl->stats.discards_issued++; 452 else 453 ctrl->stats.reads_issued++; 454 } 455 456 /* Let HW know we've queued commands. */ 457 if (cmds_pending) { 458 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth); 459 mod_timer(&ctrl->activity_timer, 460 jiffies + DMA_ACTIVITY_TIMEOUT); 461 462 if (unlikely(ctrl->card->eeh_state)) { 463 del_timer_sync(&ctrl->activity_timer); 464 return; 465 } 466 467 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 468 } 469} 470 471static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl) 472{ 473 struct rsxx_dma *dma; 474 unsigned long flags; 475 u16 count; 476 u8 status; 477 u8 tag; 478 struct hw_status *hw_st_buf; 479 480 hw_st_buf = ctrl->status.buf; 481 482 if (unlikely(ctrl->card->halt) || 483 unlikely(ctrl->card->dma_fault) || 484 unlikely(ctrl->card->eeh_state)) 485 return; 486 487 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 488 489 while (count == ctrl->e_cnt) { 490 /* 491 * The read memory-barrier is necessary to keep aggressive 492 * processors/optimizers (such as the PPC Apple G5) from 493 * reordering the following status-buffer tag & status read 494 * *before* the count read on subsequent iterations of the 495 * loop! 496 */ 497 rmb(); 498 499 status = hw_st_buf[ctrl->status.idx].status; 500 tag = hw_st_buf[ctrl->status.idx].tag; 501 502 dma = get_tracker_dma(ctrl->trackers, tag); 503 if (dma == NULL) { 504 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 505 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL); 506 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 507 508 dev_err(CARD_TO_DEV(ctrl->card), 509 "No tracker for tag %d " 510 "(idx %d id %d)\n", 511 tag, ctrl->status.idx, ctrl->id); 512 return; 513 } 514 515 dev_dbg(CARD_TO_DEV(ctrl->card), 516 "Completing DMA%d" 517 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n", 518 ctrl->id, dma->laddr, tag, status, count, 519 ctrl->status.idx); 520 521 atomic_dec(&ctrl->stats.hw_q_depth); 522 523 mod_timer(&ctrl->activity_timer, 524 jiffies + DMA_ACTIVITY_TIMEOUT); 525 526 if (status) 527 rsxx_handle_dma_error(ctrl, dma, status); 528 else 529 rsxx_complete_dma(ctrl, dma, 0); 530 531 push_tracker(ctrl->trackers, tag); 532 533 ctrl->status.idx = (ctrl->status.idx + 1) & 534 RSXX_CS_IDX_MASK; 535 ctrl->e_cnt++; 536 537 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 538 } 539 540 dma_intr_coal_auto_tune(ctrl->card); 541 542 if (atomic_read(&ctrl->stats.hw_q_depth) == 0) 543 del_timer_sync(&ctrl->activity_timer); 544 545 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 546 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id)); 547 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 548 549 spin_lock_bh(&ctrl->queue_lock); 550 if (ctrl->stats.sw_q_depth) 551 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work); 552 spin_unlock_bh(&ctrl->queue_lock); 553} 554 555static void rsxx_schedule_issue(struct work_struct *work) 556{ 557 struct rsxx_dma_ctrl *ctrl; 558 559 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work); 560 561 mutex_lock(&ctrl->work_lock); 562 rsxx_issue_dmas(ctrl); 563 mutex_unlock(&ctrl->work_lock); 564} 565 566static void rsxx_schedule_done(struct work_struct *work) 567{ 568 struct rsxx_dma_ctrl *ctrl; 569 570 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work); 571 572 mutex_lock(&ctrl->work_lock); 573 rsxx_dma_done(ctrl); 574 mutex_unlock(&ctrl->work_lock); 575} 576 577static int rsxx_queue_discard(struct rsxx_cardinfo *card, 578 struct list_head *q, 579 unsigned int laddr, 580 rsxx_dma_cb cb, 581 void *cb_data) 582{ 583 struct rsxx_dma *dma; 584 585 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 586 if (!dma) 587 return -ENOMEM; 588 589 dma->cmd = HW_CMD_BLK_DISCARD; 590 dma->laddr = laddr; 591 dma->dma_addr = 0; 592 dma->sub_page.off = 0; 593 dma->sub_page.cnt = 0; 594 dma->page = NULL; 595 dma->pg_off = 0; 596 dma->cb = cb; 597 dma->cb_data = cb_data; 598 599 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr); 600 601 list_add_tail(&dma->list, q); 602 603 return 0; 604} 605 606static int rsxx_queue_dma(struct rsxx_cardinfo *card, 607 struct list_head *q, 608 int dir, 609 unsigned int dma_off, 610 unsigned int dma_len, 611 unsigned int laddr, 612 struct page *page, 613 unsigned int pg_off, 614 rsxx_dma_cb cb, 615 void *cb_data) 616{ 617 struct rsxx_dma *dma; 618 619 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 620 if (!dma) 621 return -ENOMEM; 622 623 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len, 624 dir ? PCI_DMA_TODEVICE : 625 PCI_DMA_FROMDEVICE); 626 if (!dma->dma_addr) { 627 kmem_cache_free(rsxx_dma_pool, dma); 628 return -ENOMEM; 629 } 630 631 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ; 632 dma->laddr = laddr; 633 dma->sub_page.off = (dma_off >> 9); 634 dma->sub_page.cnt = (dma_len >> 9); 635 dma->page = page; 636 dma->pg_off = pg_off; 637 dma->cb = cb; 638 dma->cb_data = cb_data; 639 640 dev_dbg(CARD_TO_DEV(card), 641 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n", 642 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off, 643 dma->sub_page.cnt, dma->page, dma->pg_off); 644 645 /* Queue the DMA */ 646 list_add_tail(&dma->list, q); 647 648 return 0; 649} 650 651int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, 652 struct bio *bio, 653 atomic_t *n_dmas, 654 rsxx_dma_cb cb, 655 void *cb_data) 656{ 657 struct list_head dma_list[RSXX_MAX_TARGETS]; 658 struct bio_vec *bvec; 659 unsigned long long addr8; 660 unsigned int laddr; 661 unsigned int bv_len; 662 unsigned int bv_off; 663 unsigned int dma_off; 664 unsigned int dma_len; 665 int dma_cnt[RSXX_MAX_TARGETS]; 666 int tgt; 667 int st; 668 int i; 669 670 addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */ 671 atomic_set(n_dmas, 0); 672 673 for (i = 0; i < card->n_targets; i++) { 674 INIT_LIST_HEAD(&dma_list[i]); 675 dma_cnt[i] = 0; 676 } 677 678 if (bio->bi_rw & REQ_DISCARD) { 679 bv_len = bio->bi_size; 680 681 while (bv_len > 0) { 682 tgt = rsxx_get_dma_tgt(card, addr8); 683 laddr = rsxx_addr8_to_laddr(addr8, card); 684 685 st = rsxx_queue_discard(card, &dma_list[tgt], laddr, 686 cb, cb_data); 687 if (st) 688 goto bvec_err; 689 690 dma_cnt[tgt]++; 691 atomic_inc(n_dmas); 692 addr8 += RSXX_HW_BLK_SIZE; 693 bv_len -= RSXX_HW_BLK_SIZE; 694 } 695 } else { 696 bio_for_each_segment(bvec, bio, i) { 697 bv_len = bvec->bv_len; 698 bv_off = bvec->bv_offset; 699 700 while (bv_len > 0) { 701 tgt = rsxx_get_dma_tgt(card, addr8); 702 laddr = rsxx_addr8_to_laddr(addr8, card); 703 dma_off = addr8 & RSXX_HW_BLK_MASK; 704 dma_len = min(bv_len, 705 RSXX_HW_BLK_SIZE - dma_off); 706 707 st = rsxx_queue_dma(card, &dma_list[tgt], 708 bio_data_dir(bio), 709 dma_off, dma_len, 710 laddr, bvec->bv_page, 711 bv_off, cb, cb_data); 712 if (st) 713 goto bvec_err; 714 715 dma_cnt[tgt]++; 716 atomic_inc(n_dmas); 717 addr8 += dma_len; 718 bv_off += dma_len; 719 bv_len -= dma_len; 720 } 721 } 722 } 723 724 for (i = 0; i < card->n_targets; i++) { 725 if (!list_empty(&dma_list[i])) { 726 spin_lock_bh(&card->ctrl[i].queue_lock); 727 card->ctrl[i].stats.sw_q_depth += dma_cnt[i]; 728 list_splice_tail(&dma_list[i], &card->ctrl[i].queue); 729 spin_unlock_bh(&card->ctrl[i].queue_lock); 730 731 queue_work(card->ctrl[i].issue_wq, 732 &card->ctrl[i].issue_dma_work); 733 } 734 } 735 736 return 0; 737 738bvec_err: 739 for (i = 0; i < card->n_targets; i++) { 740 spin_lock_bh(&card->ctrl[i].queue_lock); 741 rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]); 742 spin_unlock_bh(&card->ctrl[i].queue_lock); 743 } 744 745 return st; 746} 747 748 749/*----------------- DMA Engine Initialization & Setup -------------------*/ 750int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl) 751{ 752 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8, 753 &ctrl->status.dma_addr); 754 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8, 755 &ctrl->cmd.dma_addr); 756 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL) 757 return -ENOMEM; 758 759 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8); 760 iowrite32(lower_32_bits(ctrl->status.dma_addr), 761 ctrl->regmap + SB_ADD_LO); 762 iowrite32(upper_32_bits(ctrl->status.dma_addr), 763 ctrl->regmap + SB_ADD_HI); 764 765 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8); 766 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO); 767 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI); 768 769 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT); 770 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) { 771 dev_crit(&dev->dev, "Failed reading status cnt x%x\n", 772 ctrl->status.idx); 773 return -EINVAL; 774 } 775 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT); 776 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT); 777 778 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX); 779 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) { 780 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n", 781 ctrl->status.idx); 782 return -EINVAL; 783 } 784 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX); 785 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 786 787 return 0; 788} 789 790static int rsxx_dma_ctrl_init(struct pci_dev *dev, 791 struct rsxx_dma_ctrl *ctrl) 792{ 793 int i; 794 int st; 795 796 memset(&ctrl->stats, 0, sizeof(ctrl->stats)); 797 798 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8); 799 if (!ctrl->trackers) 800 return -ENOMEM; 801 802 ctrl->trackers->head = 0; 803 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { 804 ctrl->trackers->list[i].next_tag = i + 1; 805 ctrl->trackers->list[i].dma = NULL; 806 } 807 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1; 808 spin_lock_init(&ctrl->trackers->lock); 809 810 spin_lock_init(&ctrl->queue_lock); 811 mutex_init(&ctrl->work_lock); 812 INIT_LIST_HEAD(&ctrl->queue); 813 814 setup_timer(&ctrl->activity_timer, dma_engine_stalled, 815 (unsigned long)ctrl); 816 817 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0); 818 if (!ctrl->issue_wq) 819 return -ENOMEM; 820 821 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0); 822 if (!ctrl->done_wq) 823 return -ENOMEM; 824 825 INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue); 826 INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done); 827 828 st = rsxx_hw_buffers_init(dev, ctrl); 829 if (st) 830 return st; 831 832 return 0; 833} 834 835static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card, 836 unsigned int stripe_size8) 837{ 838 if (!is_power_of_2(stripe_size8)) { 839 dev_err(CARD_TO_DEV(card), 840 "stripe_size is NOT a power of 2!\n"); 841 return -EINVAL; 842 } 843 844 card->_stripe.lower_mask = stripe_size8 - 1; 845 846 card->_stripe.upper_mask = ~(card->_stripe.lower_mask); 847 card->_stripe.upper_shift = ffs(card->n_targets) - 1; 848 849 card->_stripe.target_mask = card->n_targets - 1; 850 card->_stripe.target_shift = ffs(stripe_size8) - 1; 851 852 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n", 853 card->_stripe.lower_mask); 854 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n", 855 card->_stripe.upper_shift); 856 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n", 857 card->_stripe.upper_mask); 858 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n", 859 card->_stripe.target_mask); 860 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n", 861 card->_stripe.target_shift); 862 863 return 0; 864} 865 866int rsxx_dma_configure(struct rsxx_cardinfo *card) 867{ 868 u32 intr_coal; 869 870 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 871 card->config.data.intr_coal.count, 872 card->config.data.intr_coal.latency); 873 iowrite32(intr_coal, card->regmap + INTR_COAL); 874 875 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size); 876} 877 878int rsxx_dma_setup(struct rsxx_cardinfo *card) 879{ 880 unsigned long flags; 881 int st; 882 int i; 883 884 dev_info(CARD_TO_DEV(card), 885 "Initializing %d DMA targets\n", 886 card->n_targets); 887 888 /* Regmap is divided up into 4K chunks. One for each DMA channel */ 889 for (i = 0; i < card->n_targets; i++) 890 card->ctrl[i].regmap = card->regmap + (i * 4096); 891 892 card->dma_fault = 0; 893 894 /* Reset the DMA queues */ 895 rsxx_dma_queue_reset(card); 896 897 /************* Setup DMA Control *************/ 898 for (i = 0; i < card->n_targets; i++) { 899 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]); 900 if (st) 901 goto failed_dma_setup; 902 903 card->ctrl[i].card = card; 904 card->ctrl[i].id = i; 905 } 906 907 card->scrub_hard = 1; 908 909 if (card->config_valid) 910 rsxx_dma_configure(card); 911 912 /* Enable the interrupts after all setup has completed. */ 913 for (i = 0; i < card->n_targets; i++) { 914 spin_lock_irqsave(&card->irq_lock, flags); 915 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i)); 916 spin_unlock_irqrestore(&card->irq_lock, flags); 917 } 918 919 return 0; 920 921failed_dma_setup: 922 for (i = 0; i < card->n_targets; i++) { 923 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i]; 924 925 if (ctrl->issue_wq) { 926 destroy_workqueue(ctrl->issue_wq); 927 ctrl->issue_wq = NULL; 928 } 929 930 if (ctrl->done_wq) { 931 destroy_workqueue(ctrl->done_wq); 932 ctrl->done_wq = NULL; 933 } 934 935 if (ctrl->trackers) 936 vfree(ctrl->trackers); 937 938 if (ctrl->status.buf) 939 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 940 ctrl->status.buf, 941 ctrl->status.dma_addr); 942 if (ctrl->cmd.buf) 943 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 944 ctrl->cmd.buf, ctrl->cmd.dma_addr); 945 } 946 947 return st; 948} 949 950int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl) 951{ 952 struct rsxx_dma *dma; 953 int i; 954 int cnt = 0; 955 956 /* Clean up issued DMAs */ 957 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { 958 dma = get_tracker_dma(ctrl->trackers, i); 959 if (dma) { 960 atomic_dec(&ctrl->stats.hw_q_depth); 961 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); 962 push_tracker(ctrl->trackers, i); 963 cnt++; 964 } 965 } 966 967 return cnt; 968} 969 970void rsxx_dma_destroy(struct rsxx_cardinfo *card) 971{ 972 struct rsxx_dma_ctrl *ctrl; 973 int i; 974 975 for (i = 0; i < card->n_targets; i++) { 976 ctrl = &card->ctrl[i]; 977 978 if (ctrl->issue_wq) { 979 destroy_workqueue(ctrl->issue_wq); 980 ctrl->issue_wq = NULL; 981 } 982 983 if (ctrl->done_wq) { 984 destroy_workqueue(ctrl->done_wq); 985 ctrl->done_wq = NULL; 986 } 987 988 if (timer_pending(&ctrl->activity_timer)) 989 del_timer_sync(&ctrl->activity_timer); 990 991 /* Clean up the DMA queue */ 992 spin_lock_bh(&ctrl->queue_lock); 993 rsxx_cleanup_dma_queue(ctrl, &ctrl->queue); 994 spin_unlock_bh(&ctrl->queue_lock); 995 996 rsxx_dma_cancel(ctrl); 997 998 vfree(ctrl->trackers); 999 1000 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 1001 ctrl->status.buf, ctrl->status.dma_addr); 1002 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 1003 ctrl->cmd.buf, ctrl->cmd.dma_addr); 1004 } 1005} 1006 1007int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card) 1008{ 1009 int i; 1010 int j; 1011 int cnt; 1012 struct rsxx_dma *dma; 1013 struct list_head *issued_dmas; 1014 1015 issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets, 1016 GFP_KERNEL); 1017 if (!issued_dmas) 1018 return -ENOMEM; 1019 1020 for (i = 0; i < card->n_targets; i++) { 1021 INIT_LIST_HEAD(&issued_dmas[i]); 1022 cnt = 0; 1023 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) { 1024 dma = get_tracker_dma(card->ctrl[i].trackers, j); 1025 if (dma == NULL) 1026 continue; 1027 1028 if (dma->cmd == HW_CMD_BLK_WRITE) 1029 card->ctrl[i].stats.writes_issued--; 1030 else if (dma->cmd == HW_CMD_BLK_DISCARD) 1031 card->ctrl[i].stats.discards_issued--; 1032 else 1033 card->ctrl[i].stats.reads_issued--; 1034 1035 list_add_tail(&dma->list, &issued_dmas[i]); 1036 push_tracker(card->ctrl[i].trackers, j); 1037 cnt++; 1038 } 1039 1040 spin_lock_bh(&card->ctrl[i].queue_lock); 1041 list_splice(&issued_dmas[i], &card->ctrl[i].queue); 1042 1043 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth); 1044 card->ctrl[i].stats.sw_q_depth += cnt; 1045 card->ctrl[i].e_cnt = 0; 1046 1047 list_for_each_entry(dma, &card->ctrl[i].queue, list) { 1048 if (dma->dma_addr) 1049 pci_unmap_page(card->dev, dma->dma_addr, 1050 get_dma_size(dma), 1051 dma->cmd == HW_CMD_BLK_WRITE ? 1052 PCI_DMA_TODEVICE : 1053 PCI_DMA_FROMDEVICE); 1054 } 1055 spin_unlock_bh(&card->ctrl[i].queue_lock); 1056 } 1057 1058 kfree(issued_dmas); 1059 1060 return 0; 1061} 1062 1063int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card) 1064{ 1065 struct rsxx_dma *dma; 1066 int i; 1067 1068 for (i = 0; i < card->n_targets; i++) { 1069 spin_lock_bh(&card->ctrl[i].queue_lock); 1070 list_for_each_entry(dma, &card->ctrl[i].queue, list) { 1071 dma->dma_addr = pci_map_page(card->dev, dma->page, 1072 dma->pg_off, get_dma_size(dma), 1073 dma->cmd == HW_CMD_BLK_WRITE ? 1074 PCI_DMA_TODEVICE : 1075 PCI_DMA_FROMDEVICE); 1076 if (!dma->dma_addr) { 1077 spin_unlock_bh(&card->ctrl[i].queue_lock); 1078 kmem_cache_free(rsxx_dma_pool, dma); 1079 return -ENOMEM; 1080 } 1081 } 1082 spin_unlock_bh(&card->ctrl[i].queue_lock); 1083 } 1084 1085 return 0; 1086} 1087 1088int rsxx_dma_init(void) 1089{ 1090 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN); 1091 if (!rsxx_dma_pool) 1092 return -ENOMEM; 1093 1094 return 0; 1095} 1096 1097 1098void rsxx_dma_cleanup(void) 1099{ 1100 kmem_cache_destroy(rsxx_dma_pool); 1101} 1102 1103