dma.c revision f37912039eb04979f269de0a7dc1a601702df51a
1/* 2* Filename: dma.c 3* 4* 5* Authors: Joshua Morris <josh.h.morris@us.ibm.com> 6* Philip Kelleher <pjk1939@linux.vnet.ibm.com> 7* 8* (C) Copyright 2013 IBM Corporation 9* 10* This program is free software; you can redistribute it and/or 11* modify it under the terms of the GNU General Public License as 12* published by the Free Software Foundation; either version 2 of the 13* License, or (at your option) any later version. 14* 15* This program is distributed in the hope that it will be useful, but 16* WITHOUT ANY WARRANTY; without even the implied warranty of 17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18* General Public License for more details. 19* 20* You should have received a copy of the GNU General Public License 21* along with this program; if not, write to the Free Software Foundation, 22* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 23*/ 24 25#include <linux/slab.h> 26#include "rsxx_priv.h" 27 28struct rsxx_dma { 29 struct list_head list; 30 u8 cmd; 31 unsigned int laddr; /* Logical address on the ramsan */ 32 struct { 33 u32 off; 34 u32 cnt; 35 } sub_page; 36 dma_addr_t dma_addr; 37 struct page *page; 38 unsigned int pg_off; /* Page Offset */ 39 rsxx_dma_cb cb; 40 void *cb_data; 41}; 42 43/* This timeout is used to detect a stalled DMA channel */ 44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000) 45 46struct hw_status { 47 u8 status; 48 u8 tag; 49 __le16 count; 50 __le32 _rsvd2; 51 __le64 _rsvd3; 52} __packed; 53 54enum rsxx_dma_status { 55 DMA_SW_ERR = 0x1, 56 DMA_HW_FAULT = 0x2, 57 DMA_CANCELLED = 0x4, 58}; 59 60struct hw_cmd { 61 u8 command; 62 u8 tag; 63 u8 _rsvd; 64 u8 sub_page; /* Bit[0:2]: 512byte offset */ 65 /* Bit[4:6]: 512byte count */ 66 __le32 device_addr; 67 __le64 host_addr; 68} __packed; 69 70enum rsxx_hw_cmd { 71 HW_CMD_BLK_DISCARD = 0x70, 72 HW_CMD_BLK_WRITE = 0x80, 73 HW_CMD_BLK_READ = 0xC0, 74 HW_CMD_BLK_RECON_READ = 0xE0, 75}; 76 77enum rsxx_hw_status { 78 HW_STATUS_CRC = 0x01, 79 HW_STATUS_HARD_ERR = 0x02, 80 HW_STATUS_SOFT_ERR = 0x04, 81 HW_STATUS_FAULT = 0x08, 82}; 83 84#define STATUS_BUFFER_SIZE8 4096 85#define COMMAND_BUFFER_SIZE8 4096 86 87static struct kmem_cache *rsxx_dma_pool; 88 89struct dma_tracker { 90 int next_tag; 91 struct rsxx_dma *dma; 92}; 93 94#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \ 95 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS)) 96 97struct dma_tracker_list { 98 spinlock_t lock; 99 int head; 100 struct dma_tracker list[0]; 101}; 102 103 104/*----------------- Misc Utility Functions -------------------*/ 105static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card) 106{ 107 unsigned long long tgt_addr8; 108 109 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) & 110 card->_stripe.upper_mask) | 111 ((addr8) & card->_stripe.lower_mask); 112 do_div(tgt_addr8, RSXX_HW_BLK_SIZE); 113 return tgt_addr8; 114} 115 116static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8) 117{ 118 unsigned int tgt; 119 120 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask; 121 122 return tgt; 123} 124 125static void rsxx_dma_queue_reset(struct rsxx_cardinfo *card) 126{ 127 /* Reset all DMA Command/Status Queues */ 128 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET); 129} 130 131static unsigned int get_dma_size(struct rsxx_dma *dma) 132{ 133 if (dma->sub_page.cnt) 134 return dma->sub_page.cnt << 9; 135 else 136 return RSXX_HW_BLK_SIZE; 137} 138 139 140/*----------------- DMA Tracker -------------------*/ 141static void set_tracker_dma(struct dma_tracker_list *trackers, 142 int tag, 143 struct rsxx_dma *dma) 144{ 145 trackers->list[tag].dma = dma; 146} 147 148static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers, 149 int tag) 150{ 151 return trackers->list[tag].dma; 152} 153 154static int pop_tracker(struct dma_tracker_list *trackers) 155{ 156 int tag; 157 158 spin_lock(&trackers->lock); 159 tag = trackers->head; 160 if (tag != -1) { 161 trackers->head = trackers->list[tag].next_tag; 162 trackers->list[tag].next_tag = -1; 163 } 164 spin_unlock(&trackers->lock); 165 166 return tag; 167} 168 169static void push_tracker(struct dma_tracker_list *trackers, int tag) 170{ 171 spin_lock(&trackers->lock); 172 trackers->list[tag].next_tag = trackers->head; 173 trackers->head = tag; 174 trackers->list[tag].dma = NULL; 175 spin_unlock(&trackers->lock); 176} 177 178 179/*----------------- Interrupt Coalescing -------------*/ 180/* 181 * Interrupt Coalescing Register Format: 182 * Interrupt Timer (64ns units) [15:0] 183 * Interrupt Count [24:16] 184 * Reserved [31:25] 185*/ 186#define INTR_COAL_LATENCY_MASK (0x0000ffff) 187 188#define INTR_COAL_COUNT_SHIFT 16 189#define INTR_COAL_COUNT_BITS 9 190#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \ 191 INTR_COAL_COUNT_SHIFT) 192#define INTR_COAL_LATENCY_UNITS_NS 64 193 194 195static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency) 196{ 197 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS; 198 199 if (mode == RSXX_INTR_COAL_DISABLED) 200 return 0; 201 202 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) | 203 (latency_units & INTR_COAL_LATENCY_MASK); 204 205} 206 207static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card) 208{ 209 int i; 210 u32 q_depth = 0; 211 u32 intr_coal; 212 213 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE) 214 return; 215 216 for (i = 0; i < card->n_targets; i++) 217 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth); 218 219 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 220 q_depth / 2, 221 card->config.data.intr_coal.latency); 222 iowrite32(intr_coal, card->regmap + INTR_COAL); 223} 224 225/*----------------- RSXX DMA Handling -------------------*/ 226static void rsxx_complete_dma(struct rsxx_cardinfo *card, 227 struct rsxx_dma *dma, 228 unsigned int status) 229{ 230 if (status & DMA_SW_ERR) 231 printk_ratelimited(KERN_ERR 232 "SW Error in DMA(cmd x%02x, laddr x%08x)\n", 233 dma->cmd, dma->laddr); 234 if (status & DMA_HW_FAULT) 235 printk_ratelimited(KERN_ERR 236 "HW Fault in DMA(cmd x%02x, laddr x%08x)\n", 237 dma->cmd, dma->laddr); 238 if (status & DMA_CANCELLED) 239 printk_ratelimited(KERN_ERR 240 "DMA Cancelled(cmd x%02x, laddr x%08x)\n", 241 dma->cmd, dma->laddr); 242 243 if (dma->dma_addr) 244 pci_unmap_page(card->dev, dma->dma_addr, get_dma_size(dma), 245 dma->cmd == HW_CMD_BLK_WRITE ? 246 PCI_DMA_TODEVICE : 247 PCI_DMA_FROMDEVICE); 248 249 if (dma->cb) 250 dma->cb(card, dma->cb_data, status ? 1 : 0); 251 252 kmem_cache_free(rsxx_dma_pool, dma); 253} 254 255static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl, 256 struct rsxx_dma *dma) 257{ 258 /* 259 * Requeued DMAs go to the front of the queue so they are issued 260 * first. 261 */ 262 spin_lock(&ctrl->queue_lock); 263 list_add(&dma->list, &ctrl->queue); 264 spin_unlock(&ctrl->queue_lock); 265} 266 267static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl, 268 struct rsxx_dma *dma, 269 u8 hw_st) 270{ 271 unsigned int status = 0; 272 int requeue_cmd = 0; 273 274 dev_dbg(CARD_TO_DEV(ctrl->card), 275 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n", 276 dma->cmd, dma->laddr, hw_st); 277 278 if (hw_st & HW_STATUS_CRC) 279 ctrl->stats.crc_errors++; 280 if (hw_st & HW_STATUS_HARD_ERR) 281 ctrl->stats.hard_errors++; 282 if (hw_st & HW_STATUS_SOFT_ERR) 283 ctrl->stats.soft_errors++; 284 285 switch (dma->cmd) { 286 case HW_CMD_BLK_READ: 287 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 288 if (ctrl->card->scrub_hard) { 289 dma->cmd = HW_CMD_BLK_RECON_READ; 290 requeue_cmd = 1; 291 ctrl->stats.reads_retried++; 292 } else { 293 status |= DMA_HW_FAULT; 294 ctrl->stats.reads_failed++; 295 } 296 } else if (hw_st & HW_STATUS_FAULT) { 297 status |= DMA_HW_FAULT; 298 ctrl->stats.reads_failed++; 299 } 300 301 break; 302 case HW_CMD_BLK_RECON_READ: 303 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { 304 /* Data could not be reconstructed. */ 305 status |= DMA_HW_FAULT; 306 ctrl->stats.reads_failed++; 307 } 308 309 break; 310 case HW_CMD_BLK_WRITE: 311 status |= DMA_HW_FAULT; 312 ctrl->stats.writes_failed++; 313 314 break; 315 case HW_CMD_BLK_DISCARD: 316 status |= DMA_HW_FAULT; 317 ctrl->stats.discards_failed++; 318 319 break; 320 default: 321 dev_err(CARD_TO_DEV(ctrl->card), 322 "Unknown command in DMA!(cmd: x%02x " 323 "laddr x%08x st: x%02x\n", 324 dma->cmd, dma->laddr, hw_st); 325 status |= DMA_SW_ERR; 326 327 break; 328 } 329 330 if (requeue_cmd) 331 rsxx_requeue_dma(ctrl, dma); 332 else 333 rsxx_complete_dma(ctrl->card, dma, status); 334} 335 336static void dma_engine_stalled(unsigned long data) 337{ 338 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data; 339 340 if (atomic_read(&ctrl->stats.hw_q_depth) == 0) 341 return; 342 343 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) { 344 /* 345 * The dma engine was stalled because the SW_CMD_IDX write 346 * was lost. Issue it again to recover. 347 */ 348 dev_warn(CARD_TO_DEV(ctrl->card), 349 "SW_CMD_IDX write was lost, re-writing...\n"); 350 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 351 mod_timer(&ctrl->activity_timer, 352 jiffies + DMA_ACTIVITY_TIMEOUT); 353 } else { 354 dev_warn(CARD_TO_DEV(ctrl->card), 355 "DMA channel %d has stalled, faulting interface.\n", 356 ctrl->id); 357 ctrl->card->dma_fault = 1; 358 } 359} 360 361static void rsxx_issue_dmas(struct work_struct *work) 362{ 363 struct rsxx_dma_ctrl *ctrl; 364 struct rsxx_dma *dma; 365 int tag; 366 int cmds_pending = 0; 367 struct hw_cmd *hw_cmd_buf; 368 369 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work); 370 hw_cmd_buf = ctrl->cmd.buf; 371 372 if (unlikely(ctrl->card->halt)) 373 return; 374 375 while (1) { 376 spin_lock(&ctrl->queue_lock); 377 if (list_empty(&ctrl->queue)) { 378 spin_unlock(&ctrl->queue_lock); 379 break; 380 } 381 spin_unlock(&ctrl->queue_lock); 382 383 tag = pop_tracker(ctrl->trackers); 384 if (tag == -1) 385 break; 386 387 spin_lock(&ctrl->queue_lock); 388 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list); 389 list_del(&dma->list); 390 ctrl->stats.sw_q_depth--; 391 spin_unlock(&ctrl->queue_lock); 392 393 /* 394 * This will catch any DMAs that slipped in right before the 395 * fault, but was queued after all the other DMAs were 396 * cancelled. 397 */ 398 if (unlikely(ctrl->card->dma_fault)) { 399 push_tracker(ctrl->trackers, tag); 400 rsxx_complete_dma(ctrl->card, dma, DMA_CANCELLED); 401 continue; 402 } 403 404 set_tracker_dma(ctrl->trackers, tag, dma); 405 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd; 406 hw_cmd_buf[ctrl->cmd.idx].tag = tag; 407 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0; 408 hw_cmd_buf[ctrl->cmd.idx].sub_page = 409 ((dma->sub_page.cnt & 0x7) << 4) | 410 (dma->sub_page.off & 0x7); 411 412 hw_cmd_buf[ctrl->cmd.idx].device_addr = 413 cpu_to_le32(dma->laddr); 414 415 hw_cmd_buf[ctrl->cmd.idx].host_addr = 416 cpu_to_le64(dma->dma_addr); 417 418 dev_dbg(CARD_TO_DEV(ctrl->card), 419 "Issue DMA%d(laddr %d tag %d) to idx %d\n", 420 ctrl->id, dma->laddr, tag, ctrl->cmd.idx); 421 422 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK; 423 cmds_pending++; 424 425 if (dma->cmd == HW_CMD_BLK_WRITE) 426 ctrl->stats.writes_issued++; 427 else if (dma->cmd == HW_CMD_BLK_DISCARD) 428 ctrl->stats.discards_issued++; 429 else 430 ctrl->stats.reads_issued++; 431 } 432 433 /* Let HW know we've queued commands. */ 434 if (cmds_pending) { 435 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth); 436 mod_timer(&ctrl->activity_timer, 437 jiffies + DMA_ACTIVITY_TIMEOUT); 438 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 439 } 440} 441 442static void rsxx_dma_done(struct work_struct *work) 443{ 444 struct rsxx_dma_ctrl *ctrl; 445 struct rsxx_dma *dma; 446 unsigned long flags; 447 u16 count; 448 u8 status; 449 u8 tag; 450 struct hw_status *hw_st_buf; 451 452 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work); 453 hw_st_buf = ctrl->status.buf; 454 455 if (unlikely(ctrl->card->halt) || 456 unlikely(ctrl->card->dma_fault)) 457 return; 458 459 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 460 461 while (count == ctrl->e_cnt) { 462 /* 463 * The read memory-barrier is necessary to keep aggressive 464 * processors/optimizers (such as the PPC Apple G5) from 465 * reordering the following status-buffer tag & status read 466 * *before* the count read on subsequent iterations of the 467 * loop! 468 */ 469 rmb(); 470 471 status = hw_st_buf[ctrl->status.idx].status; 472 tag = hw_st_buf[ctrl->status.idx].tag; 473 474 dma = get_tracker_dma(ctrl->trackers, tag); 475 if (dma == NULL) { 476 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 477 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL); 478 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 479 480 dev_err(CARD_TO_DEV(ctrl->card), 481 "No tracker for tag %d " 482 "(idx %d id %d)\n", 483 tag, ctrl->status.idx, ctrl->id); 484 return; 485 } 486 487 dev_dbg(CARD_TO_DEV(ctrl->card), 488 "Completing DMA%d" 489 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n", 490 ctrl->id, dma->laddr, tag, status, count, 491 ctrl->status.idx); 492 493 atomic_dec(&ctrl->stats.hw_q_depth); 494 495 mod_timer(&ctrl->activity_timer, 496 jiffies + DMA_ACTIVITY_TIMEOUT); 497 498 if (status) 499 rsxx_handle_dma_error(ctrl, dma, status); 500 else 501 rsxx_complete_dma(ctrl->card, dma, 0); 502 503 push_tracker(ctrl->trackers, tag); 504 505 ctrl->status.idx = (ctrl->status.idx + 1) & 506 RSXX_CS_IDX_MASK; 507 ctrl->e_cnt++; 508 509 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); 510 } 511 512 dma_intr_coal_auto_tune(ctrl->card); 513 514 if (atomic_read(&ctrl->stats.hw_q_depth) == 0) 515 del_timer_sync(&ctrl->activity_timer); 516 517 spin_lock_irqsave(&ctrl->card->irq_lock, flags); 518 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id)); 519 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); 520 521 spin_lock(&ctrl->queue_lock); 522 if (ctrl->stats.sw_q_depth) 523 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work); 524 spin_unlock(&ctrl->queue_lock); 525} 526 527static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card, 528 struct list_head *q) 529{ 530 struct rsxx_dma *dma; 531 struct rsxx_dma *tmp; 532 int cnt = 0; 533 534 list_for_each_entry_safe(dma, tmp, q, list) { 535 list_del(&dma->list); 536 537 if (dma->dma_addr) 538 pci_unmap_page(card->dev, dma->dma_addr, 539 get_dma_size(dma), 540 (dma->cmd == HW_CMD_BLK_WRITE) ? 541 PCI_DMA_TODEVICE : 542 PCI_DMA_FROMDEVICE); 543 kmem_cache_free(rsxx_dma_pool, dma); 544 cnt++; 545 } 546 547 return cnt; 548} 549 550static int rsxx_queue_discard(struct rsxx_cardinfo *card, 551 struct list_head *q, 552 unsigned int laddr, 553 rsxx_dma_cb cb, 554 void *cb_data) 555{ 556 struct rsxx_dma *dma; 557 558 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 559 if (!dma) 560 return -ENOMEM; 561 562 dma->cmd = HW_CMD_BLK_DISCARD; 563 dma->laddr = laddr; 564 dma->dma_addr = 0; 565 dma->sub_page.off = 0; 566 dma->sub_page.cnt = 0; 567 dma->page = NULL; 568 dma->pg_off = 0; 569 dma->cb = cb; 570 dma->cb_data = cb_data; 571 572 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr); 573 574 list_add_tail(&dma->list, q); 575 576 return 0; 577} 578 579static int rsxx_queue_dma(struct rsxx_cardinfo *card, 580 struct list_head *q, 581 int dir, 582 unsigned int dma_off, 583 unsigned int dma_len, 584 unsigned int laddr, 585 struct page *page, 586 unsigned int pg_off, 587 rsxx_dma_cb cb, 588 void *cb_data) 589{ 590 struct rsxx_dma *dma; 591 592 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); 593 if (!dma) 594 return -ENOMEM; 595 596 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len, 597 dir ? PCI_DMA_TODEVICE : 598 PCI_DMA_FROMDEVICE); 599 if (!dma->dma_addr) { 600 kmem_cache_free(rsxx_dma_pool, dma); 601 return -ENOMEM; 602 } 603 604 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ; 605 dma->laddr = laddr; 606 dma->sub_page.off = (dma_off >> 9); 607 dma->sub_page.cnt = (dma_len >> 9); 608 dma->page = page; 609 dma->pg_off = pg_off; 610 dma->cb = cb; 611 dma->cb_data = cb_data; 612 613 dev_dbg(CARD_TO_DEV(card), 614 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n", 615 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off, 616 dma->sub_page.cnt, dma->page, dma->pg_off); 617 618 /* Queue the DMA */ 619 list_add_tail(&dma->list, q); 620 621 return 0; 622} 623 624int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, 625 struct bio *bio, 626 atomic_t *n_dmas, 627 rsxx_dma_cb cb, 628 void *cb_data) 629{ 630 struct list_head dma_list[RSXX_MAX_TARGETS]; 631 struct bio_vec *bvec; 632 unsigned long long addr8; 633 unsigned int laddr; 634 unsigned int bv_len; 635 unsigned int bv_off; 636 unsigned int dma_off; 637 unsigned int dma_len; 638 int dma_cnt[RSXX_MAX_TARGETS]; 639 int tgt; 640 int st; 641 int i; 642 643 addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */ 644 atomic_set(n_dmas, 0); 645 646 for (i = 0; i < card->n_targets; i++) { 647 INIT_LIST_HEAD(&dma_list[i]); 648 dma_cnt[i] = 0; 649 } 650 651 if (bio->bi_rw & REQ_DISCARD) { 652 bv_len = bio->bi_size; 653 654 while (bv_len > 0) { 655 tgt = rsxx_get_dma_tgt(card, addr8); 656 laddr = rsxx_addr8_to_laddr(addr8, card); 657 658 st = rsxx_queue_discard(card, &dma_list[tgt], laddr, 659 cb, cb_data); 660 if (st) 661 goto bvec_err; 662 663 dma_cnt[tgt]++; 664 atomic_inc(n_dmas); 665 addr8 += RSXX_HW_BLK_SIZE; 666 bv_len -= RSXX_HW_BLK_SIZE; 667 } 668 } else { 669 bio_for_each_segment(bvec, bio, i) { 670 bv_len = bvec->bv_len; 671 bv_off = bvec->bv_offset; 672 673 while (bv_len > 0) { 674 tgt = rsxx_get_dma_tgt(card, addr8); 675 laddr = rsxx_addr8_to_laddr(addr8, card); 676 dma_off = addr8 & RSXX_HW_BLK_MASK; 677 dma_len = min(bv_len, 678 RSXX_HW_BLK_SIZE - dma_off); 679 680 st = rsxx_queue_dma(card, &dma_list[tgt], 681 bio_data_dir(bio), 682 dma_off, dma_len, 683 laddr, bvec->bv_page, 684 bv_off, cb, cb_data); 685 if (st) 686 goto bvec_err; 687 688 dma_cnt[tgt]++; 689 atomic_inc(n_dmas); 690 addr8 += dma_len; 691 bv_off += dma_len; 692 bv_len -= dma_len; 693 } 694 } 695 } 696 697 for (i = 0; i < card->n_targets; i++) { 698 if (!list_empty(&dma_list[i])) { 699 spin_lock(&card->ctrl[i].queue_lock); 700 card->ctrl[i].stats.sw_q_depth += dma_cnt[i]; 701 list_splice_tail(&dma_list[i], &card->ctrl[i].queue); 702 spin_unlock(&card->ctrl[i].queue_lock); 703 704 queue_work(card->ctrl[i].issue_wq, 705 &card->ctrl[i].issue_dma_work); 706 } 707 } 708 709 return 0; 710 711bvec_err: 712 for (i = 0; i < card->n_targets; i++) 713 rsxx_cleanup_dma_queue(card, &dma_list[i]); 714 715 return st; 716} 717 718 719/*----------------- DMA Engine Initialization & Setup -------------------*/ 720static int rsxx_dma_ctrl_init(struct pci_dev *dev, 721 struct rsxx_dma_ctrl *ctrl) 722{ 723 int i; 724 725 memset(&ctrl->stats, 0, sizeof(ctrl->stats)); 726 727 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8, 728 &ctrl->status.dma_addr); 729 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8, 730 &ctrl->cmd.dma_addr); 731 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL) 732 return -ENOMEM; 733 734 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8); 735 if (!ctrl->trackers) 736 return -ENOMEM; 737 738 ctrl->trackers->head = 0; 739 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { 740 ctrl->trackers->list[i].next_tag = i + 1; 741 ctrl->trackers->list[i].dma = NULL; 742 } 743 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1; 744 spin_lock_init(&ctrl->trackers->lock); 745 746 spin_lock_init(&ctrl->queue_lock); 747 INIT_LIST_HEAD(&ctrl->queue); 748 749 setup_timer(&ctrl->activity_timer, dma_engine_stalled, 750 (unsigned long)ctrl); 751 752 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0); 753 if (!ctrl->issue_wq) 754 return -ENOMEM; 755 756 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0); 757 if (!ctrl->done_wq) 758 return -ENOMEM; 759 760 INIT_WORK(&ctrl->issue_dma_work, rsxx_issue_dmas); 761 INIT_WORK(&ctrl->dma_done_work, rsxx_dma_done); 762 763 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8); 764 iowrite32(lower_32_bits(ctrl->status.dma_addr), 765 ctrl->regmap + SB_ADD_LO); 766 iowrite32(upper_32_bits(ctrl->status.dma_addr), 767 ctrl->regmap + SB_ADD_HI); 768 769 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8); 770 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO); 771 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI); 772 773 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT); 774 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) { 775 dev_crit(&dev->dev, "Failed reading status cnt x%x\n", 776 ctrl->status.idx); 777 return -EINVAL; 778 } 779 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT); 780 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT); 781 782 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX); 783 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) { 784 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n", 785 ctrl->status.idx); 786 return -EINVAL; 787 } 788 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX); 789 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); 790 791 return 0; 792} 793 794static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card, 795 unsigned int stripe_size8) 796{ 797 if (!is_power_of_2(stripe_size8)) { 798 dev_err(CARD_TO_DEV(card), 799 "stripe_size is NOT a power of 2!\n"); 800 return -EINVAL; 801 } 802 803 card->_stripe.lower_mask = stripe_size8 - 1; 804 805 card->_stripe.upper_mask = ~(card->_stripe.lower_mask); 806 card->_stripe.upper_shift = ffs(card->n_targets) - 1; 807 808 card->_stripe.target_mask = card->n_targets - 1; 809 card->_stripe.target_shift = ffs(stripe_size8) - 1; 810 811 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n", 812 card->_stripe.lower_mask); 813 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n", 814 card->_stripe.upper_shift); 815 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n", 816 card->_stripe.upper_mask); 817 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n", 818 card->_stripe.target_mask); 819 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n", 820 card->_stripe.target_shift); 821 822 return 0; 823} 824 825static int rsxx_dma_configure(struct rsxx_cardinfo *card) 826{ 827 u32 intr_coal; 828 829 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, 830 card->config.data.intr_coal.count, 831 card->config.data.intr_coal.latency); 832 iowrite32(intr_coal, card->regmap + INTR_COAL); 833 834 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size); 835} 836 837int rsxx_dma_setup(struct rsxx_cardinfo *card) 838{ 839 unsigned long flags; 840 int st; 841 int i; 842 843 dev_info(CARD_TO_DEV(card), 844 "Initializing %d DMA targets\n", 845 card->n_targets); 846 847 /* Regmap is divided up into 4K chunks. One for each DMA channel */ 848 for (i = 0; i < card->n_targets; i++) 849 card->ctrl[i].regmap = card->regmap + (i * 4096); 850 851 card->dma_fault = 0; 852 853 /* Reset the DMA queues */ 854 rsxx_dma_queue_reset(card); 855 856 /************* Setup DMA Control *************/ 857 for (i = 0; i < card->n_targets; i++) { 858 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]); 859 if (st) 860 goto failed_dma_setup; 861 862 card->ctrl[i].card = card; 863 card->ctrl[i].id = i; 864 } 865 866 card->scrub_hard = 1; 867 868 if (card->config_valid) 869 rsxx_dma_configure(card); 870 871 /* Enable the interrupts after all setup has completed. */ 872 for (i = 0; i < card->n_targets; i++) { 873 spin_lock_irqsave(&card->irq_lock, flags); 874 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i)); 875 spin_unlock_irqrestore(&card->irq_lock, flags); 876 } 877 878 return 0; 879 880failed_dma_setup: 881 for (i = 0; i < card->n_targets; i++) { 882 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i]; 883 884 if (ctrl->issue_wq) { 885 destroy_workqueue(ctrl->issue_wq); 886 ctrl->issue_wq = NULL; 887 } 888 889 if (ctrl->done_wq) { 890 destroy_workqueue(ctrl->done_wq); 891 ctrl->done_wq = NULL; 892 } 893 894 if (ctrl->trackers) 895 vfree(ctrl->trackers); 896 897 if (ctrl->status.buf) 898 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 899 ctrl->status.buf, 900 ctrl->status.dma_addr); 901 if (ctrl->cmd.buf) 902 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 903 ctrl->cmd.buf, ctrl->cmd.dma_addr); 904 } 905 906 return st; 907} 908 909 910void rsxx_dma_destroy(struct rsxx_cardinfo *card) 911{ 912 struct rsxx_dma_ctrl *ctrl; 913 struct rsxx_dma *dma; 914 int i, j; 915 int cnt = 0; 916 917 for (i = 0; i < card->n_targets; i++) { 918 ctrl = &card->ctrl[i]; 919 920 if (ctrl->issue_wq) { 921 destroy_workqueue(ctrl->issue_wq); 922 ctrl->issue_wq = NULL; 923 } 924 925 if (ctrl->done_wq) { 926 destroy_workqueue(ctrl->done_wq); 927 ctrl->done_wq = NULL; 928 } 929 930 if (timer_pending(&ctrl->activity_timer)) 931 del_timer_sync(&ctrl->activity_timer); 932 933 /* Clean up the DMA queue */ 934 spin_lock(&ctrl->queue_lock); 935 cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue); 936 spin_unlock(&ctrl->queue_lock); 937 938 if (cnt) 939 dev_info(CARD_TO_DEV(card), 940 "Freed %d queued DMAs on channel %d\n", 941 cnt, i); 942 943 /* Clean up issued DMAs */ 944 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) { 945 dma = get_tracker_dma(ctrl->trackers, j); 946 if (dma) { 947 pci_unmap_page(card->dev, dma->dma_addr, 948 get_dma_size(dma), 949 (dma->cmd == HW_CMD_BLK_WRITE) ? 950 PCI_DMA_TODEVICE : 951 PCI_DMA_FROMDEVICE); 952 kmem_cache_free(rsxx_dma_pool, dma); 953 cnt++; 954 } 955 } 956 957 if (cnt) 958 dev_info(CARD_TO_DEV(card), 959 "Freed %d pending DMAs on channel %d\n", 960 cnt, i); 961 962 vfree(ctrl->trackers); 963 964 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, 965 ctrl->status.buf, ctrl->status.dma_addr); 966 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, 967 ctrl->cmd.buf, ctrl->cmd.dma_addr); 968 } 969} 970 971 972int rsxx_dma_init(void) 973{ 974 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN); 975 if (!rsxx_dma_pool) 976 return -ENOMEM; 977 978 return 0; 979} 980 981 982void rsxx_dma_cleanup(void) 983{ 984 kmem_cache_destroy(rsxx_dma_pool); 985} 986 987