1/* 2 * offload engine driver for the Intel Xscale series of i/o processors 3 * Copyright © 2006, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 */ 19 20/* 21 * This driver supports the asynchrounous DMA copy and RAID engines available 22 * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 23 */ 24 25#include <linux/init.h> 26#include <linux/module.h> 27#include <linux/delay.h> 28#include <linux/dma-mapping.h> 29#include <linux/spinlock.h> 30#include <linux/interrupt.h> 31#include <linux/platform_device.h> 32#include <linux/memory.h> 33#include <linux/ioport.h> 34#include <linux/raid/pq.h> 35#include <linux/slab.h> 36 37#include <mach/adma.h> 38 39#include "dmaengine.h" 40 41#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) 42#define to_iop_adma_device(dev) \ 43 container_of(dev, struct iop_adma_device, common) 44#define tx_to_iop_adma_slot(tx) \ 45 container_of(tx, struct iop_adma_desc_slot, async_tx) 46 47/** 48 * iop_adma_free_slots - flags descriptor slots for reuse 49 * @slot: Slot to free 50 * Caller must hold &iop_chan->lock while calling this function 51 */ 52static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) 53{ 54 int stride = slot->slots_per_op; 55 56 while (stride--) { 57 slot->slots_per_op = 0; 58 slot = list_entry(slot->slot_node.next, 59 struct iop_adma_desc_slot, 60 slot_node); 61 } 62} 63 64static dma_cookie_t 65iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 66 struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 67{ 68 struct dma_async_tx_descriptor *tx = &desc->async_tx; 69 70 BUG_ON(tx->cookie < 0); 71 if (tx->cookie > 0) { 72 cookie = tx->cookie; 73 tx->cookie = 0; 74 75 /* call the callback (must not sleep or submit new 76 * operations to this channel) 77 */ 78 if (tx->callback) 79 tx->callback(tx->callback_param); 80 81 dma_descriptor_unmap(tx); 82 if (desc->group_head) 83 desc->group_head = NULL; 84 } 85 86 /* run dependent operations */ 87 dma_run_dependencies(tx); 88 89 return cookie; 90} 91 92static int 93iop_adma_clean_slot(struct iop_adma_desc_slot *desc, 94 struct iop_adma_chan *iop_chan) 95{ 96 /* the client is allowed to attach dependent operations 97 * until 'ack' is set 98 */ 99 if (!async_tx_test_ack(&desc->async_tx)) 100 return 0; 101 102 /* leave the last descriptor in the chain 103 * so we can append to it 104 */ 105 if (desc->chain_node.next == &iop_chan->chain) 106 return 1; 107 108 dev_dbg(iop_chan->device->common.dev, 109 "\tfree slot: %d slots_per_op: %d\n", 110 desc->idx, desc->slots_per_op); 111 112 list_del(&desc->chain_node); 113 iop_adma_free_slots(desc); 114 115 return 0; 116} 117 118static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 119{ 120 struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; 121 dma_cookie_t cookie = 0; 122 u32 current_desc = iop_chan_get_current_descriptor(iop_chan); 123 int busy = iop_chan_is_busy(iop_chan); 124 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 125 126 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 127 /* free completed slots from the chain starting with 128 * the oldest descriptor 129 */ 130 list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 131 chain_node) { 132 pr_debug("\tcookie: %d slot: %d busy: %d " 133 "this_desc: %#x next_desc: %#x ack: %d\n", 134 iter->async_tx.cookie, iter->idx, busy, 135 iter->async_tx.phys, iop_desc_get_next_desc(iter), 136 async_tx_test_ack(&iter->async_tx)); 137 prefetch(_iter); 138 prefetch(&_iter->async_tx); 139 140 /* do not advance past the current descriptor loaded into the 141 * hardware channel, subsequent descriptors are either in 142 * process or have not been submitted 143 */ 144 if (seen_current) 145 break; 146 147 /* stop the search if we reach the current descriptor and the 148 * channel is busy, or if it appears that the current descriptor 149 * needs to be re-read (i.e. has been appended to) 150 */ 151 if (iter->async_tx.phys == current_desc) { 152 BUG_ON(seen_current++); 153 if (busy || iop_desc_get_next_desc(iter)) 154 break; 155 } 156 157 /* detect the start of a group transaction */ 158 if (!slot_cnt && !slots_per_op) { 159 slot_cnt = iter->slot_cnt; 160 slots_per_op = iter->slots_per_op; 161 if (slot_cnt <= slots_per_op) { 162 slot_cnt = 0; 163 slots_per_op = 0; 164 } 165 } 166 167 if (slot_cnt) { 168 pr_debug("\tgroup++\n"); 169 if (!grp_start) 170 grp_start = iter; 171 slot_cnt -= slots_per_op; 172 } 173 174 /* all the members of a group are complete */ 175 if (slots_per_op != 0 && slot_cnt == 0) { 176 struct iop_adma_desc_slot *grp_iter, *_grp_iter; 177 int end_of_chain = 0; 178 pr_debug("\tgroup end\n"); 179 180 /* collect the total results */ 181 if (grp_start->xor_check_result) { 182 u32 zero_sum_result = 0; 183 slot_cnt = grp_start->slot_cnt; 184 grp_iter = grp_start; 185 186 list_for_each_entry_from(grp_iter, 187 &iop_chan->chain, chain_node) { 188 zero_sum_result |= 189 iop_desc_get_zero_result(grp_iter); 190 pr_debug("\titer%d result: %d\n", 191 grp_iter->idx, zero_sum_result); 192 slot_cnt -= slots_per_op; 193 if (slot_cnt == 0) 194 break; 195 } 196 pr_debug("\tgrp_start->xor_check_result: %p\n", 197 grp_start->xor_check_result); 198 *grp_start->xor_check_result = zero_sum_result; 199 } 200 201 /* clean up the group */ 202 slot_cnt = grp_start->slot_cnt; 203 grp_iter = grp_start; 204 list_for_each_entry_safe_from(grp_iter, _grp_iter, 205 &iop_chan->chain, chain_node) { 206 cookie = iop_adma_run_tx_complete_actions( 207 grp_iter, iop_chan, cookie); 208 209 slot_cnt -= slots_per_op; 210 end_of_chain = iop_adma_clean_slot(grp_iter, 211 iop_chan); 212 213 if (slot_cnt == 0 || end_of_chain) 214 break; 215 } 216 217 /* the group should be complete at this point */ 218 BUG_ON(slot_cnt); 219 220 slots_per_op = 0; 221 grp_start = NULL; 222 if (end_of_chain) 223 break; 224 else 225 continue; 226 } else if (slots_per_op) /* wait for group completion */ 227 continue; 228 229 /* write back zero sum results (single descriptor case) */ 230 if (iter->xor_check_result && iter->async_tx.cookie) 231 *iter->xor_check_result = 232 iop_desc_get_zero_result(iter); 233 234 cookie = iop_adma_run_tx_complete_actions( 235 iter, iop_chan, cookie); 236 237 if (iop_adma_clean_slot(iter, iop_chan)) 238 break; 239 } 240 241 if (cookie > 0) { 242 iop_chan->common.completed_cookie = cookie; 243 pr_debug("\tcompleted cookie %d\n", cookie); 244 } 245} 246 247static void 248iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 249{ 250 spin_lock_bh(&iop_chan->lock); 251 __iop_adma_slot_cleanup(iop_chan); 252 spin_unlock_bh(&iop_chan->lock); 253} 254 255static void iop_adma_tasklet(unsigned long data) 256{ 257 struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data; 258 259 /* lockdep will flag depedency submissions as potentially 260 * recursive locking, this is not the case as a dependency 261 * submission will never recurse a channels submit routine. 262 * There are checks in async_tx.c to prevent this. 263 */ 264 spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); 265 __iop_adma_slot_cleanup(iop_chan); 266 spin_unlock(&iop_chan->lock); 267} 268 269static struct iop_adma_desc_slot * 270iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, 271 int slots_per_op) 272{ 273 struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; 274 LIST_HEAD(chain); 275 int slots_found, retry = 0; 276 277 /* start search from the last allocated descrtiptor 278 * if a contiguous allocation can not be found start searching 279 * from the beginning of the list 280 */ 281retry: 282 slots_found = 0; 283 if (retry == 0) 284 iter = iop_chan->last_used; 285 else 286 iter = list_entry(&iop_chan->all_slots, 287 struct iop_adma_desc_slot, 288 slot_node); 289 290 list_for_each_entry_safe_continue( 291 iter, _iter, &iop_chan->all_slots, slot_node) { 292 prefetch(_iter); 293 prefetch(&_iter->async_tx); 294 if (iter->slots_per_op) { 295 /* give up after finding the first busy slot 296 * on the second pass through the list 297 */ 298 if (retry) 299 break; 300 301 slots_found = 0; 302 continue; 303 } 304 305 /* start the allocation if the slot is correctly aligned */ 306 if (!slots_found++) { 307 if (iop_desc_is_aligned(iter, slots_per_op)) 308 alloc_start = iter; 309 else { 310 slots_found = 0; 311 continue; 312 } 313 } 314 315 if (slots_found == num_slots) { 316 struct iop_adma_desc_slot *alloc_tail = NULL; 317 struct iop_adma_desc_slot *last_used = NULL; 318 iter = alloc_start; 319 while (num_slots) { 320 int i; 321 dev_dbg(iop_chan->device->common.dev, 322 "allocated slot: %d " 323 "(desc %p phys: %#x) slots_per_op %d\n", 324 iter->idx, iter->hw_desc, 325 iter->async_tx.phys, slots_per_op); 326 327 /* pre-ack all but the last descriptor */ 328 if (num_slots != slots_per_op) 329 async_tx_ack(&iter->async_tx); 330 331 list_add_tail(&iter->chain_node, &chain); 332 alloc_tail = iter; 333 iter->async_tx.cookie = 0; 334 iter->slot_cnt = num_slots; 335 iter->xor_check_result = NULL; 336 for (i = 0; i < slots_per_op; i++) { 337 iter->slots_per_op = slots_per_op - i; 338 last_used = iter; 339 iter = list_entry(iter->slot_node.next, 340 struct iop_adma_desc_slot, 341 slot_node); 342 } 343 num_slots -= slots_per_op; 344 } 345 alloc_tail->group_head = alloc_start; 346 alloc_tail->async_tx.cookie = -EBUSY; 347 list_splice(&chain, &alloc_tail->tx_list); 348 iop_chan->last_used = last_used; 349 iop_desc_clear_next_desc(alloc_start); 350 iop_desc_clear_next_desc(alloc_tail); 351 return alloc_tail; 352 } 353 } 354 if (!retry++) 355 goto retry; 356 357 /* perform direct reclaim if the allocation fails */ 358 __iop_adma_slot_cleanup(iop_chan); 359 360 return NULL; 361} 362 363static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) 364{ 365 dev_dbg(iop_chan->device->common.dev, "pending: %d\n", 366 iop_chan->pending); 367 368 if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { 369 iop_chan->pending = 0; 370 iop_chan_append(iop_chan); 371 } 372} 373 374static dma_cookie_t 375iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) 376{ 377 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); 378 struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); 379 struct iop_adma_desc_slot *grp_start, *old_chain_tail; 380 int slot_cnt; 381 int slots_per_op; 382 dma_cookie_t cookie; 383 dma_addr_t next_dma; 384 385 grp_start = sw_desc->group_head; 386 slot_cnt = grp_start->slot_cnt; 387 slots_per_op = grp_start->slots_per_op; 388 389 spin_lock_bh(&iop_chan->lock); 390 cookie = dma_cookie_assign(tx); 391 392 old_chain_tail = list_entry(iop_chan->chain.prev, 393 struct iop_adma_desc_slot, chain_node); 394 list_splice_init(&sw_desc->tx_list, 395 &old_chain_tail->chain_node); 396 397 /* fix up the hardware chain */ 398 next_dma = grp_start->async_tx.phys; 399 iop_desc_set_next_desc(old_chain_tail, next_dma); 400 BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ 401 402 /* check for pre-chained descriptors */ 403 iop_paranoia(iop_desc_get_next_desc(sw_desc)); 404 405 /* increment the pending count by the number of slots 406 * memcpy operations have a 1:1 (slot:operation) relation 407 * other operations are heavier and will pop the threshold 408 * more often. 409 */ 410 iop_chan->pending += slot_cnt; 411 iop_adma_check_threshold(iop_chan); 412 spin_unlock_bh(&iop_chan->lock); 413 414 dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", 415 __func__, sw_desc->async_tx.cookie, sw_desc->idx); 416 417 return cookie; 418} 419 420static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); 421static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); 422 423/** 424 * iop_adma_alloc_chan_resources - returns the number of allocated descriptors 425 * @chan - allocate descriptor resources for this channel 426 * @client - current client requesting the channel be ready for requests 427 * 428 * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To 429 * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be 430 * greater than 2x the number slots needed to satisfy a device->max_xor 431 * request. 432 * */ 433static int iop_adma_alloc_chan_resources(struct dma_chan *chan) 434{ 435 char *hw_desc; 436 int idx; 437 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 438 struct iop_adma_desc_slot *slot = NULL; 439 int init = iop_chan->slots_allocated ? 0 : 1; 440 struct iop_adma_platform_data *plat_data = 441 dev_get_platdata(&iop_chan->device->pdev->dev); 442 int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; 443 444 /* Allocate descriptor slots */ 445 do { 446 idx = iop_chan->slots_allocated; 447 if (idx == num_descs_in_pool) 448 break; 449 450 slot = kzalloc(sizeof(*slot), GFP_KERNEL); 451 if (!slot) { 452 printk(KERN_INFO "IOP ADMA Channel only initialized" 453 " %d descriptor slots", idx); 454 break; 455 } 456 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; 457 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; 458 459 dma_async_tx_descriptor_init(&slot->async_tx, chan); 460 slot->async_tx.tx_submit = iop_adma_tx_submit; 461 INIT_LIST_HEAD(&slot->tx_list); 462 INIT_LIST_HEAD(&slot->chain_node); 463 INIT_LIST_HEAD(&slot->slot_node); 464 hw_desc = (char *) iop_chan->device->dma_desc_pool; 465 slot->async_tx.phys = 466 (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; 467 slot->idx = idx; 468 469 spin_lock_bh(&iop_chan->lock); 470 iop_chan->slots_allocated++; 471 list_add_tail(&slot->slot_node, &iop_chan->all_slots); 472 spin_unlock_bh(&iop_chan->lock); 473 } while (iop_chan->slots_allocated < num_descs_in_pool); 474 475 if (idx && !iop_chan->last_used) 476 iop_chan->last_used = list_entry(iop_chan->all_slots.next, 477 struct iop_adma_desc_slot, 478 slot_node); 479 480 dev_dbg(iop_chan->device->common.dev, 481 "allocated %d descriptor slots last_used: %p\n", 482 iop_chan->slots_allocated, iop_chan->last_used); 483 484 /* initialize the channel and the chain with a null operation */ 485 if (init) { 486 if (dma_has_cap(DMA_MEMCPY, 487 iop_chan->device->common.cap_mask)) 488 iop_chan_start_null_memcpy(iop_chan); 489 else if (dma_has_cap(DMA_XOR, 490 iop_chan->device->common.cap_mask)) 491 iop_chan_start_null_xor(iop_chan); 492 else 493 BUG(); 494 } 495 496 return (idx > 0) ? idx : -ENOMEM; 497} 498 499static struct dma_async_tx_descriptor * 500iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 501{ 502 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 503 struct iop_adma_desc_slot *sw_desc, *grp_start; 504 int slot_cnt, slots_per_op; 505 506 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 507 508 spin_lock_bh(&iop_chan->lock); 509 slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); 510 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 511 if (sw_desc) { 512 grp_start = sw_desc->group_head; 513 iop_desc_init_interrupt(grp_start, iop_chan); 514 sw_desc->async_tx.flags = flags; 515 } 516 spin_unlock_bh(&iop_chan->lock); 517 518 return sw_desc ? &sw_desc->async_tx : NULL; 519} 520 521static struct dma_async_tx_descriptor * 522iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 523 dma_addr_t dma_src, size_t len, unsigned long flags) 524{ 525 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 526 struct iop_adma_desc_slot *sw_desc, *grp_start; 527 int slot_cnt, slots_per_op; 528 529 if (unlikely(!len)) 530 return NULL; 531 BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); 532 533 dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", 534 __func__, len); 535 536 spin_lock_bh(&iop_chan->lock); 537 slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); 538 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 539 if (sw_desc) { 540 grp_start = sw_desc->group_head; 541 iop_desc_init_memcpy(grp_start, flags); 542 iop_desc_set_byte_count(grp_start, iop_chan, len); 543 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 544 iop_desc_set_memcpy_src_addr(grp_start, dma_src); 545 sw_desc->async_tx.flags = flags; 546 } 547 spin_unlock_bh(&iop_chan->lock); 548 549 return sw_desc ? &sw_desc->async_tx : NULL; 550} 551 552static struct dma_async_tx_descriptor * 553iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, 554 dma_addr_t *dma_src, unsigned int src_cnt, size_t len, 555 unsigned long flags) 556{ 557 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 558 struct iop_adma_desc_slot *sw_desc, *grp_start; 559 int slot_cnt, slots_per_op; 560 561 if (unlikely(!len)) 562 return NULL; 563 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 564 565 dev_dbg(iop_chan->device->common.dev, 566 "%s src_cnt: %d len: %u flags: %lx\n", 567 __func__, src_cnt, len, flags); 568 569 spin_lock_bh(&iop_chan->lock); 570 slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); 571 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 572 if (sw_desc) { 573 grp_start = sw_desc->group_head; 574 iop_desc_init_xor(grp_start, src_cnt, flags); 575 iop_desc_set_byte_count(grp_start, iop_chan, len); 576 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 577 sw_desc->async_tx.flags = flags; 578 while (src_cnt--) 579 iop_desc_set_xor_src_addr(grp_start, src_cnt, 580 dma_src[src_cnt]); 581 } 582 spin_unlock_bh(&iop_chan->lock); 583 584 return sw_desc ? &sw_desc->async_tx : NULL; 585} 586 587static struct dma_async_tx_descriptor * 588iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, 589 unsigned int src_cnt, size_t len, u32 *result, 590 unsigned long flags) 591{ 592 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 593 struct iop_adma_desc_slot *sw_desc, *grp_start; 594 int slot_cnt, slots_per_op; 595 596 if (unlikely(!len)) 597 return NULL; 598 599 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", 600 __func__, src_cnt, len); 601 602 spin_lock_bh(&iop_chan->lock); 603 slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); 604 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 605 if (sw_desc) { 606 grp_start = sw_desc->group_head; 607 iop_desc_init_zero_sum(grp_start, src_cnt, flags); 608 iop_desc_set_zero_sum_byte_count(grp_start, len); 609 grp_start->xor_check_result = result; 610 pr_debug("\t%s: grp_start->xor_check_result: %p\n", 611 __func__, grp_start->xor_check_result); 612 sw_desc->async_tx.flags = flags; 613 while (src_cnt--) 614 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, 615 dma_src[src_cnt]); 616 } 617 spin_unlock_bh(&iop_chan->lock); 618 619 return sw_desc ? &sw_desc->async_tx : NULL; 620} 621 622static struct dma_async_tx_descriptor * 623iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 624 unsigned int src_cnt, const unsigned char *scf, size_t len, 625 unsigned long flags) 626{ 627 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 628 struct iop_adma_desc_slot *sw_desc, *g; 629 int slot_cnt, slots_per_op; 630 int continue_srcs; 631 632 if (unlikely(!len)) 633 return NULL; 634 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 635 636 dev_dbg(iop_chan->device->common.dev, 637 "%s src_cnt: %d len: %u flags: %lx\n", 638 __func__, src_cnt, len, flags); 639 640 if (dmaf_p_disabled_continue(flags)) 641 continue_srcs = 1+src_cnt; 642 else if (dmaf_continue(flags)) 643 continue_srcs = 3+src_cnt; 644 else 645 continue_srcs = 0+src_cnt; 646 647 spin_lock_bh(&iop_chan->lock); 648 slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); 649 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 650 if (sw_desc) { 651 int i; 652 653 g = sw_desc->group_head; 654 iop_desc_set_byte_count(g, iop_chan, len); 655 656 /* even if P is disabled its destination address (bits 657 * [3:0]) must match Q. It is ok if P points to an 658 * invalid address, it won't be written. 659 */ 660 if (flags & DMA_PREP_PQ_DISABLE_P) 661 dst[0] = dst[1] & 0x7; 662 663 iop_desc_set_pq_addr(g, dst); 664 sw_desc->async_tx.flags = flags; 665 for (i = 0; i < src_cnt; i++) 666 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); 667 668 /* if we are continuing a previous operation factor in 669 * the old p and q values, see the comment for dma_maxpq 670 * in include/linux/dmaengine.h 671 */ 672 if (dmaf_p_disabled_continue(flags)) 673 iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 674 else if (dmaf_continue(flags)) { 675 iop_desc_set_pq_src_addr(g, i++, dst[0], 0); 676 iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 677 iop_desc_set_pq_src_addr(g, i++, dst[1], 0); 678 } 679 iop_desc_init_pq(g, i, flags); 680 } 681 spin_unlock_bh(&iop_chan->lock); 682 683 return sw_desc ? &sw_desc->async_tx : NULL; 684} 685 686static struct dma_async_tx_descriptor * 687iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 688 unsigned int src_cnt, const unsigned char *scf, 689 size_t len, enum sum_check_flags *pqres, 690 unsigned long flags) 691{ 692 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 693 struct iop_adma_desc_slot *sw_desc, *g; 694 int slot_cnt, slots_per_op; 695 696 if (unlikely(!len)) 697 return NULL; 698 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 699 700 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", 701 __func__, src_cnt, len); 702 703 spin_lock_bh(&iop_chan->lock); 704 slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); 705 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 706 if (sw_desc) { 707 /* for validate operations p and q are tagged onto the 708 * end of the source list 709 */ 710 int pq_idx = src_cnt; 711 712 g = sw_desc->group_head; 713 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); 714 iop_desc_set_pq_zero_sum_byte_count(g, len); 715 g->pq_check_result = pqres; 716 pr_debug("\t%s: g->pq_check_result: %p\n", 717 __func__, g->pq_check_result); 718 sw_desc->async_tx.flags = flags; 719 while (src_cnt--) 720 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, 721 src[src_cnt], 722 scf[src_cnt]); 723 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); 724 } 725 spin_unlock_bh(&iop_chan->lock); 726 727 return sw_desc ? &sw_desc->async_tx : NULL; 728} 729 730static void iop_adma_free_chan_resources(struct dma_chan *chan) 731{ 732 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 733 struct iop_adma_desc_slot *iter, *_iter; 734 int in_use_descs = 0; 735 736 iop_adma_slot_cleanup(iop_chan); 737 738 spin_lock_bh(&iop_chan->lock); 739 list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 740 chain_node) { 741 in_use_descs++; 742 list_del(&iter->chain_node); 743 } 744 list_for_each_entry_safe_reverse( 745 iter, _iter, &iop_chan->all_slots, slot_node) { 746 list_del(&iter->slot_node); 747 kfree(iter); 748 iop_chan->slots_allocated--; 749 } 750 iop_chan->last_used = NULL; 751 752 dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", 753 __func__, iop_chan->slots_allocated); 754 spin_unlock_bh(&iop_chan->lock); 755 756 /* one is ok since we left it on there on purpose */ 757 if (in_use_descs > 1) 758 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", 759 in_use_descs - 1); 760} 761 762/** 763 * iop_adma_status - poll the status of an ADMA transaction 764 * @chan: ADMA channel handle 765 * @cookie: ADMA transaction identifier 766 * @txstate: a holder for the current state of the channel or NULL 767 */ 768static enum dma_status iop_adma_status(struct dma_chan *chan, 769 dma_cookie_t cookie, 770 struct dma_tx_state *txstate) 771{ 772 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 773 int ret; 774 775 ret = dma_cookie_status(chan, cookie, txstate); 776 if (ret == DMA_COMPLETE) 777 return ret; 778 779 iop_adma_slot_cleanup(iop_chan); 780 781 return dma_cookie_status(chan, cookie, txstate); 782} 783 784static irqreturn_t iop_adma_eot_handler(int irq, void *data) 785{ 786 struct iop_adma_chan *chan = data; 787 788 dev_dbg(chan->device->common.dev, "%s\n", __func__); 789 790 tasklet_schedule(&chan->irq_tasklet); 791 792 iop_adma_device_clear_eot_status(chan); 793 794 return IRQ_HANDLED; 795} 796 797static irqreturn_t iop_adma_eoc_handler(int irq, void *data) 798{ 799 struct iop_adma_chan *chan = data; 800 801 dev_dbg(chan->device->common.dev, "%s\n", __func__); 802 803 tasklet_schedule(&chan->irq_tasklet); 804 805 iop_adma_device_clear_eoc_status(chan); 806 807 return IRQ_HANDLED; 808} 809 810static irqreturn_t iop_adma_err_handler(int irq, void *data) 811{ 812 struct iop_adma_chan *chan = data; 813 unsigned long status = iop_chan_get_status(chan); 814 815 dev_err(chan->device->common.dev, 816 "error ( %s%s%s%s%s%s%s)\n", 817 iop_is_err_int_parity(status, chan) ? "int_parity " : "", 818 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", 819 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", 820 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", 821 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", 822 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", 823 iop_is_err_split_tx(status, chan) ? "split_tx " : ""); 824 825 iop_adma_device_clear_err_status(chan); 826 827 BUG(); 828 829 return IRQ_HANDLED; 830} 831 832static void iop_adma_issue_pending(struct dma_chan *chan) 833{ 834 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 835 836 if (iop_chan->pending) { 837 iop_chan->pending = 0; 838 iop_chan_append(iop_chan); 839 } 840} 841 842/* 843 * Perform a transaction to verify the HW works. 844 */ 845#define IOP_ADMA_TEST_SIZE 2000 846 847static int iop_adma_memcpy_self_test(struct iop_adma_device *device) 848{ 849 int i; 850 void *src, *dest; 851 dma_addr_t src_dma, dest_dma; 852 struct dma_chan *dma_chan; 853 dma_cookie_t cookie; 854 struct dma_async_tx_descriptor *tx; 855 int err = 0; 856 struct iop_adma_chan *iop_chan; 857 858 dev_dbg(device->common.dev, "%s\n", __func__); 859 860 src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 861 if (!src) 862 return -ENOMEM; 863 dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 864 if (!dest) { 865 kfree(src); 866 return -ENOMEM; 867 } 868 869 /* Fill in src buffer */ 870 for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) 871 ((u8 *) src)[i] = (u8)i; 872 873 /* Start copy, using first DMA channel */ 874 dma_chan = container_of(device->common.channels.next, 875 struct dma_chan, 876 device_node); 877 if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 878 err = -ENODEV; 879 goto out; 880 } 881 882 dest_dma = dma_map_single(dma_chan->device->dev, dest, 883 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 884 src_dma = dma_map_single(dma_chan->device->dev, src, 885 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); 886 tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 887 IOP_ADMA_TEST_SIZE, 888 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 889 890 cookie = iop_adma_tx_submit(tx); 891 iop_adma_issue_pending(dma_chan); 892 msleep(1); 893 894 if (iop_adma_status(dma_chan, cookie, NULL) != 895 DMA_COMPLETE) { 896 dev_err(dma_chan->device->dev, 897 "Self-test copy timed out, disabling\n"); 898 err = -ENODEV; 899 goto free_resources; 900 } 901 902 iop_chan = to_iop_adma_chan(dma_chan); 903 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 904 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 905 if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { 906 dev_err(dma_chan->device->dev, 907 "Self-test copy failed compare, disabling\n"); 908 err = -ENODEV; 909 goto free_resources; 910 } 911 912free_resources: 913 iop_adma_free_chan_resources(dma_chan); 914out: 915 kfree(src); 916 kfree(dest); 917 return err; 918} 919 920#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ 921static int 922iop_adma_xor_val_self_test(struct iop_adma_device *device) 923{ 924 int i, src_idx; 925 struct page *dest; 926 struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; 927 struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 928 dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 929 dma_addr_t dest_dma; 930 struct dma_async_tx_descriptor *tx; 931 struct dma_chan *dma_chan; 932 dma_cookie_t cookie; 933 u8 cmp_byte = 0; 934 u32 cmp_word; 935 u32 zero_sum_result; 936 int err = 0; 937 struct iop_adma_chan *iop_chan; 938 939 dev_dbg(device->common.dev, "%s\n", __func__); 940 941 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 942 xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 943 if (!xor_srcs[src_idx]) { 944 while (src_idx--) 945 __free_page(xor_srcs[src_idx]); 946 return -ENOMEM; 947 } 948 } 949 950 dest = alloc_page(GFP_KERNEL); 951 if (!dest) { 952 while (src_idx--) 953 __free_page(xor_srcs[src_idx]); 954 return -ENOMEM; 955 } 956 957 /* Fill in src buffers */ 958 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 959 u8 *ptr = page_address(xor_srcs[src_idx]); 960 for (i = 0; i < PAGE_SIZE; i++) 961 ptr[i] = (1 << src_idx); 962 } 963 964 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) 965 cmp_byte ^= (u8) (1 << src_idx); 966 967 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 968 (cmp_byte << 8) | cmp_byte; 969 970 memset(page_address(dest), 0, PAGE_SIZE); 971 972 dma_chan = container_of(device->common.channels.next, 973 struct dma_chan, 974 device_node); 975 if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 976 err = -ENODEV; 977 goto out; 978 } 979 980 /* test xor */ 981 dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, 982 PAGE_SIZE, DMA_FROM_DEVICE); 983 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 984 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 985 0, PAGE_SIZE, DMA_TO_DEVICE); 986 tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 987 IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, 988 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 989 990 cookie = iop_adma_tx_submit(tx); 991 iop_adma_issue_pending(dma_chan); 992 msleep(8); 993 994 if (iop_adma_status(dma_chan, cookie, NULL) != 995 DMA_COMPLETE) { 996 dev_err(dma_chan->device->dev, 997 "Self-test xor timed out, disabling\n"); 998 err = -ENODEV; 999 goto free_resources; 1000 } 1001 1002 iop_chan = to_iop_adma_chan(dma_chan); 1003 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 1004 PAGE_SIZE, DMA_FROM_DEVICE); 1005 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 1006 u32 *ptr = page_address(dest); 1007 if (ptr[i] != cmp_word) { 1008 dev_err(dma_chan->device->dev, 1009 "Self-test xor failed compare, disabling\n"); 1010 err = -ENODEV; 1011 goto free_resources; 1012 } 1013 } 1014 dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, 1015 PAGE_SIZE, DMA_TO_DEVICE); 1016 1017 /* skip zero sum if the capability is not present */ 1018 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) 1019 goto free_resources; 1020 1021 /* zero sum the sources with the destintation page */ 1022 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1023 zero_sum_srcs[i] = xor_srcs[i]; 1024 zero_sum_srcs[i] = dest; 1025 1026 zero_sum_result = 1; 1027 1028 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 1029 dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1030 zero_sum_srcs[i], 0, PAGE_SIZE, 1031 DMA_TO_DEVICE); 1032 tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1033 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1034 &zero_sum_result, 1035 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1036 1037 cookie = iop_adma_tx_submit(tx); 1038 iop_adma_issue_pending(dma_chan); 1039 msleep(8); 1040 1041 if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1042 dev_err(dma_chan->device->dev, 1043 "Self-test zero sum timed out, disabling\n"); 1044 err = -ENODEV; 1045 goto free_resources; 1046 } 1047 1048 if (zero_sum_result != 0) { 1049 dev_err(dma_chan->device->dev, 1050 "Self-test zero sum failed compare, disabling\n"); 1051 err = -ENODEV; 1052 goto free_resources; 1053 } 1054 1055 /* test for non-zero parity sum */ 1056 zero_sum_result = 0; 1057 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 1058 dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1059 zero_sum_srcs[i], 0, PAGE_SIZE, 1060 DMA_TO_DEVICE); 1061 tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1062 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1063 &zero_sum_result, 1064 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1065 1066 cookie = iop_adma_tx_submit(tx); 1067 iop_adma_issue_pending(dma_chan); 1068 msleep(8); 1069 1070 if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1071 dev_err(dma_chan->device->dev, 1072 "Self-test non-zero sum timed out, disabling\n"); 1073 err = -ENODEV; 1074 goto free_resources; 1075 } 1076 1077 if (zero_sum_result != 1) { 1078 dev_err(dma_chan->device->dev, 1079 "Self-test non-zero sum failed compare, disabling\n"); 1080 err = -ENODEV; 1081 goto free_resources; 1082 } 1083 1084free_resources: 1085 iop_adma_free_chan_resources(dma_chan); 1086out: 1087 src_idx = IOP_ADMA_NUM_SRC_TEST; 1088 while (src_idx--) 1089 __free_page(xor_srcs[src_idx]); 1090 __free_page(dest); 1091 return err; 1092} 1093 1094#ifdef CONFIG_RAID6_PQ 1095static int 1096iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) 1097{ 1098 /* combined sources, software pq results, and extra hw pq results */ 1099 struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; 1100 /* ptr to the extra hw pq buffers defined above */ 1101 struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; 1102 /* address conversion buffers (dma_map / page_address) */ 1103 void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; 1104 dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; 1105 dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; 1106 1107 int i; 1108 struct dma_async_tx_descriptor *tx; 1109 struct dma_chan *dma_chan; 1110 dma_cookie_t cookie; 1111 u32 zero_sum_result; 1112 int err = 0; 1113 struct device *dev; 1114 1115 dev_dbg(device->common.dev, "%s\n", __func__); 1116 1117 for (i = 0; i < ARRAY_SIZE(pq); i++) { 1118 pq[i] = alloc_page(GFP_KERNEL); 1119 if (!pq[i]) { 1120 while (i--) 1121 __free_page(pq[i]); 1122 return -ENOMEM; 1123 } 1124 } 1125 1126 /* Fill in src buffers */ 1127 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { 1128 pq_sw[i] = page_address(pq[i]); 1129 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); 1130 } 1131 pq_sw[i] = page_address(pq[i]); 1132 pq_sw[i+1] = page_address(pq[i+1]); 1133 1134 dma_chan = container_of(device->common.channels.next, 1135 struct dma_chan, 1136 device_node); 1137 if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 1138 err = -ENODEV; 1139 goto out; 1140 } 1141 1142 dev = dma_chan->device->dev; 1143 1144 /* initialize the dests */ 1145 memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); 1146 memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); 1147 1148 /* test pq */ 1149 pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1150 pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1151 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1152 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1153 DMA_TO_DEVICE); 1154 1155 tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, 1156 IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, 1157 PAGE_SIZE, 1158 DMA_PREP_INTERRUPT | 1159 DMA_CTRL_ACK); 1160 1161 cookie = iop_adma_tx_submit(tx); 1162 iop_adma_issue_pending(dma_chan); 1163 msleep(8); 1164 1165 if (iop_adma_status(dma_chan, cookie, NULL) != 1166 DMA_COMPLETE) { 1167 dev_err(dev, "Self-test pq timed out, disabling\n"); 1168 err = -ENODEV; 1169 goto free_resources; 1170 } 1171 1172 raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); 1173 1174 if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], 1175 page_address(pq_hw[0]), PAGE_SIZE) != 0) { 1176 dev_err(dev, "Self-test p failed compare, disabling\n"); 1177 err = -ENODEV; 1178 goto free_resources; 1179 } 1180 if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], 1181 page_address(pq_hw[1]), PAGE_SIZE) != 0) { 1182 dev_err(dev, "Self-test q failed compare, disabling\n"); 1183 err = -ENODEV; 1184 goto free_resources; 1185 } 1186 1187 /* test correct zero sum using the software generated pq values */ 1188 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1189 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1190 DMA_TO_DEVICE); 1191 1192 zero_sum_result = ~0; 1193 tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1194 pq_src, IOP_ADMA_NUM_SRC_TEST, 1195 raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1196 DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1197 1198 cookie = iop_adma_tx_submit(tx); 1199 iop_adma_issue_pending(dma_chan); 1200 msleep(8); 1201 1202 if (iop_adma_status(dma_chan, cookie, NULL) != 1203 DMA_COMPLETE) { 1204 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); 1205 err = -ENODEV; 1206 goto free_resources; 1207 } 1208 1209 if (zero_sum_result != 0) { 1210 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", 1211 zero_sum_result); 1212 err = -ENODEV; 1213 goto free_resources; 1214 } 1215 1216 /* test incorrect zero sum */ 1217 i = IOP_ADMA_NUM_SRC_TEST; 1218 memset(pq_sw[i] + 100, 0, 100); 1219 memset(pq_sw[i+1] + 200, 0, 200); 1220 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1221 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1222 DMA_TO_DEVICE); 1223 1224 zero_sum_result = 0; 1225 tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1226 pq_src, IOP_ADMA_NUM_SRC_TEST, 1227 raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1228 DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1229 1230 cookie = iop_adma_tx_submit(tx); 1231 iop_adma_issue_pending(dma_chan); 1232 msleep(8); 1233 1234 if (iop_adma_status(dma_chan, cookie, NULL) != 1235 DMA_COMPLETE) { 1236 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); 1237 err = -ENODEV; 1238 goto free_resources; 1239 } 1240 1241 if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { 1242 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", 1243 zero_sum_result); 1244 err = -ENODEV; 1245 goto free_resources; 1246 } 1247 1248free_resources: 1249 iop_adma_free_chan_resources(dma_chan); 1250out: 1251 i = ARRAY_SIZE(pq); 1252 while (i--) 1253 __free_page(pq[i]); 1254 return err; 1255} 1256#endif 1257 1258static int iop_adma_remove(struct platform_device *dev) 1259{ 1260 struct iop_adma_device *device = platform_get_drvdata(dev); 1261 struct dma_chan *chan, *_chan; 1262 struct iop_adma_chan *iop_chan; 1263 struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev); 1264 1265 dma_async_device_unregister(&device->common); 1266 1267 dma_free_coherent(&dev->dev, plat_data->pool_size, 1268 device->dma_desc_pool_virt, device->dma_desc_pool); 1269 1270 list_for_each_entry_safe(chan, _chan, &device->common.channels, 1271 device_node) { 1272 iop_chan = to_iop_adma_chan(chan); 1273 list_del(&chan->device_node); 1274 kfree(iop_chan); 1275 } 1276 kfree(device); 1277 1278 return 0; 1279} 1280 1281static int iop_adma_probe(struct platform_device *pdev) 1282{ 1283 struct resource *res; 1284 int ret = 0, i; 1285 struct iop_adma_device *adev; 1286 struct iop_adma_chan *iop_chan; 1287 struct dma_device *dma_dev; 1288 struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev); 1289 1290 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1291 if (!res) 1292 return -ENODEV; 1293 1294 if (!devm_request_mem_region(&pdev->dev, res->start, 1295 resource_size(res), pdev->name)) 1296 return -EBUSY; 1297 1298 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 1299 if (!adev) 1300 return -ENOMEM; 1301 dma_dev = &adev->common; 1302 1303 /* allocate coherent memory for hardware descriptors 1304 * note: writecombine gives slightly better performance, but 1305 * requires that we explicitly flush the writes 1306 */ 1307 if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev, 1308 plat_data->pool_size, 1309 &adev->dma_desc_pool, 1310 GFP_KERNEL)) == NULL) { 1311 ret = -ENOMEM; 1312 goto err_free_adev; 1313 } 1314 1315 dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n", 1316 __func__, adev->dma_desc_pool_virt, 1317 (void *) adev->dma_desc_pool); 1318 1319 adev->id = plat_data->hw_id; 1320 1321 /* discover transaction capabilites from the platform data */ 1322 dma_dev->cap_mask = plat_data->cap_mask; 1323 1324 adev->pdev = pdev; 1325 platform_set_drvdata(pdev, adev); 1326 1327 INIT_LIST_HEAD(&dma_dev->channels); 1328 1329 /* set base routines */ 1330 dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; 1331 dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; 1332 dma_dev->device_tx_status = iop_adma_status; 1333 dma_dev->device_issue_pending = iop_adma_issue_pending; 1334 dma_dev->dev = &pdev->dev; 1335 1336 /* set prep routines based on capability */ 1337 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1338 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; 1339 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1340 dma_dev->max_xor = iop_adma_get_max_xor(); 1341 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; 1342 } 1343 if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) 1344 dma_dev->device_prep_dma_xor_val = 1345 iop_adma_prep_dma_xor_val; 1346 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1347 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); 1348 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; 1349 } 1350 if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) 1351 dma_dev->device_prep_dma_pq_val = 1352 iop_adma_prep_dma_pq_val; 1353 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1354 dma_dev->device_prep_dma_interrupt = 1355 iop_adma_prep_dma_interrupt; 1356 1357 iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); 1358 if (!iop_chan) { 1359 ret = -ENOMEM; 1360 goto err_free_dma; 1361 } 1362 iop_chan->device = adev; 1363 1364 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, 1365 resource_size(res)); 1366 if (!iop_chan->mmr_base) { 1367 ret = -ENOMEM; 1368 goto err_free_iop_chan; 1369 } 1370 tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long) 1371 iop_chan); 1372 1373 /* clear errors before enabling interrupts */ 1374 iop_adma_device_clear_err_status(iop_chan); 1375 1376 for (i = 0; i < 3; i++) { 1377 irq_handler_t handler[] = { iop_adma_eot_handler, 1378 iop_adma_eoc_handler, 1379 iop_adma_err_handler }; 1380 int irq = platform_get_irq(pdev, i); 1381 if (irq < 0) { 1382 ret = -ENXIO; 1383 goto err_free_iop_chan; 1384 } else { 1385 ret = devm_request_irq(&pdev->dev, irq, 1386 handler[i], 0, pdev->name, iop_chan); 1387 if (ret) 1388 goto err_free_iop_chan; 1389 } 1390 } 1391 1392 spin_lock_init(&iop_chan->lock); 1393 INIT_LIST_HEAD(&iop_chan->chain); 1394 INIT_LIST_HEAD(&iop_chan->all_slots); 1395 iop_chan->common.device = dma_dev; 1396 dma_cookie_init(&iop_chan->common); 1397 list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); 1398 1399 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1400 ret = iop_adma_memcpy_self_test(adev); 1401 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1402 if (ret) 1403 goto err_free_iop_chan; 1404 } 1405 1406 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1407 ret = iop_adma_xor_val_self_test(adev); 1408 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1409 if (ret) 1410 goto err_free_iop_chan; 1411 } 1412 1413 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && 1414 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { 1415 #ifdef CONFIG_RAID6_PQ 1416 ret = iop_adma_pq_zero_sum_self_test(adev); 1417 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); 1418 #else 1419 /* can not test raid6, so do not publish capability */ 1420 dma_cap_clear(DMA_PQ, dma_dev->cap_mask); 1421 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); 1422 ret = 0; 1423 #endif 1424 if (ret) 1425 goto err_free_iop_chan; 1426 } 1427 1428 dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n", 1429 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", 1430 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", 1431 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1432 dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", 1433 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1434 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1435 1436 dma_async_device_register(dma_dev); 1437 goto out; 1438 1439 err_free_iop_chan: 1440 kfree(iop_chan); 1441 err_free_dma: 1442 dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, 1443 adev->dma_desc_pool_virt, adev->dma_desc_pool); 1444 err_free_adev: 1445 kfree(adev); 1446 out: 1447 return ret; 1448} 1449 1450static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) 1451{ 1452 struct iop_adma_desc_slot *sw_desc, *grp_start; 1453 dma_cookie_t cookie; 1454 int slot_cnt, slots_per_op; 1455 1456 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1457 1458 spin_lock_bh(&iop_chan->lock); 1459 slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); 1460 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1461 if (sw_desc) { 1462 grp_start = sw_desc->group_head; 1463 1464 list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1465 async_tx_ack(&sw_desc->async_tx); 1466 iop_desc_init_memcpy(grp_start, 0); 1467 iop_desc_set_byte_count(grp_start, iop_chan, 0); 1468 iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1469 iop_desc_set_memcpy_src_addr(grp_start, 0); 1470 1471 cookie = dma_cookie_assign(&sw_desc->async_tx); 1472 1473 /* initialize the completed cookie to be less than 1474 * the most recently used cookie 1475 */ 1476 iop_chan->common.completed_cookie = cookie - 1; 1477 1478 /* channel should not be busy */ 1479 BUG_ON(iop_chan_is_busy(iop_chan)); 1480 1481 /* clear any prior error-status bits */ 1482 iop_adma_device_clear_err_status(iop_chan); 1483 1484 /* disable operation */ 1485 iop_chan_disable(iop_chan); 1486 1487 /* set the descriptor address */ 1488 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1489 1490 /* 1/ don't add pre-chained descriptors 1491 * 2/ dummy read to flush next_desc write 1492 */ 1493 BUG_ON(iop_desc_get_next_desc(sw_desc)); 1494 1495 /* run the descriptor */ 1496 iop_chan_enable(iop_chan); 1497 } else 1498 dev_err(iop_chan->device->common.dev, 1499 "failed to allocate null descriptor\n"); 1500 spin_unlock_bh(&iop_chan->lock); 1501} 1502 1503static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) 1504{ 1505 struct iop_adma_desc_slot *sw_desc, *grp_start; 1506 dma_cookie_t cookie; 1507 int slot_cnt, slots_per_op; 1508 1509 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1510 1511 spin_lock_bh(&iop_chan->lock); 1512 slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); 1513 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1514 if (sw_desc) { 1515 grp_start = sw_desc->group_head; 1516 list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1517 async_tx_ack(&sw_desc->async_tx); 1518 iop_desc_init_null_xor(grp_start, 2, 0); 1519 iop_desc_set_byte_count(grp_start, iop_chan, 0); 1520 iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1521 iop_desc_set_xor_src_addr(grp_start, 0, 0); 1522 iop_desc_set_xor_src_addr(grp_start, 1, 0); 1523 1524 cookie = dma_cookie_assign(&sw_desc->async_tx); 1525 1526 /* initialize the completed cookie to be less than 1527 * the most recently used cookie 1528 */ 1529 iop_chan->common.completed_cookie = cookie - 1; 1530 1531 /* channel should not be busy */ 1532 BUG_ON(iop_chan_is_busy(iop_chan)); 1533 1534 /* clear any prior error-status bits */ 1535 iop_adma_device_clear_err_status(iop_chan); 1536 1537 /* disable operation */ 1538 iop_chan_disable(iop_chan); 1539 1540 /* set the descriptor address */ 1541 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1542 1543 /* 1/ don't add pre-chained descriptors 1544 * 2/ dummy read to flush next_desc write 1545 */ 1546 BUG_ON(iop_desc_get_next_desc(sw_desc)); 1547 1548 /* run the descriptor */ 1549 iop_chan_enable(iop_chan); 1550 } else 1551 dev_err(iop_chan->device->common.dev, 1552 "failed to allocate null descriptor\n"); 1553 spin_unlock_bh(&iop_chan->lock); 1554} 1555 1556static struct platform_driver iop_adma_driver = { 1557 .probe = iop_adma_probe, 1558 .remove = iop_adma_remove, 1559 .driver = { 1560 .owner = THIS_MODULE, 1561 .name = "iop-adma", 1562 }, 1563}; 1564 1565module_platform_driver(iop_adma_driver); 1566 1567MODULE_AUTHOR("Intel Corporation"); 1568MODULE_DESCRIPTION("IOP ADMA Engine Driver"); 1569MODULE_LICENSE("GPL"); 1570MODULE_ALIAS("platform:iop-adma"); 1571