1/** 2 * Copyright (C) ST-Ericsson SA 2010 3 * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson. 4 * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson. 5 * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson. 6 * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson. 7 * Author: Jonas Linde <jonas.linde@stericsson.com> for ST-Ericsson. 8 * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson. 9 * License terms: GNU General Public License (GPL) version 2 10 */ 11 12#include <linux/clk.h> 13#include <linux/completion.h> 14#include <linux/crypto.h> 15#include <linux/dmaengine.h> 16#include <linux/err.h> 17#include <linux/errno.h> 18#include <linux/interrupt.h> 19#include <linux/io.h> 20#include <linux/irqreturn.h> 21#include <linux/klist.h> 22#include <linux/module.h> 23#include <linux/platform_device.h> 24#include <linux/regulator/consumer.h> 25#include <linux/semaphore.h> 26#include <linux/platform_data/dma-ste-dma40.h> 27 28#include <crypto/aes.h> 29#include <crypto/algapi.h> 30#include <crypto/ctr.h> 31#include <crypto/des.h> 32#include <crypto/scatterwalk.h> 33 34#include <linux/platform_data/crypto-ux500.h> 35 36#include "cryp_p.h" 37#include "cryp.h" 38 39#define CRYP_MAX_KEY_SIZE 32 40#define BYTES_PER_WORD 4 41 42static int cryp_mode; 43static atomic_t session_id; 44 45static struct stedma40_chan_cfg *mem_to_engine; 46static struct stedma40_chan_cfg *engine_to_mem; 47 48/** 49 * struct cryp_driver_data - data specific to the driver. 50 * 51 * @device_list: A list of registered devices to choose from. 52 * @device_allocation: A semaphore initialized with number of devices. 53 */ 54struct cryp_driver_data { 55 struct klist device_list; 56 struct semaphore device_allocation; 57}; 58 59/** 60 * struct cryp_ctx - Crypto context 61 * @config: Crypto mode. 62 * @key[CRYP_MAX_KEY_SIZE]: Key. 63 * @keylen: Length of key. 64 * @iv: Pointer to initialization vector. 65 * @indata: Pointer to indata. 66 * @outdata: Pointer to outdata. 67 * @datalen: Length of indata. 68 * @outlen: Length of outdata. 69 * @blocksize: Size of blocks. 70 * @updated: Updated flag. 71 * @dev_ctx: Device dependent context. 72 * @device: Pointer to the device. 73 */ 74struct cryp_ctx { 75 struct cryp_config config; 76 u8 key[CRYP_MAX_KEY_SIZE]; 77 u32 keylen; 78 u8 *iv; 79 const u8 *indata; 80 u8 *outdata; 81 u32 datalen; 82 u32 outlen; 83 u32 blocksize; 84 u8 updated; 85 struct cryp_device_context dev_ctx; 86 struct cryp_device_data *device; 87 u32 session_id; 88}; 89 90static struct cryp_driver_data driver_data; 91 92/** 93 * uint8p_to_uint32_be - 4*uint8 to uint32 big endian 94 * @in: Data to convert. 95 */ 96static inline u32 uint8p_to_uint32_be(u8 *in) 97{ 98 u32 *data = (u32 *)in; 99 100 return cpu_to_be32p(data); 101} 102 103/** 104 * swap_bits_in_byte - mirror the bits in a byte 105 * @b: the byte to be mirrored 106 * 107 * The bits are swapped the following way: 108 * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and 109 * nibble 2 (n2) bits 4-7. 110 * 111 * Nibble 1 (n1): 112 * (The "old" (moved) bit is replaced with a zero) 113 * 1. Move bit 6 and 7, 4 positions to the left. 114 * 2. Move bit 3 and 5, 2 positions to the left. 115 * 3. Move bit 1-4, 1 position to the left. 116 * 117 * Nibble 2 (n2): 118 * 1. Move bit 0 and 1, 4 positions to the right. 119 * 2. Move bit 2 and 4, 2 positions to the right. 120 * 3. Move bit 3-6, 1 position to the right. 121 * 122 * Combine the two nibbles to a complete and swapped byte. 123 */ 124 125static inline u8 swap_bits_in_byte(u8 b) 126{ 127#define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */ 128#define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5, 129 right shift 2 */ 130#define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4, 131 right shift 1 */ 132#define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */ 133#define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4, 134 left shift 2 */ 135#define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6, 136 left shift 1 */ 137 138 u8 n1; 139 u8 n2; 140 141 /* Swap most significant nibble */ 142 /* Right shift 4, bits 6 and 7 */ 143 n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4)); 144 /* Right shift 2, bits 3 and 5 */ 145 n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2)); 146 /* Right shift 1, bits 1-4 */ 147 n1 = (n1 & R_SHIFT_1_MASK) >> 1; 148 149 /* Swap least significant nibble */ 150 /* Left shift 4, bits 0 and 1 */ 151 n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4)); 152 /* Left shift 2, bits 2 and 4 */ 153 n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2)); 154 /* Left shift 1, bits 3-6 */ 155 n2 = (n2 & L_SHIFT_1_MASK) << 1; 156 157 return n1 | n2; 158} 159 160static inline void swap_words_in_key_and_bits_in_byte(const u8 *in, 161 u8 *out, u32 len) 162{ 163 unsigned int i = 0; 164 int j; 165 int index = 0; 166 167 j = len - BYTES_PER_WORD; 168 while (j >= 0) { 169 for (i = 0; i < BYTES_PER_WORD; i++) { 170 index = len - j - BYTES_PER_WORD + i; 171 out[j + i] = 172 swap_bits_in_byte(in[index]); 173 } 174 j -= BYTES_PER_WORD; 175 } 176} 177 178static void add_session_id(struct cryp_ctx *ctx) 179{ 180 /* 181 * We never want 0 to be a valid value, since this is the default value 182 * for the software context. 183 */ 184 if (unlikely(atomic_inc_and_test(&session_id))) 185 atomic_inc(&session_id); 186 187 ctx->session_id = atomic_read(&session_id); 188} 189 190static irqreturn_t cryp_interrupt_handler(int irq, void *param) 191{ 192 struct cryp_ctx *ctx; 193 int count; 194 struct cryp_device_data *device_data; 195 196 if (param == NULL) { 197 BUG_ON(!param); 198 return IRQ_HANDLED; 199 } 200 201 /* The device is coming from the one found in hw_crypt_noxts. */ 202 device_data = (struct cryp_device_data *)param; 203 204 ctx = device_data->current_ctx; 205 206 if (ctx == NULL) { 207 BUG_ON(!ctx); 208 return IRQ_HANDLED; 209 } 210 211 dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen, 212 cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ? 213 "out" : "in"); 214 215 if (cryp_pending_irq_src(device_data, 216 CRYP_IRQ_SRC_OUTPUT_FIFO)) { 217 if (ctx->outlen / ctx->blocksize > 0) { 218 count = ctx->blocksize / 4; 219 220 readsl(&device_data->base->dout, ctx->outdata, count); 221 ctx->outdata += count; 222 ctx->outlen -= count; 223 224 if (ctx->outlen == 0) { 225 cryp_disable_irq_src(device_data, 226 CRYP_IRQ_SRC_OUTPUT_FIFO); 227 } 228 } 229 } else if (cryp_pending_irq_src(device_data, 230 CRYP_IRQ_SRC_INPUT_FIFO)) { 231 if (ctx->datalen / ctx->blocksize > 0) { 232 count = ctx->blocksize / 4; 233 234 writesl(&device_data->base->din, ctx->indata, count); 235 236 ctx->indata += count; 237 ctx->datalen -= count; 238 239 if (ctx->datalen == 0) 240 cryp_disable_irq_src(device_data, 241 CRYP_IRQ_SRC_INPUT_FIFO); 242 243 if (ctx->config.algomode == CRYP_ALGO_AES_XTS) { 244 CRYP_PUT_BITS(&device_data->base->cr, 245 CRYP_START_ENABLE, 246 CRYP_CR_START_POS, 247 CRYP_CR_START_MASK); 248 249 cryp_wait_until_done(device_data); 250 } 251 } 252 } 253 254 return IRQ_HANDLED; 255} 256 257static int mode_is_aes(enum cryp_algo_mode mode) 258{ 259 return CRYP_ALGO_AES_ECB == mode || 260 CRYP_ALGO_AES_CBC == mode || 261 CRYP_ALGO_AES_CTR == mode || 262 CRYP_ALGO_AES_XTS == mode; 263} 264 265static int cfg_iv(struct cryp_device_data *device_data, u32 left, u32 right, 266 enum cryp_init_vector_index index) 267{ 268 struct cryp_init_vector_value vector_value; 269 270 dev_dbg(device_data->dev, "[%s]", __func__); 271 272 vector_value.init_value_left = left; 273 vector_value.init_value_right = right; 274 275 return cryp_configure_init_vector(device_data, 276 index, 277 vector_value); 278} 279 280static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx) 281{ 282 int i; 283 int status = 0; 284 int num_of_regs = ctx->blocksize / 8; 285 u32 iv[AES_BLOCK_SIZE / 4]; 286 287 dev_dbg(device_data->dev, "[%s]", __func__); 288 289 /* 290 * Since we loop on num_of_regs we need to have a check in case 291 * someone provides an incorrect blocksize which would force calling 292 * cfg_iv with i greater than 2 which is an error. 293 */ 294 if (num_of_regs > 2) { 295 dev_err(device_data->dev, "[%s] Incorrect blocksize %d", 296 __func__, ctx->blocksize); 297 return -EINVAL; 298 } 299 300 for (i = 0; i < ctx->blocksize / 4; i++) 301 iv[i] = uint8p_to_uint32_be(ctx->iv + i*4); 302 303 for (i = 0; i < num_of_regs; i++) { 304 status = cfg_iv(device_data, iv[i*2], iv[i*2+1], 305 (enum cryp_init_vector_index) i); 306 if (status != 0) 307 return status; 308 } 309 return status; 310} 311 312static int set_key(struct cryp_device_data *device_data, 313 u32 left_key, 314 u32 right_key, 315 enum cryp_key_reg_index index) 316{ 317 struct cryp_key_value key_value; 318 int cryp_error; 319 320 dev_dbg(device_data->dev, "[%s]", __func__); 321 322 key_value.key_value_left = left_key; 323 key_value.key_value_right = right_key; 324 325 cryp_error = cryp_configure_key_values(device_data, 326 index, 327 key_value); 328 if (cryp_error != 0) 329 dev_err(device_data->dev, "[%s]: " 330 "cryp_configure_key_values() failed!", __func__); 331 332 return cryp_error; 333} 334 335static int cfg_keys(struct cryp_ctx *ctx) 336{ 337 int i; 338 int num_of_regs = ctx->keylen / 8; 339 u32 swapped_key[CRYP_MAX_KEY_SIZE / 4]; 340 int cryp_error = 0; 341 342 dev_dbg(ctx->device->dev, "[%s]", __func__); 343 344 if (mode_is_aes(ctx->config.algomode)) { 345 swap_words_in_key_and_bits_in_byte((u8 *)ctx->key, 346 (u8 *)swapped_key, 347 ctx->keylen); 348 } else { 349 for (i = 0; i < ctx->keylen / 4; i++) 350 swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4); 351 } 352 353 for (i = 0; i < num_of_regs; i++) { 354 cryp_error = set_key(ctx->device, 355 *(((u32 *)swapped_key)+i*2), 356 *(((u32 *)swapped_key)+i*2+1), 357 (enum cryp_key_reg_index) i); 358 359 if (cryp_error != 0) { 360 dev_err(ctx->device->dev, "[%s]: set_key() failed!", 361 __func__); 362 return cryp_error; 363 } 364 } 365 return cryp_error; 366} 367 368static int cryp_setup_context(struct cryp_ctx *ctx, 369 struct cryp_device_data *device_data) 370{ 371 u32 control_register = CRYP_CR_DEFAULT; 372 373 switch (cryp_mode) { 374 case CRYP_MODE_INTERRUPT: 375 writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc); 376 break; 377 378 case CRYP_MODE_DMA: 379 writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr); 380 break; 381 382 default: 383 break; 384 } 385 386 if (ctx->updated == 0) { 387 cryp_flush_inoutfifo(device_data); 388 if (cfg_keys(ctx) != 0) { 389 dev_err(ctx->device->dev, "[%s]: cfg_keys failed!", 390 __func__); 391 return -EINVAL; 392 } 393 394 if (ctx->iv && 395 CRYP_ALGO_AES_ECB != ctx->config.algomode && 396 CRYP_ALGO_DES_ECB != ctx->config.algomode && 397 CRYP_ALGO_TDES_ECB != ctx->config.algomode) { 398 if (cfg_ivs(device_data, ctx) != 0) 399 return -EPERM; 400 } 401 402 cryp_set_configuration(device_data, &ctx->config, 403 &control_register); 404 add_session_id(ctx); 405 } else if (ctx->updated == 1 && 406 ctx->session_id != atomic_read(&session_id)) { 407 cryp_flush_inoutfifo(device_data); 408 cryp_restore_device_context(device_data, &ctx->dev_ctx); 409 410 add_session_id(ctx); 411 control_register = ctx->dev_ctx.cr; 412 } else 413 control_register = ctx->dev_ctx.cr; 414 415 writel(control_register | 416 (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS), 417 &device_data->base->cr); 418 419 return 0; 420} 421 422static int cryp_get_device_data(struct cryp_ctx *ctx, 423 struct cryp_device_data **device_data) 424{ 425 int ret; 426 struct klist_iter device_iterator; 427 struct klist_node *device_node; 428 struct cryp_device_data *local_device_data = NULL; 429 pr_debug(DEV_DBG_NAME " [%s]", __func__); 430 431 /* Wait until a device is available */ 432 ret = down_interruptible(&driver_data.device_allocation); 433 if (ret) 434 return ret; /* Interrupted */ 435 436 /* Select a device */ 437 klist_iter_init(&driver_data.device_list, &device_iterator); 438 439 device_node = klist_next(&device_iterator); 440 while (device_node) { 441 local_device_data = container_of(device_node, 442 struct cryp_device_data, list_node); 443 spin_lock(&local_device_data->ctx_lock); 444 /* current_ctx allocates a device, NULL = unallocated */ 445 if (local_device_data->current_ctx) { 446 device_node = klist_next(&device_iterator); 447 } else { 448 local_device_data->current_ctx = ctx; 449 ctx->device = local_device_data; 450 spin_unlock(&local_device_data->ctx_lock); 451 break; 452 } 453 spin_unlock(&local_device_data->ctx_lock); 454 } 455 klist_iter_exit(&device_iterator); 456 457 if (!device_node) { 458 /** 459 * No free device found. 460 * Since we allocated a device with down_interruptible, this 461 * should not be able to happen. 462 * Number of available devices, which are contained in 463 * device_allocation, is therefore decremented by not doing 464 * an up(device_allocation). 465 */ 466 return -EBUSY; 467 } 468 469 *device_data = local_device_data; 470 471 return 0; 472} 473 474static void cryp_dma_setup_channel(struct cryp_device_data *device_data, 475 struct device *dev) 476{ 477 struct dma_slave_config mem2cryp = { 478 .direction = DMA_MEM_TO_DEV, 479 .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO, 480 .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, 481 .dst_maxburst = 4, 482 }; 483 struct dma_slave_config cryp2mem = { 484 .direction = DMA_DEV_TO_MEM, 485 .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO, 486 .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, 487 .src_maxburst = 4, 488 }; 489 490 dma_cap_zero(device_data->dma.mask); 491 dma_cap_set(DMA_SLAVE, device_data->dma.mask); 492 493 device_data->dma.cfg_mem2cryp = mem_to_engine; 494 device_data->dma.chan_mem2cryp = 495 dma_request_channel(device_data->dma.mask, 496 stedma40_filter, 497 device_data->dma.cfg_mem2cryp); 498 499 device_data->dma.cfg_cryp2mem = engine_to_mem; 500 device_data->dma.chan_cryp2mem = 501 dma_request_channel(device_data->dma.mask, 502 stedma40_filter, 503 device_data->dma.cfg_cryp2mem); 504 505 dmaengine_slave_config(device_data->dma.chan_mem2cryp, &mem2cryp); 506 dmaengine_slave_config(device_data->dma.chan_cryp2mem, &cryp2mem); 507 508 init_completion(&device_data->dma.cryp_dma_complete); 509} 510 511static void cryp_dma_out_callback(void *data) 512{ 513 struct cryp_ctx *ctx = (struct cryp_ctx *) data; 514 dev_dbg(ctx->device->dev, "[%s]: ", __func__); 515 516 complete(&ctx->device->dma.cryp_dma_complete); 517} 518 519static int cryp_set_dma_transfer(struct cryp_ctx *ctx, 520 struct scatterlist *sg, 521 int len, 522 enum dma_data_direction direction) 523{ 524 struct dma_async_tx_descriptor *desc; 525 struct dma_chan *channel = NULL; 526 dma_cookie_t cookie; 527 528 dev_dbg(ctx->device->dev, "[%s]: ", __func__); 529 530 if (unlikely(!IS_ALIGNED((u32)sg, 4))) { 531 dev_err(ctx->device->dev, "[%s]: Data in sg list isn't " 532 "aligned! Addr: 0x%08x", __func__, (u32)sg); 533 return -EFAULT; 534 } 535 536 switch (direction) { 537 case DMA_TO_DEVICE: 538 channel = ctx->device->dma.chan_mem2cryp; 539 ctx->device->dma.sg_src = sg; 540 ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev, 541 ctx->device->dma.sg_src, 542 ctx->device->dma.nents_src, 543 direction); 544 545 if (!ctx->device->dma.sg_src_len) { 546 dev_dbg(ctx->device->dev, 547 "[%s]: Could not map the sg list (TO_DEVICE)", 548 __func__); 549 return -EFAULT; 550 } 551 552 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer " 553 "(TO_DEVICE)", __func__); 554 555 desc = dmaengine_prep_slave_sg(channel, 556 ctx->device->dma.sg_src, 557 ctx->device->dma.sg_src_len, 558 direction, DMA_CTRL_ACK); 559 break; 560 561 case DMA_FROM_DEVICE: 562 channel = ctx->device->dma.chan_cryp2mem; 563 ctx->device->dma.sg_dst = sg; 564 ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev, 565 ctx->device->dma.sg_dst, 566 ctx->device->dma.nents_dst, 567 direction); 568 569 if (!ctx->device->dma.sg_dst_len) { 570 dev_dbg(ctx->device->dev, 571 "[%s]: Could not map the sg list (FROM_DEVICE)", 572 __func__); 573 return -EFAULT; 574 } 575 576 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer " 577 "(FROM_DEVICE)", __func__); 578 579 desc = dmaengine_prep_slave_sg(channel, 580 ctx->device->dma.sg_dst, 581 ctx->device->dma.sg_dst_len, 582 direction, 583 DMA_CTRL_ACK | 584 DMA_PREP_INTERRUPT); 585 586 desc->callback = cryp_dma_out_callback; 587 desc->callback_param = ctx; 588 break; 589 590 default: 591 dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction", 592 __func__); 593 return -EFAULT; 594 } 595 596 cookie = dmaengine_submit(desc); 597 dma_async_issue_pending(channel); 598 599 return 0; 600} 601 602static void cryp_dma_done(struct cryp_ctx *ctx) 603{ 604 struct dma_chan *chan; 605 606 dev_dbg(ctx->device->dev, "[%s]: ", __func__); 607 608 chan = ctx->device->dma.chan_mem2cryp; 609 dmaengine_device_control(chan, DMA_TERMINATE_ALL, 0); 610 dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_src, 611 ctx->device->dma.sg_src_len, DMA_TO_DEVICE); 612 613 chan = ctx->device->dma.chan_cryp2mem; 614 dmaengine_device_control(chan, DMA_TERMINATE_ALL, 0); 615 dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_dst, 616 ctx->device->dma.sg_dst_len, DMA_FROM_DEVICE); 617} 618 619static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg, 620 int len) 621{ 622 int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE); 623 dev_dbg(ctx->device->dev, "[%s]: ", __func__); 624 625 if (error) { 626 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() " 627 "failed", __func__); 628 return error; 629 } 630 631 return len; 632} 633 634static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len) 635{ 636 int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE); 637 if (error) { 638 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() " 639 "failed", __func__); 640 return error; 641 } 642 643 return len; 644} 645 646static void cryp_polling_mode(struct cryp_ctx *ctx, 647 struct cryp_device_data *device_data) 648{ 649 int len = ctx->blocksize / BYTES_PER_WORD; 650 int remaining_length = ctx->datalen; 651 u32 *indata = (u32 *)ctx->indata; 652 u32 *outdata = (u32 *)ctx->outdata; 653 654 while (remaining_length > 0) { 655 writesl(&device_data->base->din, indata, len); 656 indata += len; 657 remaining_length -= (len * BYTES_PER_WORD); 658 cryp_wait_until_done(device_data); 659 660 readsl(&device_data->base->dout, outdata, len); 661 outdata += len; 662 cryp_wait_until_done(device_data); 663 } 664} 665 666static int cryp_disable_power(struct device *dev, 667 struct cryp_device_data *device_data, 668 bool save_device_context) 669{ 670 int ret = 0; 671 672 dev_dbg(dev, "[%s]", __func__); 673 674 spin_lock(&device_data->power_state_spinlock); 675 if (!device_data->power_state) 676 goto out; 677 678 spin_lock(&device_data->ctx_lock); 679 if (save_device_context && device_data->current_ctx) { 680 cryp_save_device_context(device_data, 681 &device_data->current_ctx->dev_ctx, 682 cryp_mode); 683 device_data->restore_dev_ctx = true; 684 } 685 spin_unlock(&device_data->ctx_lock); 686 687 clk_disable(device_data->clk); 688 ret = regulator_disable(device_data->pwr_regulator); 689 if (ret) 690 dev_err(dev, "[%s]: " 691 "regulator_disable() failed!", 692 __func__); 693 694 device_data->power_state = false; 695 696out: 697 spin_unlock(&device_data->power_state_spinlock); 698 699 return ret; 700} 701 702static int cryp_enable_power( 703 struct device *dev, 704 struct cryp_device_data *device_data, 705 bool restore_device_context) 706{ 707 int ret = 0; 708 709 dev_dbg(dev, "[%s]", __func__); 710 711 spin_lock(&device_data->power_state_spinlock); 712 if (!device_data->power_state) { 713 ret = regulator_enable(device_data->pwr_regulator); 714 if (ret) { 715 dev_err(dev, "[%s]: regulator_enable() failed!", 716 __func__); 717 goto out; 718 } 719 720 ret = clk_enable(device_data->clk); 721 if (ret) { 722 dev_err(dev, "[%s]: clk_enable() failed!", 723 __func__); 724 regulator_disable(device_data->pwr_regulator); 725 goto out; 726 } 727 device_data->power_state = true; 728 } 729 730 if (device_data->restore_dev_ctx) { 731 spin_lock(&device_data->ctx_lock); 732 if (restore_device_context && device_data->current_ctx) { 733 device_data->restore_dev_ctx = false; 734 cryp_restore_device_context(device_data, 735 &device_data->current_ctx->dev_ctx); 736 } 737 spin_unlock(&device_data->ctx_lock); 738 } 739out: 740 spin_unlock(&device_data->power_state_spinlock); 741 742 return ret; 743} 744 745static int hw_crypt_noxts(struct cryp_ctx *ctx, 746 struct cryp_device_data *device_data) 747{ 748 int ret = 0; 749 750 const u8 *indata = ctx->indata; 751 u8 *outdata = ctx->outdata; 752 u32 datalen = ctx->datalen; 753 u32 outlen = datalen; 754 755 pr_debug(DEV_DBG_NAME " [%s]", __func__); 756 757 ctx->outlen = ctx->datalen; 758 759 if (unlikely(!IS_ALIGNED((u32)indata, 4))) { 760 pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: " 761 "0x%08x", __func__, (u32)indata); 762 return -EINVAL; 763 } 764 765 ret = cryp_setup_context(ctx, device_data); 766 767 if (ret) 768 goto out; 769 770 if (cryp_mode == CRYP_MODE_INTERRUPT) { 771 cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO | 772 CRYP_IRQ_SRC_OUTPUT_FIFO); 773 774 /* 775 * ctx->outlen is decremented in the cryp_interrupt_handler 776 * function. We had to add cpu_relax() (barrier) to make sure 777 * that gcc didn't optimze away this variable. 778 */ 779 while (ctx->outlen > 0) 780 cpu_relax(); 781 } else if (cryp_mode == CRYP_MODE_POLLING || 782 cryp_mode == CRYP_MODE_DMA) { 783 /* 784 * The reason for having DMA in this if case is that if we are 785 * running cryp_mode = 2, then we separate DMA routines for 786 * handling cipher/plaintext > blocksize, except when 787 * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use 788 * the polling mode. Overhead of doing DMA setup eats up the 789 * benefits using it. 790 */ 791 cryp_polling_mode(ctx, device_data); 792 } else { 793 dev_err(ctx->device->dev, "[%s]: Invalid operation mode!", 794 __func__); 795 ret = -EPERM; 796 goto out; 797 } 798 799 cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode); 800 ctx->updated = 1; 801 802out: 803 ctx->indata = indata; 804 ctx->outdata = outdata; 805 ctx->datalen = datalen; 806 ctx->outlen = outlen; 807 808 return ret; 809} 810 811static int get_nents(struct scatterlist *sg, int nbytes) 812{ 813 int nents = 0; 814 815 while (nbytes > 0) { 816 nbytes -= sg->length; 817 sg = scatterwalk_sg_next(sg); 818 nents++; 819 } 820 821 return nents; 822} 823 824static int ablk_dma_crypt(struct ablkcipher_request *areq) 825{ 826 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 827 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 828 struct cryp_device_data *device_data; 829 830 int bytes_written = 0; 831 int bytes_read = 0; 832 int ret; 833 834 pr_debug(DEV_DBG_NAME " [%s]", __func__); 835 836 ctx->datalen = areq->nbytes; 837 ctx->outlen = areq->nbytes; 838 839 ret = cryp_get_device_data(ctx, &device_data); 840 if (ret) 841 return ret; 842 843 ret = cryp_setup_context(ctx, device_data); 844 if (ret) 845 goto out; 846 847 /* We have the device now, so store the nents in the dma struct. */ 848 ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen); 849 ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen); 850 851 /* Enable DMA in- and output. */ 852 cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS); 853 854 bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen); 855 bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written); 856 857 wait_for_completion(&ctx->device->dma.cryp_dma_complete); 858 cryp_dma_done(ctx); 859 860 cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode); 861 ctx->updated = 1; 862 863out: 864 spin_lock(&device_data->ctx_lock); 865 device_data->current_ctx = NULL; 866 ctx->device = NULL; 867 spin_unlock(&device_data->ctx_lock); 868 869 /* 870 * The down_interruptible part for this semaphore is called in 871 * cryp_get_device_data. 872 */ 873 up(&driver_data.device_allocation); 874 875 if (unlikely(bytes_written != bytes_read)) 876 return -EPERM; 877 878 return 0; 879} 880 881static int ablk_crypt(struct ablkcipher_request *areq) 882{ 883 struct ablkcipher_walk walk; 884 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 885 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 886 struct cryp_device_data *device_data; 887 unsigned long src_paddr; 888 unsigned long dst_paddr; 889 int ret; 890 int nbytes; 891 892 pr_debug(DEV_DBG_NAME " [%s]", __func__); 893 894 ret = cryp_get_device_data(ctx, &device_data); 895 if (ret) 896 goto out; 897 898 ablkcipher_walk_init(&walk, areq->dst, areq->src, areq->nbytes); 899 ret = ablkcipher_walk_phys(areq, &walk); 900 901 if (ret) { 902 pr_err(DEV_DBG_NAME "[%s]: ablkcipher_walk_phys() failed!", 903 __func__); 904 goto out; 905 } 906 907 while ((nbytes = walk.nbytes) > 0) { 908 ctx->iv = walk.iv; 909 src_paddr = (page_to_phys(walk.src.page) + walk.src.offset); 910 ctx->indata = phys_to_virt(src_paddr); 911 912 dst_paddr = (page_to_phys(walk.dst.page) + walk.dst.offset); 913 ctx->outdata = phys_to_virt(dst_paddr); 914 915 ctx->datalen = nbytes - (nbytes % ctx->blocksize); 916 917 ret = hw_crypt_noxts(ctx, device_data); 918 if (ret) 919 goto out; 920 921 nbytes -= ctx->datalen; 922 ret = ablkcipher_walk_done(areq, &walk, nbytes); 923 if (ret) 924 goto out; 925 } 926 ablkcipher_walk_complete(&walk); 927 928out: 929 /* Release the device */ 930 spin_lock(&device_data->ctx_lock); 931 device_data->current_ctx = NULL; 932 ctx->device = NULL; 933 spin_unlock(&device_data->ctx_lock); 934 935 /* 936 * The down_interruptible part for this semaphore is called in 937 * cryp_get_device_data. 938 */ 939 up(&driver_data.device_allocation); 940 941 return ret; 942} 943 944static int aes_ablkcipher_setkey(struct crypto_ablkcipher *cipher, 945 const u8 *key, unsigned int keylen) 946{ 947 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 948 u32 *flags = &cipher->base.crt_flags; 949 950 pr_debug(DEV_DBG_NAME " [%s]", __func__); 951 952 switch (keylen) { 953 case AES_KEYSIZE_128: 954 ctx->config.keysize = CRYP_KEY_SIZE_128; 955 break; 956 957 case AES_KEYSIZE_192: 958 ctx->config.keysize = CRYP_KEY_SIZE_192; 959 break; 960 961 case AES_KEYSIZE_256: 962 ctx->config.keysize = CRYP_KEY_SIZE_256; 963 break; 964 965 default: 966 pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__); 967 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 968 return -EINVAL; 969 } 970 971 memcpy(ctx->key, key, keylen); 972 ctx->keylen = keylen; 973 974 ctx->updated = 0; 975 976 return 0; 977} 978 979static int des_ablkcipher_setkey(struct crypto_ablkcipher *cipher, 980 const u8 *key, unsigned int keylen) 981{ 982 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 983 u32 *flags = &cipher->base.crt_flags; 984 u32 tmp[DES_EXPKEY_WORDS]; 985 int ret; 986 987 pr_debug(DEV_DBG_NAME " [%s]", __func__); 988 if (keylen != DES_KEY_SIZE) { 989 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 990 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN", 991 __func__); 992 return -EINVAL; 993 } 994 995 ret = des_ekey(tmp, key); 996 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 997 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 998 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY", 999 __func__); 1000 return -EINVAL; 1001 } 1002 1003 memcpy(ctx->key, key, keylen); 1004 ctx->keylen = keylen; 1005 1006 ctx->updated = 0; 1007 return 0; 1008} 1009 1010static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher, 1011 const u8 *key, unsigned int keylen) 1012{ 1013 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1014 u32 *flags = &cipher->base.crt_flags; 1015 const u32 *K = (const u32 *)key; 1016 u32 tmp[DES3_EDE_EXPKEY_WORDS]; 1017 int i, ret; 1018 1019 pr_debug(DEV_DBG_NAME " [%s]", __func__); 1020 if (keylen != DES3_EDE_KEY_SIZE) { 1021 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 1022 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN", 1023 __func__); 1024 return -EINVAL; 1025 } 1026 1027 /* Checking key interdependency for weak key detection. */ 1028 if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || 1029 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && 1030 (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 1031 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 1032 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY", 1033 __func__); 1034 return -EINVAL; 1035 } 1036 for (i = 0; i < 3; i++) { 1037 ret = des_ekey(tmp, key + i*DES_KEY_SIZE); 1038 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 1039 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 1040 pr_debug(DEV_DBG_NAME " [%s]: " 1041 "CRYPTO_TFM_REQ_WEAK_KEY", __func__); 1042 return -EINVAL; 1043 } 1044 } 1045 1046 memcpy(ctx->key, key, keylen); 1047 ctx->keylen = keylen; 1048 1049 ctx->updated = 0; 1050 return 0; 1051} 1052 1053static int cryp_blk_encrypt(struct ablkcipher_request *areq) 1054{ 1055 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1056 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1057 1058 pr_debug(DEV_DBG_NAME " [%s]", __func__); 1059 1060 ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT; 1061 1062 /* 1063 * DMA does not work for DES due to a hw bug */ 1064 if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode)) 1065 return ablk_dma_crypt(areq); 1066 1067 /* For everything except DMA, we run the non DMA version. */ 1068 return ablk_crypt(areq); 1069} 1070 1071static int cryp_blk_decrypt(struct ablkcipher_request *areq) 1072{ 1073 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1074 struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1075 1076 pr_debug(DEV_DBG_NAME " [%s]", __func__); 1077 1078 ctx->config.algodir = CRYP_ALGORITHM_DECRYPT; 1079 1080 /* DMA does not work for DES due to a hw bug */ 1081 if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode)) 1082 return ablk_dma_crypt(areq); 1083 1084 /* For everything except DMA, we run the non DMA version. */ 1085 return ablk_crypt(areq); 1086} 1087 1088struct cryp_algo_template { 1089 enum cryp_algo_mode algomode; 1090 struct crypto_alg crypto; 1091}; 1092 1093static int cryp_cra_init(struct crypto_tfm *tfm) 1094{ 1095 struct cryp_ctx *ctx = crypto_tfm_ctx(tfm); 1096 struct crypto_alg *alg = tfm->__crt_alg; 1097 struct cryp_algo_template *cryp_alg = container_of(alg, 1098 struct cryp_algo_template, 1099 crypto); 1100 1101 ctx->config.algomode = cryp_alg->algomode; 1102 ctx->blocksize = crypto_tfm_alg_blocksize(tfm); 1103 1104 return 0; 1105} 1106 1107static struct cryp_algo_template cryp_algs[] = { 1108 { 1109 .algomode = CRYP_ALGO_AES_ECB, 1110 .crypto = { 1111 .cra_name = "aes", 1112 .cra_driver_name = "aes-ux500", 1113 .cra_priority = 300, 1114 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1115 CRYPTO_ALG_ASYNC, 1116 .cra_blocksize = AES_BLOCK_SIZE, 1117 .cra_ctxsize = sizeof(struct cryp_ctx), 1118 .cra_alignmask = 3, 1119 .cra_type = &crypto_ablkcipher_type, 1120 .cra_init = cryp_cra_init, 1121 .cra_module = THIS_MODULE, 1122 .cra_u = { 1123 .ablkcipher = { 1124 .min_keysize = AES_MIN_KEY_SIZE, 1125 .max_keysize = AES_MAX_KEY_SIZE, 1126 .setkey = aes_ablkcipher_setkey, 1127 .encrypt = cryp_blk_encrypt, 1128 .decrypt = cryp_blk_decrypt 1129 } 1130 } 1131 } 1132 }, 1133 { 1134 .algomode = CRYP_ALGO_AES_ECB, 1135 .crypto = { 1136 .cra_name = "ecb(aes)", 1137 .cra_driver_name = "ecb-aes-ux500", 1138 .cra_priority = 300, 1139 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1140 CRYPTO_ALG_ASYNC, 1141 .cra_blocksize = AES_BLOCK_SIZE, 1142 .cra_ctxsize = sizeof(struct cryp_ctx), 1143 .cra_alignmask = 3, 1144 .cra_type = &crypto_ablkcipher_type, 1145 .cra_init = cryp_cra_init, 1146 .cra_module = THIS_MODULE, 1147 .cra_u = { 1148 .ablkcipher = { 1149 .min_keysize = AES_MIN_KEY_SIZE, 1150 .max_keysize = AES_MAX_KEY_SIZE, 1151 .setkey = aes_ablkcipher_setkey, 1152 .encrypt = cryp_blk_encrypt, 1153 .decrypt = cryp_blk_decrypt, 1154 } 1155 } 1156 } 1157 }, 1158 { 1159 .algomode = CRYP_ALGO_AES_CBC, 1160 .crypto = { 1161 .cra_name = "cbc(aes)", 1162 .cra_driver_name = "cbc-aes-ux500", 1163 .cra_priority = 300, 1164 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1165 CRYPTO_ALG_ASYNC, 1166 .cra_blocksize = AES_BLOCK_SIZE, 1167 .cra_ctxsize = sizeof(struct cryp_ctx), 1168 .cra_alignmask = 3, 1169 .cra_type = &crypto_ablkcipher_type, 1170 .cra_init = cryp_cra_init, 1171 .cra_module = THIS_MODULE, 1172 .cra_u = { 1173 .ablkcipher = { 1174 .min_keysize = AES_MIN_KEY_SIZE, 1175 .max_keysize = AES_MAX_KEY_SIZE, 1176 .setkey = aes_ablkcipher_setkey, 1177 .encrypt = cryp_blk_encrypt, 1178 .decrypt = cryp_blk_decrypt, 1179 .ivsize = AES_BLOCK_SIZE, 1180 } 1181 } 1182 } 1183 }, 1184 { 1185 .algomode = CRYP_ALGO_AES_CTR, 1186 .crypto = { 1187 .cra_name = "ctr(aes)", 1188 .cra_driver_name = "ctr-aes-ux500", 1189 .cra_priority = 300, 1190 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1191 CRYPTO_ALG_ASYNC, 1192 .cra_blocksize = AES_BLOCK_SIZE, 1193 .cra_ctxsize = sizeof(struct cryp_ctx), 1194 .cra_alignmask = 3, 1195 .cra_type = &crypto_ablkcipher_type, 1196 .cra_init = cryp_cra_init, 1197 .cra_module = THIS_MODULE, 1198 .cra_u = { 1199 .ablkcipher = { 1200 .min_keysize = AES_MIN_KEY_SIZE, 1201 .max_keysize = AES_MAX_KEY_SIZE, 1202 .setkey = aes_ablkcipher_setkey, 1203 .encrypt = cryp_blk_encrypt, 1204 .decrypt = cryp_blk_decrypt, 1205 .ivsize = AES_BLOCK_SIZE, 1206 } 1207 } 1208 } 1209 }, 1210 { 1211 .algomode = CRYP_ALGO_DES_ECB, 1212 .crypto = { 1213 .cra_name = "des", 1214 .cra_driver_name = "des-ux500", 1215 .cra_priority = 300, 1216 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1217 CRYPTO_ALG_ASYNC, 1218 .cra_blocksize = DES_BLOCK_SIZE, 1219 .cra_ctxsize = sizeof(struct cryp_ctx), 1220 .cra_alignmask = 3, 1221 .cra_type = &crypto_ablkcipher_type, 1222 .cra_init = cryp_cra_init, 1223 .cra_module = THIS_MODULE, 1224 .cra_u = { 1225 .ablkcipher = { 1226 .min_keysize = DES_KEY_SIZE, 1227 .max_keysize = DES_KEY_SIZE, 1228 .setkey = des_ablkcipher_setkey, 1229 .encrypt = cryp_blk_encrypt, 1230 .decrypt = cryp_blk_decrypt 1231 } 1232 } 1233 } 1234 1235 }, 1236 { 1237 .algomode = CRYP_ALGO_TDES_ECB, 1238 .crypto = { 1239 .cra_name = "des3_ede", 1240 .cra_driver_name = "des3_ede-ux500", 1241 .cra_priority = 300, 1242 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1243 CRYPTO_ALG_ASYNC, 1244 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1245 .cra_ctxsize = sizeof(struct cryp_ctx), 1246 .cra_alignmask = 3, 1247 .cra_type = &crypto_ablkcipher_type, 1248 .cra_init = cryp_cra_init, 1249 .cra_module = THIS_MODULE, 1250 .cra_u = { 1251 .ablkcipher = { 1252 .min_keysize = DES3_EDE_KEY_SIZE, 1253 .max_keysize = DES3_EDE_KEY_SIZE, 1254 .setkey = des_ablkcipher_setkey, 1255 .encrypt = cryp_blk_encrypt, 1256 .decrypt = cryp_blk_decrypt 1257 } 1258 } 1259 } 1260 }, 1261 { 1262 .algomode = CRYP_ALGO_DES_ECB, 1263 .crypto = { 1264 .cra_name = "ecb(des)", 1265 .cra_driver_name = "ecb-des-ux500", 1266 .cra_priority = 300, 1267 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1268 CRYPTO_ALG_ASYNC, 1269 .cra_blocksize = DES_BLOCK_SIZE, 1270 .cra_ctxsize = sizeof(struct cryp_ctx), 1271 .cra_alignmask = 3, 1272 .cra_type = &crypto_ablkcipher_type, 1273 .cra_init = cryp_cra_init, 1274 .cra_module = THIS_MODULE, 1275 .cra_u = { 1276 .ablkcipher = { 1277 .min_keysize = DES_KEY_SIZE, 1278 .max_keysize = DES_KEY_SIZE, 1279 .setkey = des_ablkcipher_setkey, 1280 .encrypt = cryp_blk_encrypt, 1281 .decrypt = cryp_blk_decrypt, 1282 } 1283 } 1284 } 1285 }, 1286 { 1287 .algomode = CRYP_ALGO_TDES_ECB, 1288 .crypto = { 1289 .cra_name = "ecb(des3_ede)", 1290 .cra_driver_name = "ecb-des3_ede-ux500", 1291 .cra_priority = 300, 1292 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1293 CRYPTO_ALG_ASYNC, 1294 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1295 .cra_ctxsize = sizeof(struct cryp_ctx), 1296 .cra_alignmask = 3, 1297 .cra_type = &crypto_ablkcipher_type, 1298 .cra_init = cryp_cra_init, 1299 .cra_module = THIS_MODULE, 1300 .cra_u = { 1301 .ablkcipher = { 1302 .min_keysize = DES3_EDE_KEY_SIZE, 1303 .max_keysize = DES3_EDE_KEY_SIZE, 1304 .setkey = des3_ablkcipher_setkey, 1305 .encrypt = cryp_blk_encrypt, 1306 .decrypt = cryp_blk_decrypt, 1307 } 1308 } 1309 } 1310 }, 1311 { 1312 .algomode = CRYP_ALGO_DES_CBC, 1313 .crypto = { 1314 .cra_name = "cbc(des)", 1315 .cra_driver_name = "cbc-des-ux500", 1316 .cra_priority = 300, 1317 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1318 CRYPTO_ALG_ASYNC, 1319 .cra_blocksize = DES_BLOCK_SIZE, 1320 .cra_ctxsize = sizeof(struct cryp_ctx), 1321 .cra_alignmask = 3, 1322 .cra_type = &crypto_ablkcipher_type, 1323 .cra_init = cryp_cra_init, 1324 .cra_module = THIS_MODULE, 1325 .cra_u = { 1326 .ablkcipher = { 1327 .min_keysize = DES_KEY_SIZE, 1328 .max_keysize = DES_KEY_SIZE, 1329 .setkey = des_ablkcipher_setkey, 1330 .encrypt = cryp_blk_encrypt, 1331 .decrypt = cryp_blk_decrypt, 1332 } 1333 } 1334 } 1335 }, 1336 { 1337 .algomode = CRYP_ALGO_TDES_CBC, 1338 .crypto = { 1339 .cra_name = "cbc(des3_ede)", 1340 .cra_driver_name = "cbc-des3_ede-ux500", 1341 .cra_priority = 300, 1342 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1343 CRYPTO_ALG_ASYNC, 1344 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1345 .cra_ctxsize = sizeof(struct cryp_ctx), 1346 .cra_alignmask = 3, 1347 .cra_type = &crypto_ablkcipher_type, 1348 .cra_init = cryp_cra_init, 1349 .cra_module = THIS_MODULE, 1350 .cra_u = { 1351 .ablkcipher = { 1352 .min_keysize = DES3_EDE_KEY_SIZE, 1353 .max_keysize = DES3_EDE_KEY_SIZE, 1354 .setkey = des3_ablkcipher_setkey, 1355 .encrypt = cryp_blk_encrypt, 1356 .decrypt = cryp_blk_decrypt, 1357 .ivsize = DES3_EDE_BLOCK_SIZE, 1358 } 1359 } 1360 } 1361 } 1362}; 1363 1364/** 1365 * cryp_algs_register_all - 1366 */ 1367static int cryp_algs_register_all(void) 1368{ 1369 int ret; 1370 int i; 1371 int count; 1372 1373 pr_debug("[%s]", __func__); 1374 1375 for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) { 1376 ret = crypto_register_alg(&cryp_algs[i].crypto); 1377 if (ret) { 1378 count = i; 1379 pr_err("[%s] alg registration failed", 1380 cryp_algs[i].crypto.cra_driver_name); 1381 goto unreg; 1382 } 1383 } 1384 return 0; 1385unreg: 1386 for (i = 0; i < count; i++) 1387 crypto_unregister_alg(&cryp_algs[i].crypto); 1388 return ret; 1389} 1390 1391/** 1392 * cryp_algs_unregister_all - 1393 */ 1394static void cryp_algs_unregister_all(void) 1395{ 1396 int i; 1397 1398 pr_debug(DEV_DBG_NAME " [%s]", __func__); 1399 1400 for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) 1401 crypto_unregister_alg(&cryp_algs[i].crypto); 1402} 1403 1404static int ux500_cryp_probe(struct platform_device *pdev) 1405{ 1406 int ret; 1407 int cryp_error = 0; 1408 struct resource *res = NULL; 1409 struct resource *res_irq = NULL; 1410 struct cryp_device_data *device_data; 1411 struct cryp_protection_config prot = { 1412 .privilege_access = CRYP_STATE_ENABLE 1413 }; 1414 struct device *dev = &pdev->dev; 1415 1416 dev_dbg(dev, "[%s]", __func__); 1417 device_data = kzalloc(sizeof(struct cryp_device_data), GFP_ATOMIC); 1418 if (!device_data) { 1419 dev_err(dev, "[%s]: kzalloc() failed!", __func__); 1420 ret = -ENOMEM; 1421 goto out; 1422 } 1423 1424 device_data->dev = dev; 1425 device_data->current_ctx = NULL; 1426 1427 /* Grab the DMA configuration from platform data. */ 1428 mem_to_engine = &((struct cryp_platform_data *) 1429 dev->platform_data)->mem_to_engine; 1430 engine_to_mem = &((struct cryp_platform_data *) 1431 dev->platform_data)->engine_to_mem; 1432 1433 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1434 if (!res) { 1435 dev_err(dev, "[%s]: platform_get_resource() failed", 1436 __func__); 1437 ret = -ENODEV; 1438 goto out_kfree; 1439 } 1440 1441 res = request_mem_region(res->start, resource_size(res), pdev->name); 1442 if (res == NULL) { 1443 dev_err(dev, "[%s]: request_mem_region() failed", 1444 __func__); 1445 ret = -EBUSY; 1446 goto out_kfree; 1447 } 1448 1449 device_data->phybase = res->start; 1450 device_data->base = ioremap(res->start, resource_size(res)); 1451 if (!device_data->base) { 1452 dev_err(dev, "[%s]: ioremap failed!", __func__); 1453 ret = -ENOMEM; 1454 goto out_free_mem; 1455 } 1456 1457 spin_lock_init(&device_data->ctx_lock); 1458 spin_lock_init(&device_data->power_state_spinlock); 1459 1460 /* Enable power for CRYP hardware block */ 1461 device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape"); 1462 if (IS_ERR(device_data->pwr_regulator)) { 1463 dev_err(dev, "[%s]: could not get cryp regulator", __func__); 1464 ret = PTR_ERR(device_data->pwr_regulator); 1465 device_data->pwr_regulator = NULL; 1466 goto out_unmap; 1467 } 1468 1469 /* Enable the clk for CRYP hardware block */ 1470 device_data->clk = clk_get(&pdev->dev, NULL); 1471 if (IS_ERR(device_data->clk)) { 1472 dev_err(dev, "[%s]: clk_get() failed!", __func__); 1473 ret = PTR_ERR(device_data->clk); 1474 goto out_regulator; 1475 } 1476 1477 ret = clk_prepare(device_data->clk); 1478 if (ret) { 1479 dev_err(dev, "[%s]: clk_prepare() failed!", __func__); 1480 goto out_clk; 1481 } 1482 1483 /* Enable device power (and clock) */ 1484 ret = cryp_enable_power(device_data->dev, device_data, false); 1485 if (ret) { 1486 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__); 1487 goto out_clk_unprepare; 1488 } 1489 1490 cryp_error = cryp_check(device_data); 1491 if (cryp_error != 0) { 1492 dev_err(dev, "[%s]: cryp_init() failed!", __func__); 1493 ret = -EINVAL; 1494 goto out_power; 1495 } 1496 1497 cryp_error = cryp_configure_protection(device_data, &prot); 1498 if (cryp_error != 0) { 1499 dev_err(dev, "[%s]: cryp_configure_protection() failed!", 1500 __func__); 1501 ret = -EINVAL; 1502 goto out_power; 1503 } 1504 1505 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1506 if (!res_irq) { 1507 dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable", 1508 __func__); 1509 ret = -ENODEV; 1510 goto out_power; 1511 } 1512 1513 ret = request_irq(res_irq->start, 1514 cryp_interrupt_handler, 1515 0, 1516 "cryp1", 1517 device_data); 1518 if (ret) { 1519 dev_err(dev, "[%s]: Unable to request IRQ", __func__); 1520 goto out_power; 1521 } 1522 1523 if (cryp_mode == CRYP_MODE_DMA) 1524 cryp_dma_setup_channel(device_data, dev); 1525 1526 platform_set_drvdata(pdev, device_data); 1527 1528 /* Put the new device into the device list... */ 1529 klist_add_tail(&device_data->list_node, &driver_data.device_list); 1530 1531 /* ... and signal that a new device is available. */ 1532 up(&driver_data.device_allocation); 1533 1534 atomic_set(&session_id, 1); 1535 1536 ret = cryp_algs_register_all(); 1537 if (ret) { 1538 dev_err(dev, "[%s]: cryp_algs_register_all() failed!", 1539 __func__); 1540 goto out_power; 1541 } 1542 1543 dev_info(dev, "successfully registered\n"); 1544 1545 return 0; 1546 1547out_power: 1548 cryp_disable_power(device_data->dev, device_data, false); 1549 1550out_clk_unprepare: 1551 clk_unprepare(device_data->clk); 1552 1553out_clk: 1554 clk_put(device_data->clk); 1555 1556out_regulator: 1557 regulator_put(device_data->pwr_regulator); 1558 1559out_unmap: 1560 iounmap(device_data->base); 1561 1562out_free_mem: 1563 release_mem_region(res->start, resource_size(res)); 1564 1565out_kfree: 1566 kfree(device_data); 1567out: 1568 return ret; 1569} 1570 1571static int ux500_cryp_remove(struct platform_device *pdev) 1572{ 1573 struct resource *res = NULL; 1574 struct resource *res_irq = NULL; 1575 struct cryp_device_data *device_data; 1576 1577 dev_dbg(&pdev->dev, "[%s]", __func__); 1578 device_data = platform_get_drvdata(pdev); 1579 if (!device_data) { 1580 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!", 1581 __func__); 1582 return -ENOMEM; 1583 } 1584 1585 /* Try to decrease the number of available devices. */ 1586 if (down_trylock(&driver_data.device_allocation)) 1587 return -EBUSY; 1588 1589 /* Check that the device is free */ 1590 spin_lock(&device_data->ctx_lock); 1591 /* current_ctx allocates a device, NULL = unallocated */ 1592 if (device_data->current_ctx) { 1593 /* The device is busy */ 1594 spin_unlock(&device_data->ctx_lock); 1595 /* Return the device to the pool. */ 1596 up(&driver_data.device_allocation); 1597 return -EBUSY; 1598 } 1599 1600 spin_unlock(&device_data->ctx_lock); 1601 1602 /* Remove the device from the list */ 1603 if (klist_node_attached(&device_data->list_node)) 1604 klist_remove(&device_data->list_node); 1605 1606 /* If this was the last device, remove the services */ 1607 if (list_empty(&driver_data.device_list.k_list)) 1608 cryp_algs_unregister_all(); 1609 1610 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1611 if (!res_irq) 1612 dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable", 1613 __func__); 1614 else { 1615 disable_irq(res_irq->start); 1616 free_irq(res_irq->start, device_data); 1617 } 1618 1619 if (cryp_disable_power(&pdev->dev, device_data, false)) 1620 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed", 1621 __func__); 1622 1623 clk_unprepare(device_data->clk); 1624 clk_put(device_data->clk); 1625 regulator_put(device_data->pwr_regulator); 1626 1627 iounmap(device_data->base); 1628 1629 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1630 if (res) 1631 release_mem_region(res->start, resource_size(res)); 1632 1633 kfree(device_data); 1634 1635 return 0; 1636} 1637 1638static void ux500_cryp_shutdown(struct platform_device *pdev) 1639{ 1640 struct resource *res_irq = NULL; 1641 struct cryp_device_data *device_data; 1642 1643 dev_dbg(&pdev->dev, "[%s]", __func__); 1644 1645 device_data = platform_get_drvdata(pdev); 1646 if (!device_data) { 1647 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!", 1648 __func__); 1649 return; 1650 } 1651 1652 /* Check that the device is free */ 1653 spin_lock(&device_data->ctx_lock); 1654 /* current_ctx allocates a device, NULL = unallocated */ 1655 if (!device_data->current_ctx) { 1656 if (down_trylock(&driver_data.device_allocation)) 1657 dev_dbg(&pdev->dev, "[%s]: Cryp still in use!" 1658 "Shutting down anyway...", __func__); 1659 /** 1660 * (Allocate the device) 1661 * Need to set this to non-null (dummy) value, 1662 * to avoid usage if context switching. 1663 */ 1664 device_data->current_ctx++; 1665 } 1666 spin_unlock(&device_data->ctx_lock); 1667 1668 /* Remove the device from the list */ 1669 if (klist_node_attached(&device_data->list_node)) 1670 klist_remove(&device_data->list_node); 1671 1672 /* If this was the last device, remove the services */ 1673 if (list_empty(&driver_data.device_list.k_list)) 1674 cryp_algs_unregister_all(); 1675 1676 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1677 if (!res_irq) 1678 dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable", 1679 __func__); 1680 else { 1681 disable_irq(res_irq->start); 1682 free_irq(res_irq->start, device_data); 1683 } 1684 1685 if (cryp_disable_power(&pdev->dev, device_data, false)) 1686 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed", 1687 __func__); 1688 1689} 1690 1691static int ux500_cryp_suspend(struct device *dev) 1692{ 1693 int ret; 1694 struct platform_device *pdev = to_platform_device(dev); 1695 struct cryp_device_data *device_data; 1696 struct resource *res_irq; 1697 struct cryp_ctx *temp_ctx = NULL; 1698 1699 dev_dbg(dev, "[%s]", __func__); 1700 1701 /* Handle state? */ 1702 device_data = platform_get_drvdata(pdev); 1703 if (!device_data) { 1704 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__); 1705 return -ENOMEM; 1706 } 1707 1708 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1709 if (!res_irq) 1710 dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__); 1711 else 1712 disable_irq(res_irq->start); 1713 1714 spin_lock(&device_data->ctx_lock); 1715 if (!device_data->current_ctx) 1716 device_data->current_ctx++; 1717 spin_unlock(&device_data->ctx_lock); 1718 1719 if (device_data->current_ctx == ++temp_ctx) { 1720 if (down_interruptible(&driver_data.device_allocation)) 1721 dev_dbg(dev, "[%s]: down_interruptible() failed", 1722 __func__); 1723 ret = cryp_disable_power(dev, device_data, false); 1724 1725 } else 1726 ret = cryp_disable_power(dev, device_data, true); 1727 1728 if (ret) 1729 dev_err(dev, "[%s]: cryp_disable_power()", __func__); 1730 1731 return ret; 1732} 1733 1734static int ux500_cryp_resume(struct device *dev) 1735{ 1736 int ret = 0; 1737 struct platform_device *pdev = to_platform_device(dev); 1738 struct cryp_device_data *device_data; 1739 struct resource *res_irq; 1740 struct cryp_ctx *temp_ctx = NULL; 1741 1742 dev_dbg(dev, "[%s]", __func__); 1743 1744 device_data = platform_get_drvdata(pdev); 1745 if (!device_data) { 1746 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__); 1747 return -ENOMEM; 1748 } 1749 1750 spin_lock(&device_data->ctx_lock); 1751 if (device_data->current_ctx == ++temp_ctx) 1752 device_data->current_ctx = NULL; 1753 spin_unlock(&device_data->ctx_lock); 1754 1755 1756 if (!device_data->current_ctx) 1757 up(&driver_data.device_allocation); 1758 else 1759 ret = cryp_enable_power(dev, device_data, true); 1760 1761 if (ret) 1762 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__); 1763 else { 1764 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1765 if (res_irq) 1766 enable_irq(res_irq->start); 1767 } 1768 1769 return ret; 1770} 1771 1772static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume); 1773 1774static const struct of_device_id ux500_cryp_match[] = { 1775 { .compatible = "stericsson,ux500-cryp" }, 1776 { }, 1777}; 1778 1779static struct platform_driver cryp_driver = { 1780 .probe = ux500_cryp_probe, 1781 .remove = ux500_cryp_remove, 1782 .shutdown = ux500_cryp_shutdown, 1783 .driver = { 1784 .owner = THIS_MODULE, 1785 .name = "cryp1", 1786 .of_match_table = ux500_cryp_match, 1787 .pm = &ux500_cryp_pm, 1788 } 1789}; 1790 1791static int __init ux500_cryp_mod_init(void) 1792{ 1793 pr_debug("[%s] is called!", __func__); 1794 klist_init(&driver_data.device_list, NULL, NULL); 1795 /* Initialize the semaphore to 0 devices (locked state) */ 1796 sema_init(&driver_data.device_allocation, 0); 1797 return platform_driver_register(&cryp_driver); 1798} 1799 1800static void __exit ux500_cryp_mod_fini(void) 1801{ 1802 pr_debug("[%s] is called!", __func__); 1803 platform_driver_unregister(&cryp_driver); 1804 return; 1805} 1806 1807module_init(ux500_cryp_mod_init); 1808module_exit(ux500_cryp_mod_fini); 1809 1810module_param(cryp_mode, int, 0); 1811 1812MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 CRYP crypto engine."); 1813MODULE_ALIAS("aes-all"); 1814MODULE_ALIAS("des-all"); 1815 1816MODULE_LICENSE("GPL"); 1817