14a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen/* 24a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * DMA Engine test module 34a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * 44a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * Copyright (C) 2007 Atmel Corporation 5851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko * Copyright (C) 2013 Intel Corporation 64a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * 74a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * This program is free software; you can redistribute it and/or modify 84a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * it under the terms of the GNU General Public License version 2 as 94a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * published by the Free Software Foundation. 104a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen */ 11872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams 134a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/delay.h> 14b7f080cfe223b3b7424872639d153695615a9255Alexey Dobriyan#include <linux/dma-mapping.h> 154a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/dmaengine.h> 16981ed70d8e4faf3689dbf3c48868a31d5b004d7aGuennadi Liakhovetski#include <linux/freezer.h> 174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/init.h> 184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/kthread.h> 194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/module.h> 204a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/moduleparam.h> 214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/random.h> 225a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/wait.h> 244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 254a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned int test_buf_size = 16384; 26a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 274a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 284a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 2906190d8415219d9eef7d8f04b52a109e34575a76Kay Sieversstatic char test_channel[20]; 30a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param_string(channel, test_channel, sizeof(test_channel), 31a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenko S_IRUGO | S_IWUSR); 324a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 334a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 34a85159fece07f4ff3e266da619af050928dceca1Guennadi Liakhovetskistatic char test_device[32]; 35a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param_string(device, test_device, sizeof(test_device), 36a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenko S_IRUGO | S_IWUSR); 374a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 384a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 394a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned int threads_per_chan = 1; 40a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); 414a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(threads_per_chan, 424a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen "Number of threads to start per channel (default: 1)"); 434a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 444a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned int max_channels; 45a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(max_channels, uint, S_IRUGO | S_IWUSR); 4633df8ca068123457db56c316946a3c0e4ef787d6Dan WilliamsMODULE_PARM_DESC(max_channels, 474a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen "Maximum number of channels to use (default: all)"); 484a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 490a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferrestatic unsigned int iterations; 50a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(iterations, uint, S_IRUGO | S_IWUSR); 510a2ff57d6fba92842272889b4bca447344cd9d36Nicolas FerreMODULE_PARM_DESC(iterations, 520a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre "Iterations before stopping test (default: infinite)"); 530a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre 54b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williamsstatic unsigned int xor_sources = 3; 55a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(xor_sources, uint, S_IRUGO | S_IWUSR); 56b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan WilliamsMODULE_PARM_DESC(xor_sources, 57b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams "Number of xor source buffers (default: 3)"); 58b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 5958691d64c44ae41ddf098ecb31e9a994026e3cffDan Williamsstatic unsigned int pq_sources = 3; 60a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(pq_sources, uint, S_IRUGO | S_IWUSR); 6158691d64c44ae41ddf098ecb31e9a994026e3cffDan WilliamsMODULE_PARM_DESC(pq_sources, 6258691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams "Number of p+q source buffers (default: 3)"); 6358691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams 64d42efe6bfb4eed8314c8ce3547f21954a4140399Viresh Kumarstatic int timeout = 3000; 65a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(timeout, uint, S_IRUGO | S_IWUSR); 6685ee7a1d39d75d23d21f3871f6dc9b87d572747aJoe PerchesMODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 6785ee7a1d39d75d23d21f3871f6dc9b87d572747aJoe Perches "Pass -1 for infinite timeout"); 68d42efe6bfb4eed8314c8ce3547f21954a4140399Viresh Kumar 69e3b9c347316fe243bea6abd08681050c43ca22eeDan Williamsstatic bool noverify; 70e3b9c347316fe243bea6abd08681050c43ca22eeDan Williamsmodule_param(noverify, bool, S_IRUGO | S_IWUSR); 71e3b9c347316fe243bea6abd08681050c43ca22eeDan WilliamsMODULE_PARM_DESC(noverify, "Disable random data setup and verification"); 724a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 7350137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williamsstatic bool verbose; 7450137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williamsmodule_param(verbose, bool, S_IRUGO | S_IWUSR); 7550137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan WilliamsMODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)"); 764a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 77e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko/** 7815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko * struct dmatest_params - test parameters. 79e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @buf_size: size of the memcpy test buffer 80e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @channel: bus ID of the channel to test 81e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @device: bus ID of the DMA Engine to test 82e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @threads_per_chan: number of threads to start per channel 83e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @max_channels: maximum number of channels to use 84e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @iterations: iterations before stopping test 85e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @xor_sources: number of xor source buffers 86e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @pq_sources: number of p+q source buffers 87e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @timeout: transfer timeout in msec, -1 for infinite timeout 88e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko */ 8915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenkostruct dmatest_params { 90e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int buf_size; 91e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko char channel[20]; 92a85159fece07f4ff3e266da619af050928dceca1Guennadi Liakhovetski char device[32]; 93e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int threads_per_chan; 94e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int max_channels; 95e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int iterations; 96e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int xor_sources; 97e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int pq_sources; 98e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko int timeout; 99e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams bool noverify; 10015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko}; 10115b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko 10215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko/** 10315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko * struct dmatest_info - test information. 10415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko * @params: test parameters 105851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko * @lock: access protection to the fields of this structure 10615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko */ 107a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic struct dmatest_info { 10815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko /* Test parameters */ 10915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params params; 110838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko 111838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko /* Internal state */ 112838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko struct list_head channels; 113838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko unsigned int nr_channels; 114851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko struct mutex lock; 115a310d037b8d06755c62bb4878c00d19490af5550Dan Williams bool did_init; 116a310d037b8d06755c62bb4878c00d19490af5550Dan Williams} test_info = { 117a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .channels = LIST_HEAD_INIT(test_info.channels), 118a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .lock = __MUTEX_INITIALIZER(test_info.lock), 119a310d037b8d06755c62bb4878c00d19490af5550Dan Williams}; 120851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 121a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_set(const char *val, const struct kernel_param *kp); 122a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_get(char *val, const struct kernel_param *kp); 123a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic struct kernel_param_ops run_ops = { 124a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .set = dmatest_run_set, 125a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .get = dmatest_run_get, 126e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko}; 127a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic bool dmatest_run; 128a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsmodule_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR); 129a310d037b8d06755c62bb4878c00d19490af5550Dan WilliamsMODULE_PARM_DESC(run, "Run the test (default: false)"); 130e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 131a310d037b8d06755c62bb4878c00d19490af5550Dan Williams/* Maximum amount of mismatched bytes in buffer to print */ 132a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define MAX_ERROR_COUNT 32 133a310d037b8d06755c62bb4878c00d19490af5550Dan Williams 134a310d037b8d06755c62bb4878c00d19490af5550Dan Williams/* 135a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * Initialization patterns. All bytes in the source buffer has bit 7 136a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * set, all bytes in the destination buffer has bit 7 cleared. 137a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * 138a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * Bit 6 is set for all bytes which are to be copied by the DMA 139a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * engine. Bit 5 is set for all bytes which are to be overwritten by 140a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * the DMA engine. 141a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * 142a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * The remaining bits are the inverse of a counter which increments by 143a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * one for each byte address. 144a310d037b8d06755c62bb4878c00d19490af5550Dan Williams */ 145a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_SRC 0x80 146a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_DST 0x00 147a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_COPY 0x40 148a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_OVERWRITE 0x20 149a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_COUNT_MASK 0x1f 150851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 151a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstruct dmatest_thread { 152a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct list_head node; 153a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dmatest_info *info; 154a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct task_struct *task; 155a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dma_chan *chan; 156a310d037b8d06755c62bb4878c00d19490af5550Dan Williams u8 **srcs; 157a310d037b8d06755c62bb4878c00d19490af5550Dan Williams u8 **dsts; 158a310d037b8d06755c62bb4878c00d19490af5550Dan Williams enum dma_transaction_type type; 159a310d037b8d06755c62bb4878c00d19490af5550Dan Williams bool done; 160a310d037b8d06755c62bb4878c00d19490af5550Dan Williams}; 16195019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 162a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstruct dmatest_chan { 163a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct list_head node; 164a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dma_chan *chan; 165a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct list_head threads; 166e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko}; 167e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 1682d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williamsstatic DECLARE_WAIT_QUEUE_HEAD(thread_wait); 1692d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williamsstatic bool wait; 1702d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1712d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williamsstatic bool is_threaded_test_run(struct dmatest_info *info) 1722d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams{ 1732d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams struct dmatest_chan *dtc; 1742d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1752d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams list_for_each_entry(dtc, &info->channels, node) { 1762d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams struct dmatest_thread *thread; 1772d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1782d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams list_for_each_entry(thread, &dtc->threads, node) { 1792d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams if (!thread->done) 1802d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams return true; 1812d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams } 1822d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams } 1832d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1842d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams return false; 1852d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams} 1862d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1872d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williamsstatic int dmatest_wait_get(char *val, const struct kernel_param *kp) 1882d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams{ 1892d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams struct dmatest_info *info = &test_info; 1902d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams struct dmatest_params *params = &info->params; 1912d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1922d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams if (params->iterations) 1932d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams wait_event(thread_wait, !is_threaded_test_run(info)); 1942d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams wait = true; 1952d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams return param_get_bool(val, kp); 1962d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams} 1972d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams 1982d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williamsstatic struct kernel_param_ops wait_ops = { 1992d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams .get = dmatest_wait_get, 2002d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams .set = param_set_bool, 2012d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams}; 2022d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williamsmodule_param_cb(wait, &wait_ops, &wait, S_IRUGO); 2032d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan WilliamsMODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)"); 204e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 20515b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenkostatic bool dmatest_match_channel(struct dmatest_params *params, 206e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dma_chan *chan) 2074a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 20815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->channel[0] == '\0') 2094a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return true; 21015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko return strcmp(dma_chan_name(chan), params->channel) == 0; 2114a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2124a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 21315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenkostatic bool dmatest_match_device(struct dmatest_params *params, 214e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dma_device *device) 2154a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 21615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->device[0] == '\0') 2174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return true; 21815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko return strcmp(dev_name(device->dev), params->device) == 0; 2194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2204a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 2214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned long dmatest_random(void) 2224a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 2234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned long buf; 2244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 225be9fa5a43641103bf13cd1bb8101a1453da03616Dan Williams prandom_bytes(&buf, sizeof(buf)); 2264a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return buf; 2274a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2284a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 229e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len, 230e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int buf_size) 2314a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 2324a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int i; 233b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf; 234b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 235b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (; (buf = *bufs); bufs++) { 236b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < start; i++) 237b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 238b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for ( ; i < start + len; i++) 239b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_SRC | PATTERN_COPY 240c019894efc9c9ba5939948caa78c133b1ec8ae63Joe Perches | (~i & PATTERN_COUNT_MASK); 241e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko for ( ; i < buf_size; i++) 242b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 243b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf++; 244b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 2454a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2464a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 247e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len, 248e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int buf_size) 2494a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 2504a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int i; 251b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf; 252b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 253b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (; (buf = *bufs); bufs++) { 254b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < start; i++) 255b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 256b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for ( ; i < start + len; i++) 257b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_DST | PATTERN_OVERWRITE 258b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams | (~i & PATTERN_COUNT_MASK); 259e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko for ( ; i < buf_size; i++) 260b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 261b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 2624a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2634a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 2647b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williamsstatic void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 2657b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams unsigned int counter, bool is_srcbuf) 2667b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams{ 2677b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams u8 diff = actual ^ pattern; 2687b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 2697b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams const char *thread_name = current->comm; 2707b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams 2717b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams if (is_srcbuf) 2727b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n", 2737b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2747b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams else if ((pattern & PATTERN_COPY) 2757b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 2767b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n", 2777b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2787b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams else if (diff & PATTERN_SRC) 2797b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n", 2807b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2817b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams else 2827b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n", 2837b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2847b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams} 2857b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams 2867b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williamsstatic unsigned int dmatest_verify(u8 **bufs, unsigned int start, 2877b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams unsigned int end, unsigned int counter, u8 pattern, 2887b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams bool is_srcbuf) 2894a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 2904a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int i; 2914a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int error_count = 0; 2924a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen u8 actual; 293b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 expected; 294b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf; 295b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams unsigned int counter_orig = counter; 296b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 297b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (; (buf = *bufs); bufs++) { 298b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams counter = counter_orig; 299b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = start; i < end; i++) { 300b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams actual = buf[i]; 301b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams expected = pattern | (~counter & PATTERN_COUNT_MASK); 302b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (actual != expected) { 3037b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams if (error_count < MAX_ERROR_COUNT) 3047b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams dmatest_mismatch(actual, pattern, i, 3057b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams counter, is_srcbuf); 306b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams error_count++; 307b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 308b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams counter++; 3094a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 3104a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 3114a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 31274b5c07a515b2986c9bdfe649213b8e358d32ad2Andy Shevchenko if (error_count > MAX_ERROR_COUNT) 3137b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: %u errors suppressed\n", 31474b5c07a515b2986c9bdfe649213b8e358d32ad2Andy Shevchenko current->comm, error_count - MAX_ERROR_COUNT); 3154a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 3164a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return error_count; 3174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 3184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 319adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo/* poor man's completion - we want to use wait_event_freezable() on it */ 320adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heostruct dmatest_done { 321adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo bool done; 322adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo wait_queue_head_t *wait; 323adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo}; 324adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo 325adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heostatic void dmatest_callback(void *arg) 326e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams{ 327adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo struct dmatest_done *done = arg; 328adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo 329adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo done->done = true; 330adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo wake_up_all(done->wait); 331e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams} 332e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 3338be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mitastatic unsigned int min_odd(unsigned int x, unsigned int y) 3348be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita{ 3358be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita unsigned int val = min(x, y); 3368be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita 3378be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita return val % 2 ? val : val - 1; 3388be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita} 3398be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita 340872f05c6e9a37e9358fd58eb54deee7337863496Dan Williamsstatic void result(const char *err, unsigned int n, unsigned int src_off, 341872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams unsigned int dst_off, unsigned int len, unsigned long data) 342d86b2f298e6de124984f5d5817ed1e6e759b3adaAndy Shevchenko{ 3432acec15034cc6a64b9fcac376e56b9071463812bJerome Blin pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", 344872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams current->comm, n, err, src_off, dst_off, len, data); 345d86b2f298e6de124984f5d5817ed1e6e759b3adaAndy Shevchenko} 346d86b2f298e6de124984f5d5817ed1e6e759b3adaAndy Shevchenko 347872f05c6e9a37e9358fd58eb54deee7337863496Dan Williamsstatic void dbg_result(const char *err, unsigned int n, unsigned int src_off, 348872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams unsigned int dst_off, unsigned int len, 349872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams unsigned long data) 35095019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko{ 3512acec15034cc6a64b9fcac376e56b9071463812bJerome Blin pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", 35250137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams current->comm, n, err, src_off, dst_off, len, data); 35395019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko} 35495019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 35550137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams#define verbose_result(err, n, src_off, dst_off, len, data) ({ \ 35650137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams if (verbose) \ 35750137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams result(err, n, src_off, dst_off, len, data); \ 35850137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams else \ 35950137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams dbg_result(err, n, src_off, dst_off, len, data); \ 36050137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams}) 36195019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 36286727443a04fdb25397041188efd2527f2b7237bDan Williamsstatic unsigned long long dmatest_persec(s64 runtime, unsigned int val) 363d86b2f298e6de124984f5d5817ed1e6e759b3adaAndy Shevchenko{ 36486727443a04fdb25397041188efd2527f2b7237bDan Williams unsigned long long per_sec = 1000000; 365d86b2f298e6de124984f5d5817ed1e6e759b3adaAndy Shevchenko 36686727443a04fdb25397041188efd2527f2b7237bDan Williams if (runtime <= 0) 36786727443a04fdb25397041188efd2527f2b7237bDan Williams return 0; 36895019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 36986727443a04fdb25397041188efd2527f2b7237bDan Williams /* drop precision until runtime is 32-bits */ 37086727443a04fdb25397041188efd2527f2b7237bDan Williams while (runtime > UINT_MAX) { 37186727443a04fdb25397041188efd2527f2b7237bDan Williams runtime >>= 1; 37286727443a04fdb25397041188efd2527f2b7237bDan Williams per_sec <<= 1; 37395019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko } 37495019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 37586727443a04fdb25397041188efd2527f2b7237bDan Williams per_sec *= val; 37686727443a04fdb25397041188efd2527f2b7237bDan Williams do_div(per_sec, runtime); 37786727443a04fdb25397041188efd2527f2b7237bDan Williams return per_sec; 37895019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko} 37995019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 38086727443a04fdb25397041188efd2527f2b7237bDan Williamsstatic unsigned long long dmatest_KBs(s64 runtime, unsigned long long len) 38195019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko{ 38286727443a04fdb25397041188efd2527f2b7237bDan Williams return dmatest_persec(runtime, len >> 10); 38395019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko} 38495019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 3854a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen/* 3864a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * This function repeatedly tests DMA transfers of various lengths and 387b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * offsets for a given operation type until it is told to exit by 388b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * kthread_stop(). There may be multiple threads running this function 389b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * in parallel for a single channel, and there may be multiple channels 390b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * being tested in parallel. 3914a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * 3924a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * Before each test, the source and destination buffer is initialized 3934a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * with a known pattern. This pattern is different depending on 3944a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * whether it's in an area which is supposed to be copied or 3954a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * overwritten, and different in the source and destination buffers. 3964a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * So if the DMA engine doesn't copy exactly what we tell it to copy, 3974a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * we'll notice. 3984a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen */ 3994a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic int dmatest_func(void *data) 4004a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 401adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait); 4024a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dmatest_thread *thread = data; 403adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo struct dmatest_done done = { .wait = &done_wait }; 404e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_info *info; 40515b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params *params; 4064a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dma_chan *chan; 4078be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita struct dma_device *dev; 4084a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int src_off, dst_off, len; 4094a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int error_count; 4104a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int failed_tests = 0; 4114a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int total_tests = 0; 4124a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen dma_cookie_t cookie; 4134a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen enum dma_status status; 414b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams enum dma_ctrl_flags flags; 415945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko u8 *pq_coefs = NULL; 4164a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen int ret; 417b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams int src_cnt; 418b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams int dst_cnt; 419b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams int i; 42086727443a04fdb25397041188efd2527f2b7237bDan Williams ktime_t ktime; 42186727443a04fdb25397041188efd2527f2b7237bDan Williams s64 runtime = 0; 42286727443a04fdb25397041188efd2527f2b7237bDan Williams unsigned long long total_len = 0; 4234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 424adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo set_freezable(); 4254a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 4264a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen ret = -ENOMEM; 4274a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 4284a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen smp_rmb(); 429e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko info = thread->info; 43015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko params = &info->params; 4314a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen chan = thread->chan; 4328be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita dev = chan->device; 433b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (thread->type == DMA_MEMCPY) 434b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams src_cnt = dst_cnt = 1; 435b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else if (thread->type == DMA_XOR) { 4368be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita /* force odd to ensure dst = src */ 43715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); 438b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dst_cnt = 1; 43958691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams } else if (thread->type == DMA_PQ) { 4408be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita /* force odd to ensure dst = src */ 44115b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); 44258691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams dst_cnt = 2; 443945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko 44415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL); 445945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko if (!pq_coefs) 446945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko goto err_thread_type; 447945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko 44894de648d72c8bc833590523f22386d4babbea988Anatolij Gustschin for (i = 0; i < src_cnt; i++) 44958691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams pq_coefs[i] = 1; 450b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } else 451945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko goto err_thread_type; 452b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 453b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 454b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->srcs) 455b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_srcs; 456b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < src_cnt; i++) { 45715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL); 458b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->srcs[i]) 459b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_srcbuf; 460b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 461b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->srcs[i] = NULL; 462b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 463b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 464b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->dsts) 465b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_dsts; 466b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < dst_cnt; i++) { 46715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL); 468b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->dsts[i]) 469b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_dstbuf; 470b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 471b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->dsts[i] = NULL; 472b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 473e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams set_user_nice(current, 10); 474e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 475b203bd3f6b9c3db3b1979c2ff79bb2b9be8f03a3Ira Snyder /* 476d1cab34c039584ebe76b04d2f2109e0d87d344e1Bartlomiej Zolnierkiewicz * src and dst buffers are freed by ourselves below 477b203bd3f6b9c3db3b1979c2ff79bb2b9be8f03a3Ira Snyder */ 4780776ae7b89782124ddd72eafe0b1e0fdcdabe32eBartlomiej Zolnierkiewicz flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; 4794a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 48086727443a04fdb25397041188efd2527f2b7237bDan Williams ktime = ktime_get(); 4810a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre while (!kthread_should_stop() 48215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko && !(params->iterations && total_tests >= params->iterations)) { 483b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dma_async_tx_descriptor *tx = NULL; 4844076e755dbec078c85352a8f77cec4c10181da4eDan Williams struct dmaengine_unmap_data *um; 4854076e755dbec078c85352a8f77cec4c10181da4eDan Williams dma_addr_t srcs[src_cnt]; 4864076e755dbec078c85352a8f77cec4c10181da4eDan Williams dma_addr_t *dsts; 48783544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams u8 align = 0; 488d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto 4894a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen total_tests++; 4904a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 49183544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams /* honor alignment restrictions */ 49283544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams if (thread->type == DMA_MEMCPY) 49383544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams align = dev->copy_align; 49483544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams else if (thread->type == DMA_XOR) 49583544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams align = dev->xor_align; 49683544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams else if (thread->type == DMA_PQ) 49783544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams align = dev->pq_align; 49883544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams 49915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (1 << align > params->buf_size) { 500cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski pr_err("%u-byte buffer too small for %d-byte alignment\n", 50115b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko params->buf_size, 1 << align); 502cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski break; 503cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski } 504cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski 505e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams if (params->noverify) { 506e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams len = params->buf_size; 507e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams src_off = 0; 508e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams dst_off = 0; 509e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams } else { 510e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams len = dmatest_random() % params->buf_size + 1; 511e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams len = (len >> align) << align; 512e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams if (!len) 513e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams len = 1 << align; 514e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams src_off = dmatest_random() % (params->buf_size - len + 1); 515e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams dst_off = dmatest_random() % (params->buf_size - len + 1); 516e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams 517e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams src_off = (src_off >> align) << align; 518e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams dst_off = (dst_off >> align) << align; 519e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams 520e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams dmatest_init_srcs(thread->srcs, src_off, len, 521e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams params->buf_size); 522e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams dmatest_init_dsts(thread->dsts, dst_off, len, 523e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams params->buf_size); 524e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams } 525e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams 52683544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams len = (len >> align) << align; 527cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski if (!len) 528cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski len = 1 << align; 52986727443a04fdb25397041188efd2527f2b7237bDan Williams total_len += len; 530cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski 5314076e755dbec078c85352a8f77cec4c10181da4eDan Williams um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt, 5324076e755dbec078c85352a8f77cec4c10181da4eDan Williams GFP_KERNEL); 5334076e755dbec078c85352a8f77cec4c10181da4eDan Williams if (!um) { 5344076e755dbec078c85352a8f77cec4c10181da4eDan Williams failed_tests++; 5354076e755dbec078c85352a8f77cec4c10181da4eDan Williams result("unmap data NULL", total_tests, 5364076e755dbec078c85352a8f77cec4c10181da4eDan Williams src_off, dst_off, len, ret); 5374076e755dbec078c85352a8f77cec4c10181da4eDan Williams continue; 5384076e755dbec078c85352a8f77cec4c10181da4eDan Williams } 5394a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 5404076e755dbec078c85352a8f77cec4c10181da4eDan Williams um->len = params->buf_size; 541b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < src_cnt; i++) { 542745c00daf9a75bacb53d0fe8635a132673ab0b46Dan Williams void *buf = thread->srcs[i]; 5434076e755dbec078c85352a8f77cec4c10181da4eDan Williams struct page *pg = virt_to_page(buf); 544745c00daf9a75bacb53d0fe8635a132673ab0b46Dan Williams unsigned pg_off = (unsigned long) buf & ~PAGE_MASK; 5454076e755dbec078c85352a8f77cec4c10181da4eDan Williams 5464076e755dbec078c85352a8f77cec4c10181da4eDan Williams um->addr[i] = dma_map_page(dev->dev, pg, pg_off, 5474076e755dbec078c85352a8f77cec4c10181da4eDan Williams um->len, DMA_TO_DEVICE); 5484076e755dbec078c85352a8f77cec4c10181da4eDan Williams srcs[i] = um->addr[i] + src_off; 5494076e755dbec078c85352a8f77cec4c10181da4eDan Williams ret = dma_mapping_error(dev->dev, um->addr[i]); 550afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko if (ret) { 5514076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 552872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("src mapping error", total_tests, 553872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams src_off, dst_off, len, ret); 554afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko failed_tests++; 555afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko continue; 556afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko } 5574076e755dbec078c85352a8f77cec4c10181da4eDan Williams um->to_cnt++; 558b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 559d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 5604076e755dbec078c85352a8f77cec4c10181da4eDan Williams dsts = &um->addr[src_cnt]; 561b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < dst_cnt; i++) { 562745c00daf9a75bacb53d0fe8635a132673ab0b46Dan Williams void *buf = thread->dsts[i]; 5634076e755dbec078c85352a8f77cec4c10181da4eDan Williams struct page *pg = virt_to_page(buf); 564745c00daf9a75bacb53d0fe8635a132673ab0b46Dan Williams unsigned pg_off = (unsigned long) buf & ~PAGE_MASK; 5654076e755dbec078c85352a8f77cec4c10181da4eDan Williams 5664076e755dbec078c85352a8f77cec4c10181da4eDan Williams dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len, 5674076e755dbec078c85352a8f77cec4c10181da4eDan Williams DMA_BIDIRECTIONAL); 5684076e755dbec078c85352a8f77cec4c10181da4eDan Williams ret = dma_mapping_error(dev->dev, dsts[i]); 569afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko if (ret) { 5704076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 571872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("dst mapping error", total_tests, 572872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams src_off, dst_off, len, ret); 573afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko failed_tests++; 574afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko continue; 575afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko } 5764076e755dbec078c85352a8f77cec4c10181da4eDan Williams um->bidi_cnt++; 577b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 578b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 579b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (thread->type == DMA_MEMCPY) 580b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams tx = dev->device_prep_dma_memcpy(chan, 5814076e755dbec078c85352a8f77cec4c10181da4eDan Williams dsts[0] + dst_off, 5824076e755dbec078c85352a8f77cec4c10181da4eDan Williams srcs[0], len, flags); 583b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else if (thread->type == DMA_XOR) 584b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams tx = dev->device_prep_dma_xor(chan, 5854076e755dbec078c85352a8f77cec4c10181da4eDan Williams dsts[0] + dst_off, 5864076e755dbec078c85352a8f77cec4c10181da4eDan Williams srcs, src_cnt, 587b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams len, flags); 58858691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams else if (thread->type == DMA_PQ) { 58958691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams dma_addr_t dma_pq[dst_cnt]; 59058691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams 59158691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams for (i = 0; i < dst_cnt; i++) 5924076e755dbec078c85352a8f77cec4c10181da4eDan Williams dma_pq[i] = dsts[i] + dst_off; 5934076e755dbec078c85352a8f77cec4c10181da4eDan Williams tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, 59494de648d72c8bc833590523f22386d4babbea988Anatolij Gustschin src_cnt, pq_coefs, 59558691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams len, flags); 59658691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams } 597d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto 598d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto if (!tx) { 5994076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 600872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("prep error", total_tests, src_off, 601872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dst_off, len, ret); 602d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto msleep(100); 603d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto failed_tests++; 604d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto continue; 605d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto } 606e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 607adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo done.done = false; 608e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams tx->callback = dmatest_callback; 609adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo tx->callback_param = &done; 610d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto cookie = tx->tx_submit(tx); 611d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto 6124a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (dma_submit_error(cookie)) { 6134076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 614872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("submit error", total_tests, src_off, 615872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dst_off, len, ret); 6164a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen msleep(100); 6174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen failed_tests++; 6184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen continue; 6194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 620b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_async_issue_pending(chan); 6214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 622bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko wait_event_freezable_timeout(done_wait, done.done, 62315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko msecs_to_jiffies(params->timeout)); 624981ed70d8e4faf3689dbf3c48868a31d5b004d7aGuennadi Liakhovetski 625e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 6264a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 627adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo if (!done.done) { 628adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo /* 629adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * We're leaving the timed out dma operation with 630adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * dangling pointer to done_wait. To make this 631adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * correct, we'll need to allocate wait_done for 632adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * each test iteration and perform "who's gonna 633adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * free it this time?" dancing. For now, just 634adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * leave it dangling. 635adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo */ 6364076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 637872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("test timed out", total_tests, src_off, dst_off, 638872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams len, 0); 639e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams failed_tests++; 640e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams continue; 64119e9f99f273bdefef8c88465933907e242a40d66Vinod Koul } else if (status != DMA_COMPLETE) { 6424076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 643872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result(status == DMA_ERROR ? 644872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams "completion error status" : 645872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams "completion busy status", total_tests, src_off, 646872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dst_off, len, ret); 6474a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen failed_tests++; 6484a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen continue; 6494a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 650e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 6514076e755dbec078c85352a8f77cec4c10181da4eDan Williams dmaengine_unmap_put(um); 6524a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 653e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams if (params->noverify) { 65450137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams verbose_result("test passed", total_tests, src_off, 65550137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams dst_off, len, 0); 656e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams continue; 657e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams } 6584a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 659872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_debug("%s: verifying source buffer...\n", current->comm); 660e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams error_count = dmatest_verify(thread->srcs, 0, src_off, 6614a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 0, PATTERN_SRC, true); 6627b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->srcs, src_off, 6637b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams src_off + len, src_off, 6647b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_SRC | PATTERN_COPY, true); 6657b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->srcs, src_off + len, 6667b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams params->buf_size, src_off + len, 6677b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_SRC, true); 6687b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams 669872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_debug("%s: verifying dest buffer...\n", current->comm); 6707b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->dsts, 0, dst_off, 6714a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 0, PATTERN_DST, false); 6727b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->dsts, dst_off, 6737b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams dst_off + len, src_off, 6747b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_SRC | PATTERN_COPY, false); 6757b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->dsts, dst_off + len, 6767b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams params->buf_size, dst_off + len, 6777b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_DST, false); 6784a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 6794a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (error_count) { 680872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("data error", total_tests, src_off, dst_off, 681872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams len, error_count); 6824a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen failed_tests++; 6834a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } else { 68450137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams verbose_result("test passed", total_tests, src_off, 68550137a7df982f3767fe0b3b0cd0b9cfaf09c2cd9Dan Williams dst_off, len, 0); 6864a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 6874a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 68886727443a04fdb25397041188efd2527f2b7237bDan Williams runtime = ktime_us_delta(ktime_get(), ktime); 6894a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 6904a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen ret = 0; 6918e1f50d7433b9a6e52cb296943f53c3ce8189ea5Andy Shevchenkoerr_dstbuf: 692b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; thread->dsts[i]; i++) 693b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->dsts[i]); 694b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->dsts); 695b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williamserr_dsts: 6968e1f50d7433b9a6e52cb296943f53c3ce8189ea5Andy Shevchenkoerr_srcbuf: 697b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; thread->srcs[i]; i++) 698b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->srcs[i]); 699b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->srcs); 700b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williamserr_srcs: 701945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko kfree(pq_coefs); 702945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenkoerr_thread_type: 70386727443a04fdb25397041188efd2527f2b7237bDan Williams pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n", 70486727443a04fdb25397041188efd2527f2b7237bDan Williams current->comm, total_tests, failed_tests, 70586727443a04fdb25397041188efd2527f2b7237bDan Williams dmatest_persec(runtime, total_tests), 70686727443a04fdb25397041188efd2527f2b7237bDan Williams dmatest_KBs(runtime, total_len), ret); 7070a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre 7089704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar /* terminate all transfers on specified channels */ 7095e034f7b659be9d94e64aaaa985ab530dd847fdbShiraz Hashim if (ret) 7105e034f7b659be9d94e64aaaa985ab530dd847fdbShiraz Hashim dmaengine_terminate_all(chan); 7115e034f7b659be9d94e64aaaa985ab530dd847fdbShiraz Hashim 7123e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko thread->done = true; 7132d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams wake_up(&thread_wait); 7140a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre 7154a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return ret; 7164a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 7174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 7184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic void dmatest_cleanup_channel(struct dmatest_chan *dtc) 7194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 7204a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dmatest_thread *thread; 7214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dmatest_thread *_thread; 7224a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen int ret; 7234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 7244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 7254a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen ret = kthread_stop(thread->task); 7260adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_debug("thread %s exited with status %d\n", 7270adff800662f52d0ffc3e420db231769cb3fff13Dan Williams thread->task->comm, ret); 7284a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen list_del(&thread->node); 7292d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams put_task_struct(thread->task); 7304a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen kfree(thread); 7314a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 7329704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar 7339704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar /* terminate all transfers on specified channels */ 734944ea4dd38b8575e30a5699633c81945bff1864dJon Mason dmaengine_terminate_all(dtc->chan); 7359704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar 7364a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen kfree(dtc); 7374a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 7384a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 739e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic int dmatest_add_threads(struct dmatest_info *info, 740e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_chan *dtc, enum dma_transaction_type type) 7414a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 74215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params *params = &info->params; 743b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dmatest_thread *thread; 744b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dma_chan *chan = dtc->chan; 745b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams char *op; 746b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams unsigned int i; 7474a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 748b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (type == DMA_MEMCPY) 749b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams op = "copy"; 750b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else if (type == DMA_XOR) 751b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams op = "xor"; 75258691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams else if (type == DMA_PQ) 75358691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams op = "pq"; 754b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else 755b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams return -EINVAL; 7564a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 75715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko for (i = 0; i < params->threads_per_chan; i++) { 7584a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 7594a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (!thread) { 7600adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_warn("No memory for %s-%s%u\n", 7610adff800662f52d0ffc3e420db231769cb3fff13Dan Williams dma_chan_name(chan), op, i); 7624a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen break; 7634a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 764e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko thread->info = info; 7654a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen thread->chan = dtc->chan; 766b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->type = type; 7674a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen smp_wmb(); 7682d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", 769b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_chan_name(chan), op, i); 7704a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (IS_ERR(thread->task)) { 7712d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams pr_warn("Failed to create thread %s-%s%u\n", 7720adff800662f52d0ffc3e420db231769cb3fff13Dan Williams dma_chan_name(chan), op, i); 7734a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen kfree(thread); 7744a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen break; 7754a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 7764a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 7774a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen /* srcbuf and dstbuf are allocated by the thread itself */ 7782d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams get_task_struct(thread->task); 7794a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen list_add_tail(&thread->node, &dtc->threads); 7802d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams wake_up_process(thread->task); 7814a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 7824a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 783b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams return i; 784b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams} 785b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 786e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic int dmatest_add_channel(struct dmatest_info *info, 787e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dma_chan *chan) 788b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams{ 789b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dmatest_chan *dtc; 790b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dma_device *dma_dev = chan->device; 791b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams unsigned int thread_count = 0; 792b9033e682e86f3c6a66763f9b6a3935c5c64e145Kulikov Vasiliy int cnt; 793b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 794b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 795b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!dtc) { 7960adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_warn("No memory for %s\n", dma_chan_name(chan)); 797b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams return -ENOMEM; 798b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 799b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 800b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dtc->chan = chan; 801b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams INIT_LIST_HEAD(&dtc->threads); 802b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 803b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 804e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); 805f1aef8b6e6abf32a3a269542f95a19e2cb319f6cNicolas Ferre thread_count += cnt > 0 ? cnt : 0; 806b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 807b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 808e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko cnt = dmatest_add_threads(info, dtc, DMA_XOR); 809f1aef8b6e6abf32a3a269542f95a19e2cb319f6cNicolas Ferre thread_count += cnt > 0 ? cnt : 0; 810b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 81158691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 812e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko cnt = dmatest_add_threads(info, dtc, DMA_PQ); 813d07a74a546981a09ba490936645fbf0d1340b96cDr. David Alan Gilbert thread_count += cnt > 0 ? cnt : 0; 81458691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams } 815b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 8160adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_info("Started %u threads using %s\n", 817b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread_count, dma_chan_name(chan)); 8184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 819838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko list_add_tail(&dtc->node, &info->channels); 820838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko info->nr_channels++; 8214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 82233df8ca068123457db56c316946a3c0e4ef787d6Dan Williams return 0; 8234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 8244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 8257dd602510128d7a64b11ff3b7d4f30ac8e3946ceDan Williamsstatic bool filter(struct dma_chan *chan, void *param) 8264a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 82715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params *params = param; 828e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 82915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (!dmatest_match_channel(params, chan) || 83015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko !dmatest_match_device(params, chan->device)) 8317dd602510128d7a64b11ff3b7d4f30ac8e3946ceDan Williams return false; 83233df8ca068123457db56c316946a3c0e4ef787d6Dan Williams else 8337dd602510128d7a64b11ff3b7d4f30ac8e3946ceDan Williams return true; 8344a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 8354a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 836a9e554957de406d6adc581731f571b8a1503f6b0Dan Williamsstatic void request_channels(struct dmatest_info *info, 837a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams enum dma_transaction_type type) 8384a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 83933df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dma_cap_mask_t mask; 84033df8ca068123457db56c316946a3c0e4ef787d6Dan Williams 84133df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dma_cap_zero(mask); 842a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams dma_cap_set(type, mask); 84333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams for (;;) { 844a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams struct dmatest_params *params = &info->params; 845a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams struct dma_chan *chan; 846a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams 84715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko chan = dma_request_channel(mask, filter, params); 84833df8ca068123457db56c316946a3c0e4ef787d6Dan Williams if (chan) { 849a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams if (dmatest_add_channel(info, chan)) { 85033df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dma_release_channel(chan); 85133df8ca068123457db56c316946a3c0e4ef787d6Dan Williams break; /* add_channel failed, punt */ 85233df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } 85333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } else 85433df8ca068123457db56c316946a3c0e4ef787d6Dan Williams break; /* no more channels available */ 85515b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->max_channels && 85615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko info->nr_channels >= params->max_channels) 85733df8ca068123457db56c316946a3c0e4ef787d6Dan Williams break; /* we have all we need */ 85833df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } 8594a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 8604a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 861a9e554957de406d6adc581731f571b8a1503f6b0Dan Williamsstatic void run_threaded_test(struct dmatest_info *info) 862851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko{ 863a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams struct dmatest_params *params = &info->params; 864851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 865a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams /* Copy test parameters */ 866a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->buf_size = test_buf_size; 867a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); 868a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams strlcpy(params->device, strim(test_device), sizeof(params->device)); 869a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->threads_per_chan = threads_per_chan; 870a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->max_channels = max_channels; 871a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->iterations = iterations; 872a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->xor_sources = xor_sources; 873a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->pq_sources = pq_sources; 874a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->timeout = timeout; 875e3b9c347316fe243bea6abd08681050c43ca22eeDan Williams params->noverify = noverify; 876a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams 877a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams request_channels(info, DMA_MEMCPY); 878a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams request_channels(info, DMA_XOR); 879a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams request_channels(info, DMA_PQ); 880851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 881851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 882a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic void stop_threaded_test(struct dmatest_info *info) 8834a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 88433df8ca068123457db56c316946a3c0e4ef787d6Dan Williams struct dmatest_chan *dtc, *_dtc; 8857cbd4877e5b167b56a3d6033b926a9f925186e12Dan Williams struct dma_chan *chan; 88633df8ca068123457db56c316946a3c0e4ef787d6Dan Williams 887838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { 88833df8ca068123457db56c316946a3c0e4ef787d6Dan Williams list_del(&dtc->node); 8897cbd4877e5b167b56a3d6033b926a9f925186e12Dan Williams chan = dtc->chan; 89033df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dmatest_cleanup_channel(dtc); 8910adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_debug("dropped channel %s\n", dma_chan_name(chan)); 8927cbd4877e5b167b56a3d6033b926a9f925186e12Dan Williams dma_release_channel(chan); 89333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } 894838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko 895838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko info->nr_channels = 0; 8964a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 897e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 898a9e554957de406d6adc581731f571b8a1503f6b0Dan Williamsstatic void restart_threaded_test(struct dmatest_info *info, bool run) 899851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko{ 900a310d037b8d06755c62bb4878c00d19490af5550Dan Williams /* we might be called early to set run=, defer running until all 901a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * parameters have been evaluated 902a310d037b8d06755c62bb4878c00d19490af5550Dan Williams */ 903a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (!info->did_init) 904a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams return; 905851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 906851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko /* Stop any running test first */ 907a310d037b8d06755c62bb4878c00d19490af5550Dan Williams stop_threaded_test(info); 908851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 909851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko /* Run test with new parameters */ 910a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams run_threaded_test(info); 911851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 912851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 913a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_get(char *val, const struct kernel_param *kp) 914851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko{ 915a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dmatest_info *info = &test_info; 916851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 917851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko mutex_lock(&info->lock); 918a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (is_threaded_test_run(info)) { 919a310d037b8d06755c62bb4878c00d19490af5550Dan Williams dmatest_run = true; 9203e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko } else { 921a310d037b8d06755c62bb4878c00d19490af5550Dan Williams stop_threaded_test(info); 922a310d037b8d06755c62bb4878c00d19490af5550Dan Williams dmatest_run = false; 9233e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko } 924851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko mutex_unlock(&info->lock); 925851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 926a310d037b8d06755c62bb4878c00d19490af5550Dan Williams return param_get_bool(val, kp); 927851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 928851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 929a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_set(const char *val, const struct kernel_param *kp) 93095019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko{ 931a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dmatest_info *info = &test_info; 932a310d037b8d06755c62bb4878c00d19490af5550Dan Williams int ret; 93395019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 934a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_lock(&info->lock); 935a310d037b8d06755c62bb4878c00d19490af5550Dan Williams ret = param_set_bool(val, kp); 936a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (ret) { 937851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko mutex_unlock(&info->lock); 938a310d037b8d06755c62bb4878c00d19490af5550Dan Williams return ret; 93995019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko } 94095019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 941a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (is_threaded_test_run(info)) 942a310d037b8d06755c62bb4878c00d19490af5550Dan Williams ret = -EBUSY; 943a310d037b8d06755c62bb4878c00d19490af5550Dan Williams else if (dmatest_run) 944a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams restart_threaded_test(info, dmatest_run); 945851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 946a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_unlock(&info->lock); 947851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 948a310d037b8d06755c62bb4878c00d19490af5550Dan Williams return ret; 949851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 950851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 951e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic int __init dmatest_init(void) 952e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko{ 953e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_info *info = &test_info; 9542d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams struct dmatest_params *params = &info->params; 955e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 956a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (dmatest_run) { 957a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_lock(&info->lock); 958a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams run_threaded_test(info); 959a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_unlock(&info->lock); 960a310d037b8d06755c62bb4878c00d19490af5550Dan Williams } 961838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko 9622d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams if (params->iterations && wait) 9632d88ce76eb98c4ac4411dcb299cf61ca8999d2b9Dan Williams wait_event(thread_wait, !is_threaded_test_run(info)); 96495019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 965a310d037b8d06755c62bb4878c00d19490af5550Dan Williams /* module parameters are stable, inittime tests are started, 966a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * let userspace take over 'run' control 967a310d037b8d06755c62bb4878c00d19490af5550Dan Williams */ 968a310d037b8d06755c62bb4878c00d19490af5550Dan Williams info->did_init = true; 969851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 970851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko return 0; 971e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko} 972e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko/* when compiled-in wait for drivers to load first */ 973e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkolate_initcall(dmatest_init); 974e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 975e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic void __exit dmatest_exit(void) 976e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko{ 977e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_info *info = &test_info; 978e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 979a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_lock(&info->lock); 980e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko stop_threaded_test(info); 981a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_unlock(&info->lock); 982e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko} 9834a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenmodule_exit(dmatest_exit); 9844a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 985e05503ef1186ad33dfe56794407891eb1dd93ef6Jean DelvareMODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 9864a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_LICENSE("GPL v2"); 987