1/* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#include <linux/errno.h> 35#include <linux/slab.h> 36#include <linux/mm.h> 37#include <linux/export.h> 38#include <linux/bitmap.h> 39#include <linux/dma-mapping.h> 40#include <linux/vmalloc.h> 41 42#include "mlx4.h" 43 44u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 45{ 46 u32 obj; 47 48 spin_lock(&bitmap->lock); 49 50 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 51 if (obj >= bitmap->max) { 52 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 53 & bitmap->mask; 54 obj = find_first_zero_bit(bitmap->table, bitmap->max); 55 } 56 57 if (obj < bitmap->max) { 58 set_bit(obj, bitmap->table); 59 bitmap->last = (obj + 1); 60 if (bitmap->last == bitmap->max) 61 bitmap->last = 0; 62 obj |= bitmap->top; 63 } else 64 obj = -1; 65 66 if (obj != -1) 67 --bitmap->avail; 68 69 spin_unlock(&bitmap->lock); 70 71 return obj; 72} 73 74void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 75{ 76 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 77} 78 79u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align) 80{ 81 u32 obj; 82 83 if (likely(cnt == 1 && align == 1)) 84 return mlx4_bitmap_alloc(bitmap); 85 86 spin_lock(&bitmap->lock); 87 88 obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 89 bitmap->last, cnt, align - 1); 90 if (obj >= bitmap->max) { 91 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 92 & bitmap->mask; 93 obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 94 0, cnt, align - 1); 95 } 96 97 if (obj < bitmap->max) { 98 bitmap_set(bitmap->table, obj, cnt); 99 if (obj == bitmap->last) { 100 bitmap->last = (obj + cnt); 101 if (bitmap->last >= bitmap->max) 102 bitmap->last = 0; 103 } 104 obj |= bitmap->top; 105 } else 106 obj = -1; 107 108 if (obj != -1) 109 bitmap->avail -= cnt; 110 111 spin_unlock(&bitmap->lock); 112 113 return obj; 114} 115 116u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 117{ 118 return bitmap->avail; 119} 120 121void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 122 int use_rr) 123{ 124 obj &= bitmap->max + bitmap->reserved_top - 1; 125 126 spin_lock(&bitmap->lock); 127 if (!use_rr) { 128 bitmap->last = min(bitmap->last, obj); 129 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 130 & bitmap->mask; 131 } 132 bitmap_clear(bitmap->table, obj, cnt); 133 bitmap->avail += cnt; 134 spin_unlock(&bitmap->lock); 135} 136 137int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 138 u32 reserved_bot, u32 reserved_top) 139{ 140 /* num must be a power of 2 */ 141 if (num != roundup_pow_of_two(num)) 142 return -EINVAL; 143 144 bitmap->last = 0; 145 bitmap->top = 0; 146 bitmap->max = num - reserved_top; 147 bitmap->mask = mask; 148 bitmap->reserved_top = reserved_top; 149 bitmap->avail = num - reserved_top - reserved_bot; 150 spin_lock_init(&bitmap->lock); 151 bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 152 sizeof (long), GFP_KERNEL); 153 if (!bitmap->table) 154 return -ENOMEM; 155 156 bitmap_set(bitmap->table, 0, reserved_bot); 157 158 return 0; 159} 160 161void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 162{ 163 kfree(bitmap->table); 164} 165 166/* 167 * Handling for queue buffers -- we allocate a bunch of memory and 168 * register it in a memory region at HCA virtual address 0. If the 169 * requested size is > max_direct, we split the allocation into 170 * multiple pages, so we don't require too much contiguous memory. 171 */ 172 173int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 174 struct mlx4_buf *buf, gfp_t gfp) 175{ 176 dma_addr_t t; 177 178 if (size <= max_direct) { 179 buf->nbufs = 1; 180 buf->npages = 1; 181 buf->page_shift = get_order(size) + PAGE_SHIFT; 182 buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 183 size, &t, gfp); 184 if (!buf->direct.buf) 185 return -ENOMEM; 186 187 buf->direct.map = t; 188 189 while (t & ((1 << buf->page_shift) - 1)) { 190 --buf->page_shift; 191 buf->npages *= 2; 192 } 193 194 memset(buf->direct.buf, 0, size); 195 } else { 196 int i; 197 198 buf->direct.buf = NULL; 199 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 200 buf->npages = buf->nbufs; 201 buf->page_shift = PAGE_SHIFT; 202 buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 203 gfp); 204 if (!buf->page_list) 205 return -ENOMEM; 206 207 for (i = 0; i < buf->nbufs; ++i) { 208 buf->page_list[i].buf = 209 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 210 &t, gfp); 211 if (!buf->page_list[i].buf) 212 goto err_free; 213 214 buf->page_list[i].map = t; 215 216 memset(buf->page_list[i].buf, 0, PAGE_SIZE); 217 } 218 219 if (BITS_PER_LONG == 64) { 220 struct page **pages; 221 pages = kmalloc(sizeof *pages * buf->nbufs, gfp); 222 if (!pages) 223 goto err_free; 224 for (i = 0; i < buf->nbufs; ++i) 225 pages[i] = virt_to_page(buf->page_list[i].buf); 226 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 227 kfree(pages); 228 if (!buf->direct.buf) 229 goto err_free; 230 } 231 } 232 233 return 0; 234 235err_free: 236 mlx4_buf_free(dev, size, buf); 237 238 return -ENOMEM; 239} 240EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 241 242void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 243{ 244 int i; 245 246 if (buf->nbufs == 1) 247 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 248 buf->direct.map); 249 else { 250 if (BITS_PER_LONG == 64 && buf->direct.buf) 251 vunmap(buf->direct.buf); 252 253 for (i = 0; i < buf->nbufs; ++i) 254 if (buf->page_list[i].buf) 255 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 256 buf->page_list[i].buf, 257 buf->page_list[i].map); 258 kfree(buf->page_list); 259 } 260} 261EXPORT_SYMBOL_GPL(mlx4_buf_free); 262 263static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device, 264 gfp_t gfp) 265{ 266 struct mlx4_db_pgdir *pgdir; 267 268 pgdir = kzalloc(sizeof *pgdir, gfp); 269 if (!pgdir) 270 return NULL; 271 272 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 273 pgdir->bits[0] = pgdir->order0; 274 pgdir->bits[1] = pgdir->order1; 275 pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 276 &pgdir->db_dma, gfp); 277 if (!pgdir->db_page) { 278 kfree(pgdir); 279 return NULL; 280 } 281 282 return pgdir; 283} 284 285static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 286 struct mlx4_db *db, int order) 287{ 288 int o; 289 int i; 290 291 for (o = order; o <= 1; ++o) { 292 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 293 if (i < MLX4_DB_PER_PAGE >> o) 294 goto found; 295 } 296 297 return -ENOMEM; 298 299found: 300 clear_bit(i, pgdir->bits[o]); 301 302 i <<= o; 303 304 if (o > order) 305 set_bit(i ^ 1, pgdir->bits[order]); 306 307 db->u.pgdir = pgdir; 308 db->index = i; 309 db->db = pgdir->db_page + db->index; 310 db->dma = pgdir->db_dma + db->index * 4; 311 db->order = order; 312 313 return 0; 314} 315 316int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp) 317{ 318 struct mlx4_priv *priv = mlx4_priv(dev); 319 struct mlx4_db_pgdir *pgdir; 320 int ret = 0; 321 322 mutex_lock(&priv->pgdir_mutex); 323 324 list_for_each_entry(pgdir, &priv->pgdir_list, list) 325 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 326 goto out; 327 328 pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev), gfp); 329 if (!pgdir) { 330 ret = -ENOMEM; 331 goto out; 332 } 333 334 list_add(&pgdir->list, &priv->pgdir_list); 335 336 /* This should never fail -- we just allocated an empty page: */ 337 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 338 339out: 340 mutex_unlock(&priv->pgdir_mutex); 341 342 return ret; 343} 344EXPORT_SYMBOL_GPL(mlx4_db_alloc); 345 346void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 347{ 348 struct mlx4_priv *priv = mlx4_priv(dev); 349 int o; 350 int i; 351 352 mutex_lock(&priv->pgdir_mutex); 353 354 o = db->order; 355 i = db->index; 356 357 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 358 clear_bit(i ^ 1, db->u.pgdir->order0); 359 ++o; 360 } 361 i >>= o; 362 set_bit(i, db->u.pgdir->bits[o]); 363 364 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 365 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 366 db->u.pgdir->db_page, db->u.pgdir->db_dma); 367 list_del(&db->u.pgdir->list); 368 kfree(db->u.pgdir); 369 } 370 371 mutex_unlock(&priv->pgdir_mutex); 372} 373EXPORT_SYMBOL_GPL(mlx4_db_free); 374 375int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 376 int size, int max_direct) 377{ 378 int err; 379 380 err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL); 381 if (err) 382 return err; 383 384 *wqres->db.db = 0; 385 386 err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL); 387 if (err) 388 goto err_db; 389 390 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 391 &wqres->mtt); 392 if (err) 393 goto err_buf; 394 395 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL); 396 if (err) 397 goto err_mtt; 398 399 return 0; 400 401err_mtt: 402 mlx4_mtt_cleanup(dev, &wqres->mtt); 403err_buf: 404 mlx4_buf_free(dev, size, &wqres->buf); 405err_db: 406 mlx4_db_free(dev, &wqres->db); 407 408 return err; 409} 410EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 411 412void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 413 int size) 414{ 415 mlx4_mtt_cleanup(dev, &wqres->mtt); 416 mlx4_buf_free(dev, size, &wqres->buf); 417 mlx4_db_free(dev, &wqres->db); 418} 419EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 420