1/* 2 * GPL HEADER START 3 * 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 only, 8 * as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License version 2 for more details (a copy is included 14 * in the LICENSE file that accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License 17 * version 2 along with this program; If not, see 18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf 19 * 20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, 21 * CA 95054 USA or visit www.sun.com if you need additional information or 22 * have any questions. 23 * 24 * GPL HEADER END 25 */ 26/* 27 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Use is subject to license terms. 29 * 30 * Copyright (c) 2012, Intel Corporation. 31 */ 32/* 33 * This file is part of Lustre, http://www.lustre.org/ 34 * Lustre is a trademark of Sun Microsystems, Inc. 35 * 36 * lnet/lnet/lib-md.c 37 * 38 * Memory Descriptor management routines 39 */ 40 41#define DEBUG_SUBSYSTEM S_LNET 42 43#include "../../include/linux/lnet/lib-lnet.h" 44 45/* must be called with lnet_res_lock held */ 46void 47lnet_md_unlink(lnet_libmd_t *md) 48{ 49 if ((md->md_flags & LNET_MD_FLAG_ZOMBIE) == 0) { 50 /* first unlink attempt... */ 51 lnet_me_t *me = md->md_me; 52 53 md->md_flags |= LNET_MD_FLAG_ZOMBIE; 54 55 /* Disassociate from ME (if any), 56 * and unlink it if it was created 57 * with LNET_UNLINK */ 58 if (me != NULL) { 59 /* detach MD from portal */ 60 lnet_ptl_detach_md(me, md); 61 if (me->me_unlink == LNET_UNLINK) 62 lnet_me_unlink(me); 63 } 64 65 /* ensure all future handle lookups fail */ 66 lnet_res_lh_invalidate(&md->md_lh); 67 } 68 69 if (md->md_refcount != 0) { 70 CDEBUG(D_NET, "Queueing unlink of md %p\n", md); 71 return; 72 } 73 74 CDEBUG(D_NET, "Unlinking md %p\n", md); 75 76 if (md->md_eq != NULL) { 77 int cpt = lnet_cpt_of_cookie(md->md_lh.lh_cookie); 78 79 LASSERT(*md->md_eq->eq_refs[cpt] > 0); 80 (*md->md_eq->eq_refs[cpt])--; 81 } 82 83 LASSERT(!list_empty(&md->md_list)); 84 list_del_init(&md->md_list); 85 lnet_md_free_locked(md); 86} 87 88static int 89lnet_md_build(lnet_libmd_t *lmd, lnet_md_t *umd, int unlink) 90{ 91 int i; 92 unsigned int niov; 93 int total_length = 0; 94 95 lmd->md_me = NULL; 96 lmd->md_start = umd->start; 97 lmd->md_offset = 0; 98 lmd->md_max_size = umd->max_size; 99 lmd->md_options = umd->options; 100 lmd->md_user_ptr = umd->user_ptr; 101 lmd->md_eq = NULL; 102 lmd->md_threshold = umd->threshold; 103 lmd->md_refcount = 0; 104 lmd->md_flags = (unlink == LNET_UNLINK) ? LNET_MD_FLAG_AUTO_UNLINK : 0; 105 106 if ((umd->options & LNET_MD_IOVEC) != 0) { 107 108 if ((umd->options & LNET_MD_KIOV) != 0) /* Can't specify both */ 109 return -EINVAL; 110 111 lmd->md_niov = niov = umd->length; 112 memcpy(lmd->md_iov.iov, umd->start, 113 niov * sizeof(lmd->md_iov.iov[0])); 114 115 for (i = 0; i < (int)niov; i++) { 116 /* We take the base address on trust */ 117 /* invalid length */ 118 if (lmd->md_iov.iov[i].iov_len <= 0) 119 return -EINVAL; 120 121 total_length += lmd->md_iov.iov[i].iov_len; 122 } 123 124 lmd->md_length = total_length; 125 126 if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ 127 (umd->max_size < 0 || 128 umd->max_size > total_length)) /* illegal max_size */ 129 return -EINVAL; 130 131 } else if ((umd->options & LNET_MD_KIOV) != 0) { 132 lmd->md_niov = niov = umd->length; 133 memcpy(lmd->md_iov.kiov, umd->start, 134 niov * sizeof(lmd->md_iov.kiov[0])); 135 136 for (i = 0; i < (int)niov; i++) { 137 /* We take the page pointer on trust */ 138 if (lmd->md_iov.kiov[i].kiov_offset + 139 lmd->md_iov.kiov[i].kiov_len > PAGE_CACHE_SIZE) 140 return -EINVAL; /* invalid length */ 141 142 total_length += lmd->md_iov.kiov[i].kiov_len; 143 } 144 145 lmd->md_length = total_length; 146 147 if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ 148 (umd->max_size < 0 || 149 umd->max_size > total_length)) /* illegal max_size */ 150 return -EINVAL; 151 } else { /* contiguous */ 152 lmd->md_length = umd->length; 153 lmd->md_niov = niov = 1; 154 lmd->md_iov.iov[0].iov_base = umd->start; 155 lmd->md_iov.iov[0].iov_len = umd->length; 156 157 if ((umd->options & LNET_MD_MAX_SIZE) != 0 && /* max size used */ 158 (umd->max_size < 0 || 159 umd->max_size > (int)umd->length)) /* illegal max_size */ 160 return -EINVAL; 161 } 162 163 return 0; 164} 165 166/* must be called with resource lock held */ 167static int 168lnet_md_link(lnet_libmd_t *md, lnet_handle_eq_t eq_handle, int cpt) 169{ 170 struct lnet_res_container *container = the_lnet.ln_md_containers[cpt]; 171 172 /* NB we are passed an allocated, but inactive md. 173 * if we return success, caller may lnet_md_unlink() it. 174 * otherwise caller may only lnet_md_free() it. 175 */ 176 /* This implementation doesn't know how to create START events or 177 * disable END events. Best to LASSERT our caller is compliant so 178 * we find out quickly... */ 179 /* TODO - reevaluate what should be here in light of 180 * the removal of the start and end events 181 * maybe there we shouldn't even allow LNET_EQ_NONE!) 182 * LASSERT (eq == NULL); 183 */ 184 if (!LNetHandleIsInvalid(eq_handle)) { 185 md->md_eq = lnet_handle2eq(&eq_handle); 186 187 if (md->md_eq == NULL) 188 return -ENOENT; 189 190 (*md->md_eq->eq_refs[cpt])++; 191 } 192 193 lnet_res_lh_initialize(container, &md->md_lh); 194 195 LASSERT(list_empty(&md->md_list)); 196 list_add(&md->md_list, &container->rec_active); 197 198 return 0; 199} 200 201/* must be called with lnet_res_lock held */ 202void 203lnet_md_deconstruct(lnet_libmd_t *lmd, lnet_md_t *umd) 204{ 205 /* NB this doesn't copy out all the iov entries so when a 206 * discontiguous MD is copied out, the target gets to know the 207 * original iov pointer (in start) and the number of entries it had 208 * and that's all. 209 */ 210 umd->start = lmd->md_start; 211 umd->length = ((lmd->md_options & 212 (LNET_MD_IOVEC | LNET_MD_KIOV)) == 0) ? 213 lmd->md_length : lmd->md_niov; 214 umd->threshold = lmd->md_threshold; 215 umd->max_size = lmd->md_max_size; 216 umd->options = lmd->md_options; 217 umd->user_ptr = lmd->md_user_ptr; 218 lnet_eq2handle(&umd->eq_handle, lmd->md_eq); 219} 220 221int 222lnet_md_validate(lnet_md_t *umd) 223{ 224 if (umd->start == NULL && umd->length != 0) { 225 CERROR("MD start pointer can not be NULL with length %u\n", 226 umd->length); 227 return -EINVAL; 228 } 229 230 if ((umd->options & (LNET_MD_KIOV | LNET_MD_IOVEC)) != 0 && 231 umd->length > LNET_MAX_IOV) { 232 CERROR("Invalid option: too many fragments %u, %d max\n", 233 umd->length, LNET_MAX_IOV); 234 return -EINVAL; 235 } 236 237 return 0; 238} 239 240/** 241 * Create a memory descriptor and attach it to a ME 242 * 243 * \param meh A handle for a ME to associate the new MD with. 244 * \param umd Provides initial values for the user-visible parts of a MD. 245 * Other than its use for initialization, there is no linkage between this 246 * structure and the MD maintained by the LNet. 247 * \param unlink A flag to indicate whether the MD is automatically unlinked 248 * when it becomes inactive, either because the operation threshold drops to 249 * zero or because the available memory becomes less than \a umd.max_size. 250 * (Note that the check for unlinking a MD only occurs after the completion 251 * of a successful operation on the MD.) The value LNET_UNLINK enables auto 252 * unlinking; the value LNET_RETAIN disables it. 253 * \param handle On successful returns, a handle to the newly created MD is 254 * saved here. This handle can be used later in LNetMDUnlink(). 255 * 256 * \retval 0 On success. 257 * \retval -EINVAL If \a umd is not valid. 258 * \retval -ENOMEM If new MD cannot be allocated. 259 * \retval -ENOENT Either \a meh or \a umd.eq_handle does not point to a 260 * valid object. Note that it's OK to supply a NULL \a umd.eq_handle by 261 * calling LNetInvalidateHandle() on it. 262 * \retval -EBUSY If the ME pointed to by \a meh is already associated with 263 * a MD. 264 */ 265int 266LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd, 267 lnet_unlink_t unlink, lnet_handle_md_t *handle) 268{ 269 LIST_HEAD(matches); 270 LIST_HEAD(drops); 271 struct lnet_me *me; 272 struct lnet_libmd *md; 273 int cpt; 274 int rc; 275 276 LASSERT(the_lnet.ln_init); 277 LASSERT(the_lnet.ln_refcount > 0); 278 279 if (lnet_md_validate(&umd) != 0) 280 return -EINVAL; 281 282 if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) == 0) { 283 CERROR("Invalid option: no MD_OP set\n"); 284 return -EINVAL; 285 } 286 287 md = lnet_md_alloc(&umd); 288 if (md == NULL) 289 return -ENOMEM; 290 291 rc = lnet_md_build(md, &umd, unlink); 292 cpt = lnet_cpt_of_cookie(meh.cookie); 293 294 lnet_res_lock(cpt); 295 if (rc != 0) 296 goto failed; 297 298 me = lnet_handle2me(&meh); 299 if (me == NULL) 300 rc = -ENOENT; 301 else if (me->me_md != NULL) 302 rc = -EBUSY; 303 else 304 rc = lnet_md_link(md, umd.eq_handle, cpt); 305 306 if (rc != 0) 307 goto failed; 308 309 /* attach this MD to portal of ME and check if it matches any 310 * blocked msgs on this portal */ 311 lnet_ptl_attach_md(me, md, &matches, &drops); 312 313 lnet_md2handle(handle, md); 314 315 lnet_res_unlock(cpt); 316 317 lnet_drop_delayed_msg_list(&drops, "Bad match"); 318 lnet_recv_delayed_msg_list(&matches); 319 320 return 0; 321 322 failed: 323 lnet_md_free_locked(md); 324 325 lnet_res_unlock(cpt); 326 return rc; 327} 328EXPORT_SYMBOL(LNetMDAttach); 329 330/** 331 * Create a "free floating" memory descriptor - a MD that is not associated 332 * with a ME. Such MDs are usually used in LNetPut() and LNetGet() operations. 333 * 334 * \param umd,unlink See the discussion for LNetMDAttach(). 335 * \param handle On successful returns, a handle to the newly created MD is 336 * saved here. This handle can be used later in LNetMDUnlink(), LNetPut(), 337 * and LNetGet() operations. 338 * 339 * \retval 0 On success. 340 * \retval -EINVAL If \a umd is not valid. 341 * \retval -ENOMEM If new MD cannot be allocated. 342 * \retval -ENOENT \a umd.eq_handle does not point to a valid EQ. Note that 343 * it's OK to supply a NULL \a umd.eq_handle by calling 344 * LNetInvalidateHandle() on it. 345 */ 346int 347LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle) 348{ 349 lnet_libmd_t *md; 350 int cpt; 351 int rc; 352 353 LASSERT(the_lnet.ln_init); 354 LASSERT(the_lnet.ln_refcount > 0); 355 356 if (lnet_md_validate(&umd) != 0) 357 return -EINVAL; 358 359 if ((umd.options & (LNET_MD_OP_GET | LNET_MD_OP_PUT)) != 0) { 360 CERROR("Invalid option: GET|PUT illegal on active MDs\n"); 361 return -EINVAL; 362 } 363 364 md = lnet_md_alloc(&umd); 365 if (md == NULL) 366 return -ENOMEM; 367 368 rc = lnet_md_build(md, &umd, unlink); 369 370 cpt = lnet_res_lock_current(); 371 if (rc != 0) 372 goto failed; 373 374 rc = lnet_md_link(md, umd.eq_handle, cpt); 375 if (rc != 0) 376 goto failed; 377 378 lnet_md2handle(handle, md); 379 380 lnet_res_unlock(cpt); 381 return 0; 382 383 failed: 384 lnet_md_free_locked(md); 385 386 lnet_res_unlock(cpt); 387 return rc; 388} 389EXPORT_SYMBOL(LNetMDBind); 390 391/** 392 * Unlink the memory descriptor from any ME it may be linked to and release 393 * the internal resources associated with it. As a result, active messages 394 * associated with the MD may get aborted. 395 * 396 * This function does not free the memory region associated with the MD; 397 * i.e., the memory the user allocated for this MD. If the ME associated with 398 * this MD is not NULL and was created with auto unlink enabled, the ME is 399 * unlinked as well (see LNetMEAttach()). 400 * 401 * Explicitly unlinking a MD via this function call has the same behavior as 402 * a MD that has been automatically unlinked, except that no LNET_EVENT_UNLINK 403 * is generated in the latter case. 404 * 405 * An unlinked event can be reported in two ways: 406 * - If there's no pending operations on the MD, it's unlinked immediately 407 * and an LNET_EVENT_UNLINK event is logged before this function returns. 408 * - Otherwise, the MD is only marked for deletion when this function 409 * returns, and the unlinked event will be piggybacked on the event of 410 * the completion of the last operation by setting the unlinked field of 411 * the event. No dedicated LNET_EVENT_UNLINK event is generated. 412 * 413 * Note that in both cases the unlinked field of the event is always set; no 414 * more event will happen on the MD after such an event is logged. 415 * 416 * \param mdh A handle for the MD to be unlinked. 417 * 418 * \retval 0 On success. 419 * \retval -ENOENT If \a mdh does not point to a valid MD object. 420 */ 421int 422LNetMDUnlink(lnet_handle_md_t mdh) 423{ 424 lnet_event_t ev; 425 lnet_libmd_t *md; 426 int cpt; 427 428 LASSERT(the_lnet.ln_init); 429 LASSERT(the_lnet.ln_refcount > 0); 430 431 cpt = lnet_cpt_of_cookie(mdh.cookie); 432 lnet_res_lock(cpt); 433 434 md = lnet_handle2md(&mdh); 435 if (md == NULL) { 436 lnet_res_unlock(cpt); 437 return -ENOENT; 438 } 439 440 md->md_flags |= LNET_MD_FLAG_ABORTED; 441 /* If the MD is busy, lnet_md_unlink just marks it for deletion, and 442 * when the LND is done, the completion event flags that the MD was 443 * unlinked. Otherwise, we enqueue an event now... */ 444 if (md->md_eq != NULL && md->md_refcount == 0) { 445 lnet_build_unlink_event(md, &ev); 446 lnet_eq_enqueue_event(md->md_eq, &ev); 447 } 448 449 lnet_md_unlink(md); 450 451 lnet_res_unlock(cpt); 452 return 0; 453} 454EXPORT_SYMBOL(LNetMDUnlink); 455