1/* 2 * GPL HEADER START 3 * 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 only, 8 * as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License version 2 for more details (a copy is included 14 * in the LICENSE file that accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License 17 * version 2 along with this program; If not, see 18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf 19 * 20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, 21 * CA 95054 USA or visit www.sun.com if you need additional information or 22 * have any questions. 23 * 24 * GPL HEADER END 25 */ 26/* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Use is subject to license terms. 29 * 30 * Copyright (c) 2012, Intel Corporation. 31 */ 32/* 33 * This file is part of Lustre, http://www.lustre.org/ 34 * Lustre is a trademark of Sun Microsystems, Inc. 35 * 36 * Implementation of cl_device and cl_device_type for LOV layer. 37 * 38 * Author: Nikita Danilov <nikita.danilov@sun.com> 39 */ 40 41#define DEBUG_SUBSYSTEM S_LOV 42 43/* class_name2obd() */ 44#include "../include/obd_class.h" 45 46#include "lov_cl_internal.h" 47#include "lov_internal.h" 48 49 50struct kmem_cache *lov_lock_kmem; 51struct kmem_cache *lov_object_kmem; 52struct kmem_cache *lov_thread_kmem; 53struct kmem_cache *lov_session_kmem; 54struct kmem_cache *lov_req_kmem; 55 56struct kmem_cache *lovsub_lock_kmem; 57struct kmem_cache *lovsub_object_kmem; 58struct kmem_cache *lovsub_req_kmem; 59 60struct kmem_cache *lov_lock_link_kmem; 61 62/** Lock class of lov_device::ld_mutex. */ 63struct lock_class_key cl_lov_device_mutex_class; 64 65struct lu_kmem_descr lov_caches[] = { 66 { 67 .ckd_cache = &lov_lock_kmem, 68 .ckd_name = "lov_lock_kmem", 69 .ckd_size = sizeof(struct lov_lock) 70 }, 71 { 72 .ckd_cache = &lov_object_kmem, 73 .ckd_name = "lov_object_kmem", 74 .ckd_size = sizeof(struct lov_object) 75 }, 76 { 77 .ckd_cache = &lov_thread_kmem, 78 .ckd_name = "lov_thread_kmem", 79 .ckd_size = sizeof(struct lov_thread_info) 80 }, 81 { 82 .ckd_cache = &lov_session_kmem, 83 .ckd_name = "lov_session_kmem", 84 .ckd_size = sizeof(struct lov_session) 85 }, 86 { 87 .ckd_cache = &lov_req_kmem, 88 .ckd_name = "lov_req_kmem", 89 .ckd_size = sizeof(struct lov_req) 90 }, 91 { 92 .ckd_cache = &lovsub_lock_kmem, 93 .ckd_name = "lovsub_lock_kmem", 94 .ckd_size = sizeof(struct lovsub_lock) 95 }, 96 { 97 .ckd_cache = &lovsub_object_kmem, 98 .ckd_name = "lovsub_object_kmem", 99 .ckd_size = sizeof(struct lovsub_object) 100 }, 101 { 102 .ckd_cache = &lovsub_req_kmem, 103 .ckd_name = "lovsub_req_kmem", 104 .ckd_size = sizeof(struct lovsub_req) 105 }, 106 { 107 .ckd_cache = &lov_lock_link_kmem, 108 .ckd_name = "lov_lock_link_kmem", 109 .ckd_size = sizeof(struct lov_lock_link) 110 }, 111 { 112 .ckd_cache = NULL 113 } 114}; 115 116/***************************************************************************** 117 * 118 * Lov transfer operations. 119 * 120 */ 121 122static void lov_req_completion(const struct lu_env *env, 123 const struct cl_req_slice *slice, int ioret) 124{ 125 struct lov_req *lr; 126 127 lr = cl2lov_req(slice); 128 OBD_SLAB_FREE_PTR(lr, lov_req_kmem); 129} 130 131static const struct cl_req_operations lov_req_ops = { 132 .cro_completion = lov_req_completion 133}; 134 135/***************************************************************************** 136 * 137 * Lov device and device type functions. 138 * 139 */ 140 141static void *lov_key_init(const struct lu_context *ctx, 142 struct lu_context_key *key) 143{ 144 struct lov_thread_info *info; 145 146 OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, GFP_NOFS); 147 if (info != NULL) 148 INIT_LIST_HEAD(&info->lti_closure.clc_list); 149 else 150 info = ERR_PTR(-ENOMEM); 151 return info; 152} 153 154static void lov_key_fini(const struct lu_context *ctx, 155 struct lu_context_key *key, void *data) 156{ 157 struct lov_thread_info *info = data; 158 LINVRNT(list_empty(&info->lti_closure.clc_list)); 159 OBD_SLAB_FREE_PTR(info, lov_thread_kmem); 160} 161 162struct lu_context_key lov_key = { 163 .lct_tags = LCT_CL_THREAD, 164 .lct_init = lov_key_init, 165 .lct_fini = lov_key_fini 166}; 167 168static void *lov_session_key_init(const struct lu_context *ctx, 169 struct lu_context_key *key) 170{ 171 struct lov_session *info; 172 173 OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, GFP_NOFS); 174 if (info == NULL) 175 info = ERR_PTR(-ENOMEM); 176 return info; 177} 178 179static void lov_session_key_fini(const struct lu_context *ctx, 180 struct lu_context_key *key, void *data) 181{ 182 struct lov_session *info = data; 183 OBD_SLAB_FREE_PTR(info, lov_session_kmem); 184} 185 186struct lu_context_key lov_session_key = { 187 .lct_tags = LCT_SESSION, 188 .lct_init = lov_session_key_init, 189 .lct_fini = lov_session_key_fini 190}; 191 192/* type constructor/destructor: lov_type_{init,fini,start,stop}() */ 193LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key); 194 195static struct lu_device *lov_device_fini(const struct lu_env *env, 196 struct lu_device *d) 197{ 198 int i; 199 struct lov_device *ld = lu2lov_dev(d); 200 201 LASSERT(ld->ld_lov != NULL); 202 if (ld->ld_target == NULL) 203 return NULL; 204 205 lov_foreach_target(ld, i) { 206 struct lovsub_device *lsd; 207 208 lsd = ld->ld_target[i]; 209 if (lsd != NULL) { 210 cl_stack_fini(env, lovsub2cl_dev(lsd)); 211 ld->ld_target[i] = NULL; 212 } 213 } 214 return NULL; 215} 216 217static int lov_device_init(const struct lu_env *env, struct lu_device *d, 218 const char *name, struct lu_device *next) 219{ 220 struct lov_device *ld = lu2lov_dev(d); 221 int i; 222 int rc = 0; 223 224 LASSERT(d->ld_site != NULL); 225 if (ld->ld_target == NULL) 226 return rc; 227 228 lov_foreach_target(ld, i) { 229 struct lovsub_device *lsd; 230 struct cl_device *cl; 231 struct lov_tgt_desc *desc; 232 233 desc = ld->ld_lov->lov_tgts[i]; 234 if (desc == NULL) 235 continue; 236 237 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type, 238 desc->ltd_obd->obd_lu_dev); 239 if (IS_ERR(cl)) { 240 rc = PTR_ERR(cl); 241 break; 242 } 243 lsd = cl2lovsub_dev(cl); 244 lsd->acid_idx = i; 245 lsd->acid_super = ld; 246 ld->ld_target[i] = lsd; 247 } 248 249 if (rc) 250 lov_device_fini(env, d); 251 else 252 ld->ld_flags |= LOV_DEV_INITIALIZED; 253 254 return rc; 255} 256 257static int lov_req_init(const struct lu_env *env, struct cl_device *dev, 258 struct cl_req *req) 259{ 260 struct lov_req *lr; 261 int result; 262 263 OBD_SLAB_ALLOC_PTR_GFP(lr, lov_req_kmem, GFP_NOFS); 264 if (lr != NULL) { 265 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops); 266 result = 0; 267 } else 268 result = -ENOMEM; 269 return result; 270} 271 272static const struct cl_device_operations lov_cl_ops = { 273 .cdo_req_init = lov_req_init 274}; 275 276static void lov_emerg_free(struct lov_device_emerg **emrg, int nr) 277{ 278 int i; 279 280 for (i = 0; i < nr; ++i) { 281 struct lov_device_emerg *em; 282 283 em = emrg[i]; 284 if (em != NULL) { 285 LASSERT(em->emrg_page_list.pl_nr == 0); 286 if (em->emrg_env != NULL) 287 cl_env_put(em->emrg_env, &em->emrg_refcheck); 288 OBD_FREE_PTR(em); 289 } 290 } 291 OBD_FREE(emrg, nr * sizeof(emrg[0])); 292} 293 294static struct lu_device *lov_device_free(const struct lu_env *env, 295 struct lu_device *d) 296{ 297 struct lov_device *ld = lu2lov_dev(d); 298 const int nr = ld->ld_target_nr; 299 300 cl_device_fini(lu2cl_dev(d)); 301 if (ld->ld_target != NULL) 302 OBD_FREE(ld->ld_target, nr * sizeof(ld->ld_target[0])); 303 if (ld->ld_emrg != NULL) 304 lov_emerg_free(ld->ld_emrg, nr); 305 OBD_FREE_PTR(ld); 306 return NULL; 307} 308 309static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev, 310 __u32 index) 311{ 312 struct lov_device *ld = lu2lov_dev(dev); 313 314 if (ld->ld_target[index] != NULL) { 315 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index])); 316 ld->ld_target[index] = NULL; 317 } 318} 319 320static struct lov_device_emerg **lov_emerg_alloc(int nr) 321{ 322 struct lov_device_emerg **emerg; 323 int i; 324 int result; 325 326 OBD_ALLOC(emerg, nr * sizeof(emerg[0])); 327 if (emerg == NULL) 328 return ERR_PTR(-ENOMEM); 329 for (result = i = 0; i < nr && result == 0; i++) { 330 struct lov_device_emerg *em; 331 332 OBD_ALLOC_PTR(em); 333 if (em != NULL) { 334 emerg[i] = em; 335 cl_page_list_init(&em->emrg_page_list); 336 em->emrg_env = cl_env_alloc(&em->emrg_refcheck, 337 LCT_REMEMBER|LCT_NOREF); 338 if (!IS_ERR(em->emrg_env)) 339 em->emrg_env->le_ctx.lc_cookie = 0x2; 340 else { 341 result = PTR_ERR(em->emrg_env); 342 em->emrg_env = NULL; 343 } 344 } else 345 result = -ENOMEM; 346 } 347 if (result != 0) { 348 lov_emerg_free(emerg, nr); 349 emerg = ERR_PTR(result); 350 } 351 return emerg; 352} 353 354static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev) 355{ 356 int result; 357 __u32 tgt_size; 358 __u32 sub_size; 359 360 result = 0; 361 tgt_size = dev->ld_lov->lov_tgt_size; 362 sub_size = dev->ld_target_nr; 363 if (sub_size < tgt_size) { 364 struct lovsub_device **newd; 365 struct lov_device_emerg **emerg; 366 const size_t sz = sizeof(newd[0]); 367 368 emerg = lov_emerg_alloc(tgt_size); 369 if (IS_ERR(emerg)) 370 return PTR_ERR(emerg); 371 372 OBD_ALLOC(newd, tgt_size * sz); 373 if (newd != NULL) { 374 mutex_lock(&dev->ld_mutex); 375 if (sub_size > 0) { 376 memcpy(newd, dev->ld_target, sub_size * sz); 377 OBD_FREE(dev->ld_target, sub_size * sz); 378 } 379 dev->ld_target = newd; 380 dev->ld_target_nr = tgt_size; 381 382 if (dev->ld_emrg != NULL) 383 lov_emerg_free(dev->ld_emrg, sub_size); 384 dev->ld_emrg = emerg; 385 mutex_unlock(&dev->ld_mutex); 386 } else { 387 lov_emerg_free(emerg, tgt_size); 388 result = -ENOMEM; 389 } 390 } 391 return result; 392} 393 394static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev, 395 __u32 index) 396{ 397 struct obd_device *obd = dev->ld_obd; 398 struct lov_device *ld = lu2lov_dev(dev); 399 struct lov_tgt_desc *tgt; 400 struct lovsub_device *lsd; 401 struct cl_device *cl; 402 int rc; 403 404 obd_getref(obd); 405 406 tgt = obd->u.lov.lov_tgts[index]; 407 LASSERT(tgt != NULL); 408 LASSERT(tgt->ltd_obd != NULL); 409 410 if (!tgt->ltd_obd->obd_set_up) { 411 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid)); 412 return -EINVAL; 413 } 414 415 rc = lov_expand_targets(env, ld); 416 if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) { 417 LASSERT(dev->ld_site != NULL); 418 419 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type, 420 tgt->ltd_obd->obd_lu_dev); 421 if (!IS_ERR(cl)) { 422 lsd = cl2lovsub_dev(cl); 423 lsd->acid_idx = index; 424 lsd->acid_super = ld; 425 ld->ld_target[index] = lsd; 426 } else { 427 CERROR("add failed (%d), deleting %s\n", rc, 428 obd_uuid2str(&tgt->ltd_uuid)); 429 lov_cl_del_target(env, dev, index); 430 rc = PTR_ERR(cl); 431 } 432 } 433 obd_putref(obd); 434 return rc; 435} 436 437static int lov_process_config(const struct lu_env *env, 438 struct lu_device *d, struct lustre_cfg *cfg) 439{ 440 struct obd_device *obd = d->ld_obd; 441 int cmd; 442 int rc; 443 int gen; 444 __u32 index; 445 446 obd_getref(obd); 447 448 cmd = cfg->lcfg_command; 449 rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen); 450 if (rc == 0) { 451 switch (cmd) { 452 case LCFG_LOV_ADD_OBD: 453 case LCFG_LOV_ADD_INA: 454 rc = lov_cl_add_target(env, d, index); 455 if (rc != 0) 456 lov_del_target(d->ld_obd, index, NULL, 0); 457 break; 458 case LCFG_LOV_DEL_OBD: 459 lov_cl_del_target(env, d, index); 460 break; 461 } 462 } 463 obd_putref(obd); 464 return rc; 465} 466 467static const struct lu_device_operations lov_lu_ops = { 468 .ldo_object_alloc = lov_object_alloc, 469 .ldo_process_config = lov_process_config, 470}; 471 472static struct lu_device *lov_device_alloc(const struct lu_env *env, 473 struct lu_device_type *t, 474 struct lustre_cfg *cfg) 475{ 476 struct lu_device *d; 477 struct lov_device *ld; 478 struct obd_device *obd; 479 int rc; 480 481 OBD_ALLOC_PTR(ld); 482 if (ld == NULL) 483 return ERR_PTR(-ENOMEM); 484 485 cl_device_init(&ld->ld_cl, t); 486 d = lov2lu_dev(ld); 487 d->ld_ops = &lov_lu_ops; 488 ld->ld_cl.cd_ops = &lov_cl_ops; 489 490 mutex_init(&ld->ld_mutex); 491 lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class); 492 493 /* setup the LOV OBD */ 494 obd = class_name2obd(lustre_cfg_string(cfg, 0)); 495 LASSERT(obd != NULL); 496 rc = lov_setup(obd, cfg); 497 if (rc) { 498 lov_device_free(env, d); 499 return ERR_PTR(rc); 500 } 501 502 ld->ld_lov = &obd->u.lov; 503 return d; 504} 505 506static const struct lu_device_type_operations lov_device_type_ops = { 507 .ldto_init = lov_type_init, 508 .ldto_fini = lov_type_fini, 509 510 .ldto_start = lov_type_start, 511 .ldto_stop = lov_type_stop, 512 513 .ldto_device_alloc = lov_device_alloc, 514 .ldto_device_free = lov_device_free, 515 516 .ldto_device_init = lov_device_init, 517 .ldto_device_fini = lov_device_fini 518}; 519 520struct lu_device_type lov_device_type = { 521 .ldt_tags = LU_DEVICE_CL, 522 .ldt_name = LUSTRE_LOV_NAME, 523 .ldt_ops = &lov_device_type_ops, 524 .ldt_ctx_tags = LCT_CL_THREAD 525}; 526EXPORT_SYMBOL(lov_device_type); 527 528/** @} lov */ 529