1/* 2 drbd_actlog.c 3 4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 5 6 Copyright (C) 2003-2008, LINBIT Information Technologies GmbH. 7 Copyright (C) 2003-2008, Philipp Reisner <philipp.reisner@linbit.com>. 8 Copyright (C) 2003-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 9 10 drbd is free software; you can redistribute it and/or modify 11 it under the terms of the GNU General Public License as published by 12 the Free Software Foundation; either version 2, or (at your option) 13 any later version. 14 15 drbd is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU General Public License for more details. 19 20 You should have received a copy of the GNU General Public License 21 along with drbd; see the file COPYING. If not, write to 22 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 23 24 */ 25 26#include <linux/slab.h> 27#include <linux/crc32c.h> 28#include <linux/drbd.h> 29#include <linux/drbd_limits.h> 30#include <linux/dynamic_debug.h> 31#include "drbd_int.h" 32 33 34enum al_transaction_types { 35 AL_TR_UPDATE = 0, 36 AL_TR_INITIALIZED = 0xffff 37}; 38/* all fields on disc in big endian */ 39struct __packed al_transaction_on_disk { 40 /* don't we all like magic */ 41 __be32 magic; 42 43 /* to identify the most recent transaction block 44 * in the on disk ring buffer */ 45 __be32 tr_number; 46 47 /* checksum on the full 4k block, with this field set to 0. */ 48 __be32 crc32c; 49 50 /* type of transaction, special transaction types like: 51 * purge-all, set-all-idle, set-all-active, ... to-be-defined 52 * see also enum al_transaction_types */ 53 __be16 transaction_type; 54 55 /* we currently allow only a few thousand extents, 56 * so 16bit will be enough for the slot number. */ 57 58 /* how many updates in this transaction */ 59 __be16 n_updates; 60 61 /* maximum slot number, "al-extents" in drbd.conf speak. 62 * Having this in each transaction should make reconfiguration 63 * of that parameter easier. */ 64 __be16 context_size; 65 66 /* slot number the context starts with */ 67 __be16 context_start_slot_nr; 68 69 /* Some reserved bytes. Expected usage is a 64bit counter of 70 * sectors-written since device creation, and other data generation tag 71 * supporting usage */ 72 __be32 __reserved[4]; 73 74 /* --- 36 byte used --- */ 75 76 /* Reserve space for up to AL_UPDATES_PER_TRANSACTION changes 77 * in one transaction, then use the remaining byte in the 4k block for 78 * context information. "Flexible" number of updates per transaction 79 * does not help, as we have to account for the case when all update 80 * slots are used anyways, so it would only complicate code without 81 * additional benefit. 82 */ 83 __be16 update_slot_nr[AL_UPDATES_PER_TRANSACTION]; 84 85 /* but the extent number is 32bit, which at an extent size of 4 MiB 86 * allows to cover device sizes of up to 2**54 Byte (16 PiB) */ 87 __be32 update_extent_nr[AL_UPDATES_PER_TRANSACTION]; 88 89 /* --- 420 bytes used (36 + 64*6) --- */ 90 91 /* 4096 - 420 = 3676 = 919 * 4 */ 92 __be32 context[AL_CONTEXT_PER_TRANSACTION]; 93}; 94 95void *drbd_md_get_buffer(struct drbd_device *device, const char *intent) 96{ 97 int r; 98 99 wait_event(device->misc_wait, 100 (r = atomic_cmpxchg(&device->md_io.in_use, 0, 1)) == 0 || 101 device->state.disk <= D_FAILED); 102 103 if (r) 104 return NULL; 105 106 device->md_io.current_use = intent; 107 device->md_io.start_jif = jiffies; 108 device->md_io.submit_jif = device->md_io.start_jif - 1; 109 return page_address(device->md_io.page); 110} 111 112void drbd_md_put_buffer(struct drbd_device *device) 113{ 114 if (atomic_dec_and_test(&device->md_io.in_use)) 115 wake_up(&device->misc_wait); 116} 117 118void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_backing_dev *bdev, 119 unsigned int *done) 120{ 121 long dt; 122 123 rcu_read_lock(); 124 dt = rcu_dereference(bdev->disk_conf)->disk_timeout; 125 rcu_read_unlock(); 126 dt = dt * HZ / 10; 127 if (dt == 0) 128 dt = MAX_SCHEDULE_TIMEOUT; 129 130 dt = wait_event_timeout(device->misc_wait, 131 *done || test_bit(FORCE_DETACH, &device->flags), dt); 132 if (dt == 0) { 133 drbd_err(device, "meta-data IO operation timed out\n"); 134 drbd_chk_io_error(device, 1, DRBD_FORCE_DETACH); 135 } 136} 137 138static int _drbd_md_sync_page_io(struct drbd_device *device, 139 struct drbd_backing_dev *bdev, 140 sector_t sector, int rw) 141{ 142 struct bio *bio; 143 /* we do all our meta data IO in aligned 4k blocks. */ 144 const int size = 4096; 145 int err; 146 147 device->md_io.done = 0; 148 device->md_io.error = -ENODEV; 149 150 if ((rw & WRITE) && !test_bit(MD_NO_FUA, &device->flags)) 151 rw |= REQ_FUA | REQ_FLUSH; 152 rw |= REQ_SYNC | REQ_NOIDLE; 153 154 bio = bio_alloc_drbd(GFP_NOIO); 155 bio->bi_bdev = bdev->md_bdev; 156 bio->bi_iter.bi_sector = sector; 157 err = -EIO; 158 if (bio_add_page(bio, device->md_io.page, size, 0) != size) 159 goto out; 160 bio->bi_private = device; 161 bio->bi_end_io = drbd_md_endio; 162 bio->bi_rw = rw; 163 164 if (!(rw & WRITE) && device->state.disk == D_DISKLESS && device->ldev == NULL) 165 /* special case, drbd_md_read() during drbd_adm_attach(): no get_ldev */ 166 ; 167 else if (!get_ldev_if_state(device, D_ATTACHING)) { 168 /* Corresponding put_ldev in drbd_md_endio() */ 169 drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in _drbd_md_sync_page_io()\n"); 170 err = -ENODEV; 171 goto out; 172 } 173 174 bio_get(bio); /* one bio_put() is in the completion handler */ 175 atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ 176 device->md_io.submit_jif = jiffies; 177 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) 178 bio_endio(bio, -EIO); 179 else 180 submit_bio(rw, bio); 181 wait_until_done_or_force_detached(device, bdev, &device->md_io.done); 182 if (bio_flagged(bio, BIO_UPTODATE)) 183 err = device->md_io.error; 184 185 out: 186 bio_put(bio); 187 return err; 188} 189 190int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev, 191 sector_t sector, int rw) 192{ 193 int err; 194 D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1); 195 196 BUG_ON(!bdev->md_bdev); 197 198 dynamic_drbd_dbg(device, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n", 199 current->comm, current->pid, __func__, 200 (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", 201 (void*)_RET_IP_ ); 202 203 if (sector < drbd_md_first_sector(bdev) || 204 sector + 7 > drbd_md_last_sector(bdev)) 205 drbd_alert(device, "%s [%d]:%s(,%llus,%s) out of range md access!\n", 206 current->comm, current->pid, __func__, 207 (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ"); 208 209 err = _drbd_md_sync_page_io(device, bdev, sector, rw); 210 if (err) { 211 drbd_err(device, "drbd_md_sync_page_io(,%llus,%s) failed with error %d\n", 212 (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", err); 213 } 214 return err; 215} 216 217static struct bm_extent *find_active_resync_extent(struct drbd_device *device, unsigned int enr) 218{ 219 struct lc_element *tmp; 220 tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT); 221 if (unlikely(tmp != NULL)) { 222 struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce); 223 if (test_bit(BME_NO_WRITES, &bm_ext->flags)) 224 return bm_ext; 225 } 226 return NULL; 227} 228 229static struct lc_element *_al_get(struct drbd_device *device, unsigned int enr, bool nonblock) 230{ 231 struct lc_element *al_ext; 232 struct bm_extent *bm_ext; 233 int wake; 234 235 spin_lock_irq(&device->al_lock); 236 bm_ext = find_active_resync_extent(device, enr); 237 if (bm_ext) { 238 wake = !test_and_set_bit(BME_PRIORITY, &bm_ext->flags); 239 spin_unlock_irq(&device->al_lock); 240 if (wake) 241 wake_up(&device->al_wait); 242 return NULL; 243 } 244 if (nonblock) 245 al_ext = lc_try_get(device->act_log, enr); 246 else 247 al_ext = lc_get(device->act_log, enr); 248 spin_unlock_irq(&device->al_lock); 249 return al_ext; 250} 251 252bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i) 253{ 254 /* for bios crossing activity log extent boundaries, 255 * we may need to activate two extents in one go */ 256 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); 257 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); 258 259 D_ASSERT(device, (unsigned)(last - first) <= 1); 260 D_ASSERT(device, atomic_read(&device->local_cnt) > 0); 261 262 /* FIXME figure out a fast path for bios crossing AL extent boundaries */ 263 if (first != last) 264 return false; 265 266 return _al_get(device, first, true); 267} 268 269bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i) 270{ 271 /* for bios crossing activity log extent boundaries, 272 * we may need to activate two extents in one go */ 273 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); 274 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); 275 unsigned enr; 276 bool need_transaction = false; 277 278 D_ASSERT(device, first <= last); 279 D_ASSERT(device, atomic_read(&device->local_cnt) > 0); 280 281 for (enr = first; enr <= last; enr++) { 282 struct lc_element *al_ext; 283 wait_event(device->al_wait, 284 (al_ext = _al_get(device, enr, false)) != NULL); 285 if (al_ext->lc_number != enr) 286 need_transaction = true; 287 } 288 return need_transaction; 289} 290 291static int al_write_transaction(struct drbd_device *device); 292 293void drbd_al_begin_io_commit(struct drbd_device *device) 294{ 295 bool locked = false; 296 297 /* Serialize multiple transactions. 298 * This uses test_and_set_bit, memory barrier is implicit. 299 */ 300 wait_event(device->al_wait, 301 device->act_log->pending_changes == 0 || 302 (locked = lc_try_lock_for_transaction(device->act_log))); 303 304 if (locked) { 305 /* Double check: it may have been committed by someone else, 306 * while we have been waiting for the lock. */ 307 if (device->act_log->pending_changes) { 308 bool write_al_updates; 309 310 rcu_read_lock(); 311 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates; 312 rcu_read_unlock(); 313 314 if (write_al_updates) 315 al_write_transaction(device); 316 spin_lock_irq(&device->al_lock); 317 /* FIXME 318 if (err) 319 we need an "lc_cancel" here; 320 */ 321 lc_committed(device->act_log); 322 spin_unlock_irq(&device->al_lock); 323 } 324 lc_unlock(device->act_log); 325 wake_up(&device->al_wait); 326 } 327} 328 329/* 330 * @delegate: delegate activity log I/O to the worker thread 331 */ 332void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i) 333{ 334 if (drbd_al_begin_io_prepare(device, i)) 335 drbd_al_begin_io_commit(device); 336} 337 338int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i) 339{ 340 struct lru_cache *al = device->act_log; 341 /* for bios crossing activity log extent boundaries, 342 * we may need to activate two extents in one go */ 343 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); 344 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); 345 unsigned nr_al_extents; 346 unsigned available_update_slots; 347 unsigned enr; 348 349 D_ASSERT(device, first <= last); 350 351 nr_al_extents = 1 + last - first; /* worst case: all touched extends are cold. */ 352 available_update_slots = min(al->nr_elements - al->used, 353 al->max_pending_changes - al->pending_changes); 354 355 /* We want all necessary updates for a given request within the same transaction 356 * We could first check how many updates are *actually* needed, 357 * and use that instead of the worst-case nr_al_extents */ 358 if (available_update_slots < nr_al_extents) { 359 /* Too many activity log extents are currently "hot". 360 * 361 * If we have accumulated pending changes already, 362 * we made progress. 363 * 364 * If we cannot get even a single pending change through, 365 * stop the fast path until we made some progress, 366 * or requests to "cold" extents could be starved. */ 367 if (!al->pending_changes) 368 __set_bit(__LC_STARVING, &device->act_log->flags); 369 return -ENOBUFS; 370 } 371 372 /* Is resync active in this area? */ 373 for (enr = first; enr <= last; enr++) { 374 struct lc_element *tmp; 375 tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT); 376 if (unlikely(tmp != NULL)) { 377 struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce); 378 if (test_bit(BME_NO_WRITES, &bm_ext->flags)) { 379 if (!test_and_set_bit(BME_PRIORITY, &bm_ext->flags)) 380 return -EBUSY; 381 return -EWOULDBLOCK; 382 } 383 } 384 } 385 386 /* Checkout the refcounts. 387 * Given that we checked for available elements and update slots above, 388 * this has to be successful. */ 389 for (enr = first; enr <= last; enr++) { 390 struct lc_element *al_ext; 391 al_ext = lc_get_cumulative(device->act_log, enr); 392 if (!al_ext) 393 drbd_info(device, "LOGIC BUG for enr=%u\n", enr); 394 } 395 return 0; 396} 397 398void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i) 399{ 400 /* for bios crossing activity log extent boundaries, 401 * we may need to activate two extents in one go */ 402 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); 403 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); 404 unsigned enr; 405 struct lc_element *extent; 406 unsigned long flags; 407 408 D_ASSERT(device, first <= last); 409 spin_lock_irqsave(&device->al_lock, flags); 410 411 for (enr = first; enr <= last; enr++) { 412 extent = lc_find(device->act_log, enr); 413 if (!extent) { 414 drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr); 415 continue; 416 } 417 lc_put(device->act_log, extent); 418 } 419 spin_unlock_irqrestore(&device->al_lock, flags); 420 wake_up(&device->al_wait); 421} 422 423#if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT) 424/* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT 425 * are still coupled, or assume too much about their relation. 426 * Code below will not work if this is violated. 427 * Will be cleaned up with some followup patch. 428 */ 429# error FIXME 430#endif 431 432static unsigned int al_extent_to_bm_page(unsigned int al_enr) 433{ 434 return al_enr >> 435 /* bit to page */ 436 ((PAGE_SHIFT + 3) - 437 /* al extent number to bit */ 438 (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)); 439} 440 441static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device) 442{ 443 const unsigned int stripes = device->ldev->md.al_stripes; 444 const unsigned int stripe_size_4kB = device->ldev->md.al_stripe_size_4k; 445 446 /* transaction number, modulo on-disk ring buffer wrap around */ 447 unsigned int t = device->al_tr_number % (device->ldev->md.al_size_4k); 448 449 /* ... to aligned 4k on disk block */ 450 t = ((t % stripes) * stripe_size_4kB) + t/stripes; 451 452 /* ... to 512 byte sector in activity log */ 453 t *= 8; 454 455 /* ... plus offset to the on disk position */ 456 return device->ldev->md.md_offset + device->ldev->md.al_offset + t; 457} 458 459int al_write_transaction(struct drbd_device *device) 460{ 461 struct al_transaction_on_disk *buffer; 462 struct lc_element *e; 463 sector_t sector; 464 int i, mx; 465 unsigned extent_nr; 466 unsigned crc = 0; 467 int err = 0; 468 469 if (!get_ldev(device)) { 470 drbd_err(device, "disk is %s, cannot start al transaction\n", 471 drbd_disk_str(device->state.disk)); 472 return -EIO; 473 } 474 475 /* The bitmap write may have failed, causing a state change. */ 476 if (device->state.disk < D_INCONSISTENT) { 477 drbd_err(device, 478 "disk is %s, cannot write al transaction\n", 479 drbd_disk_str(device->state.disk)); 480 put_ldev(device); 481 return -EIO; 482 } 483 484 /* protects md_io_buffer, al_tr_cycle, ... */ 485 buffer = drbd_md_get_buffer(device, __func__); 486 if (!buffer) { 487 drbd_err(device, "disk failed while waiting for md_io buffer\n"); 488 put_ldev(device); 489 return -ENODEV; 490 } 491 492 memset(buffer, 0, sizeof(*buffer)); 493 buffer->magic = cpu_to_be32(DRBD_AL_MAGIC); 494 buffer->tr_number = cpu_to_be32(device->al_tr_number); 495 496 i = 0; 497 498 /* Even though no one can start to change this list 499 * once we set the LC_LOCKED -- from drbd_al_begin_io(), 500 * lc_try_lock_for_transaction() --, someone may still 501 * be in the process of changing it. */ 502 spin_lock_irq(&device->al_lock); 503 list_for_each_entry(e, &device->act_log->to_be_changed, list) { 504 if (i == AL_UPDATES_PER_TRANSACTION) { 505 i++; 506 break; 507 } 508 buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index); 509 buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number); 510 if (e->lc_number != LC_FREE) 511 drbd_bm_mark_for_writeout(device, 512 al_extent_to_bm_page(e->lc_number)); 513 i++; 514 } 515 spin_unlock_irq(&device->al_lock); 516 BUG_ON(i > AL_UPDATES_PER_TRANSACTION); 517 518 buffer->n_updates = cpu_to_be16(i); 519 for ( ; i < AL_UPDATES_PER_TRANSACTION; i++) { 520 buffer->update_slot_nr[i] = cpu_to_be16(-1); 521 buffer->update_extent_nr[i] = cpu_to_be32(LC_FREE); 522 } 523 524 buffer->context_size = cpu_to_be16(device->act_log->nr_elements); 525 buffer->context_start_slot_nr = cpu_to_be16(device->al_tr_cycle); 526 527 mx = min_t(int, AL_CONTEXT_PER_TRANSACTION, 528 device->act_log->nr_elements - device->al_tr_cycle); 529 for (i = 0; i < mx; i++) { 530 unsigned idx = device->al_tr_cycle + i; 531 extent_nr = lc_element_by_index(device->act_log, idx)->lc_number; 532 buffer->context[i] = cpu_to_be32(extent_nr); 533 } 534 for (; i < AL_CONTEXT_PER_TRANSACTION; i++) 535 buffer->context[i] = cpu_to_be32(LC_FREE); 536 537 device->al_tr_cycle += AL_CONTEXT_PER_TRANSACTION; 538 if (device->al_tr_cycle >= device->act_log->nr_elements) 539 device->al_tr_cycle = 0; 540 541 sector = al_tr_number_to_on_disk_sector(device); 542 543 crc = crc32c(0, buffer, 4096); 544 buffer->crc32c = cpu_to_be32(crc); 545 546 if (drbd_bm_write_hinted(device)) 547 err = -EIO; 548 else { 549 bool write_al_updates; 550 rcu_read_lock(); 551 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates; 552 rcu_read_unlock(); 553 if (write_al_updates) { 554 if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) { 555 err = -EIO; 556 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR); 557 } else { 558 device->al_tr_number++; 559 device->al_writ_cnt++; 560 } 561 } 562 } 563 564 drbd_md_put_buffer(device); 565 put_ldev(device); 566 567 return err; 568} 569 570static int _try_lc_del(struct drbd_device *device, struct lc_element *al_ext) 571{ 572 int rv; 573 574 spin_lock_irq(&device->al_lock); 575 rv = (al_ext->refcnt == 0); 576 if (likely(rv)) 577 lc_del(device->act_log, al_ext); 578 spin_unlock_irq(&device->al_lock); 579 580 return rv; 581} 582 583/** 584 * drbd_al_shrink() - Removes all active extents form the activity log 585 * @device: DRBD device. 586 * 587 * Removes all active extents form the activity log, waiting until 588 * the reference count of each entry dropped to 0 first, of course. 589 * 590 * You need to lock device->act_log with lc_try_lock() / lc_unlock() 591 */ 592void drbd_al_shrink(struct drbd_device *device) 593{ 594 struct lc_element *al_ext; 595 int i; 596 597 D_ASSERT(device, test_bit(__LC_LOCKED, &device->act_log->flags)); 598 599 for (i = 0; i < device->act_log->nr_elements; i++) { 600 al_ext = lc_element_by_index(device->act_log, i); 601 if (al_ext->lc_number == LC_FREE) 602 continue; 603 wait_event(device->al_wait, _try_lc_del(device, al_ext)); 604 } 605 606 wake_up(&device->al_wait); 607} 608 609int drbd_initialize_al(struct drbd_device *device, void *buffer) 610{ 611 struct al_transaction_on_disk *al = buffer; 612 struct drbd_md *md = &device->ldev->md; 613 sector_t al_base = md->md_offset + md->al_offset; 614 int al_size_4k = md->al_stripes * md->al_stripe_size_4k; 615 int i; 616 617 memset(al, 0, 4096); 618 al->magic = cpu_to_be32(DRBD_AL_MAGIC); 619 al->transaction_type = cpu_to_be16(AL_TR_INITIALIZED); 620 al->crc32c = cpu_to_be32(crc32c(0, al, 4096)); 621 622 for (i = 0; i < al_size_4k; i++) { 623 int err = drbd_md_sync_page_io(device, device->ldev, al_base + i * 8, WRITE); 624 if (err) 625 return err; 626 } 627 return 0; 628} 629 630static const char *drbd_change_sync_fname[] = { 631 [RECORD_RS_FAILED] = "drbd_rs_failed_io", 632 [SET_IN_SYNC] = "drbd_set_in_sync", 633 [SET_OUT_OF_SYNC] = "drbd_set_out_of_sync" 634}; 635 636/* ATTENTION. The AL's extents are 4MB each, while the extents in the 637 * resync LRU-cache are 16MB each. 638 * The caller of this function has to hold an get_ldev() reference. 639 * 640 * Adjusts the caching members ->rs_left (success) or ->rs_failed (!success), 641 * potentially pulling in (and recounting the corresponding bits) 642 * this resync extent into the resync extent lru cache. 643 * 644 * Returns whether all bits have been cleared for this resync extent, 645 * precisely: (rs_left <= rs_failed) 646 * 647 * TODO will be obsoleted once we have a caching lru of the on disk bitmap 648 */ 649static bool update_rs_extent(struct drbd_device *device, 650 unsigned int enr, int count, 651 enum update_sync_bits_mode mode) 652{ 653 struct lc_element *e; 654 655 D_ASSERT(device, atomic_read(&device->local_cnt)); 656 657 /* When setting out-of-sync bits, 658 * we don't need it cached (lc_find). 659 * But if it is present in the cache, 660 * we should update the cached bit count. 661 * Otherwise, that extent should be in the resync extent lru cache 662 * already -- or we want to pull it in if necessary -- (lc_get), 663 * then update and check rs_left and rs_failed. */ 664 if (mode == SET_OUT_OF_SYNC) 665 e = lc_find(device->resync, enr); 666 else 667 e = lc_get(device->resync, enr); 668 if (e) { 669 struct bm_extent *ext = lc_entry(e, struct bm_extent, lce); 670 if (ext->lce.lc_number == enr) { 671 if (mode == SET_IN_SYNC) 672 ext->rs_left -= count; 673 else if (mode == SET_OUT_OF_SYNC) 674 ext->rs_left += count; 675 else 676 ext->rs_failed += count; 677 if (ext->rs_left < ext->rs_failed) { 678 drbd_warn(device, "BAD! enr=%u rs_left=%d " 679 "rs_failed=%d count=%d cstate=%s\n", 680 ext->lce.lc_number, ext->rs_left, 681 ext->rs_failed, count, 682 drbd_conn_str(device->state.conn)); 683 684 /* We don't expect to be able to clear more bits 685 * than have been set when we originally counted 686 * the set bits to cache that value in ext->rs_left. 687 * Whatever the reason (disconnect during resync, 688 * delayed local completion of an application write), 689 * try to fix it up by recounting here. */ 690 ext->rs_left = drbd_bm_e_weight(device, enr); 691 } 692 } else { 693 /* Normally this element should be in the cache, 694 * since drbd_rs_begin_io() pulled it already in. 695 * 696 * But maybe an application write finished, and we set 697 * something outside the resync lru_cache in sync. 698 */ 699 int rs_left = drbd_bm_e_weight(device, enr); 700 if (ext->flags != 0) { 701 drbd_warn(device, "changing resync lce: %d[%u;%02lx]" 702 " -> %d[%u;00]\n", 703 ext->lce.lc_number, ext->rs_left, 704 ext->flags, enr, rs_left); 705 ext->flags = 0; 706 } 707 if (ext->rs_failed) { 708 drbd_warn(device, "Kicking resync_lru element enr=%u " 709 "out with rs_failed=%d\n", 710 ext->lce.lc_number, ext->rs_failed); 711 } 712 ext->rs_left = rs_left; 713 ext->rs_failed = (mode == RECORD_RS_FAILED) ? count : 0; 714 /* we don't keep a persistent log of the resync lru, 715 * we can commit any change right away. */ 716 lc_committed(device->resync); 717 } 718 if (mode != SET_OUT_OF_SYNC) 719 lc_put(device->resync, &ext->lce); 720 /* no race, we are within the al_lock! */ 721 722 if (ext->rs_left <= ext->rs_failed) { 723 ext->rs_failed = 0; 724 return true; 725 } 726 } else if (mode != SET_OUT_OF_SYNC) { 727 /* be quiet if lc_find() did not find it. */ 728 drbd_err(device, "lc_get() failed! locked=%d/%d flags=%lu\n", 729 device->resync_locked, 730 device->resync->nr_elements, 731 device->resync->flags); 732 } 733 return false; 734} 735 736void drbd_advance_rs_marks(struct drbd_device *device, unsigned long still_to_go) 737{ 738 unsigned long now = jiffies; 739 unsigned long last = device->rs_mark_time[device->rs_last_mark]; 740 int next = (device->rs_last_mark + 1) % DRBD_SYNC_MARKS; 741 if (time_after_eq(now, last + DRBD_SYNC_MARK_STEP)) { 742 if (device->rs_mark_left[device->rs_last_mark] != still_to_go && 743 device->state.conn != C_PAUSED_SYNC_T && 744 device->state.conn != C_PAUSED_SYNC_S) { 745 device->rs_mark_time[next] = now; 746 device->rs_mark_left[next] = still_to_go; 747 device->rs_last_mark = next; 748 } 749 } 750} 751 752/* It is called lazy update, so don't do write-out too often. */ 753static bool lazy_bitmap_update_due(struct drbd_device *device) 754{ 755 return time_after(jiffies, device->rs_last_bcast + 2*HZ); 756} 757 758static void maybe_schedule_on_disk_bitmap_update(struct drbd_device *device, bool rs_done) 759{ 760 if (rs_done) 761 set_bit(RS_DONE, &device->flags); 762 /* and also set RS_PROGRESS below */ 763 else if (!lazy_bitmap_update_due(device)) 764 return; 765 766 drbd_device_post_work(device, RS_PROGRESS); 767} 768 769static int update_sync_bits(struct drbd_device *device, 770 unsigned long sbnr, unsigned long ebnr, 771 enum update_sync_bits_mode mode) 772{ 773 /* 774 * We keep a count of set bits per resync-extent in the ->rs_left 775 * caching member, so we need to loop and work within the resync extent 776 * alignment. Typically this loop will execute exactly once. 777 */ 778 unsigned long flags; 779 unsigned long count = 0; 780 unsigned int cleared = 0; 781 while (sbnr <= ebnr) { 782 /* set temporary boundary bit number to last bit number within 783 * the resync extent of the current start bit number, 784 * but cap at provided end bit number */ 785 unsigned long tbnr = min(ebnr, sbnr | BM_BLOCKS_PER_BM_EXT_MASK); 786 unsigned long c; 787 788 if (mode == RECORD_RS_FAILED) 789 /* Only called from drbd_rs_failed_io(), bits 790 * supposedly still set. Recount, maybe some 791 * of the bits have been successfully cleared 792 * by application IO meanwhile. 793 */ 794 c = drbd_bm_count_bits(device, sbnr, tbnr); 795 else if (mode == SET_IN_SYNC) 796 c = drbd_bm_clear_bits(device, sbnr, tbnr); 797 else /* if (mode == SET_OUT_OF_SYNC) */ 798 c = drbd_bm_set_bits(device, sbnr, tbnr); 799 800 if (c) { 801 spin_lock_irqsave(&device->al_lock, flags); 802 cleared += update_rs_extent(device, BM_BIT_TO_EXT(sbnr), c, mode); 803 spin_unlock_irqrestore(&device->al_lock, flags); 804 count += c; 805 } 806 sbnr = tbnr + 1; 807 } 808 if (count) { 809 if (mode == SET_IN_SYNC) { 810 unsigned long still_to_go = drbd_bm_total_weight(device); 811 bool rs_is_done = (still_to_go <= device->rs_failed); 812 drbd_advance_rs_marks(device, still_to_go); 813 if (cleared || rs_is_done) 814 maybe_schedule_on_disk_bitmap_update(device, rs_is_done); 815 } else if (mode == RECORD_RS_FAILED) 816 device->rs_failed += count; 817 wake_up(&device->al_wait); 818 } 819 return count; 820} 821 822/* clear the bit corresponding to the piece of storage in question: 823 * size byte of data starting from sector. Only clear a bits of the affected 824 * one ore more _aligned_ BM_BLOCK_SIZE blocks. 825 * 826 * called by worker on C_SYNC_TARGET and receiver on SyncSource. 827 * 828 */ 829int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size, 830 enum update_sync_bits_mode mode, 831 const char *file, const unsigned int line) 832{ 833 /* Is called from worker and receiver context _only_ */ 834 unsigned long sbnr, ebnr, lbnr; 835 unsigned long count = 0; 836 sector_t esector, nr_sectors; 837 838 /* This would be an empty REQ_FLUSH, be silent. */ 839 if ((mode == SET_OUT_OF_SYNC) && size == 0) 840 return 0; 841 842 if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_DISCARD_SIZE) { 843 drbd_err(device, "%s: sector=%llus size=%d nonsense!\n", 844 drbd_change_sync_fname[mode], 845 (unsigned long long)sector, size); 846 return 0; 847 } 848 849 if (!get_ldev(device)) 850 return 0; /* no disk, no metadata, no bitmap to manipulate bits in */ 851 852 nr_sectors = drbd_get_capacity(device->this_bdev); 853 esector = sector + (size >> 9) - 1; 854 855 if (!expect(sector < nr_sectors)) 856 goto out; 857 if (!expect(esector < nr_sectors)) 858 esector = nr_sectors - 1; 859 860 lbnr = BM_SECT_TO_BIT(nr_sectors-1); 861 862 if (mode == SET_IN_SYNC) { 863 /* Round up start sector, round down end sector. We make sure 864 * we only clear full, aligned, BM_BLOCK_SIZE blocks. */ 865 if (unlikely(esector < BM_SECT_PER_BIT-1)) 866 goto out; 867 if (unlikely(esector == (nr_sectors-1))) 868 ebnr = lbnr; 869 else 870 ebnr = BM_SECT_TO_BIT(esector - (BM_SECT_PER_BIT-1)); 871 sbnr = BM_SECT_TO_BIT(sector + BM_SECT_PER_BIT-1); 872 } else { 873 /* We set it out of sync, or record resync failure. 874 * Should not round anything here. */ 875 sbnr = BM_SECT_TO_BIT(sector); 876 ebnr = BM_SECT_TO_BIT(esector); 877 } 878 879 count = update_sync_bits(device, sbnr, ebnr, mode); 880out: 881 put_ldev(device); 882 return count; 883} 884 885static 886struct bm_extent *_bme_get(struct drbd_device *device, unsigned int enr) 887{ 888 struct lc_element *e; 889 struct bm_extent *bm_ext; 890 int wakeup = 0; 891 unsigned long rs_flags; 892 893 spin_lock_irq(&device->al_lock); 894 if (device->resync_locked > device->resync->nr_elements/2) { 895 spin_unlock_irq(&device->al_lock); 896 return NULL; 897 } 898 e = lc_get(device->resync, enr); 899 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL; 900 if (bm_ext) { 901 if (bm_ext->lce.lc_number != enr) { 902 bm_ext->rs_left = drbd_bm_e_weight(device, enr); 903 bm_ext->rs_failed = 0; 904 lc_committed(device->resync); 905 wakeup = 1; 906 } 907 if (bm_ext->lce.refcnt == 1) 908 device->resync_locked++; 909 set_bit(BME_NO_WRITES, &bm_ext->flags); 910 } 911 rs_flags = device->resync->flags; 912 spin_unlock_irq(&device->al_lock); 913 if (wakeup) 914 wake_up(&device->al_wait); 915 916 if (!bm_ext) { 917 if (rs_flags & LC_STARVING) 918 drbd_warn(device, "Have to wait for element" 919 " (resync LRU too small?)\n"); 920 BUG_ON(rs_flags & LC_LOCKED); 921 } 922 923 return bm_ext; 924} 925 926static int _is_in_al(struct drbd_device *device, unsigned int enr) 927{ 928 int rv; 929 930 spin_lock_irq(&device->al_lock); 931 rv = lc_is_used(device->act_log, enr); 932 spin_unlock_irq(&device->al_lock); 933 934 return rv; 935} 936 937/** 938 * drbd_rs_begin_io() - Gets an extent in the resync LRU cache and sets it to BME_LOCKED 939 * @device: DRBD device. 940 * @sector: The sector number. 941 * 942 * This functions sleeps on al_wait. Returns 0 on success, -EINTR if interrupted. 943 */ 944int drbd_rs_begin_io(struct drbd_device *device, sector_t sector) 945{ 946 unsigned int enr = BM_SECT_TO_EXT(sector); 947 struct bm_extent *bm_ext; 948 int i, sig; 949 bool sa; 950 951retry: 952 sig = wait_event_interruptible(device->al_wait, 953 (bm_ext = _bme_get(device, enr))); 954 if (sig) 955 return -EINTR; 956 957 if (test_bit(BME_LOCKED, &bm_ext->flags)) 958 return 0; 959 960 /* step aside only while we are above c-min-rate; unless disabled. */ 961 sa = drbd_rs_c_min_rate_throttle(device); 962 963 for (i = 0; i < AL_EXT_PER_BM_SECT; i++) { 964 sig = wait_event_interruptible(device->al_wait, 965 !_is_in_al(device, enr * AL_EXT_PER_BM_SECT + i) || 966 (sa && test_bit(BME_PRIORITY, &bm_ext->flags))); 967 968 if (sig || (sa && test_bit(BME_PRIORITY, &bm_ext->flags))) { 969 spin_lock_irq(&device->al_lock); 970 if (lc_put(device->resync, &bm_ext->lce) == 0) { 971 bm_ext->flags = 0; /* clears BME_NO_WRITES and eventually BME_PRIORITY */ 972 device->resync_locked--; 973 wake_up(&device->al_wait); 974 } 975 spin_unlock_irq(&device->al_lock); 976 if (sig) 977 return -EINTR; 978 if (schedule_timeout_interruptible(HZ/10)) 979 return -EINTR; 980 goto retry; 981 } 982 } 983 set_bit(BME_LOCKED, &bm_ext->flags); 984 return 0; 985} 986 987/** 988 * drbd_try_rs_begin_io() - Gets an extent in the resync LRU cache, does not sleep 989 * @device: DRBD device. 990 * @sector: The sector number. 991 * 992 * Gets an extent in the resync LRU cache, sets it to BME_NO_WRITES, then 993 * tries to set it to BME_LOCKED. Returns 0 upon success, and -EAGAIN 994 * if there is still application IO going on in this area. 995 */ 996int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector) 997{ 998 unsigned int enr = BM_SECT_TO_EXT(sector); 999 const unsigned int al_enr = enr*AL_EXT_PER_BM_SECT; 1000 struct lc_element *e; 1001 struct bm_extent *bm_ext; 1002 int i; 1003 bool throttle = drbd_rs_should_slow_down(device, sector, true); 1004 1005 /* If we need to throttle, a half-locked (only marked BME_NO_WRITES, 1006 * not yet BME_LOCKED) extent needs to be kicked out explicitly if we 1007 * need to throttle. There is at most one such half-locked extent, 1008 * which is remembered in resync_wenr. */ 1009 1010 if (throttle && device->resync_wenr != enr) 1011 return -EAGAIN; 1012 1013 spin_lock_irq(&device->al_lock); 1014 if (device->resync_wenr != LC_FREE && device->resync_wenr != enr) { 1015 /* in case you have very heavy scattered io, it may 1016 * stall the syncer undefined if we give up the ref count 1017 * when we try again and requeue. 1018 * 1019 * if we don't give up the refcount, but the next time 1020 * we are scheduled this extent has been "synced" by new 1021 * application writes, we'd miss the lc_put on the 1022 * extent we keep the refcount on. 1023 * so we remembered which extent we had to try again, and 1024 * if the next requested one is something else, we do 1025 * the lc_put here... 1026 * we also have to wake_up 1027 */ 1028 e = lc_find(device->resync, device->resync_wenr); 1029 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL; 1030 if (bm_ext) { 1031 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags)); 1032 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags)); 1033 clear_bit(BME_NO_WRITES, &bm_ext->flags); 1034 device->resync_wenr = LC_FREE; 1035 if (lc_put(device->resync, &bm_ext->lce) == 0) { 1036 bm_ext->flags = 0; 1037 device->resync_locked--; 1038 } 1039 wake_up(&device->al_wait); 1040 } else { 1041 drbd_alert(device, "LOGIC BUG\n"); 1042 } 1043 } 1044 /* TRY. */ 1045 e = lc_try_get(device->resync, enr); 1046 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL; 1047 if (bm_ext) { 1048 if (test_bit(BME_LOCKED, &bm_ext->flags)) 1049 goto proceed; 1050 if (!test_and_set_bit(BME_NO_WRITES, &bm_ext->flags)) { 1051 device->resync_locked++; 1052 } else { 1053 /* we did set the BME_NO_WRITES, 1054 * but then could not set BME_LOCKED, 1055 * so we tried again. 1056 * drop the extra reference. */ 1057 bm_ext->lce.refcnt--; 1058 D_ASSERT(device, bm_ext->lce.refcnt > 0); 1059 } 1060 goto check_al; 1061 } else { 1062 /* do we rather want to try later? */ 1063 if (device->resync_locked > device->resync->nr_elements-3) 1064 goto try_again; 1065 /* Do or do not. There is no try. -- Yoda */ 1066 e = lc_get(device->resync, enr); 1067 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL; 1068 if (!bm_ext) { 1069 const unsigned long rs_flags = device->resync->flags; 1070 if (rs_flags & LC_STARVING) 1071 drbd_warn(device, "Have to wait for element" 1072 " (resync LRU too small?)\n"); 1073 BUG_ON(rs_flags & LC_LOCKED); 1074 goto try_again; 1075 } 1076 if (bm_ext->lce.lc_number != enr) { 1077 bm_ext->rs_left = drbd_bm_e_weight(device, enr); 1078 bm_ext->rs_failed = 0; 1079 lc_committed(device->resync); 1080 wake_up(&device->al_wait); 1081 D_ASSERT(device, test_bit(BME_LOCKED, &bm_ext->flags) == 0); 1082 } 1083 set_bit(BME_NO_WRITES, &bm_ext->flags); 1084 D_ASSERT(device, bm_ext->lce.refcnt == 1); 1085 device->resync_locked++; 1086 goto check_al; 1087 } 1088check_al: 1089 for (i = 0; i < AL_EXT_PER_BM_SECT; i++) { 1090 if (lc_is_used(device->act_log, al_enr+i)) 1091 goto try_again; 1092 } 1093 set_bit(BME_LOCKED, &bm_ext->flags); 1094proceed: 1095 device->resync_wenr = LC_FREE; 1096 spin_unlock_irq(&device->al_lock); 1097 return 0; 1098 1099try_again: 1100 if (bm_ext) { 1101 if (throttle) { 1102 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags)); 1103 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags)); 1104 clear_bit(BME_NO_WRITES, &bm_ext->flags); 1105 device->resync_wenr = LC_FREE; 1106 if (lc_put(device->resync, &bm_ext->lce) == 0) { 1107 bm_ext->flags = 0; 1108 device->resync_locked--; 1109 } 1110 wake_up(&device->al_wait); 1111 } else 1112 device->resync_wenr = enr; 1113 } 1114 spin_unlock_irq(&device->al_lock); 1115 return -EAGAIN; 1116} 1117 1118void drbd_rs_complete_io(struct drbd_device *device, sector_t sector) 1119{ 1120 unsigned int enr = BM_SECT_TO_EXT(sector); 1121 struct lc_element *e; 1122 struct bm_extent *bm_ext; 1123 unsigned long flags; 1124 1125 spin_lock_irqsave(&device->al_lock, flags); 1126 e = lc_find(device->resync, enr); 1127 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL; 1128 if (!bm_ext) { 1129 spin_unlock_irqrestore(&device->al_lock, flags); 1130 if (__ratelimit(&drbd_ratelimit_state)) 1131 drbd_err(device, "drbd_rs_complete_io() called, but extent not found\n"); 1132 return; 1133 } 1134 1135 if (bm_ext->lce.refcnt == 0) { 1136 spin_unlock_irqrestore(&device->al_lock, flags); 1137 drbd_err(device, "drbd_rs_complete_io(,%llu [=%u]) called, " 1138 "but refcnt is 0!?\n", 1139 (unsigned long long)sector, enr); 1140 return; 1141 } 1142 1143 if (lc_put(device->resync, &bm_ext->lce) == 0) { 1144 bm_ext->flags = 0; /* clear BME_LOCKED, BME_NO_WRITES and BME_PRIORITY */ 1145 device->resync_locked--; 1146 wake_up(&device->al_wait); 1147 } 1148 1149 spin_unlock_irqrestore(&device->al_lock, flags); 1150} 1151 1152/** 1153 * drbd_rs_cancel_all() - Removes all extents from the resync LRU (even BME_LOCKED) 1154 * @device: DRBD device. 1155 */ 1156void drbd_rs_cancel_all(struct drbd_device *device) 1157{ 1158 spin_lock_irq(&device->al_lock); 1159 1160 if (get_ldev_if_state(device, D_FAILED)) { /* Makes sure ->resync is there. */ 1161 lc_reset(device->resync); 1162 put_ldev(device); 1163 } 1164 device->resync_locked = 0; 1165 device->resync_wenr = LC_FREE; 1166 spin_unlock_irq(&device->al_lock); 1167 wake_up(&device->al_wait); 1168} 1169 1170/** 1171 * drbd_rs_del_all() - Gracefully remove all extents from the resync LRU 1172 * @device: DRBD device. 1173 * 1174 * Returns 0 upon success, -EAGAIN if at least one reference count was 1175 * not zero. 1176 */ 1177int drbd_rs_del_all(struct drbd_device *device) 1178{ 1179 struct lc_element *e; 1180 struct bm_extent *bm_ext; 1181 int i; 1182 1183 spin_lock_irq(&device->al_lock); 1184 1185 if (get_ldev_if_state(device, D_FAILED)) { 1186 /* ok, ->resync is there. */ 1187 for (i = 0; i < device->resync->nr_elements; i++) { 1188 e = lc_element_by_index(device->resync, i); 1189 bm_ext = lc_entry(e, struct bm_extent, lce); 1190 if (bm_ext->lce.lc_number == LC_FREE) 1191 continue; 1192 if (bm_ext->lce.lc_number == device->resync_wenr) { 1193 drbd_info(device, "dropping %u in drbd_rs_del_all, apparently" 1194 " got 'synced' by application io\n", 1195 device->resync_wenr); 1196 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags)); 1197 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags)); 1198 clear_bit(BME_NO_WRITES, &bm_ext->flags); 1199 device->resync_wenr = LC_FREE; 1200 lc_put(device->resync, &bm_ext->lce); 1201 } 1202 if (bm_ext->lce.refcnt != 0) { 1203 drbd_info(device, "Retrying drbd_rs_del_all() later. " 1204 "refcnt=%d\n", bm_ext->lce.refcnt); 1205 put_ldev(device); 1206 spin_unlock_irq(&device->al_lock); 1207 return -EAGAIN; 1208 } 1209 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags)); 1210 D_ASSERT(device, !test_bit(BME_NO_WRITES, &bm_ext->flags)); 1211 lc_del(device->resync, &bm_ext->lce); 1212 } 1213 D_ASSERT(device, device->resync->used == 0); 1214 put_ldev(device); 1215 } 1216 spin_unlock_irq(&device->al_lock); 1217 wake_up(&device->al_wait); 1218 1219 return 0; 1220} 1221