vfio_pci_intrs.c revision 0bced2f7280cd12b66229ba55d05c62a4eef6cfd
1/* 2 * VFIO PCI interrupt handling 3 * 4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 5 * Author: Alex Williamson <alex.williamson@redhat.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * Derived from original vfio: 12 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 13 * Author: Tom Lyon, pugs@cisco.com 14 */ 15 16#include <linux/device.h> 17#include <linux/interrupt.h> 18#include <linux/eventfd.h> 19#include <linux/pci.h> 20#include <linux/file.h> 21#include <linux/poll.h> 22#include <linux/vfio.h> 23#include <linux/wait.h> 24#include <linux/workqueue.h> 25 26#include "vfio_pci_private.h" 27 28/* 29 * IRQfd - generic 30 */ 31struct virqfd { 32 struct vfio_pci_device *vdev; 33 struct eventfd_ctx *eventfd; 34 int (*handler)(struct vfio_pci_device *, void *); 35 void (*thread)(struct vfio_pci_device *, void *); 36 void *data; 37 struct work_struct inject; 38 wait_queue_t wait; 39 poll_table pt; 40 struct work_struct shutdown; 41 struct virqfd **pvirqfd; 42}; 43 44static struct workqueue_struct *vfio_irqfd_cleanup_wq; 45 46int __init vfio_pci_virqfd_init(void) 47{ 48 vfio_irqfd_cleanup_wq = 49 create_singlethread_workqueue("vfio-irqfd-cleanup"); 50 if (!vfio_irqfd_cleanup_wq) 51 return -ENOMEM; 52 53 return 0; 54} 55 56void vfio_pci_virqfd_exit(void) 57{ 58 destroy_workqueue(vfio_irqfd_cleanup_wq); 59} 60 61static void virqfd_deactivate(struct virqfd *virqfd) 62{ 63 queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown); 64} 65 66static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key) 67{ 68 struct virqfd *virqfd = container_of(wait, struct virqfd, wait); 69 unsigned long flags = (unsigned long)key; 70 71 if (flags & POLLIN) { 72 /* An event has been signaled, call function */ 73 if ((!virqfd->handler || 74 virqfd->handler(virqfd->vdev, virqfd->data)) && 75 virqfd->thread) 76 schedule_work(&virqfd->inject); 77 } 78 79 if (flags & POLLHUP) { 80 unsigned long flags; 81 spin_lock_irqsave(&virqfd->vdev->irqlock, flags); 82 83 /* 84 * The eventfd is closing, if the virqfd has not yet been 85 * queued for release, as determined by testing whether the 86 * vdev pointer to it is still valid, queue it now. As 87 * with kvm irqfds, we know we won't race against the virqfd 88 * going away because we hold wqh->lock to get here. 89 */ 90 if (*(virqfd->pvirqfd) == virqfd) { 91 *(virqfd->pvirqfd) = NULL; 92 virqfd_deactivate(virqfd); 93 } 94 95 spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags); 96 } 97 98 return 0; 99} 100 101static void virqfd_ptable_queue_proc(struct file *file, 102 wait_queue_head_t *wqh, poll_table *pt) 103{ 104 struct virqfd *virqfd = container_of(pt, struct virqfd, pt); 105 add_wait_queue(wqh, &virqfd->wait); 106} 107 108static void virqfd_shutdown(struct work_struct *work) 109{ 110 struct virqfd *virqfd = container_of(work, struct virqfd, shutdown); 111 u64 cnt; 112 113 eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt); 114 flush_work(&virqfd->inject); 115 eventfd_ctx_put(virqfd->eventfd); 116 117 kfree(virqfd); 118} 119 120static void virqfd_inject(struct work_struct *work) 121{ 122 struct virqfd *virqfd = container_of(work, struct virqfd, inject); 123 if (virqfd->thread) 124 virqfd->thread(virqfd->vdev, virqfd->data); 125} 126 127static int virqfd_enable(struct vfio_pci_device *vdev, 128 int (*handler)(struct vfio_pci_device *, void *), 129 void (*thread)(struct vfio_pci_device *, void *), 130 void *data, struct virqfd **pvirqfd, int fd) 131{ 132 struct file *file = NULL; 133 struct eventfd_ctx *ctx = NULL; 134 struct virqfd *virqfd; 135 int ret = 0; 136 unsigned int events; 137 138 virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL); 139 if (!virqfd) 140 return -ENOMEM; 141 142 virqfd->pvirqfd = pvirqfd; 143 virqfd->vdev = vdev; 144 virqfd->handler = handler; 145 virqfd->thread = thread; 146 virqfd->data = data; 147 148 INIT_WORK(&virqfd->shutdown, virqfd_shutdown); 149 INIT_WORK(&virqfd->inject, virqfd_inject); 150 151 file = eventfd_fget(fd); 152 if (IS_ERR(file)) { 153 ret = PTR_ERR(file); 154 goto fail; 155 } 156 157 ctx = eventfd_ctx_fileget(file); 158 if (IS_ERR(ctx)) { 159 ret = PTR_ERR(ctx); 160 goto fail; 161 } 162 163 virqfd->eventfd = ctx; 164 165 /* 166 * virqfds can be released by closing the eventfd or directly 167 * through ioctl. These are both done through a workqueue, so 168 * we update the pointer to the virqfd under lock to avoid 169 * pushing multiple jobs to release the same virqfd. 170 */ 171 spin_lock_irq(&vdev->irqlock); 172 173 if (*pvirqfd) { 174 spin_unlock_irq(&vdev->irqlock); 175 ret = -EBUSY; 176 goto fail; 177 } 178 *pvirqfd = virqfd; 179 180 spin_unlock_irq(&vdev->irqlock); 181 182 /* 183 * Install our own custom wake-up handling so we are notified via 184 * a callback whenever someone signals the underlying eventfd. 185 */ 186 init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup); 187 init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc); 188 189 events = file->f_op->poll(file, &virqfd->pt); 190 191 /* 192 * Check if there was an event already pending on the eventfd 193 * before we registered and trigger it as if we didn't miss it. 194 */ 195 if (events & POLLIN) { 196 if ((!handler || handler(vdev, data)) && thread) 197 schedule_work(&virqfd->inject); 198 } 199 200 /* 201 * Do not drop the file until the irqfd is fully initialized, 202 * otherwise we might race against the POLLHUP. 203 */ 204 fput(file); 205 206 return 0; 207 208fail: 209 if (ctx && !IS_ERR(ctx)) 210 eventfd_ctx_put(ctx); 211 212 if (file && !IS_ERR(file)) 213 fput(file); 214 215 kfree(virqfd); 216 217 return ret; 218} 219 220static void virqfd_disable(struct vfio_pci_device *vdev, 221 struct virqfd **pvirqfd) 222{ 223 unsigned long flags; 224 225 spin_lock_irqsave(&vdev->irqlock, flags); 226 227 if (*pvirqfd) { 228 virqfd_deactivate(*pvirqfd); 229 *pvirqfd = NULL; 230 } 231 232 spin_unlock_irqrestore(&vdev->irqlock, flags); 233 234 /* 235 * Block until we know all outstanding shutdown jobs have completed. 236 * Even if we don't queue the job, flush the wq to be sure it's 237 * been released. 238 */ 239 flush_workqueue(vfio_irqfd_cleanup_wq); 240} 241 242/* 243 * INTx 244 */ 245static void vfio_send_intx_eventfd(struct vfio_pci_device *vdev, void *unused) 246{ 247 if (likely(is_intx(vdev) && !vdev->virq_disabled)) 248 eventfd_signal(vdev->ctx[0].trigger, 1); 249} 250 251void vfio_pci_intx_mask(struct vfio_pci_device *vdev) 252{ 253 struct pci_dev *pdev = vdev->pdev; 254 unsigned long flags; 255 256 spin_lock_irqsave(&vdev->irqlock, flags); 257 258 /* 259 * Masking can come from interrupt, ioctl, or config space 260 * via INTx disable. The latter means this can get called 261 * even when not using intx delivery. In this case, just 262 * try to have the physical bit follow the virtual bit. 263 */ 264 if (unlikely(!is_intx(vdev))) { 265 if (vdev->pci_2_3) 266 pci_intx(pdev, 0); 267 } else if (!vdev->ctx[0].masked) { 268 /* 269 * Can't use check_and_mask here because we always want to 270 * mask, not just when something is pending. 271 */ 272 if (vdev->pci_2_3) 273 pci_intx(pdev, 0); 274 else 275 disable_irq_nosync(pdev->irq); 276 277 vdev->ctx[0].masked = true; 278 } 279 280 spin_unlock_irqrestore(&vdev->irqlock, flags); 281} 282 283/* 284 * If this is triggered by an eventfd, we can't call eventfd_signal 285 * or else we'll deadlock on the eventfd wait queue. Return >0 when 286 * a signal is necessary, which can then be handled via a work queue 287 * or directly depending on the caller. 288 */ 289static int vfio_pci_intx_unmask_handler(struct vfio_pci_device *vdev, 290 void *unused) 291{ 292 struct pci_dev *pdev = vdev->pdev; 293 unsigned long flags; 294 int ret = 0; 295 296 spin_lock_irqsave(&vdev->irqlock, flags); 297 298 /* 299 * Unmasking comes from ioctl or config, so again, have the 300 * physical bit follow the virtual even when not using INTx. 301 */ 302 if (unlikely(!is_intx(vdev))) { 303 if (vdev->pci_2_3) 304 pci_intx(pdev, 1); 305 } else if (vdev->ctx[0].masked && !vdev->virq_disabled) { 306 /* 307 * A pending interrupt here would immediately trigger, 308 * but we can avoid that overhead by just re-sending 309 * the interrupt to the user. 310 */ 311 if (vdev->pci_2_3) { 312 if (!pci_check_and_unmask_intx(pdev)) 313 ret = 1; 314 } else 315 enable_irq(pdev->irq); 316 317 vdev->ctx[0].masked = (ret > 0); 318 } 319 320 spin_unlock_irqrestore(&vdev->irqlock, flags); 321 322 return ret; 323} 324 325void vfio_pci_intx_unmask(struct vfio_pci_device *vdev) 326{ 327 if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) 328 vfio_send_intx_eventfd(vdev, NULL); 329} 330 331static irqreturn_t vfio_intx_handler(int irq, void *dev_id) 332{ 333 struct vfio_pci_device *vdev = dev_id; 334 unsigned long flags; 335 int ret = IRQ_NONE; 336 337 spin_lock_irqsave(&vdev->irqlock, flags); 338 339 if (!vdev->pci_2_3) { 340 disable_irq_nosync(vdev->pdev->irq); 341 vdev->ctx[0].masked = true; 342 ret = IRQ_HANDLED; 343 } else if (!vdev->ctx[0].masked && /* may be shared */ 344 pci_check_and_mask_intx(vdev->pdev)) { 345 vdev->ctx[0].masked = true; 346 ret = IRQ_HANDLED; 347 } 348 349 spin_unlock_irqrestore(&vdev->irqlock, flags); 350 351 if (ret == IRQ_HANDLED) 352 vfio_send_intx_eventfd(vdev, NULL); 353 354 return ret; 355} 356 357static int vfio_intx_enable(struct vfio_pci_device *vdev) 358{ 359 if (!is_irq_none(vdev)) 360 return -EINVAL; 361 362 if (!vdev->pdev->irq) 363 return -ENODEV; 364 365 vdev->ctx = kzalloc(sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); 366 if (!vdev->ctx) 367 return -ENOMEM; 368 369 vdev->num_ctx = 1; 370 371 /* 372 * If the virtual interrupt is masked, restore it. Devices 373 * supporting DisINTx can be masked at the hardware level 374 * here, non-PCI-2.3 devices will have to wait until the 375 * interrupt is enabled. 376 */ 377 vdev->ctx[0].masked = vdev->virq_disabled; 378 if (vdev->pci_2_3) 379 pci_intx(vdev->pdev, !vdev->ctx[0].masked); 380 381 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 382 383 return 0; 384} 385 386static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd) 387{ 388 struct pci_dev *pdev = vdev->pdev; 389 unsigned long irqflags = IRQF_SHARED; 390 struct eventfd_ctx *trigger; 391 unsigned long flags; 392 int ret; 393 394 if (vdev->ctx[0].trigger) { 395 free_irq(pdev->irq, vdev); 396 kfree(vdev->ctx[0].name); 397 eventfd_ctx_put(vdev->ctx[0].trigger); 398 vdev->ctx[0].trigger = NULL; 399 } 400 401 if (fd < 0) /* Disable only */ 402 return 0; 403 404 vdev->ctx[0].name = kasprintf(GFP_KERNEL, "vfio-intx(%s)", 405 pci_name(pdev)); 406 if (!vdev->ctx[0].name) 407 return -ENOMEM; 408 409 trigger = eventfd_ctx_fdget(fd); 410 if (IS_ERR(trigger)) { 411 kfree(vdev->ctx[0].name); 412 return PTR_ERR(trigger); 413 } 414 415 vdev->ctx[0].trigger = trigger; 416 417 if (!vdev->pci_2_3) 418 irqflags = 0; 419 420 ret = request_irq(pdev->irq, vfio_intx_handler, 421 irqflags, vdev->ctx[0].name, vdev); 422 if (ret) { 423 vdev->ctx[0].trigger = NULL; 424 kfree(vdev->ctx[0].name); 425 eventfd_ctx_put(trigger); 426 return ret; 427 } 428 429 /* 430 * INTx disable will stick across the new irq setup, 431 * disable_irq won't. 432 */ 433 spin_lock_irqsave(&vdev->irqlock, flags); 434 if (!vdev->pci_2_3 && vdev->ctx[0].masked) 435 disable_irq_nosync(pdev->irq); 436 spin_unlock_irqrestore(&vdev->irqlock, flags); 437 438 return 0; 439} 440 441static void vfio_intx_disable(struct vfio_pci_device *vdev) 442{ 443 vfio_intx_set_signal(vdev, -1); 444 virqfd_disable(vdev, &vdev->ctx[0].unmask); 445 virqfd_disable(vdev, &vdev->ctx[0].mask); 446 vdev->irq_type = VFIO_PCI_NUM_IRQS; 447 vdev->num_ctx = 0; 448 kfree(vdev->ctx); 449} 450 451/* 452 * MSI/MSI-X 453 */ 454static irqreturn_t vfio_msihandler(int irq, void *arg) 455{ 456 struct eventfd_ctx *trigger = arg; 457 458 eventfd_signal(trigger, 1); 459 return IRQ_HANDLED; 460} 461 462static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix) 463{ 464 struct pci_dev *pdev = vdev->pdev; 465 int ret; 466 467 if (!is_irq_none(vdev)) 468 return -EINVAL; 469 470 vdev->ctx = kzalloc(nvec * sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); 471 if (!vdev->ctx) 472 return -ENOMEM; 473 474 if (msix) { 475 int i; 476 477 vdev->msix = kzalloc(nvec * sizeof(struct msix_entry), 478 GFP_KERNEL); 479 if (!vdev->msix) { 480 kfree(vdev->ctx); 481 return -ENOMEM; 482 } 483 484 for (i = 0; i < nvec; i++) 485 vdev->msix[i].entry = i; 486 487 ret = pci_enable_msix(pdev, vdev->msix, nvec); 488 if (ret) { 489 kfree(vdev->msix); 490 kfree(vdev->ctx); 491 return ret; 492 } 493 } else { 494 ret = pci_enable_msi_block(pdev, nvec); 495 if (ret) { 496 kfree(vdev->ctx); 497 return ret; 498 } 499 } 500 501 vdev->num_ctx = nvec; 502 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 503 VFIO_PCI_MSI_IRQ_INDEX; 504 505 if (!msix) { 506 /* 507 * Compute the virtual hardware field for max msi vectors - 508 * it is the log base 2 of the number of vectors. 509 */ 510 vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 511 } 512 513 return 0; 514} 515 516static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, 517 int vector, int fd, bool msix) 518{ 519 struct pci_dev *pdev = vdev->pdev; 520 int irq = msix ? vdev->msix[vector].vector : pdev->irq + vector; 521 char *name = msix ? "vfio-msix" : "vfio-msi"; 522 struct eventfd_ctx *trigger; 523 int ret; 524 525 if (vector >= vdev->num_ctx) 526 return -EINVAL; 527 528 if (vdev->ctx[vector].trigger) { 529 free_irq(irq, vdev->ctx[vector].trigger); 530 kfree(vdev->ctx[vector].name); 531 eventfd_ctx_put(vdev->ctx[vector].trigger); 532 vdev->ctx[vector].trigger = NULL; 533 } 534 535 if (fd < 0) 536 return 0; 537 538 vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "%s[%d](%s)", 539 name, vector, pci_name(pdev)); 540 if (!vdev->ctx[vector].name) 541 return -ENOMEM; 542 543 trigger = eventfd_ctx_fdget(fd); 544 if (IS_ERR(trigger)) { 545 kfree(vdev->ctx[vector].name); 546 return PTR_ERR(trigger); 547 } 548 549 ret = request_irq(irq, vfio_msihandler, 0, 550 vdev->ctx[vector].name, trigger); 551 if (ret) { 552 kfree(vdev->ctx[vector].name); 553 eventfd_ctx_put(trigger); 554 return ret; 555 } 556 557 vdev->ctx[vector].trigger = trigger; 558 559 return 0; 560} 561 562static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start, 563 unsigned count, int32_t *fds, bool msix) 564{ 565 int i, j, ret = 0; 566 567 if (start + count > vdev->num_ctx) 568 return -EINVAL; 569 570 for (i = 0, j = start; i < count && !ret; i++, j++) { 571 int fd = fds ? fds[i] : -1; 572 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 573 } 574 575 if (ret) { 576 for (--j; j >= start; j--) 577 vfio_msi_set_vector_signal(vdev, j, -1, msix); 578 } 579 580 return ret; 581} 582 583static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix) 584{ 585 struct pci_dev *pdev = vdev->pdev; 586 int i; 587 588 vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix); 589 590 for (i = 0; i < vdev->num_ctx; i++) { 591 virqfd_disable(vdev, &vdev->ctx[i].unmask); 592 virqfd_disable(vdev, &vdev->ctx[i].mask); 593 } 594 595 if (msix) { 596 pci_disable_msix(vdev->pdev); 597 kfree(vdev->msix); 598 } else 599 pci_disable_msi(pdev); 600 601 vdev->irq_type = VFIO_PCI_NUM_IRQS; 602 vdev->num_ctx = 0; 603 kfree(vdev->ctx); 604} 605 606/* 607 * IOCTL support 608 */ 609static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev, 610 unsigned index, unsigned start, 611 unsigned count, uint32_t flags, void *data) 612{ 613 if (!is_intx(vdev) || start != 0 || count != 1) 614 return -EINVAL; 615 616 if (flags & VFIO_IRQ_SET_DATA_NONE) { 617 vfio_pci_intx_unmask(vdev); 618 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 619 uint8_t unmask = *(uint8_t *)data; 620 if (unmask) 621 vfio_pci_intx_unmask(vdev); 622 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 623 int32_t fd = *(int32_t *)data; 624 if (fd >= 0) 625 return virqfd_enable(vdev, vfio_pci_intx_unmask_handler, 626 vfio_send_intx_eventfd, NULL, 627 &vdev->ctx[0].unmask, fd); 628 629 virqfd_disable(vdev, &vdev->ctx[0].unmask); 630 } 631 632 return 0; 633} 634 635static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev, 636 unsigned index, unsigned start, 637 unsigned count, uint32_t flags, void *data) 638{ 639 if (!is_intx(vdev) || start != 0 || count != 1) 640 return -EINVAL; 641 642 if (flags & VFIO_IRQ_SET_DATA_NONE) { 643 vfio_pci_intx_mask(vdev); 644 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 645 uint8_t mask = *(uint8_t *)data; 646 if (mask) 647 vfio_pci_intx_mask(vdev); 648 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 649 return -ENOTTY; /* XXX implement me */ 650 } 651 652 return 0; 653} 654 655static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev, 656 unsigned index, unsigned start, 657 unsigned count, uint32_t flags, void *data) 658{ 659 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 660 vfio_intx_disable(vdev); 661 return 0; 662 } 663 664 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 665 return -EINVAL; 666 667 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 668 int32_t fd = *(int32_t *)data; 669 int ret; 670 671 if (is_intx(vdev)) 672 return vfio_intx_set_signal(vdev, fd); 673 674 ret = vfio_intx_enable(vdev); 675 if (ret) 676 return ret; 677 678 ret = vfio_intx_set_signal(vdev, fd); 679 if (ret) 680 vfio_intx_disable(vdev); 681 682 return ret; 683 } 684 685 if (!is_intx(vdev)) 686 return -EINVAL; 687 688 if (flags & VFIO_IRQ_SET_DATA_NONE) { 689 vfio_send_intx_eventfd(vdev, NULL); 690 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 691 uint8_t trigger = *(uint8_t *)data; 692 if (trigger) 693 vfio_send_intx_eventfd(vdev, NULL); 694 } 695 return 0; 696} 697 698static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev, 699 unsigned index, unsigned start, 700 unsigned count, uint32_t flags, void *data) 701{ 702 int i; 703 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 704 705 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 706 vfio_msi_disable(vdev, msix); 707 return 0; 708 } 709 710 if (!(irq_is(vdev, index) || is_irq_none(vdev))) 711 return -EINVAL; 712 713 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 714 int32_t *fds = data; 715 int ret; 716 717 if (vdev->irq_type == index) 718 return vfio_msi_set_block(vdev, start, count, 719 fds, msix); 720 721 ret = vfio_msi_enable(vdev, start + count, msix); 722 if (ret) 723 return ret; 724 725 ret = vfio_msi_set_block(vdev, start, count, fds, msix); 726 if (ret) 727 vfio_msi_disable(vdev, msix); 728 729 return ret; 730 } 731 732 if (!irq_is(vdev, index) || start + count > vdev->num_ctx) 733 return -EINVAL; 734 735 for (i = start; i < start + count; i++) { 736 if (!vdev->ctx[i].trigger) 737 continue; 738 if (flags & VFIO_IRQ_SET_DATA_NONE) { 739 eventfd_signal(vdev->ctx[i].trigger, 1); 740 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 741 uint8_t *bools = data; 742 if (bools[i - start]) 743 eventfd_signal(vdev->ctx[i].trigger, 1); 744 } 745 } 746 return 0; 747} 748 749static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev, 750 unsigned index, unsigned start, 751 unsigned count, uint32_t flags, void *data) 752{ 753 int32_t fd = *(int32_t *)data; 754 struct pci_dev *pdev = vdev->pdev; 755 756 if ((index != VFIO_PCI_ERR_IRQ_INDEX) || 757 !(flags & VFIO_IRQ_SET_DATA_TYPE_MASK)) 758 return -EINVAL; 759 760 /* 761 * device_lock synchronizes setting and checking of 762 * err_trigger. The vfio_pci_aer_err_detected() is also 763 * called with device_lock held. 764 */ 765 766 /* DATA_NONE/DATA_BOOL enables loopback testing */ 767 768 if (flags & VFIO_IRQ_SET_DATA_NONE) { 769 device_lock(&pdev->dev); 770 if (vdev->err_trigger) 771 eventfd_signal(vdev->err_trigger, 1); 772 device_unlock(&pdev->dev); 773 return 0; 774 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 775 uint8_t trigger = *(uint8_t *)data; 776 device_lock(&pdev->dev); 777 if (trigger && vdev->err_trigger) 778 eventfd_signal(vdev->err_trigger, 1); 779 device_unlock(&pdev->dev); 780 return 0; 781 } 782 783 /* Handle SET_DATA_EVENTFD */ 784 785 if (fd == -1) { 786 device_lock(&pdev->dev); 787 if (vdev->err_trigger) 788 eventfd_ctx_put(vdev->err_trigger); 789 vdev->err_trigger = NULL; 790 device_unlock(&pdev->dev); 791 return 0; 792 } else if (fd >= 0) { 793 struct eventfd_ctx *efdctx; 794 efdctx = eventfd_ctx_fdget(fd); 795 if (IS_ERR(efdctx)) 796 return PTR_ERR(efdctx); 797 device_lock(&pdev->dev); 798 if (vdev->err_trigger) 799 eventfd_ctx_put(vdev->err_trigger); 800 vdev->err_trigger = efdctx; 801 device_unlock(&pdev->dev); 802 return 0; 803 } else 804 return -EINVAL; 805} 806int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags, 807 unsigned index, unsigned start, unsigned count, 808 void *data) 809{ 810 int (*func)(struct vfio_pci_device *vdev, unsigned index, 811 unsigned start, unsigned count, uint32_t flags, 812 void *data) = NULL; 813 814 switch (index) { 815 case VFIO_PCI_INTX_IRQ_INDEX: 816 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 817 case VFIO_IRQ_SET_ACTION_MASK: 818 func = vfio_pci_set_intx_mask; 819 break; 820 case VFIO_IRQ_SET_ACTION_UNMASK: 821 func = vfio_pci_set_intx_unmask; 822 break; 823 case VFIO_IRQ_SET_ACTION_TRIGGER: 824 func = vfio_pci_set_intx_trigger; 825 break; 826 } 827 break; 828 case VFIO_PCI_MSI_IRQ_INDEX: 829 case VFIO_PCI_MSIX_IRQ_INDEX: 830 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 831 case VFIO_IRQ_SET_ACTION_MASK: 832 case VFIO_IRQ_SET_ACTION_UNMASK: 833 /* XXX Need masking support exported */ 834 break; 835 case VFIO_IRQ_SET_ACTION_TRIGGER: 836 func = vfio_pci_set_msi_trigger; 837 break; 838 } 839 break; 840 case VFIO_PCI_ERR_IRQ_INDEX: 841 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 842 case VFIO_IRQ_SET_ACTION_TRIGGER: 843 if (pci_is_pcie(vdev->pdev)) 844 func = vfio_pci_set_err_trigger; 845 break; 846 } 847 } 848 849 if (!func) 850 return -ENOTTY; 851 852 return func(vdev, index, start, count, flags, data); 853} 854