1/* 2 * Copyright 2010 3 * by Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> 4 * 5 * This code provides a IOMMU for Xen PV guests with PCI passthrough. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License v2.0 as published by 9 * the Free Software Foundation 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * PV guests under Xen are running in an non-contiguous memory architecture. 17 * 18 * When PCI pass-through is utilized, this necessitates an IOMMU for 19 * translating bus (DMA) to virtual and vice-versa and also providing a 20 * mechanism to have contiguous pages for device drivers operations (say DMA 21 * operations). 22 * 23 * Specifically, under Xen the Linux idea of pages is an illusion. It 24 * assumes that pages start at zero and go up to the available memory. To 25 * help with that, the Linux Xen MMU provides a lookup mechanism to 26 * translate the page frame numbers (PFN) to machine frame numbers (MFN) 27 * and vice-versa. The MFN are the "real" frame numbers. Furthermore 28 * memory is not contiguous. Xen hypervisor stitches memory for guests 29 * from different pools, which means there is no guarantee that PFN==MFN 30 * and PFN+1==MFN+1. Lastly with Xen 4.0, pages (in debug mode) are 31 * allocated in descending order (high to low), meaning the guest might 32 * never get any MFN's under the 4GB mark. 33 * 34 */ 35 36#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 37 38#include <linux/bootmem.h> 39#include <linux/dma-mapping.h> 40#include <linux/export.h> 41#include <xen/swiotlb-xen.h> 42#include <xen/page.h> 43#include <xen/xen-ops.h> 44#include <xen/hvc-console.h> 45 46#include <asm/dma-mapping.h> 47#include <asm/xen/page-coherent.h> 48 49#include <trace/events/swiotlb.h> 50/* 51 * Used to do a quick range check in swiotlb_tbl_unmap_single and 52 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this 53 * API. 54 */ 55 56#ifndef CONFIG_X86 57static unsigned long dma_alloc_coherent_mask(struct device *dev, 58 gfp_t gfp) 59{ 60 unsigned long dma_mask = 0; 61 62 dma_mask = dev->coherent_dma_mask; 63 if (!dma_mask) 64 dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32); 65 66 return dma_mask; 67} 68#endif 69 70static char *xen_io_tlb_start, *xen_io_tlb_end; 71static unsigned long xen_io_tlb_nslabs; 72/* 73 * Quick lookup value of the bus address of the IOTLB. 74 */ 75 76static u64 start_dma_addr; 77 78/* 79 * Both of these functions should avoid PFN_PHYS because phys_addr_t 80 * can be 32bit when dma_addr_t is 64bit leading to a loss in 81 * information if the shift is done before casting to 64bit. 82 */ 83static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr) 84{ 85 unsigned long mfn = pfn_to_mfn(PFN_DOWN(paddr)); 86 dma_addr_t dma = (dma_addr_t)mfn << PAGE_SHIFT; 87 88 dma |= paddr & ~PAGE_MASK; 89 90 return dma; 91} 92 93static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr) 94{ 95 unsigned long pfn = mfn_to_pfn(PFN_DOWN(baddr)); 96 dma_addr_t dma = (dma_addr_t)pfn << PAGE_SHIFT; 97 phys_addr_t paddr = dma; 98 99 BUG_ON(paddr != dma); /* truncation has occurred, should never happen */ 100 101 paddr |= baddr & ~PAGE_MASK; 102 103 return paddr; 104} 105 106static inline dma_addr_t xen_virt_to_bus(void *address) 107{ 108 return xen_phys_to_bus(virt_to_phys(address)); 109} 110 111static int check_pages_physically_contiguous(unsigned long pfn, 112 unsigned int offset, 113 size_t length) 114{ 115 unsigned long next_mfn; 116 int i; 117 int nr_pages; 118 119 next_mfn = pfn_to_mfn(pfn); 120 nr_pages = (offset + length + PAGE_SIZE-1) >> PAGE_SHIFT; 121 122 for (i = 1; i < nr_pages; i++) { 123 if (pfn_to_mfn(++pfn) != ++next_mfn) 124 return 0; 125 } 126 return 1; 127} 128 129static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) 130{ 131 unsigned long pfn = PFN_DOWN(p); 132 unsigned int offset = p & ~PAGE_MASK; 133 134 if (offset + size <= PAGE_SIZE) 135 return 0; 136 if (check_pages_physically_contiguous(pfn, offset, size)) 137 return 0; 138 return 1; 139} 140 141static int is_xen_swiotlb_buffer(dma_addr_t dma_addr) 142{ 143 unsigned long mfn = PFN_DOWN(dma_addr); 144 unsigned long pfn = mfn_to_local_pfn(mfn); 145 phys_addr_t paddr; 146 147 /* If the address is outside our domain, it CAN 148 * have the same virtual address as another address 149 * in our domain. Therefore _only_ check address within our domain. 150 */ 151 if (pfn_valid(pfn)) { 152 paddr = PFN_PHYS(pfn); 153 return paddr >= virt_to_phys(xen_io_tlb_start) && 154 paddr < virt_to_phys(xen_io_tlb_end); 155 } 156 return 0; 157} 158 159static int max_dma_bits = 32; 160 161static int 162xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) 163{ 164 int i, rc; 165 int dma_bits; 166 dma_addr_t dma_handle; 167 phys_addr_t p = virt_to_phys(buf); 168 169 dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT; 170 171 i = 0; 172 do { 173 int slabs = min(nslabs - i, (unsigned long)IO_TLB_SEGSIZE); 174 175 do { 176 rc = xen_create_contiguous_region( 177 p + (i << IO_TLB_SHIFT), 178 get_order(slabs << IO_TLB_SHIFT), 179 dma_bits, &dma_handle); 180 } while (rc && dma_bits++ < max_dma_bits); 181 if (rc) 182 return rc; 183 184 i += slabs; 185 } while (i < nslabs); 186 return 0; 187} 188static unsigned long xen_set_nslabs(unsigned long nr_tbl) 189{ 190 if (!nr_tbl) { 191 xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT); 192 xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE); 193 } else 194 xen_io_tlb_nslabs = nr_tbl; 195 196 return xen_io_tlb_nslabs << IO_TLB_SHIFT; 197} 198 199enum xen_swiotlb_err { 200 XEN_SWIOTLB_UNKNOWN = 0, 201 XEN_SWIOTLB_ENOMEM, 202 XEN_SWIOTLB_EFIXUP 203}; 204 205static const char *xen_swiotlb_error(enum xen_swiotlb_err err) 206{ 207 switch (err) { 208 case XEN_SWIOTLB_ENOMEM: 209 return "Cannot allocate Xen-SWIOTLB buffer\n"; 210 case XEN_SWIOTLB_EFIXUP: 211 return "Failed to get contiguous memory for DMA from Xen!\n"\ 212 "You either: don't have the permissions, do not have"\ 213 " enough free memory under 4GB, or the hypervisor memory"\ 214 " is too fragmented!"; 215 default: 216 break; 217 } 218 return ""; 219} 220int __ref xen_swiotlb_init(int verbose, bool early) 221{ 222 unsigned long bytes, order; 223 int rc = -ENOMEM; 224 enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN; 225 unsigned int repeat = 3; 226 227 xen_io_tlb_nslabs = swiotlb_nr_tbl(); 228retry: 229 bytes = xen_set_nslabs(xen_io_tlb_nslabs); 230 order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT); 231 /* 232 * Get IO TLB memory from any location. 233 */ 234 if (early) 235 xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes)); 236 else { 237#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) 238#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) 239 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { 240 xen_io_tlb_start = (void *)__get_free_pages(__GFP_NOWARN, order); 241 if (xen_io_tlb_start) 242 break; 243 order--; 244 } 245 if (order != get_order(bytes)) { 246 pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n", 247 (PAGE_SIZE << order) >> 20); 248 xen_io_tlb_nslabs = SLABS_PER_PAGE << order; 249 bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT; 250 } 251 } 252 if (!xen_io_tlb_start) { 253 m_ret = XEN_SWIOTLB_ENOMEM; 254 goto error; 255 } 256 xen_io_tlb_end = xen_io_tlb_start + bytes; 257 /* 258 * And replace that memory with pages under 4GB. 259 */ 260 rc = xen_swiotlb_fixup(xen_io_tlb_start, 261 bytes, 262 xen_io_tlb_nslabs); 263 if (rc) { 264 if (early) 265 free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes)); 266 else { 267 free_pages((unsigned long)xen_io_tlb_start, order); 268 xen_io_tlb_start = NULL; 269 } 270 m_ret = XEN_SWIOTLB_EFIXUP; 271 goto error; 272 } 273 start_dma_addr = xen_virt_to_bus(xen_io_tlb_start); 274 if (early) { 275 if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, 276 verbose)) 277 panic("Cannot allocate SWIOTLB buffer"); 278 rc = 0; 279 } else 280 rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs); 281 return rc; 282error: 283 if (repeat--) { 284 xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */ 285 (xen_io_tlb_nslabs >> 1)); 286 pr_info("Lowering to %luMB\n", 287 (xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20); 288 goto retry; 289 } 290 pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc); 291 if (early) 292 panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc); 293 else 294 free_pages((unsigned long)xen_io_tlb_start, order); 295 return rc; 296} 297void * 298xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, 299 dma_addr_t *dma_handle, gfp_t flags, 300 struct dma_attrs *attrs) 301{ 302 void *ret; 303 int order = get_order(size); 304 u64 dma_mask = DMA_BIT_MASK(32); 305 phys_addr_t phys; 306 dma_addr_t dev_addr; 307 308 /* 309 * Ignore region specifiers - the kernel's ideas of 310 * pseudo-phys memory layout has nothing to do with the 311 * machine physical layout. We can't allocate highmem 312 * because we can't return a pointer to it. 313 */ 314 flags &= ~(__GFP_DMA | __GFP_HIGHMEM); 315 316 if (dma_alloc_from_coherent(hwdev, size, dma_handle, &ret)) 317 return ret; 318 319 /* On ARM this function returns an ioremap'ped virtual address for 320 * which virt_to_phys doesn't return the corresponding physical 321 * address. In fact on ARM virt_to_phys only works for kernel direct 322 * mapped RAM memory. Also see comment below. 323 */ 324 ret = xen_alloc_coherent_pages(hwdev, size, dma_handle, flags, attrs); 325 326 if (!ret) 327 return ret; 328 329 if (hwdev && hwdev->coherent_dma_mask) 330 dma_mask = dma_alloc_coherent_mask(hwdev, flags); 331 332 /* At this point dma_handle is the physical address, next we are 333 * going to set it to the machine address. 334 * Do not use virt_to_phys(ret) because on ARM it doesn't correspond 335 * to *dma_handle. */ 336 phys = *dma_handle; 337 dev_addr = xen_phys_to_bus(phys); 338 if (((dev_addr + size - 1 <= dma_mask)) && 339 !range_straddles_page_boundary(phys, size)) 340 *dma_handle = dev_addr; 341 else { 342 if (xen_create_contiguous_region(phys, order, 343 fls64(dma_mask), dma_handle) != 0) { 344 xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs); 345 return NULL; 346 } 347 } 348 memset(ret, 0, size); 349 return ret; 350} 351EXPORT_SYMBOL_GPL(xen_swiotlb_alloc_coherent); 352 353void 354xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, 355 dma_addr_t dev_addr, struct dma_attrs *attrs) 356{ 357 int order = get_order(size); 358 phys_addr_t phys; 359 u64 dma_mask = DMA_BIT_MASK(32); 360 361 if (dma_release_from_coherent(hwdev, order, vaddr)) 362 return; 363 364 if (hwdev && hwdev->coherent_dma_mask) 365 dma_mask = hwdev->coherent_dma_mask; 366 367 /* do not use virt_to_phys because on ARM it doesn't return you the 368 * physical address */ 369 phys = xen_bus_to_phys(dev_addr); 370 371 if (((dev_addr + size - 1 > dma_mask)) || 372 range_straddles_page_boundary(phys, size)) 373 xen_destroy_contiguous_region(phys, order); 374 375 xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs); 376} 377EXPORT_SYMBOL_GPL(xen_swiotlb_free_coherent); 378 379 380/* 381 * Map a single buffer of the indicated size for DMA in streaming mode. The 382 * physical address to use is returned. 383 * 384 * Once the device is given the dma address, the device owns this memory until 385 * either xen_swiotlb_unmap_page or xen_swiotlb_dma_sync_single is performed. 386 */ 387dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, 388 unsigned long offset, size_t size, 389 enum dma_data_direction dir, 390 struct dma_attrs *attrs) 391{ 392 phys_addr_t map, phys = page_to_phys(page) + offset; 393 dma_addr_t dev_addr = xen_phys_to_bus(phys); 394 395 BUG_ON(dir == DMA_NONE); 396 /* 397 * If the address happens to be in the device's DMA window, 398 * we can safely return the device addr and not worry about bounce 399 * buffering it. 400 */ 401 if (dma_capable(dev, dev_addr, size) && 402 !range_straddles_page_boundary(phys, size) && !swiotlb_force) { 403 /* we are not interested in the dma_addr returned by 404 * xen_dma_map_page, only in the potential cache flushes executed 405 * by the function. */ 406 xen_dma_map_page(dev, page, offset, size, dir, attrs); 407 return dev_addr; 408 } 409 410 /* 411 * Oh well, have to allocate and map a bounce buffer. 412 */ 413 trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); 414 415 map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir); 416 if (map == SWIOTLB_MAP_ERROR) 417 return DMA_ERROR_CODE; 418 419 xen_dma_map_page(dev, pfn_to_page(map >> PAGE_SHIFT), 420 map & ~PAGE_MASK, size, dir, attrs); 421 dev_addr = xen_phys_to_bus(map); 422 423 /* 424 * Ensure that the address returned is DMA'ble 425 */ 426 if (!dma_capable(dev, dev_addr, size)) { 427 swiotlb_tbl_unmap_single(dev, map, size, dir); 428 dev_addr = 0; 429 } 430 return dev_addr; 431} 432EXPORT_SYMBOL_GPL(xen_swiotlb_map_page); 433 434/* 435 * Unmap a single streaming mode DMA translation. The dma_addr and size must 436 * match what was provided for in a previous xen_swiotlb_map_page call. All 437 * other usages are undefined. 438 * 439 * After this call, reads by the cpu to the buffer are guaranteed to see 440 * whatever the device wrote there. 441 */ 442static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, 443 size_t size, enum dma_data_direction dir, 444 struct dma_attrs *attrs) 445{ 446 phys_addr_t paddr = xen_bus_to_phys(dev_addr); 447 448 BUG_ON(dir == DMA_NONE); 449 450 xen_dma_unmap_page(hwdev, paddr, size, dir, attrs); 451 452 /* NOTE: We use dev_addr here, not paddr! */ 453 if (is_xen_swiotlb_buffer(dev_addr)) { 454 swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); 455 return; 456 } 457 458 if (dir != DMA_FROM_DEVICE) 459 return; 460 461 /* 462 * phys_to_virt doesn't work with hihgmem page but we could 463 * call dma_mark_clean() with hihgmem page here. However, we 464 * are fine since dma_mark_clean() is null on POWERPC. We can 465 * make dma_mark_clean() take a physical address if necessary. 466 */ 467 dma_mark_clean(phys_to_virt(paddr), size); 468} 469 470void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, 471 size_t size, enum dma_data_direction dir, 472 struct dma_attrs *attrs) 473{ 474 xen_unmap_single(hwdev, dev_addr, size, dir, attrs); 475} 476EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_page); 477 478/* 479 * Make physical memory consistent for a single streaming mode DMA translation 480 * after a transfer. 481 * 482 * If you perform a xen_swiotlb_map_page() but wish to interrogate the buffer 483 * using the cpu, yet do not wish to teardown the dma mapping, you must 484 * call this function before doing so. At the next point you give the dma 485 * address back to the card, you must first perform a 486 * xen_swiotlb_dma_sync_for_device, and then the device again owns the buffer 487 */ 488static void 489xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, 490 size_t size, enum dma_data_direction dir, 491 enum dma_sync_target target) 492{ 493 phys_addr_t paddr = xen_bus_to_phys(dev_addr); 494 495 BUG_ON(dir == DMA_NONE); 496 497 if (target == SYNC_FOR_CPU) 498 xen_dma_sync_single_for_cpu(hwdev, paddr, size, dir); 499 500 /* NOTE: We use dev_addr here, not paddr! */ 501 if (is_xen_swiotlb_buffer(dev_addr)) 502 swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target); 503 504 if (target == SYNC_FOR_DEVICE) 505 xen_dma_sync_single_for_cpu(hwdev, paddr, size, dir); 506 507 if (dir != DMA_FROM_DEVICE) 508 return; 509 510 dma_mark_clean(phys_to_virt(paddr), size); 511} 512 513void 514xen_swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, 515 size_t size, enum dma_data_direction dir) 516{ 517 xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); 518} 519EXPORT_SYMBOL_GPL(xen_swiotlb_sync_single_for_cpu); 520 521void 522xen_swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, 523 size_t size, enum dma_data_direction dir) 524{ 525 xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); 526} 527EXPORT_SYMBOL_GPL(xen_swiotlb_sync_single_for_device); 528 529/* 530 * Map a set of buffers described by scatterlist in streaming mode for DMA. 531 * This is the scatter-gather version of the above xen_swiotlb_map_page 532 * interface. Here the scatter gather list elements are each tagged with the 533 * appropriate dma address and length. They are obtained via 534 * sg_dma_{address,length}(SG). 535 * 536 * NOTE: An implementation may be able to use a smaller number of 537 * DMA address/length pairs than there are SG table elements. 538 * (for example via virtual mapping capabilities) 539 * The routine returns the number of addr/length pairs actually 540 * used, at most nents. 541 * 542 * Device ownership issues as mentioned above for xen_swiotlb_map_page are the 543 * same here. 544 */ 545int 546xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, 547 int nelems, enum dma_data_direction dir, 548 struct dma_attrs *attrs) 549{ 550 struct scatterlist *sg; 551 int i; 552 553 BUG_ON(dir == DMA_NONE); 554 555 for_each_sg(sgl, sg, nelems, i) { 556 phys_addr_t paddr = sg_phys(sg); 557 dma_addr_t dev_addr = xen_phys_to_bus(paddr); 558 559 if (swiotlb_force || 560 !dma_capable(hwdev, dev_addr, sg->length) || 561 range_straddles_page_boundary(paddr, sg->length)) { 562 phys_addr_t map = swiotlb_tbl_map_single(hwdev, 563 start_dma_addr, 564 sg_phys(sg), 565 sg->length, 566 dir); 567 if (map == SWIOTLB_MAP_ERROR) { 568 dev_warn(hwdev, "swiotlb buffer is full\n"); 569 /* Don't panic here, we expect map_sg users 570 to do proper error handling. */ 571 xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, 572 attrs); 573 sg_dma_len(sgl) = 0; 574 return 0; 575 } 576 xen_dma_map_page(hwdev, pfn_to_page(map >> PAGE_SHIFT), 577 map & ~PAGE_MASK, 578 sg->length, 579 dir, 580 attrs); 581 sg->dma_address = xen_phys_to_bus(map); 582 } else { 583 /* we are not interested in the dma_addr returned by 584 * xen_dma_map_page, only in the potential cache flushes executed 585 * by the function. */ 586 xen_dma_map_page(hwdev, pfn_to_page(paddr >> PAGE_SHIFT), 587 paddr & ~PAGE_MASK, 588 sg->length, 589 dir, 590 attrs); 591 sg->dma_address = dev_addr; 592 } 593 sg_dma_len(sg) = sg->length; 594 } 595 return nelems; 596} 597EXPORT_SYMBOL_GPL(xen_swiotlb_map_sg_attrs); 598 599/* 600 * Unmap a set of streaming mode DMA translations. Again, cpu read rules 601 * concerning calls here are the same as for swiotlb_unmap_page() above. 602 */ 603void 604xen_swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, 605 int nelems, enum dma_data_direction dir, 606 struct dma_attrs *attrs) 607{ 608 struct scatterlist *sg; 609 int i; 610 611 BUG_ON(dir == DMA_NONE); 612 613 for_each_sg(sgl, sg, nelems, i) 614 xen_unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs); 615 616} 617EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg_attrs); 618 619/* 620 * Make physical memory consistent for a set of streaming mode DMA translations 621 * after a transfer. 622 * 623 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules 624 * and usage. 625 */ 626static void 627xen_swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, 628 int nelems, enum dma_data_direction dir, 629 enum dma_sync_target target) 630{ 631 struct scatterlist *sg; 632 int i; 633 634 for_each_sg(sgl, sg, nelems, i) 635 xen_swiotlb_sync_single(hwdev, sg->dma_address, 636 sg_dma_len(sg), dir, target); 637} 638 639void 640xen_swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, 641 int nelems, enum dma_data_direction dir) 642{ 643 xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); 644} 645EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_cpu); 646 647void 648xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, 649 int nelems, enum dma_data_direction dir) 650{ 651 xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); 652} 653EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_device); 654 655int 656xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) 657{ 658 return !dma_addr; 659} 660EXPORT_SYMBOL_GPL(xen_swiotlb_dma_mapping_error); 661 662/* 663 * Return whether the given device DMA address mask can be supported 664 * properly. For example, if your device can only drive the low 24-bits 665 * during bus mastering, then you would pass 0x00ffffff as the mask to 666 * this function. 667 */ 668int 669xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) 670{ 671 return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask; 672} 673EXPORT_SYMBOL_GPL(xen_swiotlb_dma_supported); 674 675int 676xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask) 677{ 678 if (!dev->dma_mask || !xen_swiotlb_dma_supported(dev, dma_mask)) 679 return -EIO; 680 681 *dev->dma_mask = dma_mask; 682 683 return 0; 684} 685EXPORT_SYMBOL_GPL(xen_swiotlb_set_dma_mask); 686