Skip to content

Commit a25e796

Browse files
Christoph Hellwigmszyprow
authored andcommitted
PCI/P2PDMA: Refactor the p2pdma mapping helpers
The current scheme with a single helper to determine the P2P status and map a scatterlist segment force users to always use the map_sg helper to DMA map, which we're trying to get away from because they are very cache inefficient. Refactor the code so that there is a single helper that checks the P2P state for a page, including the result that it is not a P2P page to simplify the callers, and a second one to perform the address translation for a bus mapped P2P transfer that does not depend on the scatterlist structure. Signed-off-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Logan Gunthorpe <logang@deltatee.com> Acked-by: Bjorn Helgaas <bhelgaas@google.com> Tested-by: Jens Axboe <axboe@kernel.dk> Reviewed-by: Luis Chamberlain <mcgrof@kernel.org> Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com> Signed-off-by: Leon Romanovsky <leonro@nvidia.com> Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
1 parent 0af2f6b commit a25e796

File tree

4 files changed

+91
-88
lines changed

4 files changed

+91
-88
lines changed

drivers/iommu/dma-iommu.c

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1359,7 +1359,6 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
13591359
struct scatterlist *s, *prev = NULL;
13601360
int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
13611361
struct pci_p2pdma_map_state p2pdma_state = {};
1362-
enum pci_p2pdma_map_type map;
13631362
dma_addr_t iova;
13641363
size_t iova_len = 0;
13651364
unsigned long mask = dma_get_seg_boundary(dev);
@@ -1389,28 +1388,30 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
13891388
size_t s_length = s->length;
13901389
size_t pad_len = (mask - iova_len + 1) & mask;
13911390

1392-
if (is_pci_p2pdma_page(sg_page(s))) {
1393-
map = pci_p2pdma_map_segment(&p2pdma_state, dev, s);
1394-
switch (map) {
1395-
case PCI_P2PDMA_MAP_BUS_ADDR:
1396-
/*
1397-
* iommu_map_sg() will skip this segment as
1398-
* it is marked as a bus address,
1399-
* __finalise_sg() will copy the dma address
1400-
* into the output segment.
1401-
*/
1402-
continue;
1403-
case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1404-
/*
1405-
* Mapping through host bridge should be
1406-
* mapped with regular IOVAs, thus we
1407-
* do nothing here and continue below.
1408-
*/
1409-
break;
1410-
default:
1411-
ret = -EREMOTEIO;
1412-
goto out_restore_sg;
1413-
}
1391+
switch (pci_p2pdma_state(&p2pdma_state, dev, sg_page(s))) {
1392+
case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1393+
/*
1394+
* Mapping through host bridge should be mapped with
1395+
* regular IOVAs, thus we do nothing here and continue
1396+
* below.
1397+
*/
1398+
break;
1399+
case PCI_P2PDMA_MAP_NONE:
1400+
break;
1401+
case PCI_P2PDMA_MAP_BUS_ADDR:
1402+
/*
1403+
* iommu_map_sg() will skip this segment as it is marked
1404+
* as a bus address, __finalise_sg() will copy the dma
1405+
* address into the output segment.
1406+
*/
1407+
s->dma_address = pci_p2pdma_bus_addr_map(&p2pdma_state,
1408+
sg_phys(s));
1409+
sg_dma_len(s) = sg->length;
1410+
sg_dma_mark_bus_address(s);
1411+
continue;
1412+
default:
1413+
ret = -EREMOTEIO;
1414+
goto out_restore_sg;
14141415
}
14151416

14161417
sg_dma_address(s) = s_iova_off;

drivers/pci/p2pdma.c

Lines changed: 5 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,40 +1004,12 @@ static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct dev_pagemap *pgmap,
10041004
return type;
10051005
}
10061006

1007-
/**
1008-
* pci_p2pdma_map_segment - map an sg segment determining the mapping type
1009-
* @state: State structure that should be declared outside of the for_each_sg()
1010-
* loop and initialized to zero.
1011-
* @dev: DMA device that's doing the mapping operation
1012-
* @sg: scatterlist segment to map
1013-
*
1014-
* This is a helper to be used by non-IOMMU dma_map_sg() implementations where
1015-
* the sg segment is the same for the page_link and the dma_address.
1016-
*
1017-
* Attempt to map a single segment in an SGL with the PCI bus address.
1018-
* The segment must point to a PCI P2PDMA page and thus must be
1019-
* wrapped in a is_pci_p2pdma_page(sg_page(sg)) check.
1020-
*
1021-
* Returns the type of mapping used and maps the page if the type is
1022-
* PCI_P2PDMA_MAP_BUS_ADDR.
1023-
*/
1024-
enum pci_p2pdma_map_type
1025-
pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
1026-
struct scatterlist *sg)
1007+
void __pci_p2pdma_update_state(struct pci_p2pdma_map_state *state,
1008+
struct device *dev, struct page *page)
10271009
{
1028-
if (state->pgmap != page_pgmap(sg_page(sg))) {
1029-
state->pgmap = page_pgmap(sg_page(sg));
1030-
state->map = pci_p2pdma_map_type(state->pgmap, dev);
1031-
state->bus_off = to_p2p_pgmap(state->pgmap)->bus_offset;
1032-
}
1033-
1034-
if (state->map == PCI_P2PDMA_MAP_BUS_ADDR) {
1035-
sg->dma_address = sg_phys(sg) + state->bus_off;
1036-
sg_dma_len(sg) = sg->length;
1037-
sg_dma_mark_bus_address(sg);
1038-
}
1039-
1040-
return state->map;
1010+
state->pgmap = page_pgmap(page);
1011+
state->map = pci_p2pdma_map_type(state->pgmap, dev);
1012+
state->bus_off = to_p2p_pgmap(state->pgmap)->bus_offset;
10411013
}
10421014

10431015
/**

include/linux/dma-map-ops.h

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,11 @@ enum pci_p2pdma_map_type {
443443
*/
444444
PCI_P2PDMA_MAP_UNKNOWN = 0,
445445

446+
/*
447+
* Not a PCI P2PDMA transfer.
448+
*/
449+
PCI_P2PDMA_MAP_NONE,
450+
446451
/*
447452
* PCI_P2PDMA_MAP_NOT_SUPPORTED: Indicates the transaction will
448453
* traverse the host bridge and the host bridge is not in the
@@ -471,21 +476,47 @@ enum pci_p2pdma_map_type {
471476

472477
struct pci_p2pdma_map_state {
473478
struct dev_pagemap *pgmap;
474-
int map;
479+
enum pci_p2pdma_map_type map;
475480
u64 bus_off;
476481
};
477482

478-
#ifdef CONFIG_PCI_P2PDMA
479-
enum pci_p2pdma_map_type
480-
pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
481-
struct scatterlist *sg);
482-
#else /* CONFIG_PCI_P2PDMA */
483+
/* helper for pci_p2pdma_state(), do not use directly */
484+
void __pci_p2pdma_update_state(struct pci_p2pdma_map_state *state,
485+
struct device *dev, struct page *page);
486+
487+
/**
488+
* pci_p2pdma_state - check the P2P transfer state of a page
489+
* @state: P2P state structure
490+
* @dev: device to transfer to/from
491+
* @page: page to map
492+
*
493+
* Check if @page is a PCI P2PDMA page, and if yes of what kind. Returns the
494+
* map type, and updates @state with all information needed for a P2P transfer.
495+
*/
483496
static inline enum pci_p2pdma_map_type
484-
pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
485-
struct scatterlist *sg)
497+
pci_p2pdma_state(struct pci_p2pdma_map_state *state, struct device *dev,
498+
struct page *page)
499+
{
500+
if (IS_ENABLED(CONFIG_PCI_P2PDMA) && is_pci_p2pdma_page(page)) {
501+
if (state->pgmap != page_pgmap(page))
502+
__pci_p2pdma_update_state(state, dev, page);
503+
return state->map;
504+
}
505+
return PCI_P2PDMA_MAP_NONE;
506+
}
507+
508+
/**
509+
* pci_p2pdma_bus_addr_map - map a PCI_P2PDMA_MAP_BUS_ADDR P2P transfer
510+
* @state: P2P state structure
511+
* @paddr: physical address to map
512+
*
513+
* Map a physically contiguous PCI_P2PDMA_MAP_BUS_ADDR transfer.
514+
*/
515+
static inline dma_addr_t
516+
pci_p2pdma_bus_addr_map(struct pci_p2pdma_map_state *state, phys_addr_t paddr)
486517
{
487-
return PCI_P2PDMA_MAP_NOT_SUPPORTED;
518+
WARN_ON_ONCE(state->map != PCI_P2PDMA_MAP_BUS_ADDR);
519+
return paddr + state->bus_off;
488520
}
489-
#endif /* CONFIG_PCI_P2PDMA */
490521

491522
#endif /* _LINUX_DMA_MAP_OPS_H */

kernel/dma/direct.c

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -462,34 +462,33 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
462462
enum dma_data_direction dir, unsigned long attrs)
463463
{
464464
struct pci_p2pdma_map_state p2pdma_state = {};
465-
enum pci_p2pdma_map_type map;
466465
struct scatterlist *sg;
467466
int i, ret;
468467

469468
for_each_sg(sgl, sg, nents, i) {
470-
if (is_pci_p2pdma_page(sg_page(sg))) {
471-
map = pci_p2pdma_map_segment(&p2pdma_state, dev, sg);
472-
switch (map) {
473-
case PCI_P2PDMA_MAP_BUS_ADDR:
474-
continue;
475-
case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
476-
/*
477-
* Any P2P mapping that traverses the PCI
478-
* host bridge must be mapped with CPU physical
479-
* address and not PCI bus addresses. This is
480-
* done with dma_direct_map_page() below.
481-
*/
482-
break;
483-
default:
484-
ret = -EREMOTEIO;
469+
switch (pci_p2pdma_state(&p2pdma_state, dev, sg_page(sg))) {
470+
case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
471+
/*
472+
* Any P2P mapping that traverses the PCI host bridge
473+
* must be mapped with CPU physical address and not PCI
474+
* bus addresses.
475+
*/
476+
break;
477+
case PCI_P2PDMA_MAP_NONE:
478+
sg->dma_address = dma_direct_map_page(dev, sg_page(sg),
479+
sg->offset, sg->length, dir, attrs);
480+
if (sg->dma_address == DMA_MAPPING_ERROR) {
481+
ret = -EIO;
485482
goto out_unmap;
486483
}
487-
}
488-
489-
sg->dma_address = dma_direct_map_page(dev, sg_page(sg),
490-
sg->offset, sg->length, dir, attrs);
491-
if (sg->dma_address == DMA_MAPPING_ERROR) {
492-
ret = -EIO;
484+
break;
485+
case PCI_P2PDMA_MAP_BUS_ADDR:
486+
sg->dma_address = pci_p2pdma_bus_addr_map(&p2pdma_state,
487+
sg_phys(sg));
488+
sg_dma_mark_bus_address(sg);
489+
continue;
490+
default:
491+
ret = -EREMOTEIO;
493492
goto out_unmap;
494493
}
495494
sg_dma_len(sg) = sg->length;

0 commit comments

Comments
 (0)