Commit ba0fb44a authored by Catalin Marinas's avatar Catalin Marinas Committed by Christoph Hellwig
Browse files

dma-mapping: replace zone_dma_bits by zone_dma_limit 



The hardware DMA limit might not be power of 2. When RAM range starts
above 0, say 4GB, DMA limit of 30 bits should end at 5GB.  A single high
bit can not encode this limit.

Use a plain  address for the DMA zone limit instead.

Since the DMA zone can now potentially span beyond 4GB physical limit of
DMA32, make sure to use DMA zone for GFP_DMA32 allocations in that case.

Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Co-developed-by: default avatarBaruch Siach <baruch@tkos.co.il>
Signed-off-by: default avatarBaruch Siach <baruch@tkos.co.il>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Reviewed-by: default avatarPetr Tesarik <ptesarik@suse.com>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
parent fa3c109a

arch/arm64/mm/init.c

+15 −15
Original line number Diff line number Diff line
@@ -115,35 +115,35 @@ static void __init arch_reserve_crashkernel(void) 
}



/*

 * Return the maximum physical address for a zone accessible by the given bits

 * limit. If DRAM starts above 32-bit, expand the zone to the maximum

 * Return the maximum physical address for a zone given its limit.

 * If DRAM starts above 32-bit, expand the zone to the maximum

 * available memory, otherwise cap it at 32-bit.

 */

static phys_addr_t __init max_zone_phys(unsigned int zone_bits)

static phys_addr_t __init max_zone_phys(phys_addr_t zone_limit)

{

	phys_addr_t zone_mask = DMA_BIT_MASK(zone_bits);

	phys_addr_t phys_start = memblock_start_of_DRAM();



	if (phys_start > U32_MAX)

		zone_mask = PHYS_ADDR_MAX;

	else if (phys_start > zone_mask)

		zone_mask = U32_MAX;

		zone_limit = PHYS_ADDR_MAX;

	else if (phys_start > zone_limit)

		zone_limit = U32_MAX;



	return min(zone_mask, memblock_end_of_DRAM() - 1) + 1;

	return min(zone_limit, memblock_end_of_DRAM() - 1) + 1;

}



static void __init zone_sizes_init(void)

{

	unsigned long max_zone_pfns[MAX_NR_ZONES]  = {0};

	unsigned int __maybe_unused acpi_zone_dma_bits;

	unsigned int __maybe_unused dt_zone_dma_bits;

	phys_addr_t __maybe_unused dma32_phys_limit = max_zone_phys(32);

	phys_addr_t __maybe_unused acpi_zone_dma_limit;

	phys_addr_t __maybe_unused dt_zone_dma_limit;

	phys_addr_t __maybe_unused dma32_phys_limit =

		max_zone_phys(DMA_BIT_MASK(32));



#ifdef CONFIG_ZONE_DMA

	acpi_zone_dma_bits = fls64(acpi_iort_dma_get_max_cpu_address());

	dt_zone_dma_bits = fls64(of_dma_get_max_cpu_address(NULL));

	zone_dma_bits = min3(32U, dt_zone_dma_bits, acpi_zone_dma_bits);

	arm64_dma_phys_limit = max_zone_phys(zone_dma_bits);

	acpi_zone_dma_limit = acpi_iort_dma_get_max_cpu_address();

	dt_zone_dma_limit = of_dma_get_max_cpu_address(NULL);

	zone_dma_limit = min(dt_zone_dma_limit, acpi_zone_dma_limit);

	arm64_dma_phys_limit = max_zone_phys(zone_dma_limit);

	max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit);

#endif

#ifdef CONFIG_ZONE_DMA32

arch/powerpc/mm/mem.c

+4 −1
Original line number Diff line number Diff line
@@ -216,7 +216,7 @@ static int __init mark_nonram_nosave(void) 
 * everything else. GFP_DMA32 page allocations automatically fall back to

 * ZONE_DMA.

 *

 * By using 31-bit unconditionally, we can exploit zone_dma_bits to inform the

 * By using 31-bit unconditionally, we can exploit zone_dma_limit to inform the

 * generic DMA mapping code.  32-bit only devices (if not handled by an IOMMU

 * anyway) will take a first dip into ZONE_NORMAL and get otherwise served by

 * ZONE_DMA.
@@ -230,6 +230,7 @@ void __init paging_init(void) 
{

	unsigned long long total_ram = memblock_phys_mem_size();

	phys_addr_t top_of_ram = memblock_end_of_DRAM();

	int zone_dma_bits;



#ifdef CONFIG_HIGHMEM

	unsigned long v = __fix_to_virt(FIX_KMAP_END);
@@ -256,6 +257,8 @@ void __init paging_init(void) 
	else

		zone_dma_bits = 31;



	zone_dma_limit = DMA_BIT_MASK(zone_dma_bits);



#ifdef CONFIG_ZONE_DMA

	max_zone_pfns[ZONE_DMA]	= min(max_low_pfn,

				      1UL << (zone_dma_bits - PAGE_SHIFT));

arch/s390/mm/init.c

+1 −1
Original line number Diff line number Diff line
@@ -97,7 +97,7 @@ void __init paging_init(void) 


	vmem_map_init();

	sparse_init();

	zone_dma_bits = 31;

	zone_dma_limit = DMA_BIT_MASK(31);

	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));

	max_zone_pfns[ZONE_DMA] = virt_to_pfn(MAX_DMA_ADDRESS);

	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;

include/linux/dma-direct.h

+1 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
#include <linux/mem_encrypt.h>

#include <linux/swiotlb.h>



extern unsigned int zone_dma_bits;

extern u64 zone_dma_limit;



/*

 * Record the mapping of CPU physical to DMA addresses for a given region.

kernel/dma/direct.c

+3 −3
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ 
 * it for entirely different regions. In that case the arch code needs to

 * override the variable below for dma-direct to work properly.

 */

unsigned int zone_dma_bits __ro_after_init = 24;

u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24);



static inline dma_addr_t phys_to_dma_direct(struct device *dev,

		phys_addr_t phys)
@@ -59,7 +59,7 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit) 
	 * zones.

	 */

	*phys_limit = dma_to_phys(dev, dma_limit);

	if (*phys_limit <= DMA_BIT_MASK(zone_dma_bits))

	if (*phys_limit <= zone_dma_limit)

		return GFP_DMA;

	if (*phys_limit <= DMA_BIT_MASK(32))

		return GFP_DMA32;
@@ -580,7 +580,7 @@ int dma_direct_supported(struct device *dev, u64 mask) 
	 * part of the check.

	 */

	if (IS_ENABLED(CONFIG_ZONE_DMA))

		min_mask = min_t(u64, min_mask, DMA_BIT_MASK(zone_dma_bits));

		min_mask = min_t(u64, min_mask, zone_dma_limit);

	return mask >= phys_to_dma_unencrypted(dev, min_mask);

}