Commit 7ccaaf13 authored by James Zhu's avatar James Zhu Committed by Alex Deucher
Browse files

Revert "drm/amdkfd: return migration pages from copy function" 



This reverts commit bd6093e2.

migrate_vma_pages can fail if a CPU thread faults on the same page.
However, the page table is locked and only one of the new pages will
be inserted. The device driver will see that the MIGRATE_PFN_MIGRATE
bit is cleared if it loses the race.

Signed-off-by: default avatarJames Zhu <James.Zhu@amd.com>
Reviewed-by: default avatarPhilip Yang <Philip.Yang@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent 85442bac

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c

+65 −65
Original line number Diff line number Diff line
@@ -39,22 +39,22 @@ 
#endif

#define dev_fmt(fmt) "kfd_migrate: " fmt



static uint64_t

svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, uint64_t addr)

static u64

svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, u64 addr)

{

	return addr + amdgpu_ttm_domain_start(adev, TTM_PL_VRAM);

}



static int

svm_migrate_gart_map(struct amdgpu_ring *ring, uint64_t npages,

		     dma_addr_t *addr, uint64_t *gart_addr, uint64_t flags)

svm_migrate_gart_map(struct amdgpu_ring *ring, u64 npages,

		     dma_addr_t *addr, u64 *gart_addr, u64 flags)

{

	struct amdgpu_device *adev = ring->adev;

	struct amdgpu_job *job;

	unsigned int num_dw, num_bytes;

	struct dma_fence *fence;

	uint64_t src_addr, dst_addr;

	uint64_t pte_flags;

	u64 src_addr, dst_addr;

	u64 pte_flags;

	void *cpu_addr;

	int r;
@@ -122,15 +122,15 @@ svm_migrate_gart_map(struct amdgpu_ring *ring, uint64_t npages, 


static int

svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,

			     uint64_t *vram, uint64_t npages,

			     u64 *vram, u64 npages,

			     enum MIGRATION_COPY_DIR direction,

			     struct dma_fence **mfence)

{

	const uint64_t GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;

	const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;

	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;

	uint64_t gart_s, gart_d;

	u64 gart_s, gart_d;

	struct dma_fence *next;

	uint64_t size;

	u64 size;

	int r;



	mutex_lock(&adev->mman.gtt_window_lock);
@@ -260,30 +260,42 @@ static void svm_migrate_put_sys_page(unsigned long addr) 
	put_page(page);

}



static long

static unsigned long svm_migrate_unsuccessful_pages(struct migrate_vma *migrate)

{

	unsigned long upages = 0;

	unsigned long i;



	for (i = 0; i < migrate->npages; i++) {

		if (migrate->src[i] & MIGRATE_PFN_VALID &&

		    !(migrate->src[i] & MIGRATE_PFN_MIGRATE))

			upages++;

	}

	return upages;

}



static int

svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,

			 struct migrate_vma *migrate, struct dma_fence **mfence,

			 dma_addr_t *scratch, uint64_t ttm_res_offset)

			 dma_addr_t *scratch, u64 ttm_res_offset)

{

	uint64_t npages = migrate->npages;

	u64 npages = migrate->npages;

	struct amdgpu_device *adev = node->adev;

	struct device *dev = adev->dev;

	struct amdgpu_res_cursor cursor;

	long mpages;

	u64 mpages = 0;

	dma_addr_t *src;

	uint64_t *dst;

	uint64_t i, j;

	u64 *dst;

	u64 i, j;

	int r;



	pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,

		 prange->last, ttm_res_offset);



	src = scratch;

	dst = (uint64_t *)(scratch + npages);

	dst = (u64 *)(scratch + npages);



	amdgpu_res_first(prange->ttm_res, ttm_res_offset,

			 npages << PAGE_SHIFT, &cursor);

	mpages = 0;

	for (i = j = 0; (i < npages) && (mpages < migrate->cpages); i++) {

		struct page *spage;
@@ -344,14 +356,13 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, 
out_free_vram_pages:

	if (r) {

		pr_debug("failed %d to copy memory to vram\n", r);

		while (i-- && mpages) {

		for (i = 0; i < npages && mpages; i++) {

			if (!dst[i])

				continue;

			svm_migrate_put_vram_page(adev, dst[i]);

			migrate->dst[i] = 0;

			mpages--;

		}

		mpages = r;

	}



#ifdef DEBUG_FORCE_MIXED_DOMAINS
@@ -369,22 +380,22 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, 
	}

#endif



	return mpages;

	return r;

}



static long

svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,

			struct vm_area_struct *vma, uint64_t start,

			uint64_t end, uint32_t trigger, uint64_t ttm_res_offset)

			struct vm_area_struct *vma, u64 start,

			u64 end, uint32_t trigger, u64 ttm_res_offset)

{

	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);

	uint64_t npages = (end - start) >> PAGE_SHIFT;

	u64 npages = (end - start) >> PAGE_SHIFT;

	struct amdgpu_device *adev = node->adev;

	struct kfd_process_device *pdd;

	struct dma_fence *mfence = NULL;

	struct migrate_vma migrate = { 0 };

	unsigned long cpages = 0;

	long mpages = 0;

	unsigned long mpages = 0;

	dma_addr_t *scratch;

	void *buf;

	int r = -ENOMEM;
@@ -397,7 +408,7 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange, 
	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);



	buf = kvcalloc(npages,

		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),

		       2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),

		       GFP_KERNEL);

	if (!buf)

		goto out;
@@ -430,17 +441,15 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange, 
	else

		pr_debug("0x%lx pages collected\n", cpages);



	mpages = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);

	r = svm_migrate_copy_to_vram(node, prange, &migrate, &mfence, scratch, ttm_res_offset);

	migrate_vma_pages(&migrate);



	svm_migrate_copy_done(adev, mfence);

	migrate_vma_finalize(&migrate);



	if (mpages >= 0)

		pr_debug("migrated/collected/requested 0x%lx/0x%lx/0x%lx\n",

	mpages = cpages - svm_migrate_unsuccessful_pages(&migrate);

	pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",

		 mpages, cpages, migrate.npages);

	else

		r = mpages;



	svm_range_dma_unmap_dev(adev->dev, scratch, 0, npages);
@@ -450,13 +459,14 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange, 
				    start >> PAGE_SHIFT, end >> PAGE_SHIFT,

				    0, node->id, trigger, r);

out:

	if (!r && mpages > 0) {

	if (!r && mpages) {

		pdd = svm_range_get_pdd_by_node(prange, node);

		if (pdd)

			WRITE_ONCE(pdd->page_in, pdd->page_in + mpages);

	}



	return r ? r : mpages;

		return mpages;

	}

	return r;

}



/**
@@ -480,7 +490,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc, 
{

	unsigned long addr, start, end;

	struct vm_area_struct *vma;

	uint64_t ttm_res_offset;

	u64 ttm_res_offset;

	struct kfd_node *node;

	unsigned long mpages = 0;

	long r = 0;
@@ -567,18 +577,17 @@ static void svm_migrate_page_free(struct page *page) 
	}

}



static long

static int

svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,

			struct migrate_vma *migrate, struct dma_fence **mfence,

			dma_addr_t *scratch, uint64_t npages)

			dma_addr_t *scratch, u64 npages)

{

	struct device *dev = adev->dev;

	uint64_t *src;

	u64 *src;

	dma_addr_t *dst;

	struct page *dpage;

	long mpages;

	uint64_t i = 0, j;

	uint64_t addr;

	u64 i = 0, j;

	u64 addr;

	int r = 0;



	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
@@ -586,10 +595,9 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, 


	addr = migrate->start;



	src = (uint64_t *)(scratch + npages);

	src = (u64 *)(scratch + npages);

	dst = scratch;



	mpages = 0;

	for (i = 0, j = 0; i < npages; i++, addr += PAGE_SIZE) {

		struct page *spage;
@@ -638,7 +646,6 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, 
				     dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));



		migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));

		mpages++;

		j++;

	}
@@ -648,17 +655,13 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, 
out_oom:

	if (r) {

		pr_debug("failed %d copy to ram\n", r);

		while (i-- && mpages) {

			if (!migrate->dst[i])

				continue;

		while (i--) {

			svm_migrate_put_sys_page(dst[i]);

			migrate->dst[i] = 0;

			mpages--;

		}

		mpages = r;

	}



	return mpages;

	return r;

}



/**
@@ -680,13 +683,14 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, 
 */

static long

svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,

		       struct vm_area_struct *vma, uint64_t start, uint64_t end,

		       struct vm_area_struct *vma, u64 start, u64 end,

		       uint32_t trigger, struct page *fault_page)

{

	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);

	uint64_t npages = (end - start) >> PAGE_SHIFT;

	u64 npages = (end - start) >> PAGE_SHIFT;

	unsigned long upages = npages;

	unsigned long cpages = 0;

	long mpages = 0;

	unsigned long mpages = 0;

	struct amdgpu_device *adev = node->adev;

	struct kfd_process_device *pdd;

	struct dma_fence *mfence = NULL;
@@ -706,7 +710,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange, 
		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;



	buf = kvcalloc(npages,

		       2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t),

		       2 * sizeof(*migrate.src) + sizeof(u64) + sizeof(dma_addr_t),

		       GFP_KERNEL);

	if (!buf)

		goto out;
@@ -740,15 +744,13 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange, 
	else

		pr_debug("0x%lx pages collected\n", cpages);



	mpages = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,

	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,

				    scratch, npages);

	migrate_vma_pages(&migrate);



	if (mpages >= 0)

		pr_debug("migrated/collected/requested 0x%lx/0x%lx/0x%lx\n",

		 mpages, cpages, migrate.npages);

	else

		r = mpages;

	upages = svm_migrate_unsuccessful_pages(&migrate);

	pr_debug("unsuccessful/cpages/npages 0x%lx/0x%lx/0x%lx\n",

		 upages, cpages, migrate.npages);



	svm_migrate_copy_done(adev, mfence);

	migrate_vma_finalize(&migrate);
@@ -761,7 +763,8 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange, 
				    start >> PAGE_SHIFT, end >> PAGE_SHIFT,

				    node->id, 0, trigger, r);

out:

	if (!r && mpages > 0) {

	if (!r && cpages) {

		mpages = cpages - upages;

		pdd = svm_range_get_pdd_by_node(prange, node);

		if (pdd)

			WRITE_ONCE(pdd->page_out, pdd->page_out + mpages);
@@ -844,9 +847,6 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm, 
	}



	if (r >= 0) {

		WARN_ONCE(prange->vram_pages < mpages,

			"Recorded vram pages(0x%llx) should not be less than migration pages(0x%lx).",

			prange->vram_pages, mpages);

		prange->vram_pages -= mpages;



		/* prange does not have vram page set its actual_loc to system