Commit eea6520c authored by Dave Airlie's avatar Dave Airlie
Browse files

Merge tag 'drm-xe-fixes-2025-03-06' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-fixes 



- Remove double page flip on initial plane (Maarten)
- Properly setup userptr pfn_flags_mask (Auld)
- Fix GT "for each engine" workarounds (Tvrtko)
- Fix userptr races and missed validations (Thomas, Brost)
- Userptr invalid page access fixes (Thomas)

Signed-off-by: default avatarDave Airlie <airlied@redhat.com>

From: Rodrigo Vivi <rodrigo.vivi@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/Z8ni6w3tskCFL11O@intel.com
parents 019899b5 333b8906

drivers/gpu/drm/xe/display/xe_plane_initial.c

+0 −10
Original line number Diff line number Diff line
@@ -194,8 +194,6 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc, 
		to_intel_plane(crtc->base.primary);

	struct intel_plane_state *plane_state =

		to_intel_plane_state(plane->base.state);

	struct intel_crtc_state *crtc_state =

		to_intel_crtc_state(crtc->base.state);

	struct drm_framebuffer *fb;

	struct i915_vma *vma;
@@ -241,14 +239,6 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc, 
	atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits);



	plane_config->vma = vma;



	/*

	 * Flip to the newly created mapping ASAP, so we can re-use the

	 * first part of GGTT for WOPCM, prevent flickering, and prevent

	 * the lookup of sysmem scratch pages.

	 */

	plane->check_plane(crtc_state, plane_state);

	plane->async_flip(NULL, plane, crtc_state, plane_state, true);

	return;



nofb:

drivers/gpu/drm/xe/xe_gt.c

+2 −2
Original line number Diff line number Diff line
@@ -380,9 +380,7 @@ int xe_gt_init_early(struct xe_gt *gt) 
	if (err)

		return err;



	xe_wa_process_gt(gt);

	xe_wa_process_oob(gt);

	xe_tuning_process_gt(gt);



	xe_force_wake_init_gt(gt, gt_to_fw(gt));

	spin_lock_init(&gt->global_invl_lock);
@@ -474,6 +472,8 @@ static int all_fw_domain_init(struct xe_gt *gt) 
	}



	xe_gt_mcr_set_implicit_defaults(gt);

	xe_wa_process_gt(gt);

	xe_tuning_process_gt(gt);

	xe_reg_sr_apply_mmio(&gt->reg_sr, gt);



	err = xe_gt_clock_init(gt);

drivers/gpu/drm/xe/xe_hmm.c

+140 −48
Original line number Diff line number Diff line
@@ -19,11 +19,10 @@ static u64 xe_npages_in_range(unsigned long start, unsigned long end) 
	return (end - start) >> PAGE_SHIFT;

}



/*

/**

 * xe_mark_range_accessed() - mark a range is accessed, so core mm

 * have such information for memory eviction or write back to

 * hard disk

 *

 * @range: the range to mark

 * @write: if write to this range, we mark pages in this range

 * as dirty
@@ -43,15 +42,51 @@ static void xe_mark_range_accessed(struct hmm_range *range, bool write) 
	}

}



/*

static int xe_alloc_sg(struct xe_device *xe, struct sg_table *st,

		       struct hmm_range *range, struct rw_semaphore *notifier_sem)

{

	unsigned long i, npages, hmm_pfn;

	unsigned long num_chunks = 0;

	int ret;



	/* HMM docs says this is needed. */

	ret = down_read_interruptible(notifier_sem);

	if (ret)

		return ret;



	if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) {

		up_read(notifier_sem);

		return -EAGAIN;

	}



	npages = xe_npages_in_range(range->start, range->end);

	for (i = 0; i < npages;) {

		unsigned long len;



		hmm_pfn = range->hmm_pfns[i];

		xe_assert(xe, hmm_pfn & HMM_PFN_VALID);



		len = 1UL << hmm_pfn_to_map_order(hmm_pfn);



		/* If order > 0 the page may extend beyond range->start */

		len -= (hmm_pfn & ~HMM_PFN_FLAGS) & (len - 1);

		i += len;

		num_chunks++;

	}

	up_read(notifier_sem);



	return sg_alloc_table(st, num_chunks, GFP_KERNEL);

}



/**

 * xe_build_sg() - build a scatter gather table for all the physical pages/pfn

 * in a hmm_range. dma-map pages if necessary. dma-address is save in sg table

 * and will be used to program GPU page table later.

 *

 * @xe: the xe device who will access the dma-address in sg table

 * @range: the hmm range that we build the sg table from. range->hmm_pfns[]

 * has the pfn numbers of pages that back up this hmm address range.

 * @st: pointer to the sg table.

 * @notifier_sem: The xe notifier lock.

 * @write: whether we write to this range. This decides dma map direction

 * for system pages. If write we map it bi-diretional; otherwise

 * DMA_TO_DEVICE
@@ -78,43 +113,84 @@ static void xe_mark_range_accessed(struct hmm_range *range, bool write) 
 * Returns 0 if successful; -ENOMEM if fails to allocate memory

 */

static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,

		       struct sg_table *st, bool write)

		       struct sg_table *st,

		       struct rw_semaphore *notifier_sem,

		       bool write)

{

	unsigned long npages = xe_npages_in_range(range->start, range->end);

	struct device *dev = xe->drm.dev;

	struct page **pages;

	u64 i, npages;

	int ret;

	struct scatterlist *sgl;

	struct page *page;

	unsigned long i, j;



	npages = xe_npages_in_range(range->start, range->end);

	pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL);

	if (!pages)

		return -ENOMEM;

	lockdep_assert_held(notifier_sem);



	for (i = 0; i < npages; i++) {

		pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);

		xe_assert(xe, !is_device_private_page(pages[i]));

	}

	i = 0;

	for_each_sg(st->sgl, sgl, st->nents, j) {

		unsigned long hmm_pfn, size;



	ret = sg_alloc_table_from_pages_segment(st, pages, npages, 0, npages << PAGE_SHIFT,

						xe_sg_segment_size(dev), GFP_KERNEL);

	if (ret)

		goto free_pages;

		hmm_pfn = range->hmm_pfns[i];

		page = hmm_pfn_to_page(hmm_pfn);

		xe_assert(xe, !is_device_private_page(page));



		size = 1UL << hmm_pfn_to_map_order(hmm_pfn);

		size -= page_to_pfn(page) & (size - 1);

		i += size;



	ret = dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,

		if (unlikely(j == st->nents - 1)) {

			if (i > npages)

				size -= (i - npages);

			sg_mark_end(sgl);

		}

		sg_set_page(sgl, page, size << PAGE_SHIFT, 0);

	}

	xe_assert(xe, i == npages);



	return dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,

			       DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING);

	if (ret) {

		sg_free_table(st);

		st = NULL;

}



free_pages:

	kvfree(pages);

	return ret;

static void xe_hmm_userptr_set_mapped(struct xe_userptr_vma *uvma)

{

	struct xe_userptr *userptr = &uvma->userptr;

	struct xe_vm *vm = xe_vma_vm(&uvma->vma);



	lockdep_assert_held_write(&vm->lock);

	lockdep_assert_held(&vm->userptr.notifier_lock);



	mutex_lock(&userptr->unmap_mutex);

	xe_assert(vm->xe, !userptr->mapped);

	userptr->mapped = true;

	mutex_unlock(&userptr->unmap_mutex);

}



/*

void xe_hmm_userptr_unmap(struct xe_userptr_vma *uvma)

{

	struct xe_userptr *userptr = &uvma->userptr;

	struct xe_vma *vma = &uvma->vma;

	bool write = !xe_vma_read_only(vma);

	struct xe_vm *vm = xe_vma_vm(vma);

	struct xe_device *xe = vm->xe;



	if (!lockdep_is_held_type(&vm->userptr.notifier_lock, 0) &&

	    !lockdep_is_held_type(&vm->lock, 0) &&

	    !(vma->gpuva.flags & XE_VMA_DESTROYED)) {

		/* Don't unmap in exec critical section. */

		xe_vm_assert_held(vm);

		/* Don't unmap while mapping the sg. */

		lockdep_assert_held(&vm->lock);

	}



	mutex_lock(&userptr->unmap_mutex);

	if (userptr->sg && userptr->mapped)

		dma_unmap_sgtable(xe->drm.dev, userptr->sg,

				  write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);

	userptr->mapped = false;

	mutex_unlock(&userptr->unmap_mutex);

}



/**

 * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr

 *

 * @uvma: the userptr vma which hold the scatter gather table

 *

 * With function xe_userptr_populate_range, we allocate storage of
@@ -124,16 +200,9 @@ static int xe_build_sg(struct xe_device *xe, struct hmm_range *range, 
void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)

{

	struct xe_userptr *userptr = &uvma->userptr;

	struct xe_vma *vma = &uvma->vma;

	bool write = !xe_vma_read_only(vma);

	struct xe_vm *vm = xe_vma_vm(vma);

	struct xe_device *xe = vm->xe;

	struct device *dev = xe->drm.dev;



	xe_assert(xe, userptr->sg);

	dma_unmap_sgtable(dev, userptr->sg,

			  write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);



	xe_assert(xe_vma_vm(&uvma->vma)->xe, userptr->sg);

	xe_hmm_userptr_unmap(uvma);

	sg_free_table(userptr->sg);

	userptr->sg = NULL;

}
@@ -166,13 +235,20 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, 
{

	unsigned long timeout =

		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);

	unsigned long *pfns, flags = HMM_PFN_REQ_FAULT;

	unsigned long *pfns;

	struct xe_userptr *userptr;

	struct xe_vma *vma = &uvma->vma;

	u64 userptr_start = xe_vma_userptr(vma);

	u64 userptr_end = userptr_start + xe_vma_size(vma);

	struct xe_vm *vm = xe_vma_vm(vma);

	struct hmm_range hmm_range;

	struct hmm_range hmm_range = {

		.pfn_flags_mask = 0, /* ignore pfns */

		.default_flags = HMM_PFN_REQ_FAULT,

		.start = userptr_start,

		.end = userptr_end,

		.notifier = &uvma->userptr.notifier,

		.dev_private_owner = vm->xe,

	};

	bool write = !xe_vma_read_only(vma);

	unsigned long notifier_seq;

	u64 npages;
@@ -199,19 +275,14 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, 
		return -ENOMEM;



	if (write)

		flags |= HMM_PFN_REQ_WRITE;

		hmm_range.default_flags |= HMM_PFN_REQ_WRITE;



	if (!mmget_not_zero(userptr->notifier.mm)) {

		ret = -EFAULT;

		goto free_pfns;

	}



	hmm_range.default_flags = flags;

	hmm_range.hmm_pfns = pfns;

	hmm_range.notifier = &userptr->notifier;

	hmm_range.start = userptr_start;

	hmm_range.end = userptr_end;

	hmm_range.dev_private_owner = vm->xe;



	while (true) {

		hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier);
@@ -238,16 +309,37 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, 
	if (ret)

		goto free_pfns;



	ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt, write);

	ret = xe_alloc_sg(vm->xe, &userptr->sgt, &hmm_range, &vm->userptr.notifier_lock);

	if (ret)

		goto free_pfns;



	ret = down_read_interruptible(&vm->userptr.notifier_lock);

	if (ret)

		goto free_st;



	if (mmu_interval_read_retry(hmm_range.notifier, hmm_range.notifier_seq)) {

		ret = -EAGAIN;

		goto out_unlock;

	}



	ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt,

			  &vm->userptr.notifier_lock, write);

	if (ret)

		goto out_unlock;



	xe_mark_range_accessed(&hmm_range, write);

	userptr->sg = &userptr->sgt;

	xe_hmm_userptr_set_mapped(uvma);

	userptr->notifier_seq = hmm_range.notifier_seq;

	up_read(&vm->userptr.notifier_lock);

	kvfree(pfns);

	return 0;



out_unlock:

	up_read(&vm->userptr.notifier_lock);

free_st:

	sg_free_table(&userptr->sgt);

free_pfns:

	kvfree(pfns);

	return ret;

}

drivers/gpu/drm/xe/xe_hmm.h

+7 −0
Original line number Diff line number Diff line
@@ -3,9 +3,16 @@ 
 * Copyright © 2024 Intel Corporation

 */



#ifndef _XE_HMM_H_

#define _XE_HMM_H_



#include <linux/types.h>



struct xe_userptr_vma;



int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, bool is_mm_mmap_locked);



void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma);



void xe_hmm_userptr_unmap(struct xe_userptr_vma *uvma);

#endif

drivers/gpu/drm/xe/xe_pt.c

+49 −47
Original line number Diff line number Diff line
@@ -28,6 +28,8 @@ struct xe_pt_dir { 
	struct xe_pt pt;

	/** @children: Array of page-table child nodes */

	struct xe_ptw *children[XE_PDES];

	/** @staging: Array of page-table staging nodes */

	struct xe_ptw *staging[XE_PDES];

};



#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
@@ -48,9 +50,10 @@ static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt) 
	return container_of(pt, struct xe_pt_dir, pt);

}



static struct xe_pt *xe_pt_entry(struct xe_pt_dir *pt_dir, unsigned int index)

static struct xe_pt *

xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)

{

	return container_of(pt_dir->children[index], struct xe_pt, base);

	return container_of(pt_dir->staging[index], struct xe_pt, base);

}



static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
@@ -125,6 +128,7 @@ struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, 
	}

	pt->bo = bo;

	pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;

	pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;



	if (vm->xef)

		xe_drm_client_add_bo(vm->xef->client, pt->bo);
@@ -206,8 +210,8 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) 
		struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);



		for (i = 0; i < XE_PDES; i++) {

			if (xe_pt_entry(pt_dir, i))

				xe_pt_destroy(xe_pt_entry(pt_dir, i), flags,

			if (xe_pt_entry_staging(pt_dir, i))

				xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags,

					      deferred);

		}

	}
@@ -376,8 +380,10 @@ xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, 
		/* Continue building a non-connected subtree. */

		struct iosys_map *map = &parent->bo->vmap;



		if (unlikely(xe_child))

		if (unlikely(xe_child)) {

			parent->base.children[offset] = &xe_child->base;

			parent->base.staging[offset] = &xe_child->base;

		}



		xe_pt_write(xe_walk->vm->xe, map, offset, pte);

		parent->num_live++;
@@ -614,6 +620,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, 
			.ops = &xe_pt_stage_bind_ops,

			.shifts = xe_normal_pt_shifts,

			.max_level = XE_PT_HIGHEST_LEVEL,

			.staging = true,

		},

		.vm = xe_vma_vm(vma),

		.tile = tile,
@@ -873,7 +880,7 @@ static void xe_pt_cancel_bind(struct xe_vma *vma, 
	}

}



static void xe_pt_commit_locks_assert(struct xe_vma *vma)

static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)

{

	struct xe_vm *vm = xe_vma_vm(vma);
@@ -885,6 +892,16 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma) 
	xe_vm_assert_held(vm);

}



static void xe_pt_commit_locks_assert(struct xe_vma *vma)

{

	struct xe_vm *vm = xe_vma_vm(vma);



	xe_pt_commit_prepare_locks_assert(vma);



	if (xe_vma_is_userptr(vma))

		lockdep_assert_held_read(&vm->userptr.notifier_lock);

}



static void xe_pt_commit(struct xe_vma *vma,

			 struct xe_vm_pgtable_update *entries,

			 u32 num_entries, struct llist_head *deferred)
@@ -895,13 +912,17 @@ static void xe_pt_commit(struct xe_vma *vma, 


	for (i = 0; i < num_entries; i++) {

		struct xe_pt *pt = entries[i].pt;

		struct xe_pt_dir *pt_dir;



		if (!pt->level)

			continue;



		pt_dir = as_xe_pt_dir(pt);

		for (j = 0; j < entries[i].qwords; j++) {

			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;

			int j_ = j + entries[i].ofs;



			pt_dir->children[j_] = pt_dir->staging[j_];

			xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred);

		}

	}
@@ -913,7 +934,7 @@ static void xe_pt_abort_bind(struct xe_vma *vma, 
{

	int i, j;



	xe_pt_commit_locks_assert(vma);

	xe_pt_commit_prepare_locks_assert(vma);



	for (i = num_entries - 1; i >= 0; --i) {

		struct xe_pt *pt = entries[i].pt;
@@ -928,10 +949,10 @@ static void xe_pt_abort_bind(struct xe_vma *vma, 
		pt_dir = as_xe_pt_dir(pt);

		for (j = 0; j < entries[i].qwords; j++) {

			u32 j_ = j + entries[i].ofs;

			struct xe_pt *newpte = xe_pt_entry(pt_dir, j_);

			struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_);

			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;



			pt_dir->children[j_] = oldpte ? &oldpte->base : 0;

			pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;

			xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);

		}

	}
@@ -943,7 +964,7 @@ static void xe_pt_commit_prepare_bind(struct xe_vma *vma, 
{

	u32 i, j;



	xe_pt_commit_locks_assert(vma);

	xe_pt_commit_prepare_locks_assert(vma);



	for (i = 0; i < num_entries; i++) {

		struct xe_pt *pt = entries[i].pt;
@@ -961,10 +982,10 @@ static void xe_pt_commit_prepare_bind(struct xe_vma *vma, 
			struct xe_pt *newpte = entries[i].pt_entries[j].pt;

			struct xe_pt *oldpte = NULL;



			if (xe_pt_entry(pt_dir, j_))

				oldpte = xe_pt_entry(pt_dir, j_);

			if (xe_pt_entry_staging(pt_dir, j_))

				oldpte = xe_pt_entry_staging(pt_dir, j_);



			pt_dir->children[j_] = &newpte->base;

			pt_dir->staging[j_] = &newpte->base;

			entries[i].pt_entries[j].pt = oldpte;

		}

	}
@@ -1213,42 +1234,22 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, 
		return 0;



	uvma = to_userptr_vma(vma);

	notifier_seq = uvma->userptr.notifier_seq;

	if (xe_pt_userptr_inject_eagain(uvma))

		xe_vma_userptr_force_invalidate(uvma);



	if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))

		return 0;

	notifier_seq = uvma->userptr.notifier_seq;



	if (!mmu_interval_read_retry(&uvma->userptr.notifier,

				     notifier_seq) &&

	    !xe_pt_userptr_inject_eagain(uvma))

				     notifier_seq))

		return 0;



	if (xe_vm_in_fault_mode(vm)) {

	if (xe_vm_in_fault_mode(vm))

		return -EAGAIN;

	} else {

		spin_lock(&vm->userptr.invalidated_lock);

		list_move_tail(&uvma->userptr.invalidate_link,

			       &vm->userptr.invalidated);

		spin_unlock(&vm->userptr.invalidated_lock);



		if (xe_vm_in_preempt_fence_mode(vm)) {

			struct dma_resv_iter cursor;

			struct dma_fence *fence;

			long err;



			dma_resv_iter_begin(&cursor, xe_vm_resv(vm),

					    DMA_RESV_USAGE_BOOKKEEP);

			dma_resv_for_each_fence_unlocked(&cursor, fence)

				dma_fence_enable_sw_signaling(fence);

			dma_resv_iter_end(&cursor);



			err = dma_resv_wait_timeout(xe_vm_resv(vm),

						    DMA_RESV_USAGE_BOOKKEEP,

						    false, MAX_SCHEDULE_TIMEOUT);

			XE_WARN_ON(err <= 0);

		}

	}



	/*

	 * Just continue the operation since exec or rebind worker

	 * will take care of rebinding.

	 */

	return 0;

}
@@ -1514,6 +1515,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma, 
			.ops = &xe_pt_stage_unbind_ops,

			.shifts = xe_normal_pt_shifts,

			.max_level = XE_PT_HIGHEST_LEVEL,

			.staging = true,

		},

		.tile = tile,

		.modified_start = xe_vma_start(vma),
@@ -1555,7 +1557,7 @@ static void xe_pt_abort_unbind(struct xe_vma *vma, 
{

	int i, j;



	xe_pt_commit_locks_assert(vma);

	xe_pt_commit_prepare_locks_assert(vma);



	for (i = num_entries - 1; i >= 0; --i) {

		struct xe_vm_pgtable_update *entry = &entries[i];
@@ -1568,7 +1570,7 @@ static void xe_pt_abort_unbind(struct xe_vma *vma, 
			continue;



		for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)

			pt_dir->children[j] =

			pt_dir->staging[j] =

				entries[i].pt_entries[j - entry->ofs].pt ?

				&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;

	}
@@ -1581,7 +1583,7 @@ xe_pt_commit_prepare_unbind(struct xe_vma *vma, 
{

	int i, j;



	xe_pt_commit_locks_assert(vma);

	xe_pt_commit_prepare_locks_assert(vma);



	for (i = 0; i < num_entries; ++i) {

		struct xe_vm_pgtable_update *entry = &entries[i];
@@ -1595,8 +1597,8 @@ xe_pt_commit_prepare_unbind(struct xe_vma *vma, 
		pt_dir = as_xe_pt_dir(pt);

		for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {

			entry->pt_entries[j - entry->ofs].pt =

				xe_pt_entry(pt_dir, j);

			pt_dir->children[j] = NULL;

				xe_pt_entry_staging(pt_dir, j);

			pt_dir->staging[j] = NULL;

		}

	}

}