Commit d8b52a02 authored by Maarten Lankhorst's avatar Maarten Lankhorst Committed by Rodrigo Vivi
Browse files

drm/xe: Implement stolen memory. 



This adds support for stolen memory, with the same allocator as
vram_mgr. This allows us to skip a whole lot of copy-paste,
by re-using parts of xe_ttm_vram_mgr.

The stolen memory may be bound using VM_BIND, so it performs like any
other memory region.

We should be able to map a stolen BO directly using the physical memory
location instead of through GGTT even on old platforms, but I don't know
what the effects are on coherency.

Signed-off-by: default avatarMaarten Lankhorst <maarten.lankhorst@linux.intel.com>
Reviewed-by: default avatarMatthew Brost <matthew.brost@intel.com>
Signed-off-by: default avatarRodrigo Vivi <rodrigo.vivi@intel.com>
parent 765b65e5

drivers/gpu/drm/xe/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -93,6 +93,7 @@ xe-y += xe_bb.o \ 
	xe_sync.o \

	xe_trace.o \

	xe_ttm_gtt_mgr.o \

	xe_ttm_stolen_mgr.o \

	xe_ttm_vram_mgr.o \

	xe_tuning.o \

	xe_uc.o \

drivers/gpu/drm/xe/xe_bo.c

+147 −26
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include "xe_preempt_fence.h"

#include "xe_res_cursor.h"

#include "xe_trace.h"

#include "xe_ttm_stolen_mgr.h"

#include "xe_vm.h"



static const struct ttm_place sys_placement_flags = {
@@ -42,7 +43,12 @@ static struct ttm_placement sys_placement = { 


bool mem_type_is_vram(u32 mem_type)

{

	return mem_type >= XE_PL_VRAM0;

	return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;

}



static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)

{

	return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);

}



static bool resource_is_vram(struct ttm_resource *res)
@@ -52,7 +58,13 @@ static bool resource_is_vram(struct ttm_resource *res) 


bool xe_bo_is_vram(struct xe_bo *bo)

{

	return resource_is_vram(bo->ttm.resource);

	return resource_is_vram(bo->ttm.resource) ||

		resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);

}



bool xe_bo_is_stolen(struct xe_bo *bo)

{

	return bo->ttm.resource->mem_type == XE_PL_STOLEN;

}



static bool xe_bo_is_user(struct xe_bo *bo)
@@ -63,9 +75,9 @@ static bool xe_bo_is_user(struct xe_bo *bo) 
static struct xe_gt *

mem_type_to_gt(struct xe_device *xe, u32 mem_type)

{

	XE_BUG_ON(!mem_type_is_vram(mem_type));

	XE_BUG_ON(mem_type != XE_PL_STOLEN && !mem_type_is_vram(mem_type));



	return xe_device_get_gt(xe, mem_type - XE_PL_VRAM0);

	return xe_device_get_gt(xe, mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0));

}



static void try_add_system(struct xe_bo *bo, struct ttm_place *places,
@@ -134,6 +146,20 @@ static void try_add_vram1(struct xe_device *xe, struct xe_bo *bo, 
	}

}



static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,

			   struct ttm_place *places, u32 bo_flags, u32 *c)

{

	if (bo_flags & XE_BO_CREATE_STOLEN_BIT) {

		places[*c] = (struct ttm_place) {

			.mem_type = XE_PL_STOLEN,

			.flags = bo_flags & (XE_BO_CREATE_PINNED_BIT |

					     XE_BO_CREATE_GGTT_BIT) ?

				TTM_PL_FLAG_CONTIGUOUS : 0,

		};

		*c += 1;

	}

}



static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,

				       u32 bo_flags)

{
@@ -162,6 +188,7 @@ static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, 
		try_add_vram1(xe, bo, places, bo_flags, &c);

		try_add_system(bo, places, bo_flags, &c);

	}

	try_add_stolen(xe, bo, places, bo_flags, &c);



	if (!c)

		return -EINVAL;
@@ -209,6 +236,7 @@ static void xe_evict_flags(struct ttm_buffer_object *tbo, 
	switch (tbo->resource->mem_type) {

	case XE_PL_VRAM0:

	case XE_PL_VRAM1:

	case XE_PL_STOLEN:

	case XE_PL_TT:

	default:

		/* for now kick out to system */
@@ -362,11 +390,12 @@ static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, 
#if  !defined(CONFIG_X86)

		mem->bus.caching = ttm_write_combined;

#endif

		break;

		return 0;

	case XE_PL_STOLEN:

		return xe_ttm_stolen_io_mem_reserve(xe, mem);

	default:

		return -EINVAL;

	}

	return 0;

}



static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
@@ -673,14 +702,18 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, 


}



static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *bo,

static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,

				       unsigned long page_offset)

{

	struct xe_device *xe = ttm_to_xe_device(bo->bdev);

	struct xe_gt *gt = mem_type_to_gt(xe, bo->resource->mem_type);

	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);

	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);

	struct xe_gt *gt = mem_type_to_gt(xe, ttm_bo->resource->mem_type);

	struct xe_res_cursor cursor;



	xe_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);

	if (ttm_bo->resource->mem_type == XE_PL_STOLEN)

		return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;



	xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);

	return (gt->mem.vram.io_start + cursor.start) >> PAGE_SHIFT;

}
@@ -945,7 +978,8 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, 
			return bo;

	}



	if (flags & (XE_BO_CREATE_VRAM0_BIT | XE_BO_CREATE_VRAM1_BIT) &&

	if (flags & (XE_BO_CREATE_VRAM0_BIT | XE_BO_CREATE_VRAM1_BIT |

		     XE_BO_CREATE_STOLEN_BIT) &&

	    !(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) &&

	    xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) {

		size = ALIGN(size, SZ_64K);
@@ -973,9 +1007,11 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, 
		ctx.resv = resv;

	}



	if (!(flags & XE_BO_FIXED_PLACEMENT_BIT)) {

		err = __xe_bo_placement_for_flags(xe, bo, bo->flags);

		if (WARN_ON(err))

			return ERR_PTR(err);

	}



	/* Defer populating type_sg bos */

	placement = (type == ttm_bo_type_sg ||
@@ -993,16 +1029,73 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, 
	return bo;

}



struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt,

				  struct xe_vm *vm, size_t size,

static int __xe_bo_fixed_placement(struct xe_device *xe,

				   struct xe_bo *bo,

				   u32 flags,

				   u64 start, u64 end, u64 size)

{

	struct ttm_place *place = bo->placements;



	if (flags & (XE_BO_CREATE_USER_BIT|XE_BO_CREATE_SYSTEM_BIT))

		return -EINVAL;



	place->flags = TTM_PL_FLAG_CONTIGUOUS;

	place->fpfn = start >> PAGE_SHIFT;

	place->lpfn = end >> PAGE_SHIFT;



	switch (flags & (XE_BO_CREATE_STOLEN_BIT |

		XE_BO_CREATE_VRAM0_BIT |XE_BO_CREATE_VRAM1_BIT)) {

	case XE_BO_CREATE_VRAM0_BIT:

		place->mem_type = XE_PL_VRAM0;

		break;

	case XE_BO_CREATE_VRAM1_BIT:

		place->mem_type = XE_PL_VRAM1;

		break;

	case XE_BO_CREATE_STOLEN_BIT:

		place->mem_type = XE_PL_STOLEN;

		break;



	default:

		/* 0 or multiple of the above set */

		return -EINVAL;

	}



	bo->placement = (struct ttm_placement) {

		.num_placement = 1,

		.placement = place,

		.num_busy_placement = 1,

		.busy_placement = place,

	};



	return 0;

}



struct xe_bo *

xe_bo_create_locked_range(struct xe_device *xe,

			  struct xe_gt *gt, struct xe_vm *vm,

			  size_t size, u64 start, u64 end,

			  enum ttm_bo_type type, u32 flags)

{

	struct xe_bo *bo;

	struct xe_bo *bo = NULL;

	int err;



	if (vm)

		xe_vm_assert_held(vm);

	bo = __xe_bo_create_locked(xe, NULL, gt, vm ? &vm->resv : NULL, size,



	if (start || end != ~0ULL) {

		bo = xe_bo_alloc();

		if (IS_ERR(bo))

			return bo;



		flags |= XE_BO_FIXED_PLACEMENT_BIT;

		err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size);

		if (err) {

			xe_bo_free(bo);

			return ERR_PTR(err);

		}

	}



	bo = __xe_bo_create_locked(xe, bo, gt, vm ? &vm->resv : NULL, size,

				   type, flags);

	if (IS_ERR(bo))

		return bo;
@@ -1011,7 +1104,10 @@ struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt, 
		xe_vm_get(vm);

	bo->vm = vm;



	if (flags & XE_BO_CREATE_GGTT_BIT) {

	if (bo->flags & XE_BO_CREATE_GGTT_BIT) {

		if (!gt && flags & XE_BO_CREATE_STOLEN_BIT)

			gt = xe_device_get_gt(xe, 0);



		XE_BUG_ON(!gt);



		err = xe_ggtt_insert_bo(gt->mem.ggtt, bo);
@@ -1027,6 +1123,13 @@ struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt, 
	return ERR_PTR(err);

}



struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt,

				  struct xe_vm *vm, size_t size,

				  enum ttm_bo_type type, u32 flags)

{

	return xe_bo_create_locked_range(xe, gt, vm, size, 0, ~0ULL, type, flags);

}



struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt,

			   struct xe_vm *vm, size_t size,

			   enum ttm_bo_type type, u32 flags)
@@ -1039,13 +1142,21 @@ struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, 
	return bo;

}



struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt,

				   struct xe_vm *vm, size_t size,

struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt,

				      struct xe_vm *vm,

				      size_t size, u64 offset,

				      enum ttm_bo_type type, u32 flags)

{

	struct xe_bo *bo = xe_bo_create_locked(xe, gt, vm, size, type, flags);

	struct xe_bo *bo;

	int err;

	u64 start = offset == ~0ull ? 0 : offset;

	u64 end = offset == ~0ull ? offset : start + size;



	if (flags & XE_BO_CREATE_STOLEN_BIT &&

	    xe_ttm_stolen_inaccessible(xe))

		flags |= XE_BO_CREATE_GGTT_BIT;



	bo = xe_bo_create_locked_range(xe, gt, vm, size, start, end, type, flags);

	if (IS_ERR(bo))

		return bo;
@@ -1069,6 +1180,13 @@ struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt, 
	return ERR_PTR(err);

}



struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt,

				   struct xe_vm *vm, size_t size,

				   enum ttm_bo_type type, u32 flags)

{

	return xe_bo_create_pin_map_at(xe, gt, vm, size, ~0ull, type, flags);

}



struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt,

				     const void *data, size_t size,

				     enum ttm_bo_type type, u32 flags)
@@ -1093,6 +1211,9 @@ static uint64_t vram_region_io_offset(struct xe_bo *bo) 
	struct xe_device *xe = xe_bo_device(bo);

	struct xe_gt *gt = mem_type_to_gt(xe, bo->ttm.resource->mem_type);



	if (bo->ttm.resource->mem_type == XE_PL_STOLEN)

		return xe_ttm_stolen_gpu_offset(xe);



	return gt->mem.vram.io_start - xe->mem.vram.io_start;

}
@@ -1174,7 +1295,7 @@ int xe_bo_pin(struct xe_bo *bo) 
		bool lmem;



		XE_BUG_ON(!(place->flags & TTM_PL_FLAG_CONTIGUOUS));

		XE_BUG_ON(!mem_type_is_vram(place->mem_type));

		XE_BUG_ON(!mem_type_is_vram(place->mem_type) && place->mem_type != XE_PL_STOLEN);



		place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE, &lmem) -

			       vram_region_io_offset(bo)) >> PAGE_SHIFT;
@@ -1305,7 +1426,7 @@ dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, 


	*is_lmem = xe_bo_is_vram(bo);



	if (!*is_lmem) {

	if (!*is_lmem && !xe_bo_is_stolen(bo)) {

		XE_BUG_ON(!bo->ttm.ttm);



		xe_res_first_sg(xe_bo_get_sg(bo), page << PAGE_SHIFT,

drivers/gpu/drm/xe/xe_bo.h

+14 −2
Original line number Diff line number Diff line
@@ -12,8 +12,9 @@ 


#define XE_DEFAULT_GTT_SIZE_MB          3072ULL /* 3GB by default */



#define XE_BO_CREATE_USER_BIT		BIT(1)

#define XE_BO_CREATE_SYSTEM_BIT		BIT(2)

#define XE_BO_CREATE_USER_BIT		BIT(0)

#define XE_BO_CREATE_SYSTEM_BIT		BIT(1)

#define XE_BO_CREATE_STOLEN_BIT		BIT(2)

#define XE_BO_CREATE_VRAM0_BIT		BIT(3)

#define XE_BO_CREATE_VRAM1_BIT		BIT(4)

#define XE_BO_CREATE_VRAM_IF_DGFX(gt) \
@@ -24,6 +25,7 @@ 
#define XE_BO_CREATE_PINNED_BIT		BIT(7)

#define XE_BO_DEFER_BACKING		BIT(8)

#define XE_BO_SCANOUT_BIT		BIT(9)

#define XE_BO_FIXED_PLACEMENT_BIT	BIT(10)

/* this one is trigger internally only */

#define XE_BO_INTERNAL_TEST		BIT(30)

#define XE_BO_INTERNAL_64K		BIT(31)
@@ -64,6 +66,7 @@ 
#define XE_PL_TT		TTM_PL_TT

#define XE_PL_VRAM0		TTM_PL_VRAM

#define XE_PL_VRAM1		(XE_PL_VRAM0 + 1)

#define XE_PL_STOLEN		(TTM_NUM_MEM_TYPES - 1)



#define XE_BO_PROPS_INVALID	(-1)
@@ -76,6 +79,11 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, 
				    struct xe_gt *gt, struct dma_resv *resv,

				    size_t size, enum ttm_bo_type type,

				    u32 flags);

struct xe_bo *

xe_bo_create_locked_range(struct xe_device *xe,

			  struct xe_gt *gt, struct xe_vm *vm,

			  size_t size, u64 start, u64 end,

			  enum ttm_bo_type type, u32 flags);

struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt,

				  struct xe_vm *vm, size_t size,

				  enum ttm_bo_type type, u32 flags);
@@ -85,6 +93,9 @@ struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, 
struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt,

				   struct xe_vm *vm, size_t size,

				   enum ttm_bo_type type, u32 flags);

struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt,

				      struct xe_vm *vm, size_t size, u64 offset,

				      enum ttm_bo_type type, u32 flags);

struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt,

				     const void *data, size_t size,

				     enum ttm_bo_type type, u32 flags);
@@ -206,6 +217,7 @@ void xe_bo_vunmap(struct xe_bo *bo); 


bool mem_type_is_vram(u32 mem_type);

bool xe_bo_is_vram(struct xe_bo *bo);

bool xe_bo_is_stolen(struct xe_bo *bo);



bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type);

drivers/gpu/drm/xe/xe_debugfs.c

+4 −0
Original line number Diff line number Diff line
@@ -124,6 +124,10 @@ void xe_debugfs_register(struct xe_device *xe) 
	man = ttm_manager_type(bdev, XE_PL_TT);

	ttm_resource_manager_create_debugfs(man, root, "gtt_mm");



	man = ttm_manager_type(bdev, XE_PL_STOLEN);

	if (man)

		ttm_resource_manager_create_debugfs(man, root, "stolen_mm");



	for_each_gt(gt, xe, id)

		xe_gt_debugfs_register(gt);

}

drivers/gpu/drm/xe/xe_device.c

+4 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include "xe_pcode.h"

#include "xe_pm.h"

#include "xe_query.h"

#include "xe_ttm_stolen_mgr.h"

#include "xe_vm.h"

#include "xe_vm_madvise.h"

#include "xe_wait_user_fence.h"
@@ -256,6 +257,9 @@ int xe_device_probe(struct xe_device *xe) 
			goto err_irq_shutdown;

	}



	/* Allocate and map stolen after potential VRAM resize */

	xe_ttm_stolen_mgr_init(xe);



	for_each_gt(gt, xe, id) {

		err = xe_gt_init(gt);

		if (err)