mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git
synced 2026-04-23 05:56:14 -04:00
0b0e518d61
Register svm range with same address and size but perferred_location is changed from CPU to GPU or from GPU to CPU, trigger migration the svm range from ram to vram or from vram to ram. If svm range prefetch location is GPU with flags KFD_IOCTL_SVM_FLAG_HOST_ACCESS, validate the svm range on ram first, then migrate it from ram to vram. After migrating to vram is done, CPU access will have cpu page fault, page fault handler migrate it back to ram and resume cpu access. Migration steps: 1. migrate_vma_pages get svm range ram pages, notify the interval is invalidated and unmap from CPU page table, HMM interval notifier callback evict process queues 2. Allocate new pages in vram using TTM 3. Use svm copy memory to sdma copy data from ram to vram 4. migrate_vma_pages copy ram pages structure to vram pages structure 5. migrate_vma_finalize put ram pages to free ram pages and memory 6. Restore work wait for migration is finished, then update GPUs page table mapping to new vram pages, resume process queues If migrate_vma_setup failed to collect all ram pages of range, retry 3 times until success to start migration. Signed-off-by: Philip Yang <Philip.Yang@amd.com> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com> Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
156 lines
5.8 KiB
C
156 lines
5.8 KiB
C
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
|
|
/*
|
|
* Copyright 2020-2021 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
* OTHER DEALINGS IN THE SOFTWARE.
|
|
*
|
|
*/
|
|
|
|
#ifndef KFD_SVM_H_
|
|
#define KFD_SVM_H_
|
|
|
|
#include <linux/rwsem.h>
|
|
#include <linux/list.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/sched/mm.h>
|
|
#include <linux/hmm.h>
|
|
#include "amdgpu.h"
|
|
#include "kfd_priv.h"
|
|
|
|
struct svm_range_bo {
|
|
struct amdgpu_bo *bo;
|
|
struct kref kref;
|
|
struct list_head range_list; /* all svm ranges shared this bo */
|
|
spinlock_t list_lock;
|
|
};
|
|
|
|
enum svm_work_list_ops {
|
|
SVM_OP_NULL,
|
|
SVM_OP_UNMAP_RANGE,
|
|
SVM_OP_UPDATE_RANGE_NOTIFIER,
|
|
SVM_OP_ADD_RANGE
|
|
};
|
|
|
|
struct svm_work_list_item {
|
|
enum svm_work_list_ops op;
|
|
struct mm_struct *mm;
|
|
};
|
|
|
|
/**
|
|
* struct svm_range - shared virtual memory range
|
|
*
|
|
* @svms: list of svm ranges, structure defined in kfd_process
|
|
* @migrate_mutex: to serialize range migration, validation and mapping update
|
|
* @start: range start address in pages
|
|
* @last: range last address in pages
|
|
* @it_node: node [start, last] stored in interval tree, start, last are page
|
|
* aligned, page size is (last - start + 1)
|
|
* @list: link list node, used to scan all ranges of svms
|
|
* @update_list:link list node used to add to update_list
|
|
* @remove_list:link list node used to add to remove list
|
|
* @insert_list:link list node used to add to insert list
|
|
* @mapping: bo_va mapping structure to create and update GPU page table
|
|
* @npages: number of pages
|
|
* @dma_addr: dma mapping address on each GPU for system memory physical page
|
|
* @ttm_res: vram ttm resource map
|
|
* @offset: range start offset within mm_nodes
|
|
* @svm_bo: struct to manage splited amdgpu_bo
|
|
* @svm_bo_list:link list node, to scan all ranges which share same svm_bo
|
|
* @lock: protect prange start, last, child_list, svm_bo_list
|
|
* @saved_flags:save/restore current PF_MEMALLOC flags
|
|
* @flags: flags defined as KFD_IOCTL_SVM_FLAG_*
|
|
* @perferred_loc: perferred location, 0 for CPU, or GPU id
|
|
* @perfetch_loc: last prefetch location, 0 for CPU, or GPU id
|
|
* @actual_loc: the actual location, 0 for CPU, or GPU id
|
|
* @granularity:migration granularity, log2 num pages
|
|
* @invalid: not 0 means cpu page table is invalidated
|
|
* @notifier: register mmu interval notifier
|
|
* @work_item: deferred work item information
|
|
* @deferred_list: list header used to add range to deferred list
|
|
* @child_list: list header for split ranges which are not added to svms yet
|
|
* @bitmap_access: index bitmap of GPUs which can access the range
|
|
* @bitmap_aip: index bitmap of GPUs which can access the range in place
|
|
*
|
|
* Data structure for virtual memory range shared by CPU and GPUs, it can be
|
|
* allocated from system memory ram or device vram, and migrate from ram to vram
|
|
* or from vram to ram.
|
|
*/
|
|
struct svm_range {
|
|
struct svm_range_list *svms;
|
|
struct mutex migrate_mutex;
|
|
unsigned long start;
|
|
unsigned long last;
|
|
struct interval_tree_node it_node;
|
|
struct list_head list;
|
|
struct list_head update_list;
|
|
struct list_head remove_list;
|
|
struct list_head insert_list;
|
|
struct amdgpu_bo_va_mapping mapping;
|
|
uint64_t npages;
|
|
dma_addr_t *dma_addr[MAX_GPU_INSTANCE];
|
|
struct ttm_resource *ttm_res;
|
|
uint64_t offset;
|
|
struct svm_range_bo *svm_bo;
|
|
struct list_head svm_bo_list;
|
|
struct mutex lock;
|
|
unsigned int saved_flags;
|
|
uint32_t flags;
|
|
uint32_t preferred_loc;
|
|
uint32_t prefetch_loc;
|
|
uint32_t actual_loc;
|
|
uint8_t granularity;
|
|
atomic_t invalid;
|
|
struct mmu_interval_notifier notifier;
|
|
struct svm_work_list_item work_item;
|
|
struct list_head deferred_list;
|
|
struct list_head child_list;
|
|
DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
|
|
DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
|
|
bool validated_once;
|
|
};
|
|
|
|
static inline void svm_range_lock(struct svm_range *prange)
|
|
{
|
|
mutex_lock(&prange->lock);
|
|
prange->saved_flags = memalloc_noreclaim_save();
|
|
|
|
}
|
|
static inline void svm_range_unlock(struct svm_range *prange)
|
|
{
|
|
memalloc_noreclaim_restore(prange->saved_flags);
|
|
mutex_unlock(&prange->lock);
|
|
}
|
|
|
|
int svm_range_list_init(struct kfd_process *p);
|
|
void svm_range_list_fini(struct kfd_process *p);
|
|
int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
|
|
uint64_t size, uint32_t nattrs,
|
|
struct kfd_ioctl_svm_attribute *attrs);
|
|
struct amdgpu_device *svm_range_get_adev_by_id(struct svm_range *prange,
|
|
uint32_t id);
|
|
int svm_range_vram_node_new(struct amdgpu_device *adev,
|
|
struct svm_range *prange, bool clear);
|
|
void svm_range_vram_node_free(struct svm_range *prange);
|
|
void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr,
|
|
unsigned long offset, unsigned long npages);
|
|
void svm_range_free_dma_mappings(struct svm_range *prange);
|
|
void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm);
|
|
|
|
#endif /* KFD_SVM_H_ */
|