Commit 25c17c4b authored by Yang Shi's avatar Yang Shi Committed by Catalin Marinas
Browse files

hugetlb: arm64: add mte support 



Enable MTE support for hugetlb.

The MTE page flags will be set on the folio only.  When copying
hugetlb folio (for example, CoW), the tags for all subpages will be copied
when copying the first subpage.

When freeing hugetlb folio, the MTE flags will be cleared.

Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Reviewed-by: default avatarDavid Hildenbrand <david@redhat.com>
Signed-off-by: default avatarYang Shi <yang@os.amperecomputing.com>
Link: https://lore.kernel.org/r/20241001225220.271178-1-yang@os.amperecomputing.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 9852d85e

arch/arm64/include/asm/hugetlb.h

+8 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#define __ASM_HUGETLB_H



#include <asm/cacheflush.h>

#include <asm/mte.h>

#include <asm/page.h>



#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
@@ -21,6 +22,13 @@ extern bool arch_hugetlb_migration_supported(struct hstate *h); 
static inline void arch_clear_hugetlb_flags(struct folio *folio)

{

	clear_bit(PG_dcache_clean, &folio->flags);



#ifdef CONFIG_ARM64_MTE

	if (system_supports_mte()) {

		clear_bit(PG_mte_tagged, &folio->flags);

		clear_bit(PG_mte_lock, &folio->flags);

	}

#endif

}

#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags

arch/arm64/include/asm/mman.h

+2 −1
Original line number Diff line number Diff line
@@ -38,7 +38,8 @@ static inline unsigned long arch_calc_vm_flag_bits(unsigned long flags) 
	 * backed by tags-capable memory. The vm_flags may be overridden by a

	 * filesystem supporting MTE (RAM-based).

	 */

	if (system_supports_mte() && (flags & MAP_ANONYMOUS))

	if (system_supports_mte() &&

	    (flags & (MAP_ANONYMOUS | MAP_HUGETLB)))

		return VM_MTE_ALLOWED;



	return 0;

arch/arm64/include/asm/mte.h

+67 −0
Original line number Diff line number Diff line
@@ -41,6 +41,8 @@ void mte_free_tag_storage(char *storage); 


static inline void set_page_mte_tagged(struct page *page)

{

	VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));



	/*

	 * Ensure that the tags written prior to this function are visible

	 * before the page flags update.
@@ -53,6 +55,8 @@ static inline bool page_mte_tagged(struct page *page) 
{

	bool ret = test_bit(PG_mte_tagged, &page->flags);



	VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));



	/*

	 * If the page is tagged, ensure ordering with a likely subsequent

	 * read of the tags.
@@ -76,6 +80,8 @@ static inline bool page_mte_tagged(struct page *page) 
 */

static inline bool try_page_mte_tagging(struct page *page)

{

	VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));



	if (!test_and_set_bit(PG_mte_lock, &page->flags))

		return true;
@@ -157,6 +163,67 @@ static inline int mte_ptrace_copy_tags(struct task_struct *child, 


#endif /* CONFIG_ARM64_MTE */



#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_ARM64_MTE)

static inline void folio_set_hugetlb_mte_tagged(struct folio *folio)

{

	VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));



	/*

	 * Ensure that the tags written prior to this function are visible

	 * before the folio flags update.

	 */

	smp_wmb();

	set_bit(PG_mte_tagged, &folio->flags);



}



static inline bool folio_test_hugetlb_mte_tagged(struct folio *folio)

{

	bool ret = test_bit(PG_mte_tagged, &folio->flags);



	VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));



	/*

	 * If the folio is tagged, ensure ordering with a likely subsequent

	 * read of the tags.

	 */

	if (ret)

		smp_rmb();

	return ret;

}



static inline bool folio_try_hugetlb_mte_tagging(struct folio *folio)

{

	VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));



	if (!test_and_set_bit(PG_mte_lock, &folio->flags))

		return true;



	/*

	 * The tags are either being initialised or may have been initialised

	 * already. Check if the PG_mte_tagged flag has been set or wait

	 * otherwise.

	 */

	smp_cond_load_acquire(&folio->flags, VAL & (1UL << PG_mte_tagged));



	return false;

}

#else

static inline void folio_set_hugetlb_mte_tagged(struct folio *folio)

{

}



static inline bool folio_test_hugetlb_mte_tagged(struct folio *folio)

{

	return false;

}



static inline bool folio_try_hugetlb_mte_tagging(struct folio *folio)

{

	return false;

}

#endif



static inline void mte_disable_tco_entry(struct task_struct *task)

{

	if (!system_supports_mte())

arch/arm64/kernel/hibernate.c

+6 −0
Original line number Diff line number Diff line
@@ -266,9 +266,15 @@ static int swsusp_mte_save_tags(void) 
		max_zone_pfn = zone_end_pfn(zone);

		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {

			struct page *page = pfn_to_online_page(pfn);

			struct folio *folio;



			if (!page)

				continue;

			folio = page_folio(page);



			if (folio_test_hugetlb(folio) &&

			    !folio_test_hugetlb_mte_tagged(folio))

				continue;



			if (!page_mte_tagged(page))

				continue;

arch/arm64/kernel/mte.c

+25 −2
Original line number Diff line number Diff line
@@ -38,7 +38,24 @@ EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); 
void mte_sync_tags(pte_t pte, unsigned int nr_pages)

{

	struct page *page = pte_page(pte);

	unsigned int i;

	struct folio *folio = page_folio(page);

	unsigned long i;



	if (folio_test_hugetlb(folio)) {

		unsigned long nr = folio_nr_pages(folio);



		/* Hugetlb MTE flags are set for head page only */

		if (folio_try_hugetlb_mte_tagging(folio)) {

			for (i = 0; i < nr; i++, page++)

				mte_clear_page_tags(page_address(page));

			folio_set_hugetlb_mte_tagged(folio);

		}



		/* ensure the tags are visible before the PTE is set */

		smp_wmb();



		return;

	}



	/* if PG_mte_tagged is set, tags have already been initialised */

	for (i = 0; i < nr_pages; i++, page++) {
@@ -410,6 +427,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, 
		void *maddr;

		struct page *page = get_user_page_vma_remote(mm, addr,

							     gup_flags, &vma);

		struct folio *folio;



		if (IS_ERR(page)) {

			err = PTR_ERR(page);
@@ -428,6 +446,11 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, 
			put_page(page);

			break;

		}



		folio = page_folio(page);

		if (folio_test_hugetlb(folio))

			WARN_ON_ONCE(!folio_test_hugetlb_mte_tagged(folio));

		else

			WARN_ON_ONCE(!page_mte_tagged(page));



		/* limit access to the end of the page */