Commit 1e4df185 authored by Feng Tang's avatar Feng Tang Committed by Vlastimil Babka
Browse files

mm/slub: Move krealloc() and related code to slub.c 



This is a preparation for the following refactoring of krealloc(),
for more efficient function calling as it will call some internal
functions defined in slub.c.

Signed-off-by: default avatarFeng Tang <feng.tang@intel.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent fb5eda0d

mm/slab_common.c

+0 −84
Original line number Diff line number Diff line
@@ -1190,90 +1190,6 @@ module_init(slab_proc_init); 


#endif /* CONFIG_SLUB_DEBUG */



static __always_inline __realloc_size(2) void *

__do_krealloc(const void *p, size_t new_size, gfp_t flags)

{

	void *ret;

	size_t ks;



	/* Check for double-free before calling ksize. */

	if (likely(!ZERO_OR_NULL_PTR(p))) {

		if (!kasan_check_byte(p))

			return NULL;

		ks = ksize(p);

	} else

		ks = 0;



	/* If the object still fits, repoison it precisely. */

	if (ks >= new_size) {

		/* Zero out spare memory. */

		if (want_init_on_alloc(flags)) {

			kasan_disable_current();

			memset(kasan_reset_tag(p) + new_size, 0, ks - new_size);

			kasan_enable_current();

		}



		p = kasan_krealloc((void *)p, new_size, flags);

		return (void *)p;

	}



	ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_);

	if (ret && p) {

		/* Disable KASAN checks as the object's redzone is accessed. */

		kasan_disable_current();

		memcpy(ret, kasan_reset_tag(p), ks);

		kasan_enable_current();

	}



	return ret;

}



/**

 * krealloc - reallocate memory. The contents will remain unchanged.

 * @p: object to reallocate memory for.

 * @new_size: how many bytes of memory are required.

 * @flags: the type of memory to allocate.

 *

 * If @p is %NULL, krealloc() behaves exactly like kmalloc().  If @new_size

 * is 0 and @p is not a %NULL pointer, the object pointed to is freed.

 *

 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the

 * initial memory allocation, every subsequent call to this API for the same

 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that

 * __GFP_ZERO is not fully honored by this API.

 *

 * This is the case, since krealloc() only knows about the bucket size of an

 * allocation (but not the exact size it was allocated with) and hence

 * implements the following semantics for shrinking and growing buffers with

 * __GFP_ZERO.

 *

 *         new             bucket

 * 0       size             size

 * |--------|----------------|

 * |  keep  |      zero      |

 *

 * In any case, the contents of the object pointed to are preserved up to the

 * lesser of the new and old sizes.

 *

 * Return: pointer to the allocated memory or %NULL in case of error

 */

void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)

{

	void *ret;



	if (unlikely(!new_size)) {

		kfree(p);

		return ZERO_SIZE_PTR;

	}



	ret = __do_krealloc(p, new_size, flags);

	if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))

		kfree(p);



	return ret;

}

EXPORT_SYMBOL(krealloc_noprof);



/**

 * kfree_sensitive - Clear sensitive information in memory before freeing

 * @p: object to free memory of

mm/slub.c

+84 −0
Original line number Diff line number Diff line
@@ -4711,6 +4711,90 @@ void kfree(const void *object) 
}

EXPORT_SYMBOL(kfree);



static __always_inline __realloc_size(2) void *

__do_krealloc(const void *p, size_t new_size, gfp_t flags)

{

	void *ret;

	size_t ks;



	/* Check for double-free before calling ksize. */

	if (likely(!ZERO_OR_NULL_PTR(p))) {

		if (!kasan_check_byte(p))

			return NULL;

		ks = ksize(p);

	} else

		ks = 0;



	/* If the object still fits, repoison it precisely. */

	if (ks >= new_size) {

		/* Zero out spare memory. */

		if (want_init_on_alloc(flags)) {

			kasan_disable_current();

			memset(kasan_reset_tag(p) + new_size, 0, ks - new_size);

			kasan_enable_current();

		}



		p = kasan_krealloc((void *)p, new_size, flags);

		return (void *)p;

	}



	ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_);

	if (ret && p) {

		/* Disable KASAN checks as the object's redzone is accessed. */

		kasan_disable_current();

		memcpy(ret, kasan_reset_tag(p), ks);

		kasan_enable_current();

	}



	return ret;

}



/**

 * krealloc - reallocate memory. The contents will remain unchanged.

 * @p: object to reallocate memory for.

 * @new_size: how many bytes of memory are required.

 * @flags: the type of memory to allocate.

 *

 * If @p is %NULL, krealloc() behaves exactly like kmalloc().  If @new_size

 * is 0 and @p is not a %NULL pointer, the object pointed to is freed.

 *

 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the

 * initial memory allocation, every subsequent call to this API for the same

 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that

 * __GFP_ZERO is not fully honored by this API.

 *

 * This is the case, since krealloc() only knows about the bucket size of an

 * allocation (but not the exact size it was allocated with) and hence

 * implements the following semantics for shrinking and growing buffers with

 * __GFP_ZERO.

 *

 *         new             bucket

 * 0       size             size

 * |--------|----------------|

 * |  keep  |      zero      |

 *

 * In any case, the contents of the object pointed to are preserved up to the

 * lesser of the new and old sizes.

 *

 * Return: pointer to the allocated memory or %NULL in case of error

 */

void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)

{

	void *ret;



	if (unlikely(!new_size)) {

		kfree(p);

		return ZERO_SIZE_PTR;

	}



	ret = __do_krealloc(p, new_size, flags);

	if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))

		kfree(p);



	return ret;

}

EXPORT_SYMBOL(krealloc_noprof);



struct detached_freelist {

	struct slab *slab;

	void *tail;