Commit 8ba2f844 authored by Chengming Zhou's avatar Chengming Zhou Committed by Andrew Morton
Browse files

mm/zswap: change per-cpu mutex and buffer to per-acomp_ctx 

First of all, we need to rename acomp_ctx->dstmem field to buffer, since
we are now using for purposes other than compression.

Then we change per-cpu mutex and buffer to per-acomp_ctx, since them
belong to the acomp_ctx and are necessary parts when used in the
compress/decompress contexts.

So we can remove the old per-cpu mutex and dstmem.

Link: https://lkml.kernel.org/r/20231213-zswap-dstmem-v5-5-9382162bbf05@bytedance.com


Signed-off-by: default avatarChengming Zhou <zhouchengming@bytedance.com>
Acked-by: Chris Li <chrisl@kernel.org> (Google)
Reviewed-by: default avatarNhat Pham <nphamcs@gmail.com>
Cc: Barry Song <21cnbao@gmail.com>
Cc: Dan Streetman <ddstreet@ieee.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Seth Jennings <sjenning@redhat.com>
Cc: Vitaly Wool <vitaly.wool@konsulko.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent e947ba0b

include/linux/cpuhotplug.h

+0 −1
Original line number Diff line number Diff line
@@ -124,7 +124,6 @@ enum cpuhp_state { 
	CPUHP_ARM_BL_PREPARE,

	CPUHP_TRACE_RB_PREPARE,

	CPUHP_MM_ZS_PREPARE,

	CPUHP_MM_ZSWP_MEM_PREPARE,

	CPUHP_MM_ZSWP_POOL_PREPARE,

	CPUHP_KVM_PPC_BOOK3S_PREPARE,

	CPUHP_ZCOMP_PREPARE,

mm/zswap.c

+33 −71
Original line number Diff line number Diff line
@@ -161,8 +161,8 @@ struct crypto_acomp_ctx { 
	struct crypto_acomp *acomp;

	struct acomp_req *req;

	struct crypto_wait wait;

	u8 *dstmem;

	struct mutex *mutex;

	u8 *buffer;

	struct mutex mutex;

};



/*
@@ -688,63 +688,26 @@ static void zswap_alloc_shrinker(struct zswap_pool *pool) 
/*********************************

* per-cpu code

**********************************/

static DEFINE_PER_CPU(u8 *, zswap_dstmem);

/*

 * If users dynamically change the zpool type and compressor at runtime, i.e.

 * zswap is running, zswap can have more than one zpool on one cpu, but they

 * are sharing dtsmem. So we need this mutex to be per-cpu.

 */

static DEFINE_PER_CPU(struct mutex *, zswap_mutex);



static int zswap_dstmem_prepare(unsigned int cpu)

{

	struct mutex *mutex;

	u8 *dst;



	dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));

	if (!dst)

		return -ENOMEM;



	mutex = kmalloc_node(sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu));

	if (!mutex) {

		kfree(dst);

		return -ENOMEM;

	}



	mutex_init(mutex);

	per_cpu(zswap_dstmem, cpu) = dst;

	per_cpu(zswap_mutex, cpu) = mutex;

	return 0;

}



static int zswap_dstmem_dead(unsigned int cpu)

{

	struct mutex *mutex;

	u8 *dst;



	mutex = per_cpu(zswap_mutex, cpu);

	kfree(mutex);

	per_cpu(zswap_mutex, cpu) = NULL;



	dst = per_cpu(zswap_dstmem, cpu);

	kfree(dst);

	per_cpu(zswap_dstmem, cpu) = NULL;



	return 0;

}



static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)

{

	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);

	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);

	struct crypto_acomp *acomp;

	struct acomp_req *req;

	int ret;



	mutex_init(&acomp_ctx->mutex);



	acomp_ctx->buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));

	if (!acomp_ctx->buffer)

		return -ENOMEM;



	acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));

	if (IS_ERR(acomp)) {

		pr_err("could not alloc crypto acomp %s : %ld\n",

				pool->tfm_name, PTR_ERR(acomp));

		return PTR_ERR(acomp);

		ret = PTR_ERR(acomp);

		goto acomp_fail;

	}

	acomp_ctx->acomp = acomp;
@@ -752,8 +715,8 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 
	if (!req) {

		pr_err("could not alloc crypto acomp_request %s\n",

		       pool->tfm_name);

		crypto_free_acomp(acomp_ctx->acomp);

		return -ENOMEM;

		ret = -ENOMEM;

		goto req_fail;

	}

	acomp_ctx->req = req;
@@ -766,10 +729,13 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 
	acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,

				   crypto_req_done, &acomp_ctx->wait);



	acomp_ctx->mutex = per_cpu(zswap_mutex, cpu);

	acomp_ctx->dstmem = per_cpu(zswap_dstmem, cpu);



	return 0;



req_fail:

	crypto_free_acomp(acomp_ctx->acomp);

acomp_fail:

	kfree(acomp_ctx->buffer);

	return ret;

}



static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
@@ -782,6 +748,7 @@ static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) 
			acomp_request_free(acomp_ctx->req);

		if (!IS_ERR_OR_NULL(acomp_ctx->acomp))

			crypto_free_acomp(acomp_ctx->acomp);

		kfree(acomp_ctx->buffer);

	}



	return 0;
@@ -1391,12 +1358,12 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page) 
	u8 *src;



	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);

	mutex_lock(acomp_ctx->mutex);

	mutex_lock(&acomp_ctx->mutex);



	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);

	if (!zpool_can_sleep_mapped(zpool)) {

		memcpy(acomp_ctx->dstmem, src, entry->length);

		src = acomp_ctx->dstmem;

		memcpy(acomp_ctx->buffer, src, entry->length);

		src = acomp_ctx->buffer;

		zpool_unmap_handle(zpool, entry->handle);

	}
@@ -1406,7 +1373,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page) 
	acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);

	BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));

	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);

	mutex_unlock(acomp_ctx->mutex);

	mutex_unlock(&acomp_ctx->mutex);



	if (zpool_can_sleep_mapped(zpool))

		zpool_unmap_handle(zpool, entry->handle);
@@ -1622,13 +1589,17 @@ bool zswap_store(struct folio *folio) 
	/* compress */

	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);



	mutex_lock(acomp_ctx->mutex);

	mutex_lock(&acomp_ctx->mutex);



	dst = acomp_ctx->dstmem;

	dst = acomp_ctx->buffer;

	sg_init_table(&input, 1);

	sg_set_page(&input, page, PAGE_SIZE, 0);



	/* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */

	/*

	 * We need PAGE_SIZE * 2 here since there maybe over-compression case,

	 * and hardware-accelerators may won't check the dst buffer size, so

	 * giving the dst buffer with enough length to avoid buffer overflow.

	 */

	sg_init_one(&output, dst, PAGE_SIZE * 2);

	acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);

	/*
@@ -1668,7 +1639,7 @@ bool zswap_store(struct folio *folio) 
	buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);

	memcpy(buf, dst, dlen);

	zpool_unmap_handle(zpool, handle);

	mutex_unlock(acomp_ctx->mutex);

	mutex_unlock(&acomp_ctx->mutex);



	/* populate entry */

	entry->swpentry = swp_entry(type, offset);
@@ -1711,7 +1682,7 @@ bool zswap_store(struct folio *folio) 
	return true;



put_dstmem:

	mutex_unlock(acomp_ctx->mutex);

	mutex_unlock(&acomp_ctx->mutex);

put_pool:

	zswap_pool_put(entry->pool);

freepage:
@@ -1886,13 +1857,6 @@ static int zswap_setup(void) 
		goto cache_fail;

	}



	ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",

				zswap_dstmem_prepare, zswap_dstmem_dead);

	if (ret) {

		pr_err("dstmem alloc failed\n");

		goto dstmem_fail;

	}



	ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,

				      "mm/zswap_pool:prepare",

				      zswap_cpu_comp_prepare,
@@ -1924,8 +1888,6 @@ static int zswap_setup(void) 
	if (pool)

		zswap_pool_destroy(pool);

hp_fail:

	cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);

dstmem_fail:

	kmem_cache_destroy(zswap_entry_cache);

cache_fail:

	/* if built-in, we aren't unloaded on failure; don't allow use */