Commit f91e81d3 authored by Johannes Weiner's avatar Johannes Weiner Committed by Andrew Morton
Browse files

mm: zswap: function ordering: compress & decompress functions 

Writeback needs to decompress. Move the (de)compression API above what
will be the consolidated shrinking/writeback code.

Link: https://lkml.kernel.org/r/20240130014208.565554-18-hannes@cmpxchg.org


Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Reviewed-by: default avatarNhat Pham <nphamcs@gmail.com>
Cc: Chengming Zhou <zhouchengming@bytedance.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 36034bf6

mm/zswap.c

+105 −102
Original line number Diff line number Diff line
@@ -992,6 +992,111 @@ static void zswap_invalidate_entry(struct zswap_tree *tree, 
		zswap_entry_put(entry);

}



/*********************************

* compressed storage functions

**********************************/

static bool zswap_compress(struct folio *folio, struct zswap_entry *entry)

{

	struct crypto_acomp_ctx *acomp_ctx;

	struct scatterlist input, output;

	unsigned int dlen = PAGE_SIZE;

	unsigned long handle;

	struct zpool *zpool;

	char *buf;

	gfp_t gfp;

	int ret;

	u8 *dst;



	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);



	mutex_lock(&acomp_ctx->mutex);



	dst = acomp_ctx->buffer;

	sg_init_table(&input, 1);

	sg_set_page(&input, &folio->page, PAGE_SIZE, 0);



	/*

	 * We need PAGE_SIZE * 2 here since there maybe over-compression case,

	 * and hardware-accelerators may won't check the dst buffer size, so

	 * giving the dst buffer with enough length to avoid buffer overflow.

	 */

	sg_init_one(&output, dst, PAGE_SIZE * 2);

	acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);



	/*

	 * it maybe looks a little bit silly that we send an asynchronous request,

	 * then wait for its completion synchronously. This makes the process look

	 * synchronous in fact.

	 * Theoretically, acomp supports users send multiple acomp requests in one

	 * acomp instance, then get those requests done simultaneously. but in this

	 * case, zswap actually does store and load page by page, there is no

	 * existing method to send the second page before the first page is done

	 * in one thread doing zwap.

	 * but in different threads running on different cpu, we have different

	 * acomp instance, so multiple threads can do (de)compression in parallel.

	 */

	ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);

	dlen = acomp_ctx->req->dlen;

	if (ret) {

		zswap_reject_compress_fail++;

		goto unlock;

	}



	zpool = zswap_find_zpool(entry);

	gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;

	if (zpool_malloc_support_movable(zpool))

		gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;

	ret = zpool_malloc(zpool, dlen, gfp, &handle);

	if (ret == -ENOSPC) {

		zswap_reject_compress_poor++;

		goto unlock;

	}

	if (ret) {

		zswap_reject_alloc_fail++;

		goto unlock;

	}



	buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);

	memcpy(buf, dst, dlen);

	zpool_unmap_handle(zpool, handle);



	entry->handle = handle;

	entry->length = dlen;



unlock:

	mutex_unlock(&acomp_ctx->mutex);

	return ret == 0;

}



static void zswap_decompress(struct zswap_entry *entry, struct page *page)

{

	struct zpool *zpool = zswap_find_zpool(entry);

	struct scatterlist input, output;

	struct crypto_acomp_ctx *acomp_ctx;

	u8 *src;



	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);

	mutex_lock(&acomp_ctx->mutex);



	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);

	if (!zpool_can_sleep_mapped(zpool)) {

		memcpy(acomp_ctx->buffer, src, entry->length);

		src = acomp_ctx->buffer;

		zpool_unmap_handle(zpool, entry->handle);

	}



	sg_init_one(&input, src, entry->length);

	sg_init_table(&output, 1);

	sg_set_page(&output, page, PAGE_SIZE, 0);

	acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);

	BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));

	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);

	mutex_unlock(&acomp_ctx->mutex);



	if (zpool_can_sleep_mapped(zpool))

		zpool_unmap_handle(zpool, entry->handle);

}



/*********************************

* shrinker functions

**********************************/
@@ -1317,108 +1422,6 @@ static void shrink_worker(struct work_struct *w) 
	zswap_pool_put(pool);

}



static bool zswap_compress(struct folio *folio, struct zswap_entry *entry)

{

	struct crypto_acomp_ctx *acomp_ctx;

	struct scatterlist input, output;

	unsigned int dlen = PAGE_SIZE;

	unsigned long handle;

	struct zpool *zpool;

	char *buf;

	gfp_t gfp;

	int ret;

	u8 *dst;



	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);



	mutex_lock(&acomp_ctx->mutex);



	dst = acomp_ctx->buffer;

	sg_init_table(&input, 1);

	sg_set_page(&input, &folio->page, PAGE_SIZE, 0);



	/*

	 * We need PAGE_SIZE * 2 here since there maybe over-compression case,

	 * and hardware-accelerators may won't check the dst buffer size, so

	 * giving the dst buffer with enough length to avoid buffer overflow.

	 */

	sg_init_one(&output, dst, PAGE_SIZE * 2);

	acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);



	/*

	 * it maybe looks a little bit silly that we send an asynchronous request,

	 * then wait for its completion synchronously. This makes the process look

	 * synchronous in fact.

	 * Theoretically, acomp supports users send multiple acomp requests in one

	 * acomp instance, then get those requests done simultaneously. but in this

	 * case, zswap actually does store and load page by page, there is no

	 * existing method to send the second page before the first page is done

	 * in one thread doing zwap.

	 * but in different threads running on different cpu, we have different

	 * acomp instance, so multiple threads can do (de)compression in parallel.

	 */

	ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);

	dlen = acomp_ctx->req->dlen;

	if (ret) {

		zswap_reject_compress_fail++;

		goto unlock;

	}



	zpool = zswap_find_zpool(entry);

	gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;

	if (zpool_malloc_support_movable(zpool))

		gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;

	ret = zpool_malloc(zpool, dlen, gfp, &handle);

	if (ret == -ENOSPC) {

		zswap_reject_compress_poor++;

		goto unlock;

	}

	if (ret) {

		zswap_reject_alloc_fail++;

		goto unlock;

	}



	buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);

	memcpy(buf, dst, dlen);

	zpool_unmap_handle(zpool, handle);



	entry->handle = handle;

	entry->length = dlen;



unlock:

	mutex_unlock(&acomp_ctx->mutex);

	return ret == 0;

}



static void zswap_decompress(struct zswap_entry *entry, struct page *page)

{

	struct zpool *zpool = zswap_find_zpool(entry);

	struct scatterlist input, output;

	struct crypto_acomp_ctx *acomp_ctx;

	u8 *src;



	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);

	mutex_lock(&acomp_ctx->mutex);



	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);

	if (!zpool_can_sleep_mapped(zpool)) {

		memcpy(acomp_ctx->buffer, src, entry->length);

		src = acomp_ctx->buffer;

		zpool_unmap_handle(zpool, entry->handle);

	}



	sg_init_one(&input, src, entry->length);

	sg_init_table(&output, 1);

	sg_set_page(&output, page, PAGE_SIZE, 0);

	acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);

	BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));

	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);

	mutex_unlock(&acomp_ctx->mutex);



	if (zpool_can_sleep_mapped(zpool))

		zpool_unmap_handle(zpool, entry->handle);

}



/*********************************

* writeback code

**********************************/