git
/
linux
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CRC updates for 6.17 

Updates for the kernel's CRC (cyclic redundancy check) code:
 
  - Reorganize the architecture-optimized CRC code. It now lives in
    lib/crc/$(SRCARCH)/ rather than arch/$(SRCARCH)/lib/, and it is no
    longer artificially split into separate generic and arch modules.
    This allows better inlining and dead code elimination. The generic
    CRC code is also no longer exported, simplifying the API. (This
    mirrors the similar changes to SHA-1 and SHA-2 in lib/crypto/,
    which can be found in the "Crypto library updates" pull request.)
 
  - Improve crc32c() performance on newer x86_64 CPUs on long messages
    by enabling the VPCLMULQDQ optimized code.
 
  - Simplify the crypto_shash wrappers for crc32_le() and crc32c().
    Register just one shash algorithm for each that uses the (fully
    optimized) library functions, instead of unnecessarily providing
    direct access to the generic CRC code.
 
  - Remove unused and obsolete drivers for hardware CRC engines.
 
  - Remove CRC-32 combination functions that are no longer used.
 
  - Add kerneldoc for crc32_le(), crc32_be(), and crc32c().
 
  - Convert the crc32() macro to an inline function.
 -----BEGIN PGP SIGNATURE-----
 
 iIoEABYIADIWIQSacvsUNc7UX4ntmEPzXCl4vpKOKwUCaIZ8rRQcZWJpZ2dlcnNA
 a2VybmVsLm9yZwAKCRDzXCl4vpKOK3yOAP9OuoCirD42ZHNSgQeGTzhhZ2jCHiPN
 BPvHChwtE2MSRwEA0ddNX36aOiEKmpjog3TMllOIBz7wBrwZV7KgoX75+AU=
 =uAY8
 -----END PGP SIGNATURE-----

Merge tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull CRC updates from Eric Biggers:

 - Reorganize the architecture-optimized CRC code

   It now lives in lib/crc/$(SRCARCH)/ rather than arch/$(SRCARCH)/lib/,
   and it is no longer artificially split into separate generic and arch
   modules. This allows better inlining and dead code elimination

   The generic CRC code is also no longer exported, simplifying the API.
   (This mirrors the similar changes to SHA-1 and SHA-2 in lib/crypto/,
   which can be found in the "Crypto library updates" pull request)

 - Improve crc32c() performance on newer x86_64 CPUs on long messages by
   enabling the VPCLMULQDQ optimized code

 - Simplify the crypto_shash wrappers for crc32_le() and crc32c()

   Register just one shash algorithm for each that uses the (fully
   optimized) library functions, instead of unnecessarily providing
   direct access to the generic CRC code

 - Remove unused and obsolete drivers for hardware CRC engines

 - Remove CRC-32 combination functions that are no longer used

 - Add kerneldoc for crc32_le(), crc32_be(), and crc32c()

 - Convert the crc32() macro to an inline function

* tag 'crc-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (26 commits)
  lib/crc: x86/crc32c: Enable VPCLMULQDQ optimization where beneficial
  lib/crc: x86: Reorganize crc-pclmul static_call initialization
  lib/crc: crc64: Add include/linux/crc64.h to kernel-api.rst
  lib/crc: crc32: Change crc32() from macro to inline function and remove cast
  nvmem: layouts: Switch from crc32() to crc32_le()
  lib/crc: crc32: Document crc32_le(), crc32_be(), and crc32c()
  lib/crc: Explicitly include <linux/export.h>
  lib/crc: Remove ARCH_HAS_* kconfig symbols
  lib/crc: x86: Migrate optimized CRC code into lib/crc/
  lib/crc: sparc: Migrate optimized CRC code into lib/crc/
  lib/crc: s390: Migrate optimized CRC code into lib/crc/
  lib/crc: riscv: Migrate optimized CRC code into lib/crc/
  lib/crc: powerpc: Migrate optimized CRC code into lib/crc/
  lib/crc: mips: Migrate optimized CRC code into lib/crc/
  lib/crc: loongarch: Migrate optimized CRC code into lib/crc/
  lib/crc: arm64: Migrate optimized CRC code into lib/crc/
  lib/crc: arm: Migrate optimized CRC code into lib/crc/
  lib/crc: Prepare for arch-optimized code in subdirs of lib/crc/
  lib/crc: Move files into lib/crc/
  lib/crc32: Remove unused combination support
  ...
This commit is contained in:
Linus Torvalds 2025-07-28 17:43:29 -07:00
parent 8e736a2eea 118da22eb6
commit a578dd095d
101 changed files with 784 additions and 1704 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Documentation/core-api/kernel-api.rst
View File

@ -136,26 +136,28 @@ Arithmetic Overflow Checking
CRC Functions
-------------
.. kernel-doc:: lib/crc4.c
.. kernel-doc:: lib/crc/crc4.c
:export:
.. kernel-doc:: lib/crc7.c
.. kernel-doc:: lib/crc/crc7.c
:export:
.. kernel-doc:: lib/crc8.c
.. kernel-doc:: lib/crc/crc8.c
:export:
.. kernel-doc:: lib/crc16.c
.. kernel-doc:: lib/crc/crc16.c
:export:
.. kernel-doc:: lib/crc32.c
.. kernel-doc:: lib/crc-ccitt.c
.. kernel-doc:: lib/crc/crc-ccitt.c
:export:
.. kernel-doc:: lib/crc-itu-t.c
.. kernel-doc:: lib/crc/crc-itu-t.c
:export:
.. kernel-doc:: include/linux/crc32.h
.. kernel-doc:: include/linux/crc64.h
Base 2 log and power Functions
------------------------------

4
MAINTAINERS
View File

@ -6362,10 +6362,8 @@ L: linux-crypto@vger.kernel.org
S: Maintained
T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-next
F: Documentation/staging/crc*
F: arch/*/lib/crc*
F: include/linux/crc*
F: lib/crc*
F: lib/tests/crc_kunit.c
F: lib/crc/
F: scripts/gen-crc-consts.py
CREATIVE SB0540

2
arch/arm/Kconfig
View File

@ -8,8 +8,6 @@ config ARM
select ARCH_HAS_CACHE_LINE_SIZE if OF
select ARCH_HAS_CPU_CACHE_ALIASING
select ARCH_HAS_CPU_FINALIZE_INIT if MMU
select ARCH_HAS_CRC32 if KERNEL_MODE_NEON
select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL if MMU
select ARCH_HAS_DMA_ALLOC if MMU

1
arch/arm/configs/multi_v7_defconfig
View File

@ -1298,7 +1298,6 @@ CONFIG_CRYPTO_DEV_MARVELL_CESA=m
CONFIG_CRYPTO_DEV_QCE=m
CONFIG_CRYPTO_DEV_QCOM_RNG=m
CONFIG_CRYPTO_DEV_ROCKCHIP=m
CONFIG_CRYPTO_DEV_STM32_CRC=m
CONFIG_CRYPTO_DEV_STM32_HASH=m
CONFIG_CRYPTO_DEV_STM32_CRYP=m
CONFIG_CMA_SIZE_MBYTES=64

6
arch/arm/lib/Makefile
View File

@ -47,9 +47,3 @@ ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
endif
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o
crc32-arm-y := crc32.o crc32-core.o
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm.o
crc-t10dif-arm-y := crc-t10dif.o crc-t10dif-core.o

2
arch/arm64/Kconfig
View File

@ -21,8 +21,6 @@ config ARM64
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
select ARCH_HAS_CACHE_LINE_SIZE
select ARCH_HAS_CC_PLATFORM
select ARCH_HAS_CRC32
select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE

6
arch/arm64/lib/Makefile
View File

@ -16,12 +16,6 @@ endif
lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
obj-$(CONFIG_CRC32_ARCH) += crc32-arm64.o
crc32-arm64-y := crc32.o crc32-core.o
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm64.o
crc-t10dif-arm64-y := crc-t10dif.o crc-t10dif-core.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
obj-$(CONFIG_ARM64_MTE) += mte.o

1
arch/loongarch/Kconfig
View File

@ -15,7 +15,6 @@ config LOONGARCH
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CRC32
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VM_PGTABLE
select ARCH_HAS_FAST_MULTIPLIER

2
arch/loongarch/lib/Makefile
View File

@ -11,5 +11,3 @@ obj-$(CONFIG_ARCH_SUPPORTS_INT128) += tishift.o
obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
obj-$(CONFIG_CRC32_ARCH) += crc32-loongarch.o

1
arch/mips/Kconfig
View File

@ -2024,7 +2024,6 @@ config CPU_MIPSR5
config CPU_MIPSR6
bool
default y if CPU_MIPS32_R6 || CPU_MIPS64_R6
select ARCH_HAS_CRC32
select CPU_HAS_RIXI
select CPU_HAS_DIEI if !CPU_DIEI_BROKEN
select HAVE_ARCH_BITREVERSE

2
arch/mips/lib/Makefile
View File

@ -16,7 +16,5 @@ lib-$(CONFIG_GENERIC_CSUM) := $(filter-out csum_partial.o, $(lib-y))
obj-$(CONFIG_CPU_GENERIC_DUMP_TLB) += dump_tlb.o
obj-$(CONFIG_CPU_R3000) += r3k_dump_tlb.o
obj-$(CONFIG_CRC32_ARCH) += crc32-mips.o
# libgcc-style stuff needed in the kernel
obj-y += bswapsi.o bswapdi.o multi3.o

2
arch/powerpc/Kconfig
View File

@ -127,8 +127,6 @@ config PPC
select ARCH_ENABLE_MEMORY_HOTPLUG
select ARCH_ENABLE_MEMORY_HOTREMOVE
select ARCH_HAS_COPY_MC if PPC64
select ARCH_HAS_CRC32 if PPC64 && ALTIVEC
select ARCH_HAS_CRC_T10DIF if PPC64 && ALTIVEC
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE

6
arch/powerpc/lib/Makefile
View File

@ -80,10 +80,4 @@ CFLAGS_xor_vmx.o += -mhard-float -maltivec $(call cc-option,-mabi=altivec)
# Enable <altivec.h>
CFLAGS_xor_vmx.o += -isystem $(shell $(CC) -print-file-name=include)
obj-$(CONFIG_CRC32_ARCH) += crc32-powerpc.o
crc32-powerpc-y := crc32.o crc32c-vpmsum_asm.o
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-powerpc.o
crc-t10dif-powerpc-y := crc-t10dif.o crct10dif-vpmsum_asm.o
obj-$(CONFIG_PPC64) += $(obj64-y)

3
arch/riscv/Kconfig
View File

@ -24,9 +24,6 @@ config RISCV
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
select ARCH_HAS_BINFMT_FLAT
select ARCH_HAS_CRC32 if RISCV_ISA_ZBC
select ARCH_HAS_CRC64 if 64BIT && RISCV_ISA_ZBC
select ARCH_HAS_CRC_T10DIF if RISCV_ISA_ZBC
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL if MMU
select ARCH_HAS_DEBUG_VM_PGTABLE

6
arch/riscv/lib/Makefile
View File

@ -16,12 +16,6 @@ endif
lib-$(CONFIG_MMU) += uaccess.o
lib-$(CONFIG_64BIT) += tishift.o
lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o
obj-$(CONFIG_CRC32_ARCH) += crc32-riscv.o
crc32-riscv-y := crc32.o crc32_msb.o crc32_lsb.o
obj-$(CONFIG_CRC64_ARCH) += crc64-riscv.o
crc64-riscv-y := crc64.o crc64_msb.o crc64_lsb.o
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-riscv.o
crc-t10dif-riscv-y := crc-t10dif.o crc16_msb.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
lib-$(CONFIG_RISCV_ISA_V) += xor.o
lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o

1
arch/s390/Kconfig
View File

@ -75,7 +75,6 @@ config S390
select ARCH_ENABLE_MEMORY_HOTREMOVE
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CRC32
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE

3
arch/s390/lib/Makefile
View File

@ -25,6 +25,3 @@ obj-$(CONFIG_S390_MODULES_SANITY_TEST_HELPERS) += test_modules_helpers.o
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o
obj-$(CONFIG_CRC32_ARCH) += crc32-s390.o
crc32-s390-y := crc32.o crc32le-vx.o crc32be-vx.o

1
arch/sparc/Kconfig
View File

@ -110,7 +110,6 @@ config SPARC64
select HAVE_SETUP_PER_CPU_AREA
select NEED_PER_CPU_EMBED_FIRST_CHUNK
select NEED_PER_CPU_PAGE_FIRST_CHUNK
select ARCH_HAS_CRC32
config ARCH_PROC_KCORE_TEXT
def_bool y

2
arch/sparc/lib/Makefile
View File

@ -54,5 +54,3 @@ lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
obj-$(CONFIG_SPARC64) += iomap.o
obj-$(CONFIG_SPARC32) += atomic32.o
obj-$(CONFIG_SPARC64) += PeeCeeI.o
obj-$(CONFIG_CRC32_ARCH) += crc32-sparc.o
crc32-sparc-y := crc32.o crc32c_asm.o

3
arch/x86/Kconfig
View File

@ -79,9 +79,6 @@ config X86
select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CPU_PASID if IOMMU_SVA
select ARCH_HAS_CRC32
select ARCH_HAS_CRC64 if X86_64
select ARCH_HAS_CRC_T10DIF
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE

10
arch/x86/lib/Makefile
View File

@ -40,16 +40,6 @@ lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
lib-$(CONFIG_MITIGATION_RETPOLINE) += retpoline.o
obj-$(CONFIG_CRC32_ARCH) += crc32-x86.o
crc32-x86-y := crc32.o crc32-pclmul.o
crc32-x86-$(CONFIG_64BIT) += crc32c-3way.o
obj-$(CONFIG_CRC64_ARCH) += crc64-x86.o
crc64-x86-y := crc64.o crc64-pclmul.o
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-x86.o
crc-t10dif-x86-y := crc-t10dif.o crc16-msb-pclmul.o
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
obj-y += iomem.o

111
arch/x86/lib/crc32.c
View File

@ -1,111 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* x86-optimized CRC32 functions
*
* Copyright (C) 2008 Intel Corporation
* Copyright 2012 Xyratex Technology Limited
* Copyright 2024 Google LLC
*/
#include <linux/crc32.h>
#include <linux/module.h>
#include "crc-pclmul-template.h"
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32);
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts,
have_pclmulqdq);
return crc32_le_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);
#ifdef CONFIG_X86_64
#define CRC32_INST "crc32q %1, %q0"
#else
#define CRC32_INST "crc32l %1, %0"
#endif
/*
* Use carryless multiply version of crc32c when buffer size is >= 512 to
* account for FPU state save/restore overhead.
*/
#define CRC32C_PCLMUL_BREAKEVEN 512
asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
size_t num_longs;
if (!static_branch_likely(&have_crc32))
return crc32c_base(crc, p, len);
if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
kernel_fpu_begin();
crc = crc32c_x86_3way(crc, p, len);
kernel_fpu_end();
return crc;
}
for (num_longs = len / sizeof(unsigned long);
num_longs != 0; num_longs--, p += sizeof(unsigned long))
asm(CRC32_INST : "+r" (crc) : ASM_INPUT_RM (*(unsigned long *)p));
if (sizeof(unsigned long) > 4 && (len & 4)) {
asm("crc32l %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u32 *)p));
p += 4;
}
if (len & 2) {
asm("crc32w %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u16 *)p));
p += 2;
}
if (len & 1)
asm("crc32b %1, %0" : "+r" (crc) : ASM_INPUT_RM (*p));
return crc;
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
return crc32_be_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
static int __init crc32_x86_init(void)
{
if (boot_cpu_has(X86_FEATURE_XMM4_2))
static_branch_enable(&have_crc32);
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
static_branch_enable(&have_pclmulqdq);
INIT_CRC_PCLMUL(crc32_lsb);
}
return 0;
}
subsys_initcall(crc32_x86_init);
static void __exit crc32_x86_exit(void)
{
}
module_exit(crc32_x86_exit);
u32 crc32_optimizations(void)
{
u32 optimizations = 0;
if (static_key_enabled(&have_crc32))
optimizations |= CRC32C_OPTIMIZATION;
if (static_key_enabled(&have_pclmulqdq))
optimizations |= CRC32_LE_OPTIMIZATION;
return optimizations;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_DESCRIPTION("x86-optimized CRC32 functions");
MODULE_LICENSE("GPL");

2
crypto/Makefile
View File

@ -154,10 +154,8 @@ obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o
obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
obj-$(CONFIG_CRYPTO_CRC32C) += crc32c-cryptoapi.o
crc32c-cryptoapi-y := crc32c.o
CFLAGS_crc32c.o += -DARCH=$(ARCH)
obj-$(CONFIG_CRYPTO_CRC32) += crc32-cryptoapi.o
crc32-cryptoapi-y := crc32.o
CFLAGS_crc32.o += -DARCH=$(ARCH)
obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
obj-$(CONFIG_CRYPTO_KRB5ENC) += krb5enc.o
obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o

65
crypto/crc32.c
View File

@ -59,29 +59,12 @@ static int crc32_update(struct shash_desc *desc, const u8 *data,
{
u32 *crcp = shash_desc_ctx(desc);
*crcp = crc32_le_base(*crcp, data, len);
return 0;
}
static int crc32_update_arch(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
u32 *crcp = shash_desc_ctx(desc);
*crcp = crc32_le(*crcp, data, len);
return 0;
}
/* No final XOR 0xFFFFFFFF, like crc32_le */
static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len,
u8 *out)
{
put_unaligned_le32(crc32_le_base(*crcp, data, len), out);
return 0;
}
static int __crc32_finup_arch(u32 *crcp, const u8 *data, unsigned int len,
u8 *out)
static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
{
put_unaligned_le32(crc32_le(*crcp, data, len), out);
return 0;
@ -93,12 +76,6 @@ static int crc32_finup(struct shash_desc *desc, const u8 *data,
return __crc32_finup(shash_desc_ctx(desc), data, len, out);
}
static int crc32_finup_arch(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
return __crc32_finup_arch(shash_desc_ctx(desc), data, len, out);
}
static int crc32_final(struct shash_desc *desc, u8 *out)
{
u32 *crcp = shash_desc_ctx(desc);
@ -113,13 +90,7 @@ static int crc32_digest(struct shash_desc *desc, const u8 *data,
return __crc32_finup(crypto_shash_ctx(desc->tfm), data, len, out);
}
static int crc32_digest_arch(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
return __crc32_finup_arch(crypto_shash_ctx(desc->tfm), data, len, out);
}
static struct shash_alg algs[] = {{
static struct shash_alg alg = {
.setkey = crc32_setkey,
.init = crc32_init,
.update = crc32_update,
@ -130,46 +101,23 @@ static struct shash_alg algs[] = {{
.digestsize = CHKSUM_DIGEST_SIZE,
.base.cra_name = "crc32",
.base.cra_driver_name = "crc32-generic",
.base.cra_driver_name = "crc32-lib",
.base.cra_priority = 100,
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.base.cra_blocksize = CHKSUM_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(u32),
.base.cra_module = THIS_MODULE,
.base.cra_init = crc32_cra_init,
}, {
.setkey = crc32_setkey,
.init = crc32_init,
.update = crc32_update_arch,
.final = crc32_final,
.finup = crc32_finup_arch,
.digest = crc32_digest_arch,
.descsize = sizeof(u32),
.digestsize = CHKSUM_DIGEST_SIZE,
.base.cra_name = "crc32",
.base.cra_driver_name = "crc32-" __stringify(ARCH),
.base.cra_priority = 150,
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.base.cra_blocksize = CHKSUM_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(u32),
.base.cra_module = THIS_MODULE,
.base.cra_init = crc32_cra_init,
}};
static int num_algs;
};
static int __init crc32_mod_init(void)
{
/* register the arch flavor only if it differs from the generic one */
num_algs = 1 + ((crc32_optimizations() & CRC32_LE_OPTIMIZATION) != 0);
return crypto_register_shashes(algs, num_algs);
return crypto_register_shash(&alg);
}
static void __exit crc32_mod_fini(void)
{
crypto_unregister_shashes(algs, num_algs);
crypto_unregister_shash(&alg);
}
module_init(crc32_mod_init);
@ -179,4 +127,3 @@ MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
MODULE_DESCRIPTION("CRC32 calculations wrapper for lib/crc32");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("crc32");
MODULE_ALIAS_CRYPTO("crc32-generic");

66
crypto/crc32c.c
View File

@ -85,15 +85,6 @@ static int chksum_update(struct shash_desc *desc, const u8 *data,
{
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
ctx->crc = crc32c_base(ctx->crc, data, length);
return 0;
}
static int chksum_update_arch(struct shash_desc *desc, const u8 *data,
unsigned int length)
{
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
ctx->crc = crc32c(ctx->crc, data, length);
return 0;
}
@ -107,13 +98,6 @@ static int chksum_final(struct shash_desc *desc, u8 *out)
}
static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out)
{
put_unaligned_le32(~crc32c_base(*crcp, data, len), out);
return 0;
}
static int __chksum_finup_arch(u32 *crcp, const u8 *data, unsigned int len,
u8 *out)
{
put_unaligned_le32(~crc32c(*crcp, data, len), out);
return 0;
@ -127,14 +111,6 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data,
return __chksum_finup(&ctx->crc, data, len, out);
}
static int chksum_finup_arch(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
return __chksum_finup_arch(&ctx->crc, data, len, out);
}
static int chksum_digest(struct shash_desc *desc, const u8 *data,
unsigned int length, u8 *out)
{
@ -143,14 +119,6 @@ static int chksum_digest(struct shash_desc *desc, const u8 *data,
return __chksum_finup(&mctx->key, data, length, out);
}
static int chksum_digest_arch(struct shash_desc *desc, const u8 *data,
unsigned int length, u8 *out)
{
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
return __chksum_finup_arch(&mctx->key, data, length, out);
}
static int crc32c_cra_init(struct crypto_tfm *tfm)
{
struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
@ -159,7 +127,7 @@ static int crc32c_cra_init(struct crypto_tfm *tfm)
return 0;
}
static struct shash_alg algs[] = {{
static struct shash_alg alg = {
.digestsize = CHKSUM_DIGEST_SIZE,
.setkey = chksum_setkey,
.init = chksum_init,
@ -170,46 +138,23 @@ static struct shash_alg algs[] = {{
.descsize = sizeof(struct chksum_desc_ctx),
.base.cra_name = "crc32c",
.base.cra_driver_name = "crc32c-generic",
.base.cra_driver_name = "crc32c-lib",
.base.cra_priority = 100,
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.base.cra_blocksize = CHKSUM_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct chksum_ctx),
.base.cra_module = THIS_MODULE,
.base.cra_init = crc32c_cra_init,
}, {
.digestsize = CHKSUM_DIGEST_SIZE,
.setkey = chksum_setkey,
.init = chksum_init,
.update = chksum_update_arch,
.final = chksum_final,
.finup = chksum_finup_arch,
.digest = chksum_digest_arch,
.descsize = sizeof(struct chksum_desc_ctx),
.base.cra_name = "crc32c",
.base.cra_driver_name = "crc32c-" __stringify(ARCH),
.base.cra_priority = 150,
.base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.base.cra_blocksize = CHKSUM_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct chksum_ctx),
.base.cra_module = THIS_MODULE,
.base.cra_init = crc32c_cra_init,
}};
static int num_algs;
};
static int __init crc32c_mod_init(void)
{
/* register the arch flavor only if it differs from the generic one */
num_algs = 1 + ((crc32_optimizations() & CRC32C_OPTIMIZATION) != 0);
return crypto_register_shashes(algs, num_algs);
return crypto_register_shash(&alg);
}
static void __exit crc32c_mod_fini(void)
{
crypto_unregister_shashes(algs, num_algs);
crypto_unregister_shash(&alg);
}
module_init(crc32c_mod_init);
@ -219,4 +164,3 @@ MODULE_AUTHOR("Clay Haapala <chaapala@cisco.com>");
MODULE_DESCRIPTION("CRC32c (Castagnoli) calculations wrapper for lib/crc32c");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("crc32c");
MODULE_ALIAS_CRYPTO("crc32c-generic");

57
crypto/testmgr.c
View File

@ -3550,59 +3550,6 @@ static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
return err;
}
static int alg_test_crc32c(const struct alg_test_desc *desc,
const char *driver, u32 type, u32 mask)
{
struct crypto_shash *tfm;
__le32 val;
int err;
err = alg_test_hash(desc, driver, type, mask);
if (err)
return err;
tfm = crypto_alloc_shash(driver, type, mask);
if (IS_ERR(tfm)) {
if (PTR_ERR(tfm) == -ENOENT) {
/*
* This crc32c implementation is only available through
* ahash API, not the shash API, so the remaining part
* of the test is not applicable to it.
*/
return 0;
}
printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
"%ld\n", driver, PTR_ERR(tfm));
return PTR_ERR(tfm);
}
driver = crypto_shash_driver_name(tfm);
do {
SHASH_DESC_ON_STACK(shash, tfm);
u32 *ctx = (u32 *)shash_desc_ctx(shash);
shash->tfm = tfm;
*ctx = 420553207;
err = crypto_shash_final(shash, (u8 *)&val);
if (err) {
printk(KERN_ERR "alg: crc32c: Operation failed for "
"%s: %d\n", driver, err);
break;
}
if (val != cpu_to_le32(~420553207)) {
pr_err("alg: crc32c: Test failed for %s: %u\n",
driver, le32_to_cpu(val));
err = -EINVAL;
}
} while (0);
crypto_free_shash(tfm);
return err;
}
static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
u32 type, u32 mask)
{
@ -4555,6 +4502,7 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "crc32",
.generic_driver = "crc32-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
@ -4562,7 +4510,8 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "crc32c",
.test = alg_test_crc32c,
.generic_driver = "crc32c-lib",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
.hash = __VECS(crc32c_tv_template)

1
drivers/crypto/inside-secure/safexcel.c
View File

@ -1218,7 +1218,6 @@ static struct safexcel_alg_template *safexcel_algs[] = {
&safexcel_alg_xts_aes,
&safexcel_alg_gcm,
&safexcel_alg_ccm,
&safexcel_alg_crc32,
&safexcel_alg_cbcmac,
&safexcel_alg_xcbcmac,
&safexcel_alg_cmac,

1
drivers/crypto/inside-secure/safexcel.h
View File

@ -959,7 +959,6 @@ extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes;
extern struct safexcel_alg_template safexcel_alg_xts_aes;
extern struct safexcel_alg_template safexcel_alg_gcm;
extern struct safexcel_alg_template safexcel_alg_ccm;
extern struct safexcel_alg_template safexcel_alg_crc32;
extern struct safexcel_alg_template safexcel_alg_cbcmac;
extern struct safexcel_alg_template safexcel_alg_xcbcmac;
extern struct safexcel_alg_template safexcel_alg_cmac;

92
drivers/crypto/inside-secure/safexcel_hash.c
View File

@ -289,14 +289,8 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
return 1;
}
if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
/* Undo final XOR with 0xffffffff ...*/
*(__le32 *)areq->result = ~sreq->state[0];
} else {
memcpy(areq->result, sreq->state,
crypto_ahash_digestsize(ahash));
}
memcpy(areq->result, sreq->state,
crypto_ahash_digestsize(ahash));
}
cache_len = safexcel_queued_len(sreq);
@ -1881,88 +1875,6 @@ struct safexcel_alg_template safexcel_alg_hmac_md5 = {
},
};
static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
{
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
int ret = safexcel_ahash_cra_init(tfm);
/* Default 'key' is all zeroes */
memset(&ctx->base.ipad, 0, sizeof(u32));
return ret;
}
static int safexcel_crc32_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
memset(req, 0, sizeof(*req));
/* Start from loaded key */
req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]);
/* Set processed to non-zero to enable invalidation detection */
req->len = sizeof(u32);
req->processed = sizeof(u32);
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
req->digest = CONTEXT_CONTROL_DIGEST_XCM;
req->state_sz = sizeof(u32);
req->digest_sz = sizeof(u32);
req->block_sz = sizeof(u32);
return 0;
}
static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
if (keylen != sizeof(u32))
return -EINVAL;
memcpy(&ctx->base.ipad, key, sizeof(u32));
return 0;
}
static int safexcel_crc32_digest(struct ahash_request *areq)
{
return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_crc32 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.algo_mask = 0,
.alg.ahash = {
.init = safexcel_crc32_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_crc32_digest,
.setkey = safexcel_crc32_setkey,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = sizeof(u32),
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "crc32",
.cra_driver_name = "safexcel-crc32",
.cra_priority = SAFEXCEL_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_crc32_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};
static int safexcel_cbcmac_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));

9
drivers/crypto/stm32/Kconfig
View File

@ -1,13 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
config CRYPTO_DEV_STM32_CRC
tristate "Support for STM32 crc accelerators"
depends on ARCH_STM32
select CRYPTO_HASH
select CRC32
help
This enables support for the CRC32 hw accelerator which can be found
on STMicroelectronics STM32 SOC.
config CRYPTO_DEV_STM32_HASH
tristate "Support for STM32 hash accelerators"
depends on ARCH_STM32 || ARCH_U8500

1
drivers/crypto/stm32/Makefile
View File

@ -1,4 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o
obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o
obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o

480
drivers/crypto/stm32/stm32-crc32.c
View File

@ -1,480 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) STMicroelectronics SA 2017
* Author: Fabien Dessenne <fabien.dessenne@st.com>
*/
#include <linux/bitrev.h>
#include <linux/clk.h>
#include <linux/crc32.h>
#include <linux/crc32poly.h>
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <crypto/internal/hash.h>
#include <linux/unaligned.h>
#define DRIVER_NAME "stm32-crc32"
#define CHKSUM_DIGEST_SIZE 4
#define CHKSUM_BLOCK_SIZE 1
/* Registers */
#define CRC_DR 0x00000000
#define CRC_CR 0x00000008
#define CRC_INIT 0x00000010
#define CRC_POL 0x00000014
/* Registers values */
#define CRC_CR_RESET BIT(0)
#define CRC_CR_REV_IN_WORD (BIT(6) | BIT(5))
#define CRC_CR_REV_IN_BYTE BIT(5)
#define CRC_CR_REV_OUT BIT(7)
#define CRC32C_INIT_DEFAULT 0xFFFFFFFF
#define CRC_AUTOSUSPEND_DELAY 50
static unsigned int burst_size;
module_param(burst_size, uint, 0644);
MODULE_PARM_DESC(burst_size, "Select burst byte size (0 unlimited)");
struct stm32_crc {
struct list_head list;
struct device *dev;
void __iomem *regs;
struct clk *clk;
spinlock_t lock;
};
struct stm32_crc_list {
struct list_head dev_list;
spinlock_t lock; /* protect dev_list */
};
static struct stm32_crc_list crc_list = {
.dev_list = LIST_HEAD_INIT(crc_list.dev_list),
.lock = __SPIN_LOCK_UNLOCKED(crc_list.lock),
};
struct stm32_crc_ctx {
u32 key;
u32 poly;
};
struct stm32_crc_desc_ctx {
u32 partial; /* crc32c: partial in first 4 bytes of that struct */
};
static int stm32_crc32_cra_init(struct crypto_tfm *tfm)
{
struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm);
mctx->key = 0;
mctx->poly = CRC32_POLY_LE;
return 0;
}
static int stm32_crc32c_cra_init(struct crypto_tfm *tfm)
{
struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm);
mctx->key = CRC32C_INIT_DEFAULT;
mctx->poly = CRC32C_POLY_LE;
return 0;
}
static int stm32_crc_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)
{
struct stm32_crc_ctx *mctx = crypto_shash_ctx(tfm);
if (keylen != sizeof(u32))
return -EINVAL;
mctx->key = get_unaligned_le32(key);
return 0;
}
static struct stm32_crc *stm32_crc_get_next_crc(void)
{
struct stm32_crc *crc;
spin_lock_bh(&crc_list.lock);
crc = list_first_entry_or_null(&crc_list.dev_list, struct stm32_crc, list);
if (crc)
list_move_tail(&crc->list, &crc_list.dev_list);
spin_unlock_bh(&crc_list.lock);
return crc;
}
static int stm32_crc_init(struct shash_desc *desc)
{
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
struct stm32_crc *crc;
unsigned long flags;
crc = stm32_crc_get_next_crc();
if (!crc)
return -ENODEV;
pm_runtime_get_sync(crc->dev);
spin_lock_irqsave(&crc->lock, flags);
/* Reset, set key, poly and configure in bit reverse mode */
writel_relaxed(bitrev32(mctx->key), crc->regs + CRC_INIT);
writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL);
writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
/* Store partial result */
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
spin_unlock_irqrestore(&crc->lock, flags);
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
return 0;
}
static int burst_update(struct shash_desc *desc, const u8 *d8,
size_t length)
{
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
struct stm32_crc *crc;
crc = stm32_crc_get_next_crc();
if (!crc)
return -ENODEV;
pm_runtime_get_sync(crc->dev);
if (!spin_trylock(&crc->lock)) {
/* Hardware is busy, calculate crc32 by software */
if (mctx->poly == CRC32_POLY_LE)
ctx->partial = crc32_le(ctx->partial, d8, length);
else
ctx->partial = crc32c(ctx->partial, d8, length);
goto pm_out;
}
/*
* Restore previously calculated CRC for this context as init value
* Restore polynomial configuration
* Configure in register for word input data,
* Configure out register in reversed bit mode data.
*/
writel_relaxed(bitrev32(ctx->partial), crc->regs + CRC_INIT);
writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL);
writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
if (d8 != PTR_ALIGN(d8, sizeof(u32))) {
/* Configure for byte data */
writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
while (d8 != PTR_ALIGN(d8, sizeof(u32)) && length) {
writeb_relaxed(*d8++, crc->regs + CRC_DR);
length--;
}
/* Configure for word data */
writel_relaxed(CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
}
for (; length >= sizeof(u32); d8 += sizeof(u32), length -= sizeof(u32))
writel_relaxed(*((u32 *)d8), crc->regs + CRC_DR);
if (length) {
/* Configure for byte data */
writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT,
crc->regs + CRC_CR);
while (length--)
writeb_relaxed(*d8++, crc->regs + CRC_DR);
}
/* Store partial result */
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
spin_unlock(&crc->lock);
pm_out:
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
return 0;
}
static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
unsigned int length)
{
const unsigned int burst_sz = burst_size;
unsigned int rem_sz;
const u8 *cur;
size_t size;
int ret;
if (!burst_sz)
return burst_update(desc, d8, length);
/* Digest first bytes not 32bit aligned at first pass in the loop */
size = min_t(size_t, length, burst_sz + (size_t)d8 -
ALIGN_DOWN((size_t)d8, sizeof(u32)));
for (rem_sz = length, cur = d8; rem_sz;
rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) {
ret = burst_update(desc, cur, size);
if (ret)
return ret;
}
return 0;
}
static int stm32_crc_final(struct shash_desc *desc, u8 *out)
{
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
/* Send computed CRC */
put_unaligned_le32(mctx->poly == CRC32C_POLY_LE ?
~ctx->partial : ctx->partial, out);
return 0;
}
static int stm32_crc_finup(struct shash_desc *desc, const u8 *data,
unsigned int length, u8 *out)
{
return stm32_crc_update(desc, data, length) ?:
stm32_crc_final(desc, out);
}
static int stm32_crc_digest(struct shash_desc *desc, const u8 *data,
unsigned int length, u8 *out)
{
return stm32_crc_init(desc) ?: stm32_crc_finup(desc, data, length, out);
}
static unsigned int refcnt;
static DEFINE_MUTEX(refcnt_lock);
static struct shash_alg algs[] = {
/* CRC-32 */
{
.setkey = stm32_crc_setkey,
.init = stm32_crc_init,
.update = stm32_crc_update,
.final = stm32_crc_final,
.finup = stm32_crc_finup,
.digest = stm32_crc_digest,
.descsize = sizeof(struct stm32_crc_desc_ctx),
.digestsize = CHKSUM_DIGEST_SIZE,
.base = {
.cra_name = "crc32",
.cra_driver_name = "stm32-crc32-crc32",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.cra_blocksize = CHKSUM_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_crc_ctx),
.cra_module = THIS_MODULE,
.cra_init = stm32_crc32_cra_init,
}
},
/* CRC-32Castagnoli */
{
.setkey = stm32_crc_setkey,
.init = stm32_crc_init,
.update = stm32_crc_update,
.final = stm32_crc_final,
.finup = stm32_crc_finup,
.digest = stm32_crc_digest,
.descsize = sizeof(struct stm32_crc_desc_ctx),
.digestsize = CHKSUM_DIGEST_SIZE,
.base = {
.cra_name = "crc32c",
.cra_driver_name = "stm32-crc32-crc32c",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
.cra_blocksize = CHKSUM_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_crc_ctx),
.cra_module = THIS_MODULE,
.cra_init = stm32_crc32c_cra_init,
}
}
};
static int stm32_crc_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct stm32_crc *crc;
int ret;
crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL);
if (!crc)
return -ENOMEM;
crc->dev = dev;
crc->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(crc->regs)) {
dev_err(dev, "Cannot map CRC IO\n");
return PTR_ERR(crc->regs);
}
crc->clk = devm_clk_get(dev, NULL);
if (IS_ERR(crc->clk)) {
dev_err(dev, "Could not get clock\n");
return PTR_ERR(crc->clk);
}
ret = clk_prepare_enable(crc->clk);
if (ret) {
dev_err(crc->dev, "Failed to enable clock\n");
return ret;
}
pm_runtime_set_autosuspend_delay(dev, CRC_AUTOSUSPEND_DELAY);
pm_runtime_use_autosuspend(dev);
pm_runtime_get_noresume(dev);
pm_runtime_set_active(dev);
pm_runtime_irq_safe(dev);
pm_runtime_enable(dev);
spin_lock_init(&crc->lock);
platform_set_drvdata(pdev, crc);
spin_lock(&crc_list.lock);
list_add(&crc->list, &crc_list.dev_list);
spin_unlock(&crc_list.lock);
mutex_lock(&refcnt_lock);
if (!refcnt) {
ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
if (ret) {
mutex_unlock(&refcnt_lock);
dev_err(dev, "Failed to register\n");
clk_disable_unprepare(crc->clk);
return ret;
}
}
refcnt++;
mutex_unlock(&refcnt_lock);
dev_info(dev, "Initialized\n");
pm_runtime_put_sync(dev);
return 0;
}
static void stm32_crc_remove(struct platform_device *pdev)
{
struct stm32_crc *crc = platform_get_drvdata(pdev);
int ret = pm_runtime_get_sync(crc->dev);
spin_lock(&crc_list.lock);
list_del(&crc->list);
spin_unlock(&crc_list.lock);
mutex_lock(&refcnt_lock);
if (!--refcnt)
crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
mutex_unlock(&refcnt_lock);
pm_runtime_disable(crc->dev);
pm_runtime_put_noidle(crc->dev);
if (ret >= 0)
clk_disable(crc->clk);
clk_unprepare(crc->clk);
}
static int __maybe_unused stm32_crc_suspend(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = pm_runtime_force_suspend(dev);
if (ret)
return ret;
clk_unprepare(crc->clk);
return 0;
}
static int __maybe_unused stm32_crc_resume(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = clk_prepare(crc->clk);
if (ret) {
dev_err(crc->dev, "Failed to prepare clock\n");
return ret;
}
return pm_runtime_force_resume(dev);
}
static int __maybe_unused stm32_crc_runtime_suspend(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
clk_disable(crc->clk);
return 0;
}
static int __maybe_unused stm32_crc_runtime_resume(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = clk_enable(crc->clk);
if (ret) {
dev_err(crc->dev, "Failed to enable clock\n");
return ret;
}
return 0;
}
static const struct dev_pm_ops stm32_crc_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(stm32_crc_suspend,
stm32_crc_resume)
SET_RUNTIME_PM_OPS(stm32_crc_runtime_suspend,
stm32_crc_runtime_resume, NULL)
};
static const struct of_device_id stm32_dt_ids[] = {
{ .compatible = "st,stm32f7-crc", },
{},
};
MODULE_DEVICE_TABLE(of, stm32_dt_ids);
static struct platform_driver stm32_crc_driver = {
.probe = stm32_crc_probe,
.remove = stm32_crc_remove,
.driver = {
.name = DRIVER_NAME,
.pm = &stm32_crc_pm_ops,
.of_match_table = stm32_dt_ids,
},
};
module_platform_driver(stm32_crc_driver);
MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");
MODULE_DESCRIPTION("STMicrolectronics STM32 CRC32 hardware driver");
MODULE_LICENSE("GPL");

2
drivers/nvmem/layouts/u-boot-env.c
View File

@ -148,7 +148,7 @@ int u_boot_env_parse(struct device *dev, struct nvmem_device *nvmem,
crc32_data_len = dev_size - crc32_data_offset;
data_len = dev_size - data_offset;
calc = crc32(~0, buf + crc32_data_offset, crc32_data_len) ^ ~0L;
calc = crc32_le(~0, buf + crc32_data_offset, crc32_data_len) ^ ~0L;
if (calc != crc32) {
dev_err(dev, "Invalid calculated CRC32: 0x%08x (expected: 0x%08x)\n", calc, crc32);
err = -EINVAL;

8
fs/btrfs/disk-io.c
View File

@ -2029,14 +2029,10 @@ static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
fs_info->csum_shash = csum_shash;
/*
* Check if the checksum implementation is a fast accelerated one.
* As-is this is a bit of a hack and should be replaced once the csum
* implementations provide that information themselves.
*/
/* Check if the checksum implementation is a fast accelerated one. */
switch (csum_type) {
case BTRFS_CSUM_TYPE_CRC32:
if (!strstr(crypto_shash_driver_name(csum_shash), "generic"))
if (crc32_optimizations() & CRC32C_OPTIMIZATION)
set_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags);
break;
case BTRFS_CSUM_TYPE_XXHASH:

10
include/linux/crc-t10dif.h
View File

@ -4,15 +4,7 @@
#include <linux/types.h>
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len);
u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len);
static inline u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len)
{
if (IS_ENABLED(CONFIG_CRC_T10DIF_ARCH))
return crc_t10dif_arch(crc, p, len);
return crc_t10dif_generic(crc, p, len);
}
u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len);
static inline u16 crc_t10dif(const u8 *p, size_t len)
{

119
include/linux/crc32.h
View File

@ -5,33 +5,81 @@
#include <linux/types.h>
#include <linux/bitrev.h>
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len);
u32 crc32_le_base(u32 crc, const u8 *p, size_t len);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len);
u32 crc32_be_base(u32 crc, const u8 *p, size_t len);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len);
u32 crc32c_base(u32 crc, const u8 *p, size_t len);
/**
* crc32_le() - Compute least-significant-bit-first IEEE CRC-32
* @crc: Initial CRC value. ~0 (recommended) or 0 for a new CRC computation, or
* the previous CRC value if computing incrementally.
* @p: Pointer to the data buffer
* @len: Length of data in bytes
*
* This implements the CRC variant that is often known as the IEEE CRC-32, or
* simply CRC-32, and is widely used in Ethernet and other applications:
*
* - Polynomial: x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 +
* x^7 + x^5 + x^4 + x^2 + x^1 + x^0
* - Bit order: Least-significant-bit-first
* - Polynomial in integer form: 0xedb88320
*
* This does *not* invert the CRC at the beginning or end. The caller is
* expected to do that if it needs to. Inverting at both ends is recommended.
*
* For new applications, prefer to use CRC-32C instead. See crc32c().
*
* Context: Any context
* Return: The new CRC value
*/
u32 crc32_le(u32 crc, const void *p, size_t len);
static inline u32 crc32_le(u32 crc, const void *p, size_t len)
/* This is just an alias for crc32_le(). */
static inline u32 crc32(u32 crc, const void *p, size_t len)
{
if (IS_ENABLED(CONFIG_CRC32_ARCH))
return crc32_le_arch(crc, p, len);
return crc32_le_base(crc, p, len);
return crc32_le(crc, p, len);
}
static inline u32 crc32_be(u32 crc, const void *p, size_t len)
{
if (IS_ENABLED(CONFIG_CRC32_ARCH))
return crc32_be_arch(crc, p, len);
return crc32_be_base(crc, p, len);
}
/**
* crc32_be() - Compute most-significant-bit-first IEEE CRC-32
* @crc: Initial CRC value. ~0 (recommended) or 0 for a new CRC computation, or
* the previous CRC value if computing incrementally.
* @p: Pointer to the data buffer
* @len: Length of data in bytes
*
* crc32_be() is the same as crc32_le() except that crc32_be() computes the
* *most-significant-bit-first* variant of the CRC. I.e., within each byte, the
* most significant bit is processed first (treated as highest order polynomial
* coefficient). The same bit order is also used for the CRC value itself:
*
* - Polynomial: x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 +
* x^7 + x^5 + x^4 + x^2 + x^1 + x^0
* - Bit order: Most-significant-bit-first
* - Polynomial in integer form: 0x04c11db7
*
* Context: Any context
* Return: The new CRC value
*/
u32 crc32_be(u32 crc, const void *p, size_t len);
static inline u32 crc32c(u32 crc, const void *p, size_t len)
{
if (IS_ENABLED(CONFIG_CRC32_ARCH))
return crc32c_arch(crc, p, len);
return crc32c_base(crc, p, len);
}
/**
* crc32c() - Compute CRC-32C
* @crc: Initial CRC value. ~0 (recommended) or 0 for a new CRC computation, or
* the previous CRC value if computing incrementally.
* @p: Pointer to the data buffer
* @len: Length of data in bytes
*
* This implements CRC-32C, i.e. the Castagnoli CRC. This is the recommended
* CRC variant to use in new applications that want a 32-bit CRC.
*
* - Polynomial: x^32 + x^28 + x^27 + x^26 + x^25 + x^23 + x^22 + x^20 + x^19 +
* x^18 + x^14 + x^13 + x^11 + x^10 + x^9 + x^8 + x^6 + x^0
* - Bit order: Least-significant-bit-first
* - Polynomial in integer form: 0x82f63b78
*
* This does *not* invert the CRC at the beginning or end. The caller is
* expected to do that if it needs to. Inverting at both ends is recommended.
*
* Context: Any context
* Return: The new CRC value
*/
u32 crc32c(u32 crc, const void *p, size_t len);
/*
* crc32_optimizations() returns flags that indicate which CRC32 library
@ -48,33 +96,6 @@ u32 crc32_optimizations(void);
static inline u32 crc32_optimizations(void) { return 0; }
#endif
/**
* crc32_le_combine - Combine two crc32 check values into one. For two
* sequences of bytes, seq1 and seq2 with lengths len1
* and len2, crc32_le() check values were calculated
* for each, crc1 and crc2.
*
* @crc1: crc32 of the first block
* @crc2: crc32 of the second block
* @len2: length of the second block
*
* Return: The crc32_le() check value of seq1 and seq2 concatenated,
* requiring only crc1, crc2, and len2. Note: If seq_full denotes
* the concatenated memory area of seq1 with seq2, and crc_full
* the crc32_le() value of seq_full, then crc_full ==
* crc32_le_combine(crc1, crc2, len2) when crc_full was seeded
* with the same initializer as crc1, and crc2 seed was 0. See
* also crc32_combine_test().
*/
u32 crc32_le_shift(u32 crc, size_t len);
static inline u32 crc32_le_combine(u32 crc1, u32 crc2, size_t len2)
{
return crc32_le_shift(crc1, len2) ^ crc2;
}
#define crc32(seed, data, length) crc32_le(seed, (unsigned char const *)(data), length)
/*
* Helpers for hash table generation of ethernet nics:
*

16
include/linux/crc32poly.h
View File

@ -2,19 +2,13 @@
#ifndef _LINUX_CRC32_POLY_H
#define _LINUX_CRC32_POLY_H
/*
* There are multiple 16-bit CRC polynomials in common use, but this is
* *the* standard CRC-32 polynomial, first popularized by Ethernet.
* x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x^1+x^0
*/
/* The polynomial used by crc32_le(), in integer form. See crc32_le(). */
#define CRC32_POLY_LE 0xedb88320
/* The polynomial used by crc32_be(), in integer form. See crc32_be(). */
#define CRC32_POLY_BE 0x04c11db7
/*
* This is the CRC32c polynomial, as outlined by Castagnoli.
* x^32+x^28+x^27+x^26+x^25+x^23+x^22+x^20+x^19+x^18+x^14+x^13+x^11+x^10+x^9+
* x^8+x^6+x^0
*/
#define CRC32C_POLY_LE 0x82F63B78
/* The polynomial used by crc32c(), in integer form. See crc32c(). */
#define CRC32C_POLY_LE 0x82f63b78
#endif /* _LINUX_CRC32_POLY_H */

22
include/linux/crc64.h
View File

@ -1,17 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* See lib/crc64.c for the related specification and polynomial arithmetic.
*/
#ifndef _LINUX_CRC64_H
#define _LINUX_CRC64_H
#include <linux/types.h>
u64 crc64_be_arch(u64 crc, const u8 *p, size_t len);
u64 crc64_be_generic(u64 crc, const u8 *p, size_t len);
u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len);
u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len);
/**
* crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
* @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
@ -19,12 +11,7 @@ u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len);
* @p: pointer to buffer over which CRC64 is run
* @len: length of buffer @p
*/
static inline u64 crc64_be(u64 crc, const void *p, size_t len)
{
if (IS_ENABLED(CONFIG_CRC64_ARCH))
return crc64_be_arch(crc, p, len);
return crc64_be_generic(crc, p, len);
}
u64 crc64_be(u64 crc, const void *p, size_t len);
/**
* crc64_nvme - Calculate CRC64-NVME
@ -36,11 +23,6 @@ static inline u64 crc64_be(u64 crc, const void *p, size_t len)
* This computes the CRC64 defined in the NVME NVM Command Set Specification,
* *including the bitwise inversion at the beginning and end*.
*/
static inline u64 crc64_nvme(u64 crc, const void *p, size_t len)
{
if (IS_ENABLED(CONFIG_CRC64_ARCH))
return ~crc64_nvme_arch(~crc, p, len);
return ~crc64_nvme_generic(~crc, p, len);
}
u64 crc64_nvme(u64 crc, const void *p, size_t len);
#endif /* _LINUX_CRC64_H */

88
lib/Kconfig
View File

@ -136,95 +136,9 @@ config TRACE_MMIO_ACCESS
Create tracepoints for MMIO read/write operations. These trace events
can be used for logging all MMIO read/write operations.
source "lib/crc/Kconfig"
source "lib/crypto/Kconfig"
config CRC_CCITT
tristate
help
The CRC-CCITT library functions. Select this if your module uses any
of the functions from <linux/crc-ccitt.h>.
config CRC16
tristate
help
The CRC16 library functions. Select this if your module uses any of
the functions from <linux/crc16.h>.
config CRC_T10DIF
tristate
help
The CRC-T10DIF library functions. Select this if your module uses
any of the functions from <linux/crc-t10dif.h>.
config ARCH_HAS_CRC_T10DIF
bool
config CRC_T10DIF_ARCH
tristate
default CRC_T10DIF if ARCH_HAS_CRC_T10DIF && CRC_OPTIMIZATIONS
config CRC_ITU_T
tristate
help
The CRC-ITU-T library functions. Select this if your module uses
any of the functions from <linux/crc-itu-t.h>.
config CRC32
tristate
select BITREVERSE
help
The CRC32 library functions. Select this if your module uses any of
the functions from <linux/crc32.h> or <linux/crc32c.h>.
config ARCH_HAS_CRC32
bool
config CRC32_ARCH
tristate
default CRC32 if ARCH_HAS_CRC32 && CRC_OPTIMIZATIONS
config CRC64
tristate
help
The CRC64 library functions. Select this if your module uses any of
the functions from <linux/crc64.h>.
config ARCH_HAS_CRC64
bool
config CRC64_ARCH
tristate
default CRC64 if ARCH_HAS_CRC64 && CRC_OPTIMIZATIONS
config CRC4
tristate
help
The CRC4 library functions. Select this if your module uses any of
the functions from <linux/crc4.h>.
config CRC7
tristate
help
The CRC7 library functions. Select this if your module uses any of
the functions from <linux/crc7.h>.
config CRC8
tristate
help
The CRC8 library functions. Select this if your module uses any of
the functions from <linux/crc8.h>.
config CRC_OPTIMIZATIONS
bool "Enable optimized CRC implementations" if EXPERT
default y
help
Disabling this option reduces code size slightly by disabling the
architecture-optimized implementations of any CRC variants that are
enabled. CRC checksumming performance may get much slower.
Keep this enabled unless you're really trying to minimize the size of
the kernel.
config XXHASH
tristate

21
lib/Kconfig.debug
View File

@ -2910,27 +2910,6 @@ config HW_BREAKPOINT_KUNIT_TEST
If unsure, say N.
config CRC_KUNIT_TEST
tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS
depends on KUNIT
default KUNIT_ALL_TESTS
select CRC7
select CRC16
select CRC_T10DIF
select CRC32
select CRC64
help
Unit tests for the CRC library functions.
This is intended to help people writing architecture-specific
optimized versions. If unsure, say N.
config CRC_BENCHMARK
bool "Benchmark for the CRC functions"
depends on CRC_KUNIT_TEST
help
Include benchmarks in the KUnit test suite for the CRC functions.
config SIPHASH_KUNIT_TEST
tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS
depends on KUNIT

32
lib/Makefile
View File

@ -122,7 +122,7 @@ endif
obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
obj-y += math/ crypto/ tests/ vdso/
obj-y += math/ crc/ crypto/ tests/ vdso/
obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
@ -148,15 +148,6 @@ obj-$(CONFIG_BITREVERSE) += bitrev.o
obj-$(CONFIG_LINEAR_RANGES) += linear_ranges.o
obj-$(CONFIG_PACKING) += packing.o
obj-$(CONFIG_PACKING_KUNIT_TEST) += packing_test.o
obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
obj-$(CONFIG_CRC16) += crc16.o
obj-$(CONFIG_CRC_T10DIF)+= crc-t10dif.o
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
obj-$(CONFIG_CRC32) += crc32.o
obj-$(CONFIG_CRC64) += crc64.o
obj-$(CONFIG_CRC4) += crc4.o
obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_CRC8) += crc8.o
obj-$(CONFIG_XXHASH) += xxhash.o
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
@ -294,27 +285,6 @@ obj-$(CONFIG_ASN1_ENCODER) += asn1_encoder.o
obj-$(CONFIG_FONT_SUPPORT) += fonts/
hostprogs := gen_crc32table
hostprogs += gen_crc64table
clean-files := crc32table.h
clean-files += crc64table.h
$(obj)/crc32.o: $(obj)/crc32table.h
quiet_cmd_crc32 = GEN $@
cmd_crc32 = $< > $@
$(obj)/crc32table.h: $(obj)/gen_crc32table
$(call cmd,crc32)
$(obj)/crc64.o: $(obj)/crc64table.h
quiet_cmd_crc64 = GEN $@
cmd_crc64 = $< > $@
$(obj)/crc64table.h: $(obj)/gen_crc64table
$(call cmd,crc64)
#
# Build a fast OID lookip registry from include/linux/oid_registry.h
#

5
lib/crc/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
# SPDX-License-Identifier: GPL-2.0-only
/crc32table.h
/crc64table.h
/gen_crc32table
/gen_crc64table

119
lib/crc/Kconfig Normal file
View File

@ -0,0 +1,119 @@
# SPDX-License-Identifier: GPL-2.0-only
# Kconfig for the kernel's cyclic redundancy check (CRC) library code
config CRC4
tristate
help
The CRC4 library functions. Select this if your module uses any of
the functions from <linux/crc4.h>.
config CRC7
tristate
help
The CRC7 library functions. Select this if your module uses any of
the functions from <linux/crc7.h>.
config CRC8
tristate
help
The CRC8 library functions. Select this if your module uses any of
the functions from <linux/crc8.h>.
config CRC16
tristate
help
The CRC16 library functions. Select this if your module uses any of
the functions from <linux/crc16.h>.
config CRC_CCITT
tristate
help
The CRC-CCITT library functions. Select this if your module uses any
of the functions from <linux/crc-ccitt.h>.
config CRC_ITU_T
tristate
help
The CRC-ITU-T library functions. Select this if your module uses
any of the functions from <linux/crc-itu-t.h>.
config CRC_T10DIF
tristate
help
The CRC-T10DIF library functions. Select this if your module uses
any of the functions from <linux/crc-t10dif.h>.
config CRC_T10DIF_ARCH
bool
depends on CRC_T10DIF && CRC_OPTIMIZATIONS
default y if ARM && KERNEL_MODE_NEON
default y if ARM64 && KERNEL_MODE_NEON
default y if PPC64 && ALTIVEC
default y if RISCV && RISCV_ISA_ZBC
default y if X86
config CRC32
tristate
select BITREVERSE
help
The CRC32 library functions. Select this if your module uses any of
the functions from <linux/crc32.h> or <linux/crc32c.h>.
config CRC32_ARCH
bool
depends on CRC32 && CRC_OPTIMIZATIONS
default y if ARM && KERNEL_MODE_NEON
default y if ARM64
default y if LOONGARCH
default y if MIPS && CPU_MIPSR6
default y if PPC64 && ALTIVEC
default y if RISCV && RISCV_ISA_ZBC
default y if S390
default y if SPARC64
default y if X86
config CRC64
tristate
help
The CRC64 library functions. Select this if your module uses any of
the functions from <linux/crc64.h>.
config CRC64_ARCH
bool
depends on CRC64 && CRC_OPTIMIZATIONS
default y if RISCV && RISCV_ISA_ZBC && 64BIT
default y if X86_64
config CRC_OPTIMIZATIONS
bool "Enable optimized CRC implementations" if EXPERT
depends on !UML
default y
help
Disabling this option reduces code size slightly by disabling the
architecture-optimized implementations of any CRC variants that are
enabled. CRC checksumming performance may get much slower.
Keep this enabled unless you're really trying to minimize the size of
the kernel.
config CRC_KUNIT_TEST
tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS
depends on KUNIT
default KUNIT_ALL_TESTS
select CRC7
select CRC16
select CRC_T10DIF
select CRC32
select CRC64
help
Unit tests for the CRC library functions.
This is intended to help people writing architecture-specific
optimized versions. If unsure, say N.
config CRC_BENCHMARK
bool "Benchmark for the CRC functions"
depends on CRC_KUNIT_TEST
help
Include benchmarks in the KUnit test suite for the CRC functions.

63
lib/crc/Makefile Normal file
View File

@ -0,0 +1,63 @@
# SPDX-License-Identifier: GPL-2.0-only
# Makefile for the kernel's cyclic redundancy check (CRC) library code
obj-$(CONFIG_CRC4) += crc4.o
obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_CRC8) += crc8.o
obj-$(CONFIG_CRC16) += crc16.o
obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
obj-$(CONFIG_CRC_T10DIF) += crc-t10dif.o
crc-t10dif-y := crc-t10dif-main.o
ifeq ($(CONFIG_CRC_T10DIF_ARCH),y)
CFLAGS_crc-t10dif-main.o += -I$(src)/$(SRCARCH)
crc-t10dif-$(CONFIG_ARM) += arm/crc-t10dif-core.o
crc-t10dif-$(CONFIG_ARM64) += arm64/crc-t10dif-core.o
crc-t10dif-$(CONFIG_PPC) += powerpc/crct10dif-vpmsum_asm.o
crc-t10dif-$(CONFIG_RISCV) += riscv/crc16_msb.o
crc-t10dif-$(CONFIG_X86) += x86/crc16-msb-pclmul.o
endif
obj-$(CONFIG_CRC32) += crc32.o
crc32-y := crc32-main.o
ifeq ($(CONFIG_CRC32_ARCH),y)
CFLAGS_crc32-main.o += -I$(src)/$(SRCARCH)
crc32-$(CONFIG_ARM) += arm/crc32-core.o
crc32-$(CONFIG_ARM64) += arm64/crc32-core.o
crc32-$(CONFIG_PPC) += powerpc/crc32c-vpmsum_asm.o
crc32-$(CONFIG_RISCV) += riscv/crc32_lsb.o riscv/crc32_msb.o
crc32-$(CONFIG_S390) += s390/crc32le-vx.o s390/crc32be-vx.o
crc32-$(CONFIG_SPARC) += sparc/crc32c_asm.o
crc32-$(CONFIG_X86) += x86/crc32-pclmul.o
crc32-$(CONFIG_X86_64) += x86/crc32c-3way.o
endif
obj-$(CONFIG_CRC64) += crc64.o
crc64-y := crc64-main.o
ifeq ($(CONFIG_CRC64_ARCH),y)
CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH)
crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o
crc64-$(CONFIG_X86) += x86/crc64-pclmul.o
endif
obj-y += tests/
hostprogs := gen_crc32table gen_crc64table
clean-files := crc32table.h crc64table.h
$(obj)/crc32-main.o: $(obj)/crc32table.h
$(obj)/crc64-main.o: $(obj)/crc64table.h
quiet_cmd_crc32 = GEN $@
cmd_crc32 = $< > $@
quiet_cmd_crc64 = GEN $@
cmd_crc64 = $< > $@
$(obj)/crc32table.h: $(obj)/gen_crc32table
$(call cmd,crc32)
$(obj)/crc64table.h: $(obj)/gen_crc64table
$(call cmd,crc64)

0
arch/arm/lib/crc-t10dif-core.S → lib/crc/arm/crc-t10dif-core.S
View File

23
arch/arm/lib/crc-t10dif.c → lib/crc/arm/crc-t10dif.h
View File

@ -5,12 +5,6 @@
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/crc-t10dif.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <crypto/internal/simd.h>
#include <asm/neon.h>
@ -25,7 +19,7 @@ asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
u8 out[16]);
u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
{
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
if (static_branch_likely(&have_pmull)) {
@ -49,24 +43,13 @@ u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
}
return crc_t10dif_generic(crc, data, length);
}
EXPORT_SYMBOL(crc_t10dif_arch);
static int __init crc_t10dif_arm_init(void)
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
static inline void crc_t10dif_mod_init_arch(void)
{
if (elf_hwcap & HWCAP_NEON) {
static_branch_enable(&have_neon);
if (elf_hwcap2 & HWCAP2_PMULL)
static_branch_enable(&have_pmull);
}
return 0;
}
subsys_initcall(crc_t10dif_arm_init);
static void __exit crc_t10dif_arm_exit(void)
{
}
module_exit(crc_t10dif_arm_exit);
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions");
MODULE_LICENSE("GPL v2");

0
arch/arm/lib/crc32-core.S → lib/crc/arm/crc32-core.S
View File

38
arch/arm/lib/crc32.c → lib/crc/arm/crc32.h
View File

@ -6,11 +6,6 @@
*/
#include <linux/cpufeature.h>
#include <linux/crc32.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <crypto/internal/simd.h>
@ -29,14 +24,14 @@ asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
{
if (static_branch_likely(&have_crc32))
return crc32_armv8_le(crc, p, len);
return crc32_le_base(crc, p, len);
}
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
if (len >= PMULL_MIN_LEN + 15 &&
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
@ -57,16 +52,15 @@ u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
}
return crc32_le_scalar(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);
static u32 crc32c_scalar(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_scalar(u32 crc, const u8 *p, size_t len)
{
if (static_branch_likely(&have_crc32))
return crc32c_armv8_le(crc, p, len);
return crc32c_base(crc, p, len);
}
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
if (len >= PMULL_MIN_LEN + 15 &&
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
@ -87,37 +81,21 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
}
return crc32c_scalar(crc, p, len);
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
return crc32_be_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
static int __init crc32_arm_init(void)
#define crc32_mod_init_arch crc32_mod_init_arch
static inline void crc32_mod_init_arch(void)
{
if (elf_hwcap2 & HWCAP2_CRC32)
static_branch_enable(&have_crc32);
if (elf_hwcap2 & HWCAP2_PMULL)
static_branch_enable(&have_pmull);
return 0;
}
subsys_initcall(crc32_arm_init);
static void __exit crc32_arm_exit(void)
{
}
module_exit(crc32_arm_exit);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL))
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions");
MODULE_LICENSE("GPL v2");

0
arch/arm64/lib/crc-t10dif-core.S → lib/crc/arm64/crc-t10dif-core.S
View File

22
arch/arm64/lib/crc-t10dif.c → lib/crc/arm64/crc-t10dif.h
View File

@ -6,11 +6,6 @@
*/
#include <linux/cpufeature.h>
#include <linux/crc-t10dif.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <crypto/internal/simd.h>
@ -26,7 +21,7 @@ asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,
u8 out[16]);
asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
{
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
if (static_branch_likely(&have_pmull)) {
@ -50,24 +45,13 @@ u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
}
return crc_t10dif_generic(crc, data, length);
}
EXPORT_SYMBOL(crc_t10dif_arch);
static int __init crc_t10dif_arm64_init(void)
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
static inline void crc_t10dif_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_asimd);
if (cpu_have_named_feature(PMULL))
static_branch_enable(&have_pmull);
}
return 0;
}
subsys_initcall(crc_t10dif_arm64_init);
static void __exit crc_t10dif_arm64_exit(void)
{
}
module_exit(crc_t10dif_arm64_exit);
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions");
MODULE_LICENSE("GPL v2");

0
arch/arm64/lib/crc32-core.S → lib/crc/arm64/crc32-core.S
View File

19
arch/arm64/lib/crc32.c → lib/crc/arm64/crc32.h
View File

@ -1,9 +1,5 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/crc32.h>
#include <linux/linkage.h>
#include <linux/module.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/neon.h>
@ -22,7 +18,7 @@ asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
return crc32_le_base(crc, p, len);
@ -41,9 +37,8 @@ u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
return crc32_le_arm64(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
return crc32c_base(crc, p, len);
@ -62,9 +57,8 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
return crc32c_le_arm64(crc, p, len);
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
return crc32_be_base(crc, p, len);
@ -83,9 +77,8 @@ u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
return crc32_be_arm64(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (alternative_has_cap_likely(ARM64_HAS_CRC32))
return CRC32_LE_OPTIMIZATION |
@ -93,7 +86,3 @@ u32 crc32_optimizations(void)
CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("arm64-optimized CRC32 functions");

8
lib/crc-ccitt.c → lib/crc/crc-ccitt.c
View File

@ -1,11 +1,9 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/lib/crc-ccitt.c
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-ccitt.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
/*
* This mysterious table is just the CRC of each possible byte. It can be

5
lib/crc-itu-t.c → lib/crc/crc-itu-t.c
View File

@ -3,9 +3,10 @@
* crc-itu-t.c
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-itu-t.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
/* CRC table for the CRC ITU-T V.41 0x1021 (x^16 + x^12 + x^5 + 1) */
const u16 crc_itu_t_table[256] = {

44
lib/crc-t10dif.c → lib/crc/crc-t10dif-main.c
View File

@ -6,9 +6,10 @@
* Written by Martin K. Petersen <martin.petersen@oracle.com>
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-t10dif.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
/*
* Table generated using the following polynomial:
@ -50,16 +51,39 @@ static const u16 t10_dif_crc_table[256] = {
0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
};
u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len)
static inline u16 __maybe_unused
crc_t10dif_generic(u16 crc, const u8 *p, size_t len)
{
size_t i;
for (i = 0; i < len; i++)
crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ p[i]];
while (len--)
crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ *p++];
return crc;
}
EXPORT_SYMBOL(crc_t10dif_generic);
MODULE_DESCRIPTION("T10 DIF CRC calculation");
#ifdef CONFIG_CRC_T10DIF_ARCH
#include "crc-t10dif.h" /* $(SRCARCH)/crc-t10dif.h */
#else
#define crc_t10dif_arch crc_t10dif_generic
#endif
u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len)
{
return crc_t10dif_arch(crc, p, len);
}
EXPORT_SYMBOL(crc_t10dif_update);
#ifdef crc_t10dif_mod_init_arch
static int __init crc_t10dif_mod_init(void)
{
crc_t10dif_mod_init_arch();
return 0;
}
subsys_initcall(crc_t10dif_mod_init);
static void __exit crc_t10dif_mod_exit(void)
{
}
module_exit(crc_t10dif_mod_exit);
#endif
MODULE_DESCRIPTION("CRC-T10DIF library functions");
MODULE_LICENSE("GPL");

5
lib/crc16.c → lib/crc/crc16.c
View File

@ -3,9 +3,10 @@
* crc16.c
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc16.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
/** CRC table for the CRC-16. The poly is 0x8005 (x^16 + x^15 + x^2 + 1) */
static const u16 crc16_table[256] = {

105
lib/crc/crc32-main.c Normal file
View File

@ -0,0 +1,105 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin
* cleaned up code to current version of sparse and added the slicing-by-8
* algorithm to the closely similar existing slicing-by-4 algorithm.
*
* Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks!
* Code was from the public domain, copyright abandoned. Code was
* subsequently included in the kernel, thus was re-licensed under the
* GNU GPL v2.
*
* Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Same crc32 function was used in 5 other places in the kernel.
* I made one version, and deleted the others.
* There are various incantations of crc32(). Some use a seed of 0 or ~0.
* Some xor at the end with ~0. The generic crc32() function takes
* seed as an argument, and doesn't xor at the end. Then individual
* users can do whatever they need.
* drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0.
* fs/jffs2 uses seed 0, doesn't xor with ~0.
* fs/partitions/efi.c uses seed ~0, xor's with ~0.
*/
/* see: Documentation/staging/crc32.rst for a description of algorithms */
#include <linux/crc32.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
#include "crc32table.h"
static inline u32 __maybe_unused
crc32_le_base(u32 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc >> 8) ^ crc32table_le[(crc & 255) ^ *p++];
return crc;
}
static inline u32 __maybe_unused
crc32_be_base(u32 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc << 8) ^ crc32table_be[(crc >> 24) ^ *p++];
return crc;
}
static inline u32 __maybe_unused
crc32c_base(u32 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc >> 8) ^ crc32ctable_le[(crc & 255) ^ *p++];
return crc;
}
#ifdef CONFIG_CRC32_ARCH
#include "crc32.h" /* $(SRCARCH)/crc32.h */
u32 crc32_optimizations(void)
{
return crc32_optimizations_arch();
}
EXPORT_SYMBOL(crc32_optimizations);
#else
#define crc32_le_arch crc32_le_base
#define crc32_be_arch crc32_be_base
#define crc32c_arch crc32c_base
#endif
u32 crc32_le(u32 crc, const void *p, size_t len)
{
return crc32_le_arch(crc, p, len);
}
EXPORT_SYMBOL(crc32_le);
u32 crc32_be(u32 crc, const void *p, size_t len)
{
return crc32_be_arch(crc, p, len);
}
EXPORT_SYMBOL(crc32_be);
u32 crc32c(u32 crc, const void *p, size_t len)
{
return crc32c_arch(crc, p, len);
}
EXPORT_SYMBOL(crc32c);
#ifdef crc32_mod_init_arch
static int __init crc32_mod_init(void)
{
crc32_mod_init_arch();
return 0;
}
subsys_initcall(crc32_mod_init);
static void __exit crc32_mod_exit(void)
{
}
module_exit(crc32_mod_exit);
#endif
MODULE_DESCRIPTION("CRC32 library functions");
MODULE_LICENSE("GPL");

1
lib/crc4.c → lib/crc/crc4.c
View File

@ -4,6 +4,7 @@
*/
#include <linux/crc4.h>
#include <linux/export.h>
#include <linux/module.h>
static const uint8_t crc4_tab[] = {

51
lib/crc64.c → lib/crc/crc64-main.c
View File

@ -33,26 +33,61 @@
* Author: Coly Li <colyli@suse.de>
*/
#include <linux/crc64.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/crc64.h>
#include "crc64table.h"
MODULE_DESCRIPTION("CRC64 calculations");
MODULE_LICENSE("GPL v2");
u64 crc64_be_generic(u64 crc, const u8 *p, size_t len)
static inline u64 __maybe_unused
crc64_be_generic(u64 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++];
return crc;
}
EXPORT_SYMBOL_GPL(crc64_be_generic);
u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
static inline u64 __maybe_unused
crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++];
return crc;
}
EXPORT_SYMBOL_GPL(crc64_nvme_generic);
#ifdef CONFIG_CRC64_ARCH
#include "crc64.h" /* $(SRCARCH)/crc64.h */
#else
#define crc64_be_arch crc64_be_generic
#define crc64_nvme_arch crc64_nvme_generic
#endif
u64 crc64_be(u64 crc, const void *p, size_t len)
{
return crc64_be_arch(crc, p, len);
}
EXPORT_SYMBOL_GPL(crc64_be);
u64 crc64_nvme(u64 crc, const void *p, size_t len)
{
return ~crc64_nvme_arch(~crc, p, len);
}
EXPORT_SYMBOL_GPL(crc64_nvme);
#ifdef crc64_mod_init_arch
static int __init crc64_mod_init(void)
{
crc64_mod_init_arch();
return 0;
}
subsys_initcall(crc64_mod_init);
static void __exit crc64_mod_exit(void)
{
}
module_exit(crc64_mod_exit);
#endif
MODULE_DESCRIPTION("CRC64 library functions");
MODULE_LICENSE("GPL");

5
lib/crc7.c → lib/crc/crc7.c
View File

@ -3,9 +3,10 @@
* crc7.c
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc7.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>
/*
* Table for CRC-7 (polynomial x^7 + x^3 + 1).

3
lib/crc8.c → lib/crc/crc8.c
View File

@ -16,8 +16,9 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/crc8.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/printk.h>
/**

4
lib/gen_crc32table.c → lib/crc/gen_crc32table.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <stdio.h>
#include "../include/linux/crc32poly.h"
#include "../include/generated/autoconf.h"
#include "../../include/linux/crc32poly.h"
#include "../../include/generated/autoconf.h"
#include <inttypes.h>
static uint32_t crc32table_le[256];

11
lib/gen_crc64table.c → lib/crc/gen_crc64table.c
View File

@ -1,14 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Generate lookup table for the table-driven CRC64 calculation.
*
* gen_crc64table is executed in kernel build time and generates
* lib/crc64table.h. This header is included by lib/crc64.c for
* the table-driven CRC64 calculation.
*
* See lib/crc64.c for more information about which specification
* and polynomial arithmetic that gen_crc64table.c follows to
* generate the lookup table.
* This host program runs at kernel build time and generates the lookup tables
* used by the generic CRC64 code.
*
* Copyright 2018 SUSE Linux.
* Author: Coly Li <colyli@suse.de>

33
arch/loongarch/lib/crc32-loongarch.c → lib/crc/loongarch/crc32.h
View File

@ -10,9 +10,6 @@
*/
#include <asm/cpu-features.h>
#include <linux/crc32.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/unaligned.h>
#define _CRC32(crc, value, size, type) \
@ -29,7 +26,7 @@ do { \
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
if (!static_branch_likely(&have_crc32))
return crc32_le_base(crc, p, len);
@ -64,9 +61,8 @@ u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
return crc;
}
EXPORT_SYMBOL(crc32_le_arch);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
if (!static_branch_likely(&have_crc32))
return crc32c_base(crc, p, len);
@ -101,36 +97,19 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
return crc;
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
return crc32_be_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
static int __init crc32_loongarch_init(void)
#define crc32_mod_init_arch crc32_mod_init_arch
static inline void crc32_mod_init_arch(void)
{
if (cpu_has_crc32)
static_branch_enable(&have_crc32);
return 0;
}
subsys_initcall(crc32_loongarch_init);
static void __exit crc32_loongarch_exit(void)
{
}
module_exit(crc32_loongarch_exit);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (static_key_enabled(&have_crc32))
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
MODULE_LICENSE("GPL v2");

33
arch/mips/lib/crc32-mips.c → lib/crc/mips/crc32.h
View File

@ -9,10 +9,6 @@
*/
#include <linux/cpufeature.h>
#include <linux/crc32.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/mipsregs.h>
#include <linux/unaligned.h>
@ -64,7 +60,7 @@ do { \
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
if (!static_branch_likely(&have_crc32))
return crc32_le_base(crc, p, len);
@ -106,9 +102,8 @@ u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
return crc;
}
EXPORT_SYMBOL(crc32_le_arch);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
if (!static_branch_likely(&have_crc32))
return crc32c_base(crc, p, len);
@ -149,35 +144,19 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
}
return crc;
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
return crc32_be_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
static int __init crc32_mips_init(void)
#define crc32_mod_init_arch crc32_mod_init_arch
static inline void crc32_mod_init_arch(void)
{
if (cpu_have_feature(cpu_feature(MIPS_CRC32)))
static_branch_enable(&have_crc32);
return 0;
}
subsys_initcall(crc32_mips_init);
static void __exit crc32_mips_exit(void)
{
}
module_exit(crc32_mips_exit);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (static_key_enabled(&have_crc32))
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com");
MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions");
MODULE_LICENSE("GPL v2");

20
arch/powerpc/lib/crc-t10dif.c → lib/crc/powerpc/crc-t10dif.h
View File

@ -9,10 +9,7 @@
#include <asm/switch_to.h>
#include <crypto/internal/simd.h>
#include <linux/cpufeature.h>
#include <linux/crc-t10dif.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/preempt.h>
#include <linux/uaccess.h>
@ -25,7 +22,7 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
static inline u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
{
unsigned int prealign;
unsigned int tail;
@ -62,22 +59,11 @@ u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
return crc & 0xffff;
}
EXPORT_SYMBOL(crc_t10dif_arch);
static int __init crc_t10dif_powerpc_init(void)
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
static inline void crc_t10dif_mod_init_arch(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
static_branch_enable(&have_vec_crypto);
return 0;
}
subsys_initcall(crc_t10dif_powerpc_init);
static void __exit crc_t10dif_powerpc_exit(void)
{
}
module_exit(crc_t10dif_powerpc_exit);
MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
MODULE_LICENSE("GPL");

0
arch/powerpc/lib/crc-vpmsum-template.S → lib/crc/powerpc/crc-vpmsum-template.S
View File

38
arch/powerpc/lib/crc32.c → lib/crc/powerpc/crc32.h
View File

@ -2,10 +2,7 @@
#include <asm/switch_to.h>
#include <crypto/internal/simd.h>
#include <linux/cpufeature.h>
#include <linux/crc32.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/preempt.h>
#include <linux/uaccess.h>
@ -16,15 +13,12 @@
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
#define crc32_le_arch crc32_le_base /* not implemented on this arch */
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len);
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
return crc32_le_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
unsigned int prealign;
unsigned int tail;
@ -58,36 +52,18 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
return crc;
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
return crc32_be_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
static int __init crc32_powerpc_init(void)
#define crc32_mod_init_arch crc32_mod_init_arch
static inline void crc32_mod_init_arch(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
static_branch_enable(&have_vec_crypto);
return 0;
}
subsys_initcall(crc32_powerpc_init);
static void __exit crc32_powerpc_exit(void)
{
}
module_exit(crc32_powerpc_exit);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (static_key_enabled(&have_vec_crypto))
return CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
MODULE_LICENSE("GPL");

0
arch/powerpc/lib/crc32c-vpmsum_asm.S → lib/crc/powerpc/crc32c-vpmsum_asm.S
View File

0
arch/powerpc/lib/crct10dif-vpmsum_asm.S → lib/crc/powerpc/crct10dif-vpmsum_asm.S
View File

0
arch/riscv/lib/crc-clmul-consts.h → lib/crc/riscv/crc-clmul-consts.h
View File

0
arch/riscv/lib/crc-clmul-template.h → lib/crc/riscv/crc-clmul-template.h
View File

0
arch/riscv/lib/crc-clmul.h → lib/crc/riscv/crc-clmul.h
View File

8
arch/riscv/lib/crc-t10dif.c → lib/crc/riscv/crc-t10dif.h
View File

@ -7,18 +7,12 @@
#include <asm/hwcap.h>
#include <asm/alternative-macros.h>
#include <linux/crc-t10dif.h>
#include <linux/module.h>
#include "crc-clmul.h"
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len)
static inline u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return crc16_msb_clmul(crc, p, len, &crc16_msb_0x8bb7_consts);
return crc_t10dif_generic(crc, p, len);
}
EXPORT_SYMBOL(crc_t10dif_arch);
MODULE_DESCRIPTION("RISC-V optimized CRC-T10DIF function");
MODULE_LICENSE("GPL");

0
arch/riscv/lib/crc16_msb.c → lib/crc/riscv/crc16_msb.c
View File

17
arch/riscv/lib/crc32.c → lib/crc/riscv/crc32.h
View File

@ -7,39 +7,34 @@
#include <asm/hwcap.h>
#include <asm/alternative-macros.h>
#include <linux/crc32.h>
#include <linux/module.h>
#include "crc-clmul.h"
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return crc32_lsb_clmul(crc, p, len,
&crc32_lsb_0xedb88320_consts);
return crc32_le_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return crc32_msb_clmul(crc, p, len,
&crc32_msb_0x04c11db7_consts);
return crc32_be_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return crc32_lsb_clmul(crc, p, len,
&crc32_lsb_0x82f63b78_consts);
return crc32c_base(crc, p, len);
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return CRC32_LE_OPTIMIZATION |
@ -47,7 +42,3 @@ u32 crc32_optimizations(void)
CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_DESCRIPTION("RISC-V optimized CRC32 functions");
MODULE_LICENSE("GPL");

0
arch/riscv/lib/crc32_lsb.c → lib/crc/riscv/crc32_lsb.c
View File

0
arch/riscv/lib/crc32_msb.c → lib/crc/riscv/crc32_msb.c
View File

11
arch/riscv/lib/crc64.c → lib/crc/riscv/crc64.h
View File

@ -7,28 +7,21 @@
#include <asm/hwcap.h>
#include <asm/alternative-macros.h>
#include <linux/crc64.h>
#include <linux/module.h>
#include "crc-clmul.h"
u64 crc64_be_arch(u64 crc, const u8 *p, size_t len)
static inline u64 crc64_be_arch(u64 crc, const u8 *p, size_t len)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return crc64_msb_clmul(crc, p, len,
&crc64_msb_0x42f0e1eba9ea3693_consts);
return crc64_be_generic(crc, p, len);
}
EXPORT_SYMBOL(crc64_be_arch);
u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
{
if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC))
return crc64_lsb_clmul(crc, p, len,
&crc64_lsb_0x9a6c9329ac4bc9b5_consts);
return crc64_nvme_generic(crc, p, len);
}
EXPORT_SYMBOL(crc64_nvme_arch);
MODULE_DESCRIPTION("RISC-V optimized CRC64 functions");
MODULE_LICENSE("GPL");

0
arch/riscv/lib/crc64_lsb.c → lib/crc/riscv/crc64_lsb.c
View File

0
arch/riscv/lib/crc64_msb.c → lib/crc/riscv/crc64_msb.c
View File

0
arch/s390/lib/crc32-vx.h → lib/crc/s390/crc32-vx.h
View File

16
arch/s390/lib/crc32.c → lib/crc/s390/crc32.h
View File

@ -5,12 +5,8 @@
* Copyright IBM Corp. 2015
* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
*/
#define KMSG_COMPONENT "crc32-vx"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
#include <linux/module.h>
#include <linux/cpufeature.h>
#include <linux/crc32.h>
#include <asm/fpu.h>
#include "crc32-vx.h"
@ -27,7 +23,7 @@
* operations of VECTOR LOAD MULTIPLE instructions.
*/
#define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \
u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
static inline u32 ___fname(u32 crc, const u8 *data, size_t datalen) \
{ \
unsigned long prealign, aligned, remaining; \
DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \
@ -54,14 +50,13 @@
crc = ___crc32_sw(crc, data + aligned, remaining); \
\
return crc; \
} \
EXPORT_SYMBOL(___fname);
}
DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base)
DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base)
DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base)
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (cpu_has_vx()) {
return CRC32_LE_OPTIMIZATION |
@ -70,8 +65,3 @@ u32 crc32_optimizations(void)
}
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility");
MODULE_LICENSE("GPL");

0
arch/s390/lib/crc32be-vx.c → lib/crc/s390/crc32be-vx.c
View File

0
arch/s390/lib/crc32le-vx.c → lib/crc/s390/crc32le-vx.c
View File

42
arch/sparc/lib/crc32.c → lib/crc/sparc/crc32.h
View File

@ -8,26 +8,17 @@
* Kent Liu <kent.liu@intel.com>
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/crc32.h>
#include <asm/pstate.h>
#include <asm/elf.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32c_opcode);
u32 crc32_le_arch(u32 crc, const u8 *data, size_t len)
{
return crc32_le_base(crc, data, len);
}
EXPORT_SYMBOL(crc32_le_arch);
#define crc32_le_arch crc32_le_base /* not implemented on this arch */
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
void crc32c_sparc64(u32 *crcp, const u64 *data, size_t len);
u32 crc32c_arch(u32 crc, const u8 *data, size_t len)
static inline u32 crc32c_arch(u32 crc, const u8 *data, size_t len)
{
size_t n = -(uintptr_t)data & 7;
@ -51,43 +42,26 @@ u32 crc32c_arch(u32 crc, const u8 *data, size_t len)
crc = crc32c_base(crc, data, len);
return crc;
}
EXPORT_SYMBOL(crc32c_arch);
u32 crc32_be_arch(u32 crc, const u8 *data, size_t len)
{
return crc32_be_base(crc, data, len);
}
EXPORT_SYMBOL(crc32_be_arch);
static int __init crc32_sparc_init(void)
#define crc32_mod_init_arch crc32_mod_init_arch
static inline void crc32_mod_init_arch(void)
{
unsigned long cfr;
if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
return 0;
return;
__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
if (!(cfr & CFR_CRC32C))
return 0;
return;
static_branch_enable(&have_crc32c_opcode);
pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n");
return 0;
}
subsys_initcall(crc32_sparc_init);
static void __exit crc32_sparc_exit(void)
{
}
module_exit(crc32_sparc_exit);
u32 crc32_optimizations(void)
static inline u32 crc32_optimizations_arch(void)
{
if (static_key_enabled(&have_crc32c_opcode))
return CRC32C_OPTIMIZATION;
return 0;
}
EXPORT_SYMBOL(crc32_optimizations);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated");

0
arch/sparc/lib/crc32c_asm.S → lib/crc/sparc/crc32c_asm.S
View File

2
lib/crc/tests/Makefile Normal file
View File

@ -0,0 +1,2 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CRC_KUNIT_TEST) += crc_kunit.o

39
lib/tests/crc_kunit.c → lib/crc/tests/crc_kunit.c
View File

@ -36,14 +36,12 @@ static size_t test_buflen;
* can fit any CRC up to CRC-64. The CRC is passed in, and is expected
* to be returned in, the least significant bits of the u64. The
* function is expected to *not* invert the CRC at the beginning and end.
* @combine_func: Optional function to combine two CRCs.
*/
struct crc_variant {
int bits;
bool le;
u64 poly;
u64 (*func)(u64 crc, const u8 *p, size_t len);
u64 (*combine_func)(u64 crc1, u64 crc2, size_t len2);
};
static u32 rand32(void)
@ -144,7 +142,7 @@ static size_t generate_random_length(size_t max_length)
}
/* Test that v->func gives the same CRCs as a reference implementation. */
static void crc_main_test(struct kunit *test, const struct crc_variant *v)
static void crc_test(struct kunit *test, const struct crc_variant *v)
{
size_t i;
@ -188,35 +186,6 @@ static void crc_main_test(struct kunit *test, const struct crc_variant *v)
}
}
/* Test that CRC(concat(A, B)) == combine_CRCs(CRC(A), CRC(B), len(B)). */
static void crc_combine_test(struct kunit *test, const struct crc_variant *v)
{
int i;
for (i = 0; i < 100; i++) {
u64 init_crc = generate_random_initial_crc(v);
size_t len1 = generate_random_length(CRC_KUNIT_MAX_LEN);
size_t len2 = generate_random_length(CRC_KUNIT_MAX_LEN - len1);
u64 crc1, crc2, expected_crc, actual_crc;
prandom_bytes_state(&rng, test_buffer, len1 + len2);
crc1 = v->func(init_crc, test_buffer, len1);
crc2 = v->func(0, &test_buffer[len1], len2);
expected_crc = v->func(init_crc, test_buffer, len1 + len2);
actual_crc = v->combine_func(crc1, crc2, len2);
KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc,
"CRC combination gave wrong result with len1=%zu len2=%zu\n",
len1, len2);
}
}
static void crc_test(struct kunit *test, const struct crc_variant *v)
{
crc_main_test(test, v);
if (v->combine_func)
crc_combine_test(test, v);
}
static __always_inline void
crc_benchmark(struct kunit *test,
u64 (*crc_func)(u64 crc, const u8 *p, size_t len))
@ -337,17 +306,11 @@ static u64 crc32_le_wrapper(u64 crc, const u8 *p, size_t len)
return crc32_le(crc, p, len);
}
static u64 crc32_le_combine_wrapper(u64 crc1, u64 crc2, size_t len2)
{
return crc32_le_combine(crc1, crc2, len2);
}
static const struct crc_variant crc_variant_crc32_le = {
.bits = 32,
.le = true,
.poly = 0xedb88320,
.func = crc32_le_wrapper,
.combine_func = crc32_le_combine_wrapper,
};
static void crc32_le_test(struct kunit *test)

47
arch/x86/lib/crc-pclmul-consts.h → lib/crc/x86/crc-pclmul-consts.h
View File

@ -2,7 +2,7 @@
/*
* CRC constants generated by:
*
* ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320,crc64_msb_0x42f0e1eba9ea3693,crc64_lsb_0x9a6c9329ac4bc9b5
* ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320,crc32_lsb_0x82f63b78,crc64_msb_0x42f0e1eba9ea3693,crc64_lsb_0x9a6c9329ac4bc9b5
*
* Do not edit manually.
*/
@ -98,6 +98,51 @@ static const struct {
},
};
/*
* CRC folding constants generated for least-significant-bit-first CRC-32 using
* G(x) = x^32 + x^28 + x^27 + x^26 + x^25 + x^23 + x^22 + x^20 + x^19 + x^18 +
* x^14 + x^13 + x^11 + x^10 + x^9 + x^8 + x^6 + x^0
*/
static const struct {
u64 fold_across_2048_bits_consts[2];
u64 fold_across_1024_bits_consts[2];
u64 fold_across_512_bits_consts[2];
u64 fold_across_256_bits_consts[2];
u64 fold_across_128_bits_consts[2];
u8 shuf_table[48];
u64 barrett_reduction_consts[2];
} crc32_lsb_0x82f63b78_consts ____cacheline_aligned __maybe_unused = {
.fold_across_2048_bits_consts = {
0x00000000dcb17aa4, /* HI64_TERMS: (x^2079 mod G) * x^32 */
0x00000000b9e02b86, /* LO64_TERMS: (x^2015 mod G) * x^32 */
},
.fold_across_1024_bits_consts = {
0x000000006992cea2, /* HI64_TERMS: (x^1055 mod G) * x^32 */
0x000000000d3b6092, /* LO64_TERMS: (x^991 mod G) * x^32 */
},
.fold_across_512_bits_consts = {
0x00000000740eef02, /* HI64_TERMS: (x^543 mod G) * x^32 */
0x000000009e4addf8, /* LO64_TERMS: (x^479 mod G) * x^32 */
},
.fold_across_256_bits_consts = {
0x000000003da6d0cb, /* HI64_TERMS: (x^287 mod G) * x^32 */
0x00000000ba4fc28e, /* LO64_TERMS: (x^223 mod G) * x^32 */
},
.fold_across_128_bits_consts = {
0x00000000f20c0dfe, /* HI64_TERMS: (x^159 mod G) * x^32 */
0x00000000493c7d27, /* LO64_TERMS: (x^95 mod G) * x^32 */
},
.shuf_table = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
},
.barrett_reduction_consts = {
0x4869ec38dea713f1, /* HI64_TERMS: floor(x^95 / G) */
0x0000000105ec76f0, /* LO64_TERMS: (G - x^32) * x^31 */
},
};
/*
* CRC folding constants generated for most-significant-bit-first CRC-64 using
* G(x) = x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +

7
arch/x86/lib/crc-pclmul-template.S → lib/crc/x86/crc-pclmul-template.S
View File

@ -561,7 +561,6 @@
RET
.endm
#ifdef CONFIG_AS_VPCLMULQDQ
#define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \
SYM_FUNC_START(prefix##_pclmul_sse); \
_crc_pclmul n=bits, lsb_crc=lsb, vl=16, avx_level=0; \
@ -574,9 +573,3 @@ SYM_FUNC_END(prefix##_vpclmul_avx2); \
SYM_FUNC_START(prefix##_vpclmul_avx512); \
_crc_pclmul n=bits, lsb_crc=lsb, vl=64, avx_level=512; \
SYM_FUNC_END(prefix##_vpclmul_avx512);
#else
#define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \
SYM_FUNC_START(prefix##_pclmul_sse); \
_crc_pclmul n=bits, lsb_crc=lsb, vl=16, avx_level=0; \
SYM_FUNC_END(prefix##_pclmul_sse);
#endif // !CONFIG_AS_VPCLMULQDQ

32
arch/x86/lib/crc-pclmul-template.h → lib/crc/x86/crc-pclmul-template.h
View File

@ -25,24 +25,20 @@ crc_t prefix##_vpclmul_avx512(crc_t crc, const u8 *p, size_t len, \
const void *consts_ptr); \
DEFINE_STATIC_CALL(prefix##_pclmul, prefix##_pclmul_sse)
#define INIT_CRC_PCLMUL(prefix) \
do { \
if (IS_ENABLED(CONFIG_AS_VPCLMULQDQ) && \
boot_cpu_has(X86_FEATURE_VPCLMULQDQ) && \
boot_cpu_has(X86_FEATURE_AVX2) && \
cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL)) { \
if (boot_cpu_has(X86_FEATURE_AVX512BW) && \
boot_cpu_has(X86_FEATURE_AVX512VL) && \
!boot_cpu_has(X86_FEATURE_PREFER_YMM) && \
cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL)) { \
static_call_update(prefix##_pclmul, \
prefix##_vpclmul_avx512); \
} else { \
static_call_update(prefix##_pclmul, \
prefix##_vpclmul_avx2); \
} \
} \
} while (0)
static inline bool have_vpclmul(void)
{
return boot_cpu_has(X86_FEATURE_VPCLMULQDQ) &&
boot_cpu_has(X86_FEATURE_AVX2) &&
cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL);
}
static inline bool have_avx512(void)
{
return boot_cpu_has(X86_FEATURE_AVX512BW) &&
boot_cpu_has(X86_FEATURE_AVX512VL) &&
!boot_cpu_has(X86_FEATURE_PREFER_YMM) &&
cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL);
}
/*
* Call a [V]PCLMULQDQ optimized CRC function if the data length is at least 16

27
arch/x86/lib/crc-t10dif.c → lib/crc/x86/crc-t10dif.h
View File

@ -5,36 +5,31 @@
* Copyright 2024 Google LLC
*/
#include <linux/crc-t10dif.h>
#include <linux/module.h>
#include "crc-pclmul-template.h"
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
DECLARE_CRC_PCLMUL_FUNCS(crc16_msb, u16);
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len)
static inline u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len)
{
CRC_PCLMUL(crc, p, len, crc16_msb, crc16_msb_0x8bb7_consts,
have_pclmulqdq);
return crc_t10dif_generic(crc, p, len);
}
EXPORT_SYMBOL(crc_t10dif_arch);
static int __init crc_t10dif_x86_init(void)
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
static inline void crc_t10dif_mod_init_arch(void)
{
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
static_branch_enable(&have_pclmulqdq);
INIT_CRC_PCLMUL(crc16_msb);
if (have_vpclmul()) {
if (have_avx512())
static_call_update(crc16_msb_pclmul,
crc16_msb_vpclmul_avx512);
else
static_call_update(crc16_msb_pclmul,
crc16_msb_vpclmul_avx2);
}
}
return 0;
}
subsys_initcall(crc_t10dif_x86_init);
static void __exit crc_t10dif_x86_exit(void)
{
}
module_exit(crc_t10dif_x86_exit);
MODULE_DESCRIPTION("CRC-T10DIF using [V]PCLMULQDQ instructions");
MODULE_LICENSE("GPL");

0
arch/x86/lib/crc16-msb-pclmul.S → lib/crc/x86/crc16-msb-pclmul.S
View File

0
arch/x86/lib/crc32-pclmul.S → lib/crc/x86/crc32-pclmul.S
View File

137
lib/crc/x86/crc32.h Normal file
View File

@ -0,0 +1,137 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* x86-optimized CRC32 functions
*
* Copyright (C) 2008 Intel Corporation
* Copyright 2012 Xyratex Technology Limited
* Copyright 2024 Google LLC
*/
#include "crc-pclmul-template.h"
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vpclmul_avx512);
DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32);
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts,
have_pclmulqdq);
return crc32_le_base(crc, p, len);
}
#ifdef CONFIG_X86_64
#define CRC32_INST "crc32q %1, %q0"
#else
#define CRC32_INST "crc32l %1, %0"
#endif
/*
* Use carryless multiply version of crc32c when buffer size is >= 512 to
* account for FPU state save/restore overhead.
*/
#define CRC32C_PCLMUL_BREAKEVEN 512
asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
size_t num_longs;
if (!static_branch_likely(&have_crc32))
return crc32c_base(crc, p, len);
if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN &&
static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
/*
* Long length, the vector registers are usable, and the CPU is
* 64-bit and supports both CRC32 and PCLMULQDQ instructions.
* It is worthwhile to divide the data into multiple streams,
* CRC them independently, and combine them using PCLMULQDQ.
* crc32c_x86_3way() does this using 3 streams, which is the
* most that x86_64 CPUs have traditionally been capable of.
*
* However, due to improved VPCLMULQDQ performance on newer
* CPUs, use crc32_lsb_vpclmul_avx512() instead of
* crc32c_x86_3way() when the CPU supports VPCLMULQDQ and has a
* "good" implementation of AVX-512.
*
* Future work: the optimal strategy on Zen 3--5 is actually to
* use both crc32q and VPCLMULQDQ in parallel. Unfortunately,
* different numbers of streams and vector lengths are optimal
* on each CPU microarchitecture, making it challenging to take
* advantage of this. (Zen 5 even supports 7 parallel crc32q, a
* major upgrade.) For now, just choose between
* crc32c_x86_3way() and crc32_lsb_vpclmul_avx512(). The latter
* is needed anyway for crc32_le(), so we just reuse it here.
*/
kernel_fpu_begin();
if (static_branch_likely(&have_vpclmul_avx512))
crc = crc32_lsb_vpclmul_avx512(crc, p, len,
crc32_lsb_0x82f63b78_consts.fold_across_128_bits_consts);
else
crc = crc32c_x86_3way(crc, p, len);
kernel_fpu_end();
return crc;
}
/*
* Short length, XMM registers unusable, or the CPU is 32-bit; but the
* CPU supports CRC32 instructions. Just issue a single stream of CRC32
* instructions inline. While this doesn't use the CPU's CRC32
* throughput very well, it avoids the need to combine streams. Stream
* combination would be inefficient here.
*/
for (num_longs = len / sizeof(unsigned long);
num_longs != 0; num_longs--, p += sizeof(unsigned long))
asm(CRC32_INST : "+r" (crc) : ASM_INPUT_RM (*(unsigned long *)p));
if (sizeof(unsigned long) > 4 && (len & 4)) {
asm("crc32l %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u32 *)p));
p += 4;
}
if (len & 2) {
asm("crc32w %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u16 *)p));
p += 2;
}
if (len & 1)
asm("crc32b %1, %0" : "+r" (crc) : ASM_INPUT_RM (*p));
return crc;
}
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
#define crc32_mod_init_arch crc32_mod_init_arch
static inline void crc32_mod_init_arch(void)
{
if (boot_cpu_has(X86_FEATURE_XMM4_2))
static_branch_enable(&have_crc32);
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
static_branch_enable(&have_pclmulqdq);
if (have_vpclmul()) {
if (have_avx512()) {
static_call_update(crc32_lsb_pclmul,
crc32_lsb_vpclmul_avx512);
static_branch_enable(&have_vpclmul_avx512);
} else {
static_call_update(crc32_lsb_pclmul,
crc32_lsb_vpclmul_avx2);
}
}
}
}
static inline u32 crc32_optimizations_arch(void)
{
u32 optimizations = 0;
if (static_key_enabled(&have_crc32))
optimizations |= CRC32C_OPTIMIZATION;
if (static_key_enabled(&have_pclmulqdq))
optimizations |= CRC32_LE_OPTIMIZATION;
return optimizations;
}

0
arch/x86/lib/crc32c-3way.S → lib/crc/x86/crc32c-3way.S
View File

0
arch/x86/lib/crc64-pclmul.S → lib/crc/x86/crc64-pclmul.S
View File

36
arch/x86/lib/crc64.c → lib/crc/x86/crc64.h
View File

@ -5,8 +5,6 @@
* Copyright 2025 Google LLC
*/
#include <linux/crc64.h>
#include <linux/module.h>
#include "crc-pclmul-template.h"
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
@ -14,37 +12,37 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
DECLARE_CRC_PCLMUL_FUNCS(crc64_msb, u64);
DECLARE_CRC_PCLMUL_FUNCS(crc64_lsb, u64);
u64 crc64_be_arch(u64 crc, const u8 *p, size_t len)
static inline u64 crc64_be_arch(u64 crc, const u8 *p, size_t len)
{
CRC_PCLMUL(crc, p, len, crc64_msb, crc64_msb_0x42f0e1eba9ea3693_consts,
have_pclmulqdq);
return crc64_be_generic(crc, p, len);
}
EXPORT_SYMBOL_GPL(crc64_be_arch);
u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
{
CRC_PCLMUL(crc, p, len, crc64_lsb, crc64_lsb_0x9a6c9329ac4bc9b5_consts,
have_pclmulqdq);
return crc64_nvme_generic(crc, p, len);
}
EXPORT_SYMBOL_GPL(crc64_nvme_arch);
static int __init crc64_x86_init(void)
#define crc64_mod_init_arch crc64_mod_init_arch
static inline void crc64_mod_init_arch(void)
{
if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
static_branch_enable(&have_pclmulqdq);
INIT_CRC_PCLMUL(crc64_msb);
INIT_CRC_PCLMUL(crc64_lsb);
if (have_vpclmul()) {
if (have_avx512()) {
static_call_update(crc64_msb_pclmul,
crc64_msb_vpclmul_avx512);
static_call_update(crc64_lsb_pclmul,
crc64_lsb_vpclmul_avx512);
} else {
static_call_update(crc64_msb_pclmul,
crc64_msb_vpclmul_avx2);
static_call_update(crc64_lsb_pclmul,
crc64_lsb_vpclmul_avx2);
}
}
}
return 0;
}
subsys_initcall(crc64_x86_init);
static void __exit crc64_x86_exit(void)
{
}
module_exit(crc64_x86_exit);
MODULE_DESCRIPTION("CRC64 using [V]PCLMULQDQ instructions");
MODULE_LICENSE("GPL");

126
lib/crc32.c
View File

@ -1,126 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin
* cleaned up code to current version of sparse and added the slicing-by-8
* algorithm to the closely similar existing slicing-by-4 algorithm.
*
* Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks!
* Code was from the public domain, copyright abandoned. Code was
* subsequently included in the kernel, thus was re-licensed under the
* GNU GPL v2.
*
* Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Same crc32 function was used in 5 other places in the kernel.
* I made one version, and deleted the others.
* There are various incantations of crc32(). Some use a seed of 0 or ~0.
* Some xor at the end with ~0. The generic crc32() function takes
* seed as an argument, and doesn't xor at the end. Then individual
* users can do whatever they need.
* drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0.
* fs/jffs2 uses seed 0, doesn't xor with ~0.
* fs/partitions/efi.c uses seed ~0, xor's with ~0.
*/
/* see: Documentation/staging/crc32.rst for a description of algorithms */
#include <linux/crc32.h>
#include <linux/crc32poly.h>
#include <linux/module.h>
#include <linux/types.h>
#include "crc32table.h"
MODULE_AUTHOR("Matt Domsch <Matt_Domsch@dell.com>");
MODULE_DESCRIPTION("Various CRC32 calculations");
MODULE_LICENSE("GPL");
u32 crc32_le_base(u32 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc >> 8) ^ crc32table_le[(crc & 255) ^ *p++];
return crc;
}
EXPORT_SYMBOL(crc32_le_base);
u32 crc32c_base(u32 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc >> 8) ^ crc32ctable_le[(crc & 255) ^ *p++];
return crc;
}
EXPORT_SYMBOL(crc32c_base);
/*
* This multiplies the polynomials x and y modulo the given modulus.
* This follows the "little-endian" CRC convention that the lsbit
* represents the highest power of x, and the msbit represents x^0.
*/
static u32 gf2_multiply(u32 x, u32 y, u32 modulus)
{
u32 product = x & 1 ? y : 0;
int i;
for (i = 0; i < 31; i++) {
product = (product >> 1) ^ (product & 1 ? modulus : 0);
x >>= 1;
product ^= x & 1 ? y : 0;
}
return product;
}
/**
* crc32_generic_shift - Append @len 0 bytes to crc, in logarithmic time
* @crc: The original little-endian CRC (i.e. lsbit is x^31 coefficient)
* @len: The number of bytes. @crc is multiplied by x^(8*@len)
* @polynomial: The modulus used to reduce the result to 32 bits.
*
* It's possible to parallelize CRC computations by computing a CRC
* over separate ranges of a buffer, then summing them.
* This shifts the given CRC by 8*len bits (i.e. produces the same effect
* as appending len bytes of zero to the data), in time proportional
* to log(len).
*/
static u32 crc32_generic_shift(u32 crc, size_t len, u32 polynomial)
{
u32 power = polynomial; /* CRC of x^32 */
int i;
/* Shift up to 32 bits in the simple linear way */
for (i = 0; i < 8 * (int)(len & 3); i++)
crc = (crc >> 1) ^ (crc & 1 ? polynomial : 0);
len >>= 2;
if (!len)
return crc;
for (;;) {
/* "power" is x^(2^i), modulo the polynomial */
if (len & 1)
crc = gf2_multiply(crc, power, polynomial);
len >>= 1;
if (!len)
break;
/* Square power, advancing to x^(2^(i+1)) */
power = gf2_multiply(power, power, polynomial);
}
return crc;
}
u32 crc32_le_shift(u32 crc, size_t len)
{
return crc32_generic_shift(crc, len, CRC32_POLY_LE);
}
EXPORT_SYMBOL(crc32_le_shift);
u32 crc32_be_base(u32 crc, const u8 *p, size_t len)
{
while (len--)
crc = (crc << 8) ^ crc32table_be[(crc >> 24) ^ *p++];
return crc;
}
EXPORT_SYMBOL(crc32_be_base);

Some files were not shown because too many files have changed in this diff Show More