diff --git a/crypto/fipsmodule/ml_dsa/META.yml b/crypto/fipsmodule/ml_dsa/META.yml index 5f876e25cf..612f5a49a1 100644 --- a/crypto/fipsmodule/ml_dsa/META.yml +++ b/crypto/fipsmodule/ml_dsa/META.yml @@ -1,5 +1,5 @@ name: mldsa-native source: pq-code-package/mldsa-native.git -branch: 08d40f9403a9ca80f160118bc63e96bf36627866 -commit: 08d40f9403a9ca80f160118bc63e96bf36627866 -imported-at: 2026-06-12T17:04:44+0000 +branch: 1dbd70f78583fab6fbfc46287702d6a09d5ec5e6 +commit: 1dbd70f78583fab6fbfc46287702d6a09d5ec5e6 +imported-at: 2026-07-17T20:21:14+0000 diff --git a/crypto/fipsmodule/ml_dsa/importer.sh b/crypto/fipsmodule/ml_dsa/importer.sh index 9a4f72506c..d69f02fb39 100755 --- a/crypto/fipsmodule/ml_dsa/importer.sh +++ b/crypto/fipsmodule/ml_dsa/importer.sh @@ -78,22 +78,17 @@ mkdir $SRC find $TMP/mldsa/src -maxdepth 1 -type f -exec cp {} $SRC \; # Copy x86_64 backend -# We import all assembly (.S) files and shared headers/constants from the -# upstream x86_64 backend. The AVX2 C-intrinsic .c files (rej_uniform, -# decompose, use_hint, chknorm, polyz_unpack) are excluded — their includes -# are stripped from the BCM below. -# -# The upstream meta.h advertises both assembly and C-intrinsic operations. -# Rather than modify it, we keep a hand-maintained replacement in -# ../mldsa_x86_64_meta.h (referenced via MLD_CONFIG_ARITH_BACKEND_FILE) that -# declares only the assembly-backed subset. Upstream meta.h is not copied. +# The upstream x86_64 backend is now 100% assembly — every native operation +# (NTT, INTT, decompose, use_hint, chknorm, polyz_unpack, rej_uniform[_eta], +# pointwise, caddq) is a verified .S file, with no AVX2 C-intrinsic .c files +# remaining. The upstream meta.h therefore declares only assembly-backed +# operations and is suitable as-is, so we copy the whole backend tree +# verbatim (matching the aarch64 backend below). All imported .S files have +# verified proofs in s2n-bignum. mkdir -p $SRC/native/x86_64/src cp $TMP/mldsa/src/native/api.h $SRC/native -cp $TMP/mldsa/src/native/x86_64/src/arith_native_x86_64.h $SRC/native/x86_64/src -cp $TMP/mldsa/src/native/x86_64/src/consts.h $SRC/native/x86_64/src -cp $TMP/mldsa/src/native/x86_64/src/consts.c $SRC/native/x86_64/src -# NOTE: all imported .S files must have verified proofs in s2n-bignum. -cp $TMP/mldsa/src/native/x86_64/src/*.S $SRC/native/x86_64/src +cp $TMP/mldsa/src/native/x86_64/*.h $SRC/native/x86_64 +cp $TMP/mldsa/src/native/x86_64/src/* $SRC/native/x86_64/src # Copy aarch64 backend # Unlike x86_64, the aarch64 backend is 100% assembly — no C-intrinsic .c @@ -141,12 +136,6 @@ cp $TMP/mldsa/mldsa_native.h $SRC echo "Fixup include paths" sed "${SED_I[@]}" 's/#include "src\/\([^"]*\)"/#include "\1"/' $SRC/mldsa_native_bcm.c -# Drop #include directives for the C-intrinsic .c files we did not import. -# Only consts.c (shared with the assembly backend) is kept. -echo "Strip C-intrinsic includes from mldsa_native_bcm.c" -BCM=$SRC/mldsa_native_bcm.c -sed "${SED_I[@]}" '/^#include "native\/x86_64\/src\/[^"]*\.c"/{/consts\.c/!d;}' "$BCM" - # ================================================================ # Fixup assembly backends to use s2n-bignum macros # ================================================================ diff --git a/crypto/fipsmodule/ml_dsa/ml_dsa.c b/crypto/fipsmodule/ml_dsa/ml_dsa.c index f6b8342182..bf411e164d 100644 --- a/crypto/fipsmodule/ml_dsa/ml_dsa.c +++ b/crypto/fipsmodule/ml_dsa/ml_dsa.c @@ -92,11 +92,12 @@ int ml_dsa_44_sign(const uint8_t *private_key /* IN */, FIPS_service_indicator_lock_state(); boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa44_signature(sig, sig_len, message, message_len, + int ret = mldsa44_signature(sig, message, message_len, ctx_string, ctx_string_len, private_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { + *sig_len = MLDSA44_SIGNATURE_BYTES; FIPS_service_indicator_update_state(); return 1; } @@ -113,10 +114,11 @@ int ml_dsa_extmu_44_sign(const uint8_t *private_key /* IN */, // mu_len is ignored - extmu always uses MLDSA_CRHBYTES (64 bytes) (void)mu_len; - int ret = mldsa44_signature_extmu(sig, sig_len, mu, private_key); + int ret = mldsa44_signature_extmu(sig, mu, private_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { + *sig_len = MLDSA44_SIGNATURE_BYTES; FIPS_service_indicator_update_state(); return 1; } @@ -144,8 +146,11 @@ int ml_dsa_44_sign_internal_no_self_test(const uint8_t *private_key /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */, const uint8_t *rnd /* IN */) { - int ret = mldsa44_signature_internal(sig, sig_len, message, message_len, + int ret = mldsa44_signature_internal(sig, message, message_len, pre, pre_len, rnd, private_key, 0); + if (ret == 0) { + *sig_len = MLDSA44_SIGNATURE_BYTES; + } return (ret == 0) ? 1 : 0; } @@ -158,8 +163,11 @@ int ml_dsa_extmu_44_sign_internal(const uint8_t *private_key /* IN */, size_t pre_len /* IN */, const uint8_t *rnd /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa44_signature_internal(sig, sig_len, mu, mu_len, + int ret = mldsa44_signature_internal(sig, mu, mu_len, pre, pre_len, rnd, private_key, 1); + if (ret == 0) { + *sig_len = MLDSA44_SIGNATURE_BYTES; + } return (ret == 0) ? 1 : 0; } @@ -173,7 +181,11 @@ int ml_dsa_44_verify(const uint8_t *public_key /* IN */, FIPS_service_indicator_lock_state(); boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa44_verify(sig, sig_len, message, message_len, + if (sig_len != MLDSA44_SIGNATURE_BYTES) { + FIPS_service_indicator_unlock_state(); + return 0; + } + int ret = mldsa44_verify(sig, message, message_len, ctx_string, ctx_string_len, public_key); FIPS_service_indicator_unlock_state(); @@ -194,7 +206,11 @@ int ml_dsa_extmu_44_verify(const uint8_t *public_key /* IN */, // mu_len is ignored - extmu always uses MLDSA_CRHBYTES (64 bytes) (void)mu_len; - int ret = mldsa44_verify_extmu(sig, sig_len, mu, public_key); + if (sig_len != MLDSA44_SIGNATURE_BYTES) { + FIPS_service_indicator_unlock_state(); + return 0; + } + int ret = mldsa44_verify_extmu(sig, mu, public_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { @@ -223,7 +239,10 @@ int ml_dsa_44_verify_internal_no_self_test(const uint8_t *public_key /* IN */, size_t message_len /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */) { - int ret = mldsa44_verify_internal(sig, sig_len, message, message_len, + if (sig_len != MLDSA44_SIGNATURE_BYTES) { + return 0; + } + int ret = mldsa44_verify_internal(sig, message, message_len, pre, pre_len, public_key, 0); return (ret == 0) ? 1 : 0; } @@ -236,7 +255,10 @@ int ml_dsa_extmu_44_verify_internal(const uint8_t *public_key /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa44_verify_internal(sig, sig_len, mu, mu_len, + if (sig_len != MLDSA44_SIGNATURE_BYTES) { + return 0; + } + int ret = mldsa44_verify_internal(sig, mu, mu_len, pre, pre_len, public_key, 1); return (ret == 0) ? 1 : 0; } @@ -296,11 +318,12 @@ int ml_dsa_65_sign(const uint8_t *private_key /* IN */, FIPS_service_indicator_lock_state(); boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa65_signature(sig, sig_len, message, message_len, + int ret = mldsa65_signature(sig, message, message_len, ctx_string, ctx_string_len, private_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { + *sig_len = MLDSA65_SIGNATURE_BYTES; FIPS_service_indicator_update_state(); return 1; } @@ -317,10 +340,11 @@ int ml_dsa_extmu_65_sign(const uint8_t *private_key /* IN */, // mu_len is ignored - extmu always uses MLDSA_CRHBYTES (64 bytes) (void)mu_len; - int ret = mldsa65_signature_extmu(sig, sig_len, mu, private_key); + int ret = mldsa65_signature_extmu(sig, mu, private_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { + *sig_len = MLDSA65_SIGNATURE_BYTES; FIPS_service_indicator_update_state(); return 1; } @@ -336,8 +360,11 @@ int ml_dsa_65_sign_internal(const uint8_t *private_key /* IN */, size_t pre_len /* IN */, const uint8_t *rnd /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa65_signature_internal(sig, sig_len, message, message_len, + int ret = mldsa65_signature_internal(sig, message, message_len, pre, pre_len, rnd, private_key, 0); + if (ret == 0) { + *sig_len = MLDSA65_SIGNATURE_BYTES; + } return (ret == 0) ? 1 : 0; } @@ -350,8 +377,11 @@ int ml_dsa_extmu_65_sign_internal(const uint8_t *private_key /* IN */, size_t pre_len /* IN */, const uint8_t *rnd /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa65_signature_internal(sig, sig_len, mu, mu_len, + int ret = mldsa65_signature_internal(sig, mu, mu_len, pre, pre_len, rnd, private_key, 1); + if (ret == 0) { + *sig_len = MLDSA65_SIGNATURE_BYTES; + } return (ret == 0) ? 1 : 0; } @@ -365,7 +395,11 @@ int ml_dsa_65_verify(const uint8_t *public_key /* IN */, FIPS_service_indicator_lock_state(); boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa65_verify(sig, sig_len, message, message_len, + if (sig_len != MLDSA65_SIGNATURE_BYTES) { + FIPS_service_indicator_unlock_state(); + return 0; + } + int ret = mldsa65_verify(sig, message, message_len, ctx_string, ctx_string_len, public_key); FIPS_service_indicator_unlock_state(); @@ -386,7 +420,11 @@ int ml_dsa_extmu_65_verify(const uint8_t *public_key /* IN */, // mu_len is ignored - extmu always uses MLDSA_CRHBYTES (64 bytes) (void)mu_len; - int ret = mldsa65_verify_extmu(sig, sig_len, mu, public_key); + if (sig_len != MLDSA65_SIGNATURE_BYTES) { + FIPS_service_indicator_unlock_state(); + return 0; + } + int ret = mldsa65_verify_extmu(sig, mu, public_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { @@ -404,7 +442,10 @@ int ml_dsa_65_verify_internal(const uint8_t *public_key /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa65_verify_internal(sig, sig_len, message, message_len, + if (sig_len != MLDSA65_SIGNATURE_BYTES) { + return 0; + } + int ret = mldsa65_verify_internal(sig, message, message_len, pre, pre_len, public_key, 0); return (ret == 0) ? 1 : 0; } @@ -417,7 +458,10 @@ int ml_dsa_extmu_65_verify_internal(const uint8_t *public_key /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa65_verify_internal(sig, sig_len, mu, mu_len, + if (sig_len != MLDSA65_SIGNATURE_BYTES) { + return 0; + } + int ret = mldsa65_verify_internal(sig, mu, mu_len, pre, pre_len, public_key, 1); return (ret == 0) ? 1 : 0; } @@ -476,11 +520,12 @@ int ml_dsa_87_sign(const uint8_t *private_key /* IN */, FIPS_service_indicator_lock_state(); boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa87_signature(sig, sig_len, message, message_len, + int ret = mldsa87_signature(sig, message, message_len, ctx_string, ctx_string_len, private_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { + *sig_len = MLDSA87_SIGNATURE_BYTES; FIPS_service_indicator_update_state(); return 1; } @@ -497,10 +542,11 @@ int ml_dsa_extmu_87_sign(const uint8_t *private_key /* IN */, // mu_len is ignored - extmu always uses MLDSA_CRHBYTES (64 bytes) (void)mu_len; - int ret = mldsa87_signature_extmu(sig, sig_len, mu, private_key); + int ret = mldsa87_signature_extmu(sig, mu, private_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { + *sig_len = MLDSA87_SIGNATURE_BYTES; FIPS_service_indicator_update_state(); return 1; } @@ -516,8 +562,11 @@ int ml_dsa_87_sign_internal(const uint8_t *private_key /* IN */, size_t pre_len /* IN */, const uint8_t *rnd /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa87_signature_internal(sig, sig_len, message, message_len, + int ret = mldsa87_signature_internal(sig, message, message_len, pre, pre_len, rnd, private_key, 0); + if (ret == 0) { + *sig_len = MLDSA87_SIGNATURE_BYTES; + } return (ret == 0) ? 1 : 0; } @@ -530,8 +579,11 @@ int ml_dsa_extmu_87_sign_internal(const uint8_t *private_key /* IN */, size_t pre_len /* IN */, const uint8_t *rnd /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa87_signature_internal(sig, sig_len, mu, mu_len, + int ret = mldsa87_signature_internal(sig, mu, mu_len, pre, pre_len, rnd, private_key, 1); + if (ret == 0) { + *sig_len = MLDSA87_SIGNATURE_BYTES; + } return (ret == 0) ? 1 : 0; } @@ -545,7 +597,11 @@ int ml_dsa_87_verify(const uint8_t *public_key /* IN */, FIPS_service_indicator_lock_state(); boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa87_verify(sig, sig_len, message, message_len, + if (sig_len != MLDSA87_SIGNATURE_BYTES) { + FIPS_service_indicator_unlock_state(); + return 0; + } + int ret = mldsa87_verify(sig, message, message_len, ctx_string, ctx_string_len, public_key); FIPS_service_indicator_unlock_state(); @@ -566,7 +622,11 @@ int ml_dsa_extmu_87_verify(const uint8_t *public_key /* IN */, // mu_len is ignored - extmu always uses MLDSA_CRHBYTES (64 bytes) (void)mu_len; - int ret = mldsa87_verify_extmu(sig, sig_len, mu, public_key); + if (sig_len != MLDSA87_SIGNATURE_BYTES) { + FIPS_service_indicator_unlock_state(); + return 0; + } + int ret = mldsa87_verify_extmu(sig, mu, public_key); FIPS_service_indicator_unlock_state(); if (ret == 0) { @@ -584,7 +644,10 @@ int ml_dsa_87_verify_internal(const uint8_t *public_key /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa87_verify_internal(sig, sig_len, message, message_len, + if (sig_len != MLDSA87_SIGNATURE_BYTES) { + return 0; + } + int ret = mldsa87_verify_internal(sig, message, message_len, pre, pre_len, public_key, 0); return (ret == 0) ? 1 : 0; } @@ -597,7 +660,10 @@ int ml_dsa_extmu_87_verify_internal(const uint8_t *public_key /* IN */, const uint8_t *pre /* IN */, size_t pre_len /* IN */) { boringssl_ensure_ml_dsa_self_test(); - int ret = mldsa87_verify_internal(sig, sig_len, mu, mu_len, + if (sig_len != MLDSA87_SIGNATURE_BYTES) { + return 0; + } + int ret = mldsa87_verify_internal(sig, mu, mu_len, pre, pre_len, public_key, 1); return (ret == 0) ? 1 : 0; } diff --git a/crypto/fipsmodule/ml_dsa/mldsa/.clang-format b/crypto/fipsmodule/ml_dsa/mldsa/.clang-format index 6ebaa3b5a2..2d13fa4ef2 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/.clang-format +++ b/crypto/fipsmodule/ml_dsa/mldsa/.clang-format @@ -27,3 +27,4 @@ Macros: - __loop__(x)={} do # Make this artifically long to force line break - MLD_INTERNAL_API=void abcdefghijklmnopqrstuvwabcdefghijklmnopqrstuvwabcdefg(); + - MLD_SYSV_ABI=void abcdefghijklmnopqrstuvwabcdefghijklmnopqrstuvwabcdefg(); diff --git a/crypto/fipsmodule/ml_dsa/mldsa/cbmc.h b/crypto/fipsmodule/ml_dsa/mldsa/cbmc.h index 0a7483a85f..0fd2045cc5 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/cbmc.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/cbmc.h @@ -9,17 +9,28 @@ /*************************************************** * Basic replacements for __CPROVER_XXX contracts ***************************************************/ +/* + * The `__contract__` / `__loop__` annotation macros use a + * leading-double-underscore spelling in line with other CBMC macros. + * clang-tidy flags these as reserved identifiers; we suppress the diagnostic + * at each definition site (NOLINT) rather than disabling the check globally, + * so it stays active for the rest of the tree. + */ #ifndef CBMC -#define __contract__(x) -#define __loop__(x) +/* clang-format off */ +#define __contract__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ #define cassert(x) #else /* !CBMC */ -#define __contract__(x) x -#define __loop__(x) x +/* clang-format off */ +#define __contract__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +#define __loop__(x) x /* NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp) */ +/* clang-format on */ /* Conditionally expand to __VA_ARGS__ depending on MLD_CONFIG_REDUCE_RAM. */ #if defined(MLD_CONFIG_REDUCE_RAM) diff --git a/crypto/fipsmodule/ml_dsa/mldsa/common.h b/crypto/fipsmodule/ml_dsa/mldsa/common.h index b97c2cd94f..895dae8508 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/common.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/common.h @@ -129,15 +129,15 @@ #endif #if defined(MLD_CONFIG_NO_RANDOMIZED_API) && defined(MLD_CONFIG_KEYGEN_PCT) -#error Bad configuration: MLD_CONFIG_NO_RANDOMIZED_API is incompatible with MLD_CONFIG_KEYGEN_PCT as the current PCT implementation requires crypto_sign_signature() +#error Bad configuration: MLD_CONFIG_NO_RANDOMIZED_API is incompatible with MLD_CONFIG_KEYGEN_PCT as the current PCT implementation requires signature() #endif #if defined(MLD_CONFIG_NO_SIGN_API) && defined(MLD_CONFIG_KEYGEN_PCT) -#error Bad configuration: MLD_CONFIG_NO_SIGN_API is incompatible with MLD_CONFIG_KEYGEN_PCT as the current PCT implementation requires crypto_sign_signature() +#error Bad configuration: MLD_CONFIG_NO_SIGN_API is incompatible with MLD_CONFIG_KEYGEN_PCT as the current PCT implementation requires signature() #endif #if defined(MLD_CONFIG_NO_VERIFY_API) && defined(MLD_CONFIG_KEYGEN_PCT) -#error Bad configuration: MLD_CONFIG_NO_VERIFY_API is incompatible with MLD_CONFIG_KEYGEN_PCT as the current PCT implementation requires crypto_sign_verify() +#error Bad configuration: MLD_CONFIG_NO_VERIFY_API is incompatible with MLD_CONFIG_KEYGEN_PCT as the current PCT implementation requires verify() #endif #if defined(MLD_CONFIG_USE_NATIVE_BACKEND_ARITH) @@ -202,77 +202,31 @@ #error Bad configuration: MLD_CONFIG_CUSTOM_ALLOC_FREE must be set together with MLD_CUSTOM_ALLOC and MLD_CUSTOM_FREE #endif -/* - * If the integration wants to provide a context parameter for use in - * platform-specific hooks, then it should define this parameter. - * - * The MLD_CONTEXT_PARAMETERS_n macros are intended to be used with macros - * defining the function names and expand to either pass or discard the context - * argument as required by the current build. If there is no context parameter - * requested then these are removed from the prototypes and from all calls. - */ -#ifdef MLD_CONFIG_CONTEXT_PARAMETER -#define MLD_CONTEXT_PARAMETERS_0(context) (context) -#define MLD_CONTEXT_PARAMETERS_1(arg0, context) (arg0, context) -#define MLD_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1, context) -#define MLD_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) \ - (arg0, arg1, arg2, context) -#define MLD_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ - (arg0, arg1, arg2, arg3, context) -#define MLD_CONTEXT_PARAMETERS_5(arg0, arg1, arg2, arg3, arg4, context) \ - (arg0, arg1, arg2, arg3, arg4, context) -#define MLD_CONTEXT_PARAMETERS_6(arg0, arg1, arg2, arg3, arg4, arg5, context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, context) -#define MLD_CONTEXT_PARAMETERS_7(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ - context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, arg6, context) -#define MLD_CONTEXT_PARAMETERS_8(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ - arg7, context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, context) -#define MLD_CONTEXT_PARAMETERS_9(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ - arg7, arg8, context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, context) -#else /* MLD_CONFIG_CONTEXT_PARAMETER */ -#define MLD_CONTEXT_PARAMETERS_0(context) () -#define MLD_CONTEXT_PARAMETERS_1(arg0, context) (arg0) -#define MLD_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1) -#define MLD_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) (arg0, arg1, arg2) -#define MLD_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ - (arg0, arg1, arg2, arg3) -#define MLD_CONTEXT_PARAMETERS_5(arg0, arg1, arg2, arg3, arg4, context) \ - (arg0, arg1, arg2, arg3, arg4) -#define MLD_CONTEXT_PARAMETERS_6(arg0, arg1, arg2, arg3, arg4, arg5, context) \ - (arg0, arg1, arg2, arg3, arg4, arg5) -#define MLD_CONTEXT_PARAMETERS_7(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ - context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, arg6) -#define MLD_CONTEXT_PARAMETERS_8(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ - arg7, context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) -#define MLD_CONTEXT_PARAMETERS_9(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ - arg7, arg8, context) \ - (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) -#endif /* !MLD_CONFIG_CONTEXT_PARAMETER */ - -#if defined(MLD_CONFIG_CONTEXT_PARAMETER_TYPE) != \ - defined(MLD_CONFIG_CONTEXT_PARAMETER) -#error MLD_CONFIG_CONTEXT_PARAMETER_TYPE must be defined if and only if MLD_CONFIG_CONTEXT_PARAMETER is defined -#endif +/* Context-parameter machinery (MLD_CONTEXT_PARAMETERS_n and related config + * checks). Kept in a separate, level-generic header for readability; included + * here so it is available to the allocation macros below and to all consumers + * of common.h. */ +#include "context.h" #if !defined(MLD_CONFIG_CUSTOM_ALLOC_FREE) /* Default: stack allocation */ +/* This is a declaration macro, not an expression macro: T is a type and v is + * a declarator, neither of which can be wrapped in parentheses. The + * bugprone-macro-parentheses diagnostic is therefore a false positive here. */ #define MLD_ALLOC(v, T, N, context) \ MLD_ALIGN T mld_alloc_##v[N]; \ - T *v = mld_alloc_##v + T *v = mld_alloc_##v /* NOLINT(bugprone-macro-parentheses) */ -/* TODO: This leads to a circular dependency between common and ct.h - * It just works out before we're at the end of the file, but it's still - * prone to issues in the future. */ -#include "ct.h" +/* The MLD_FREE macro body references mld_zeroize(), which is declared in + * ct.h. We deliberately do NOT include ct.h here: doing so would create a + * circular dependency (ct.h includes common.h), and common.h itself never + * calls mld_zeroize() -- only the macro expansion does. Each translation + * unit that uses MLD_FREE therefore includes ct.h directly. */ #define MLD_FREE(v, T, N, context) \ do \ { \ + MLD_CONTEXT_UNUSED(context); \ mld_zeroize(mld_alloc_##v, sizeof(mld_alloc_##v)); \ (v) = NULL; \ } while (0) @@ -305,27 +259,33 @@ /****************************** Error codes ***********************************/ /* Generic failure condition */ -#define MLD_ERR_FAIL -1 +#define MLD_ERR_FAIL (-1) /* An allocation failed. This can only happen if MLD_CONFIG_CUSTOM_ALLOC_FREE * is defined and the provided MLD_CUSTOM_ALLOC can fail. */ -#define MLD_ERR_OUT_OF_MEMORY -2 +#define MLD_ERR_OUT_OF_MEMORY (-2) /* An rng failure occured. Might be due to insufficient entropy or * system misconfiguration. */ -#define MLD_ERR_RNG_FAIL -3 +#define MLD_ERR_RNG_FAIL (-3) /* The signing rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS iterations without producing a valid * signature. With a FIPS 204 Appendix C compliant bound (>= 814) this * has probability < 2^-256. */ -#define MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED -4 +#define MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED (-4) +/* Signing was paused before completing, at the request of a caller-provided + * MLD_CONFIG_SIGN_HOOK_ATTEMPT hook (see mldsa_native_config.h). The caller + * resumes by re-invoking signing with the same inputs; the attempt hook, + * together with MLD_CONFIG_SIGN_HOOK_RESUME, decides where to continue. */ +#define MLD_ERR_SIGNING_PAUSED (-5) /* Disjunction over the full set of MLD_ERR_XXX failure codes. * * Intended for use in top-level `ensures` clauses that admit every * possible error. Narrower contracts should enumerate only the * specific errors they can actually return. */ -#define MLD_ANY_ERROR(err) \ - ((err) == MLD_ERR_FAIL || (err) == MLD_ERR_OUT_OF_MEMORY || \ - (err) == MLD_ERR_RNG_FAIL || (err) == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED) +#define MLD_ANY_ERROR(err) \ + ((err) == MLD_ERR_FAIL || (err) == MLD_ERR_OUT_OF_MEMORY || \ + (err) == MLD_ERR_RNG_FAIL || (err) == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED || \ + (err) == MLD_ERR_SIGNING_PAUSED) #endif /* !__ASSEMBLER__ */ diff --git a/crypto/fipsmodule/ml_dsa/mldsa/context.h b/crypto/fipsmodule/ml_dsa/mldsa/context.h new file mode 100644 index 0000000000..f5e582d7c5 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/context.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ +#ifndef MLD_CONTEXT_H +#define MLD_CONTEXT_H + +/* This header is included by common.h once the configuration has been pulled + * in; it is not meant to be included directly. */ +#if !defined(__ASSEMBLER__) + +#include +#include "cbmc.h" +#include "sys.h" + +/* + * If the integration wants to provide a context parameter for use in + * platform-specific hooks, then it should define this parameter. + * + * The MLD_CONTEXT_PARAMETERS_n macros are intended to be used with macros + * defining the function names and expand to either pass or discard the context + * argument as required by the current build. If there is no context parameter + * requested then these are removed from the prototypes and from all calls. + */ +#ifdef MLD_CONFIG_CONTEXT_PARAMETER +#define MLD_CONTEXT_PARAMETERS_0(context) (context) +#define MLD_CONTEXT_PARAMETERS_1(arg0, context) (arg0, context) +#define MLD_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1, context) +#define MLD_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) \ + (arg0, arg1, arg2, context) +#define MLD_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3, context) +#define MLD_CONTEXT_PARAMETERS_5(arg0, arg1, arg2, arg3, arg4, context) \ + (arg0, arg1, arg2, arg3, arg4, context) +#define MLD_CONTEXT_PARAMETERS_6(arg0, arg1, arg2, arg3, arg4, arg5, context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, context) +#define MLD_CONTEXT_PARAMETERS_7(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ + context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, arg6, context) +#define MLD_CONTEXT_PARAMETERS_8(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ + arg7, context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, context) +#define MLD_CONTEXT_PARAMETERS_9(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ + arg7, arg8, context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, context) +#else /* MLD_CONFIG_CONTEXT_PARAMETER */ +#define MLD_CONTEXT_PARAMETERS_0(context) () +#define MLD_CONTEXT_PARAMETERS_1(arg0, context) (arg0) +#define MLD_CONTEXT_PARAMETERS_2(arg0, arg1, context) (arg0, arg1) +#define MLD_CONTEXT_PARAMETERS_3(arg0, arg1, arg2, context) (arg0, arg1, arg2) +#define MLD_CONTEXT_PARAMETERS_4(arg0, arg1, arg2, arg3, context) \ + (arg0, arg1, arg2, arg3) +#define MLD_CONTEXT_PARAMETERS_5(arg0, arg1, arg2, arg3, arg4, context) \ + (arg0, arg1, arg2, arg3, arg4) +#define MLD_CONTEXT_PARAMETERS_6(arg0, arg1, arg2, arg3, arg4, arg5, context) \ + (arg0, arg1, arg2, arg3, arg4, arg5) +#define MLD_CONTEXT_PARAMETERS_7(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ + context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, arg6) +#define MLD_CONTEXT_PARAMETERS_8(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ + arg7, context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) +#define MLD_CONTEXT_PARAMETERS_9(arg0, arg1, arg2, arg3, arg4, arg5, arg6, \ + arg7, arg8, context) \ + (arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) +#endif /* !MLD_CONFIG_CONTEXT_PARAMETER */ + +/* Consume a context parameter carried only for the integration's benefit, + * avoiding -Wunused-parameter; expands to nothing when no context is + * configured. */ +#if defined(MLD_CONFIG_CONTEXT_PARAMETER) +#define MLD_CONTEXT_UNUSED(context) ((void)(context)) +#else +#define MLD_CONTEXT_UNUSED(context) ((void)0) +#endif + +#if defined(MLD_CONFIG_CONTEXT_PARAMETER_TYPE) != \ + defined(MLD_CONFIG_CONTEXT_PARAMETER) +#error MLD_CONFIG_CONTEXT_PARAMETER_TYPE must be defined if and only if MLD_CONFIG_CONTEXT_PARAMETER is defined +#endif + +/* The signing hooks tie into the rejection-sampling loop. A pausing attempt + * hook only reproduces the uninterrupted signature if the randomness is fixed + * across calls, and thus requires the deterministic API. + * For now we impose that requirement on all three hooks uniformly: enabling any + * of them requires MLD_CONFIG_NO_RANDOMIZED_API. This also rules out + * MLD_CONFIG_KEYGEN_PCT (whose PCT needs the randomized signature(), see + * common.h). + * + * A logging-only use (attempt always returns 0; resume/finish merely observe) + * would be safe with the randomized API too, but the restriction is applied + * uniformly for now. */ +#if (defined(MLD_CONFIG_SIGN_HOOK_RESUME) || \ + defined(MLD_CONFIG_SIGN_HOOK_ATTEMPT) || \ + defined(MLD_CONFIG_SIGN_HOOK_FINISH)) && \ + !defined(MLD_CONFIG_NO_RANDOMIZED_API) +#error Signing hooks (MLD_CONFIG_SIGN_HOOK_RESUME / _ATTEMPT / _FINISH) require MLD_CONFIG_NO_RANDOMIZED_API +#endif /* (MLD_CONFIG_SIGN_HOOK_RESUME || MLD_CONFIG_SIGN_HOOK_ATTEMPT || \ + MLD_CONFIG_SIGN_HOOK_FINISH) && !MLD_CONFIG_NO_RANDOMIZED_API */ + +/* Signing hooks (MLD_CONFIG_SIGN_HOOK_RESUME / _ATTEMPT / _FINISH; documented + * in mldsa_native_config.h). The following macros route the call sites to + * mld_sign_hook_*, appending or dropping the context argument; each unset hook + * uses the dummy below. */ +#define mld_sign_resume mld_sign_hook_resume MLD_CONTEXT_PARAMETERS_0 +#define mld_sign_attempt mld_sign_hook_attempt MLD_CONTEXT_PARAMETERS_1 +#define mld_sign_finish mld_sign_hook_finish MLD_CONTEXT_PARAMETERS_1 + +/* We don't use mld_sign_resume here because MLD_CONTEXT_PARAMETERS_0 is + * unsuitable for function declarations: it misses `void` as the placeholder + * argument. */ +#if !defined(MLD_CONFIG_SIGN_HOOK_RESUME) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE uint16_t mld_sign_hook_resume( +#if defined(MLD_CONFIG_CONTEXT_PARAMETER) + MLD_CONFIG_CONTEXT_PARAMETER_TYPE context +#else + void +#endif +) +__contract__(assigns() ensures(1)) +{ + MLD_CONTEXT_UNUSED(context); + return 0; +} +#endif /* !MLD_CONFIG_SIGN_HOOK_RESUME */ + +#if !defined(MLD_CONFIG_SIGN_HOOK_ATTEMPT) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_sign_attempt( + uint16_t attempt, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__(assigns() ensures(1)) +{ + ((void)attempt); + MLD_CONTEXT_UNUSED(context); + return 0; +} +#endif /* !MLD_CONFIG_SIGN_HOOK_ATTEMPT */ + +#if !defined(MLD_CONFIG_SIGN_HOOK_FINISH) +static MLD_INLINE void mld_sign_finish( + uint16_t attempt, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) +__contract__(assigns() ensures(1)) +{ + ((void)attempt); + MLD_CONTEXT_UNUSED(context); +} +#endif /* !MLD_CONFIG_SIGN_HOOK_FINISH */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_CONTEXT_H */ diff --git a/crypto/fipsmodule/ml_dsa/mldsa/ct.h b/crypto/fipsmodule/ml_dsa/mldsa/ct.h index f7953b3b02..1a08a83fa7 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/ct.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/ct.h @@ -230,6 +230,12 @@ static MLD_INLINE uint32_t mld_ct_cmask_nonzero_u32(uint32_t x) __contract__(ensures(return_value == ((x == 0) ? 0 : 0xFFFFFFFF))) { int64_t tmp = mld_value_barrier_i64(-((int64_t)x)); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ tmp >>= 32; return mld_cast_int64_to_uint32(tmp); } @@ -259,6 +265,12 @@ __contract__( ) { int64_t tmp = mld_value_barrier_i64((int64_t)x); + /* + * PORTABILITY: Right-shift on a signed integer is + * implementation-defined for negative left argument. + * Here, we assume it's sign-preserving "arithmetic" shift right. + * See (C99 6.5.7 (5)) + */ tmp >>= 31; return mld_cast_int64_to_uint32(tmp); } diff --git a/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native.h b/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native.h index df93f2910e..155399b1f6 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native.h @@ -30,47 +30,7 @@ * in the configuration file and include this header multiple times, setting * MLD_CONFIG_PARAMETER_SET accordingly for each, and #undef'ing the MLD_H * guard to allow multiple inclusions. - * - * # Legacy configuration (deprecated) - * - * Instead of providing the config file used for the build, you can - * alternatively set the following configuration options prior to - * including this header. - * - * This method of configuration is deprecated. - * It will be removed in mldsa-native-v2. - * - * - MLD_CONFIG_API_PARAMETER_SET [required] - * - * The parameter set used for the build; 44, 65, or 87. - * - * - MLD_CONFIG_API_NAMESPACE_PREFIX [required] - * - * The namespace prefix used for the build. - * - * NOTE: - * For a multi-level build, you must include the 44/65/87 suffixes - * in MLD_CONFIG_API_NAMESPACE_PREFIX. - * - * - MLD_CONFIG_API_NO_SUPERCOP [optional] - * - * By default, this header will also expose the mldsa-native API in the - * SUPERCOP naming convention crypto_sign_xxx. If you don't want/need this, - * set MLD_CONFIG_API_NO_SUPERCOP. You must set this for a multi-level build. - * - * - MLD_CONFIG_API_CONSTANTS_ONLY [optional] - * - * If you don't want this header to expose any function declarations, - * but only constants for the sizes of key material, set - * MLD_CONFIG_API_CONSTANTS_ONLY. In this case, you don't need to set - * MLD_CONFIG_API_PARAMETER_SET or MLD_CONFIG_API_NAMESPACE_PREFIX, - * nor include a configuration. - * - * - MLD_CONFIG_API_QUALIFIER [optional] - * - * Qualifier to apply to external API. - * - ******************************************************************************/ + */ /******************************* Key sizes ************************************/ @@ -102,7 +62,13 @@ #define MLDSA65_CRHBYTES MLDSA_CRHBYTES #define MLDSA87_CRHBYTES MLDSA_CRHBYTES -/* Size of TR output in bytes (level-independent) */ +/* Size of TR in bytes (level-independent) + * + * TR = SHAKE256(pk, 64) is the hash of the public key. Callers of the + * external-mu API (signature_extmu / verify_extmu) that compute the message + * representative mu = SHAKE256(TR || M', MLDSA_CRHBYTES) themselves -- e.g. to + * sign or verify a message that is too large or streamed to hold in memory -- + * need this constant to size the TR buffer. */ #define MLDSA_TRBYTES 64 #define MLDSA44_TRBYTES MLDSA_TRBYTES #define MLDSA65_TRBYTES MLDSA_TRBYTES @@ -125,32 +91,30 @@ /****************************** Error codes ***********************************/ /* Generic failure condition */ -#define MLD_ERR_FAIL -1 +#define MLD_ERR_FAIL (-1) /* An allocation failed. This can only happen if MLD_CONFIG_CUSTOM_ALLOC_FREE * is defined and the provided MLD_CUSTOM_ALLOC can fail. */ -#define MLD_ERR_OUT_OF_MEMORY -2 +#define MLD_ERR_OUT_OF_MEMORY (-2) /* An rng failure occured. Might be due to insufficient entropy or * system misconfiguration. */ -#define MLD_ERR_RNG_FAIL -3 +#define MLD_ERR_RNG_FAIL (-3) /* The signing rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS iterations without producing a valid * signature. With a FIPS 204 Appendix C compliant bound (>= 814) this * has probability < 2^-256. */ -#define MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED -4 +#define MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED (-4) +/* Signing was paused before completing, at the request of a caller-provided + * MLD_CONFIG_SIGN_HOOK_ATTEMPT hook (see mldsa_native_config.h). The caller + * resumes by re-invoking signing with the same inputs; the attempt hook, + * together with MLD_CONFIG_SIGN_HOOK_RESUME, decides where to continue. */ +#define MLD_ERR_SIGNING_PAUSED (-5) -/****************************** Function API **********************************/ +/********************* Namespacing and Qualifiers *****************************/ #define MLD_API_CONCAT_(x, y) x##y #define MLD_API_CONCAT(x, y) MLD_API_CONCAT_(x, y) #define MLD_API_CONCAT_UNDERSCORE(x, y) MLD_API_CONCAT(MLD_API_CONCAT(x, _), y) -#if !defined(MLD_CONFIG_API_PARAMETER_SET) -/* Recommended configuration via same config file as used for the build. */ - -/* For now, we derive the legacy API configuration MLD_CONFIG_API_XXX from - * the config file. In mldsa-native-v2, this will be removed and we will - * exclusively work with MLD_CONFIG_XXX. */ - /* You need to make sure the config file is in the include path. */ #if defined(MLD_CONFIG_FILE) #include MLD_CONFIG_FILE @@ -158,53 +122,37 @@ #include "mldsa_native_config.h" #endif -#define MLD_CONFIG_API_PARAMETER_SET MLD_CONFIG_PARAMETER_SET - +/* Namespace prefix for the public API symbols. For multi-level builds, the + * parameter set is appended to disambiguate the security levels. */ #if defined(MLD_CONFIG_MULTILEVEL_BUILD) -#define MLD_CONFIG_API_NAMESPACE_PREFIX \ +#define MLD_API_NAMESPACE_PREFIX \ MLD_API_CONCAT(MLD_CONFIG_NAMESPACE_PREFIX, MLD_CONFIG_PARAMETER_SET) #else -#define MLD_CONFIG_API_NAMESPACE_PREFIX MLD_CONFIG_NAMESPACE_PREFIX -#endif - -#if defined(MLD_CONFIG_NO_SUPERCOP) -#define MLD_CONFIG_API_NO_SUPERCOP +#define MLD_API_NAMESPACE_PREFIX MLD_CONFIG_NAMESPACE_PREFIX #endif -#if defined(MLD_CONFIG_CONSTANTS_ONLY) -#define MLD_CONFIG_API_CONSTANTS_ONLY -#endif - -#if defined(MLD_CONFIG_EXTERNAL_API_QUALIFIER) -#define MLD_CONFIG_API_QUALIFIER MLD_CONFIG_EXTERNAL_API_QUALIFIER -#endif - -#else /* !MLD_CONFIG_API_PARAMETER_SET */ -#define MLD_API_LEGACY_CONFIG - -#endif /* MLD_CONFIG_API_PARAMETER_SET */ - #define MLD_API_NAMESPACE(sym) \ - MLD_API_CONCAT_UNDERSCORE(MLD_CONFIG_API_NAMESPACE_PREFIX, sym) + MLD_API_CONCAT_UNDERSCORE(MLD_API_NAMESPACE_PREFIX, sym) -#if defined(__GNUC__) || defined(clang) +#if defined(__GNUC__) || defined(__clang__) #define MLD_API_MUST_CHECK_RETURN_VALUE __attribute__((warn_unused_result)) #else #define MLD_API_MUST_CHECK_RETURN_VALUE #endif -#if defined(MLD_CONFIG_API_QUALIFIER) -#define MLD_API_QUALIFIER MLD_CONFIG_API_QUALIFIER +#if defined(MLD_CONFIG_EXTERNAL_API_QUALIFIER) +#define MLD_API_QUALIFIER MLD_CONFIG_EXTERNAL_API_QUALIFIER #else #define MLD_API_QUALIFIER #endif -#if !defined(MLD_CONFIG_API_CONSTANTS_ONLY) +/****************************** Function API **********************************/ + +#if !defined(MLD_CONFIG_CONSTANTS_ONLY) #include #include - #ifdef __cplusplus extern "C" { @@ -220,7 +168,7 @@ extern "C" * @warning The seed must be generated by a cryptographically secure random * number generator. * - * @spec{Implements @[FIPS204 Algorithm 6 (ML-DSA.KeyGen_internal)].} + * @spec{Implements @[FIPS204, Algorithm 6, ML-DSA.KeyGen_internal].} * * @param[out] pk Output public key. * @param[out] sk Output private key. @@ -245,8 +193,8 @@ extern "C" MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(keypair_internal)( - uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], - uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], + uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)], + uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t seed[MLDSA_SEEDBYTES] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , @@ -261,7 +209,7 @@ int MLD_API_NAMESPACE(keypair_internal)( * When MLD_CONFIG_KEYGEN_PCT is set, performs a Pairwise Consistency Test * (PCT) as required by FIPS 140-3 IG. * - * @spec{Implements @[FIPS204 Algorithm 1 (ML-DSA.KeyGen)].} + * @spec{Implements @[FIPS204, Algorithm 1, ML-DSA.KeyGen].} * * @param[out] pk Output public key. * @param[out] sk Output private key. @@ -284,8 +232,8 @@ int MLD_API_NAMESPACE(keypair_internal)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(keypair)( - uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], - uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)], + uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context @@ -298,13 +246,16 @@ int MLD_API_NAMESPACE(keypair)( /** * Compute signature using a caller-supplied random seed and prefix. * - * If the returned value is non-zero, then the values of *sig and *siglen - * should not be referenced. + * On error (non-zero return value), the signature buffer sig is zeroized. + * + * @spec{Implements @[FIPS204, Algorithm 7, ML-DSA.Sign_internal].} * - * @spec{Implements @[FIPS204 Algorithm 7 (ML-DSA.Sign_internal)].} + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use crypto_sign_pk_from_sk + * to validate them before signing. * - * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. + * @param[out] sig Pointer to buffer to hold the generated signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] m Pointer to message to be signed (when * externalmu == 0), or to a precomputed * message representative mu (when externalmu != 0). @@ -315,7 +266,7 @@ int MLD_API_NAMESPACE(keypair)( * @param prelen Length of prefix string. Ignored when * externalmu != 0. * @param[in] rnd Random seed. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param externalmu 0: m/mlen is the raw message; mu = H(tr, pre, m) is * computed internally. * non-zero: m points to a precomputed mu of @@ -331,15 +282,17 @@ int MLD_API_NAMESPACE(keypair)( * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS * iterations. + * @retval MLD_ERR_SIGNING_PAUSED A MLD_CONFIG_SIGN_HOOK_ATTEMPT hook + * paused signing; re-invoke to resume. * @retval MLD_ERR_FAIL Other kinds of failure. */ MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(signature_internal)( - uint8_t sig[MLDSA_BYTES(MLD_CONFIG_API_PARAMETER_SET)], size_t *siglen, - const uint8_t *m, size_t mlen, const uint8_t *pre, size_t prelen, + uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *m, + size_t mlen, const uint8_t *pre, size_t prelen, const uint8_t rnd[MLDSA_RNDBYTES], - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], + const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)], int externalmu #ifdef MLD_CONFIG_CONTEXT_PARAMETER , @@ -351,17 +304,21 @@ int MLD_API_NAMESPACE(signature_internal)( /** * Compute signature. This function implements the randomized variant of * ML-DSA. If you require the deterministic variant, use - * crypto_sign_signature_internal directly. + * signature_internal directly. + * + * @spec{Implements @[FIPS204, Algorithm 2, ML-DSA.Sign].} * - * @spec{Implements @[FIPS204 Algorithm 2 (ML-DSA.Sign)].} + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use crypto_sign_pk_from_sk + * to validate them before signing. * - * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. + * @param[out] sig Pointer to buffer to hold the generated signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] m Pointer to message to be signed. * @param mlen Length of message. * @param[in] ctx Pointer to context string. May be NULL if ctxlen == 0. * @param ctxlen Length of context string. Should be <= 255. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. @@ -379,9 +336,9 @@ int MLD_API_NAMESPACE(signature_internal)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(signature)( - uint8_t sig[MLDSA_BYTES(MLD_CONFIG_API_PARAMETER_SET)], size_t *siglen, - const uint8_t *m, size_t mlen, const uint8_t *ctx, size_t ctxlen, - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *m, + size_t mlen, const uint8_t *ctx, size_t ctxlen, + const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context @@ -395,12 +352,16 @@ int MLD_API_NAMESPACE(signature)( * 0x00 || ctxlen || ctx || msg for pure ML-DSA). This is useful when the * message is large or streamed and cannot be held in memory. * - * @spec{Implements @[FIPS204 Algorithm 2 (ML-DSA.Sign external mu variant)].} + * @spec{Implements @[FIPS204, Algorithm 2, ML-DSA.Sign external mu variant].} * - * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use crypto_sign_pk_from_sk + * to validate them before signing. + * + * @param[out] sig Pointer to buffer to hold the generated signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] mu Precomputed message representative. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. @@ -418,52 +379,15 @@ int MLD_API_NAMESPACE(signature)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(signature_extmu)( - uint8_t sig[MLDSA_BYTES(MLD_CONFIG_API_PARAMETER_SET)], size_t *siglen, + uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t mu[MLDSA_CRHBYTES], - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context #endif ); -/** - * Compute signed message. This function implements the randomized variant of - * ML-DSA. If you require the deterministic variant, use - * crypto_sign_signature_internal directly. - * - * @param[out] sm Pointer to output signed message (allocated array with - * MLDSA{44,65,87}_BYTES + mlen bytes); can be equal to m. - * @param[out] smlen Pointer to output length of signed message. - * @param[in] m Pointer to message to be signed. - * @param mlen Length of message. - * @param[in] ctx Pointer to context string. - * @param ctxlen Length of context string. - * @param[in] sk Bit-packed secret key. - * @param context Application context. Only present when - * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by - * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. - * - * @retval 0 Success. - * @retval MLD_ERR_OUT_OF_MEMORY MLD_CONFIG_CUSTOM_ALLOC_FREE was - * used and an allocation via - * MLD_CUSTOM_ALLOC returned NULL. - * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded - * MLD_CONFIG_MAX_SIGNING_ATTEMPTS - * iterations. - * @retval MLD_ERR_FAIL Other kinds of failure. - */ -MLD_API_QUALIFIER -MLD_API_MUST_CHECK_RETURN_VALUE -int MLD_API_NAMESPACE(sign)( - uint8_t *sm, size_t *smlen, const uint8_t *m, size_t mlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] -#ifdef MLD_CONFIG_CONTEXT_PARAMETER - , - MLD_CONFIG_CONTEXT_PARAMETER_TYPE context -#endif -); #endif /* !MLD_CONFIG_CORE_API_ONLY */ #endif /* !MLD_CONFIG_NO_SIGN_API */ @@ -471,10 +395,10 @@ int MLD_API_NAMESPACE(sign)( /** * Verify signature. Internal API. * - * @spec{Implements @[FIPS204 Algorithm 8 (ML-DSA.Verify_internal)].} + * @spec{Implements @[FIPS204, Algorithm 8, ML-DSA.Verify_internal].} * - * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. + * @param[in] sig Pointer to input signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] m Pointer to message (when externalmu == 0), or to a * precomputed message representative mu (when * externalmu != 0). @@ -499,9 +423,9 @@ int MLD_API_NAMESPACE(sign)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(verify_internal)( - const uint8_t *sig, size_t siglen, const uint8_t *m, size_t mlen, - const uint8_t *pre, size_t prelen, - const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], + const uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *m, + size_t mlen, const uint8_t *pre, size_t prelen, + const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)], int externalmu #ifdef MLD_CONFIG_CONTEXT_PARAMETER , @@ -513,10 +437,10 @@ int MLD_API_NAMESPACE(verify_internal)( /** * Verify signature. * - * @spec{Implements @[FIPS204 Algorithm 3 (ML-DSA.Verify)].} + * @spec{Implements @[FIPS204, Algorithm 3, ML-DSA.Verify].} * - * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. + * @param[in] sig Pointer to input signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] m Pointer to message. * @param mlen Length of message. * @param[in] ctx Pointer to context string. May be NULL if ctxlen == 0. @@ -534,9 +458,9 @@ int MLD_API_NAMESPACE(verify_internal)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(verify)( - const uint8_t *sig, size_t siglen, const uint8_t *m, size_t mlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + const uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *m, + size_t mlen, const uint8_t *ctx, size_t ctxlen, + const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context @@ -550,10 +474,10 @@ int MLD_API_NAMESPACE(verify)( * 0x00 || ctxlen || ctx || msg for pure ML-DSA). The same mu must have * been used at signing time. * - * @spec{Implements @[FIPS204 Algorithm 3 (ML-DSA.Verify external mu variant)].} + * @spec{Implements @[FIPS204, Algorithm 3, ML-DSA.Verify external mu variant].} * - * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. + * @param[in] sig Pointer to input signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] mu Precomputed message representative. * @param[in] pk Bit-packed public key. * @param context Application context. Only present when @@ -568,45 +492,15 @@ int MLD_API_NAMESPACE(verify)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(verify_extmu)( - const uint8_t *sig, size_t siglen, const uint8_t mu[MLDSA_CRHBYTES], - const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + const uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], + const uint8_t mu[MLDSA_CRHBYTES], + const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context #endif ); -/** - * Verify signed message. - * - * @param[out] m Pointer to output message (allocated array with smlen - * bytes); can be equal to sm. - * @param[out] mlen Pointer to output length of message. - * @param[in] sm Pointer to signed message. - * @param smlen Length of signed message. - * @param[in] ctx Pointer to context string. - * @param ctxlen Length of context string. - * @param[in] pk Bit-packed public key. - * @param context Application context. Only present when - * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by - * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. - * - * @retval 0 Success. - * @retval MLD_ERR_OUT_OF_MEMORY MLD_CONFIG_CUSTOM_ALLOC_FREE was used and an - * allocation via MLD_CUSTOM_ALLOC returned NULL. - * @retval MLD_ERR_FAIL Signature verification failed. - */ -MLD_API_QUALIFIER -MLD_API_MUST_CHECK_RETURN_VALUE -int MLD_API_NAMESPACE(open)( - uint8_t *m, size_t *mlen, const uint8_t *sm, size_t smlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] -#ifdef MLD_CONFIG_CONTEXT_PARAMETER - , - MLD_CONFIG_CONTEXT_PARAMETER_TYPE context -#endif -); #endif /* !MLD_CONFIG_CORE_API_ONLY */ #endif /* !MLD_CONFIG_NO_VERIFY_API */ @@ -628,8 +522,9 @@ int MLD_API_NAMESPACE(open)( #if !defined(MLD_CONFIG_CORE_API_ONLY) #if !defined(MLD_CONFIG_NO_SIGN_API) /** - * FIPS 204: Algorithm 4 HashML-DSA.Sign. Compute signature with pre-hashed - * message. + * Compute signature with pre-hashed message. + * + * @spec{Implements @[FIPS204, Algorithm 4, HashML-DSA.Sign].} * * Supported hash algorithm constants: * MLD_PREHASH_SHA2_224, MLD_PREHASH_SHA2_256, MLD_PREHASH_SHA2_384, @@ -637,17 +532,23 @@ int MLD_API_NAMESPACE(open)( * MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384, * MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256. * + * MLD_PREHASH_NONE is rejected by this API. + * * @warning This is an unstable API that may change in the future. If you need - * a stable API use crypto_sign_signature_pre_hash_shake256. + * a stable API use signature_pre_hash_shake256. + * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use crypto_sign_pk_from_sk + * to validate them before signing. * - * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. + * @param[out] sig Pointer to buffer to hold the generated signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] ph Pointer to pre-hashed message. * @param phlen Length of pre-hashed message. * @param[in] ctx Pointer to context string. * @param ctxlen Length of context string. * @param[in] rnd Random seed. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param hashalg Hash algorithm constant (one of MLD_PREHASH_*). * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by @@ -660,15 +561,17 @@ int MLD_API_NAMESPACE(open)( * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS * iterations. + * @retval MLD_ERR_SIGNING_PAUSED A MLD_CONFIG_SIGN_HOOK_ATTEMPT hook + * paused signing; re-invoke to resume. * @retval MLD_ERR_FAIL Other kinds of failure. */ MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(signature_pre_hash_internal)( - uint8_t sig[MLDSA_BYTES(MLD_CONFIG_API_PARAMETER_SET)], size_t *siglen, - const uint8_t *ph, size_t phlen, const uint8_t *ctx, size_t ctxlen, + uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *ph, + size_t phlen, const uint8_t *ctx, size_t ctxlen, const uint8_t rnd[MLDSA_RNDBYTES], - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], + const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)], int hashalg #ifdef MLD_CONFIG_CONTEXT_PARAMETER , @@ -679,8 +582,9 @@ int MLD_API_NAMESPACE(signature_pre_hash_internal)( #if !defined(MLD_CONFIG_NO_VERIFY_API) /** - * FIPS 204: Algorithm 5 HashML-DSA.Verify. Verifies signature with pre-hashed - * message. + * Verifies signature with pre-hashed message. + * + * @spec{Implements @[FIPS204, Algorithm 5, HashML-DSA.Verify].} * * Supported hash algorithm constants: * MLD_PREHASH_SHA2_224, MLD_PREHASH_SHA2_256, MLD_PREHASH_SHA2_384, @@ -688,11 +592,13 @@ int MLD_API_NAMESPACE(signature_pre_hash_internal)( * MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384, * MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256. * + * MLD_PREHASH_NONE is rejected by this API. + * * @warning This is an unstable API that may change in the future. If you need - * a stable API use crypto_sign_verify_pre_hash_shake256. + * a stable API use verify_pre_hash_shake256. * - * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. + * @param[in] sig Pointer to input signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] ph Pointer to pre-hashed message. * @param phlen Length of pre-hashed message. * @param[in] ctx Pointer to context string. @@ -711,9 +617,9 @@ int MLD_API_NAMESPACE(signature_pre_hash_internal)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(verify_pre_hash_internal)( - const uint8_t *sig, size_t siglen, const uint8_t *ph, size_t phlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], + const uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *ph, + size_t phlen, const uint8_t *ctx, size_t ctxlen, + const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)], int hashalg #ifdef MLD_CONFIG_CONTEXT_PARAMETER , @@ -724,19 +630,24 @@ int MLD_API_NAMESPACE(verify_pre_hash_internal)( #if !defined(MLD_CONFIG_NO_SIGN_API) /** - * FIPS 204: Algorithm 4 HashML-DSA.Sign with SHAKE256. - * * Compute signature with pre-hashed message using SHAKE256. This function * computes the SHAKE256 hash of the message internally. * - * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. + * @spec{Implements @[FIPS204, Algorithm 4, HashML-DSA.Sign] with SHAKE256 as + * the pre-hash.} + * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use crypto_sign_pk_from_sk + * to validate them before signing. + * + * @param[out] sig Pointer to buffer to hold the generated signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] m Pointer to message to be hashed and signed. * @param mlen Length of message. * @param[in] ctx Pointer to context string. * @param ctxlen Length of context string. * @param[in] rnd Random seed. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. @@ -748,15 +659,17 @@ int MLD_API_NAMESPACE(verify_pre_hash_internal)( * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS * iterations. + * @retval MLD_ERR_SIGNING_PAUSED A MLD_CONFIG_SIGN_HOOK_ATTEMPT hook + * paused signing; re-invoke to resume. * @retval MLD_ERR_FAIL Other kinds of failure. */ MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(signature_pre_hash_shake256)( - uint8_t sig[MLDSA_BYTES(MLD_CONFIG_API_PARAMETER_SET)], size_t *siglen, - const uint8_t *m, size_t mlen, const uint8_t *ctx, size_t ctxlen, + uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *m, + size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t rnd[MLDSA_RNDBYTES], - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context @@ -766,13 +679,14 @@ int MLD_API_NAMESPACE(signature_pre_hash_shake256)( #if !defined(MLD_CONFIG_NO_VERIFY_API) /** - * FIPS 204: Algorithm 5 HashML-DSA.Verify with SHAKE256. - * * Verify signature with pre-hashed message using SHAKE256. This function * computes the SHAKE256 hash of the message internally. * - * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. + * @spec{Implements @[FIPS204, Algorithm 5, HashML-DSA.Verify] with SHAKE256 as + * the pre-hash.} + * + * @param[in] sig Pointer to input signature of + * MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) bytes. * @param[in] m Pointer to message to be hashed and verified. * @param mlen Length of message. * @param[in] ctx Pointer to context string. @@ -790,9 +704,9 @@ int MLD_API_NAMESPACE(signature_pre_hash_shake256)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(verify_pre_hash_shake256)( - const uint8_t *sig, size_t siglen, const uint8_t *m, size_t mlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + const uint8_t sig[MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET)], const uint8_t *m, + size_t mlen, const uint8_t *ctx, size_t ctxlen, + const uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context @@ -817,15 +731,15 @@ int MLD_API_NAMESPACE(verify_pre_hash_shake256)( * This function is useful for building incremental signing APIs. * * @spec{For HashML-DSA (hashalg != MLD_PREHASH_NONE), implements - * @[FIPS204, Algorithm 4, L23]. For Pure ML-DSA (hashalg == MLD_PREHASH_NONE), - * implements + * @[FIPS204, Algorithm 4, line 23]. For Pure ML-DSA + * (hashalg == MLD_PREHASH_NONE), implements * ``` * M' <- BytesToBits(IntegerToBytes(0, 1) * || IntegerToBytes(|ctx|, 1) * || ctx * ``` - * which is part of @[FIPS204, Algorithm 2 (ML-DSA.Sign), L10] and - * @[FIPS204, Algorithm 3 (ML-DSA.Verify), L5].} + * which is part of @[FIPS204, Algorithm 2, ML-DSA.Sign, line 10] and + * @[FIPS204, Algorithm 3, ML-DSA.Verify, line 5].} * * @param[out] prefix Output domain separation prefix buffer. * @param[in] ph Pointer to pre-hashed message (ignored for pure @@ -849,7 +763,7 @@ size_t MLD_API_NAMESPACE(prepare_domain_separation_prefix)( * Perform basic validity checks on secret key, and derive public key. * * Referring to the decoding of the secret key `sk=(rho, K, tr, s1, s2, t0)` - * (cf. @[FIPS204, Algorithm 25 skDecode]), the following checks are + * (cf. @[FIPS204, Algorithm 25, skDecode]), the following checks are * performed: * - Check that s1 and s2 have coefficients in [-MLDSA_ETA, MLDSA_ETA]. * - Check that t0 and tr stored in sk match recomputed values. @@ -871,8 +785,8 @@ size_t MLD_API_NAMESPACE(prepare_domain_separation_prefix)( MLD_API_QUALIFIER MLD_API_MUST_CHECK_RETURN_VALUE int MLD_API_NAMESPACE(pk_from_sk)( - uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)], - const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET)] + uint8_t pk[MLDSA_PUBLICKEYBYTES(MLD_CONFIG_PARAMETER_SET)], + const uint8_t sk[MLDSA_SECRETKEYBYTES(MLD_CONFIG_PARAMETER_SET)] #ifdef MLD_CONFIG_CONTEXT_PARAMETER , MLD_CONFIG_CONTEXT_PARAMETER_TYPE context @@ -885,43 +799,9 @@ int MLD_API_NAMESPACE(pk_from_sk)( } #endif -/****************************** SUPERCOP API *********************************/ - -#if !defined(MLD_CONFIG_API_NO_SUPERCOP) -/* Export API in SUPERCOP naming scheme CRYPTO_xxx / crypto_sign_xxx */ -#define CRYPTO_SECRETKEYBYTES MLDSA_SECRETKEYBYTES(MLD_CONFIG_API_PARAMETER_SET) -#define CRYPTO_PUBLICKEYBYTES MLDSA_PUBLICKEYBYTES(MLD_CONFIG_API_PARAMETER_SET) -#define CRYPTO_BYTES MLDSA_BYTES(MLD_CONFIG_API_PARAMETER_SET) - -#define crypto_sign_keypair MLD_API_NAMESPACE(keypair) -#define crypto_sign_signature MLD_API_NAMESPACE(signature) -#define crypto_sign MLD_API_NAMESPACE(sign) -#define crypto_sign_verify MLD_API_NAMESPACE(verify) -#define crypto_sign_open MLD_API_NAMESPACE(open) - -#else /* !MLD_CONFIG_API_NO_SUPERCOP */ - -/* If the SUPERCOP API is not needed, we can undefine the various helper macros - * above. Otherwise, they are needed for lazy evaluation of crypto_sign_xxx. */ -#if !defined(MLD_API_LEGACY_CONFIG) -#undef MLD_CONFIG_API_PARAMETER_SET -#undef MLD_CONFIG_API_NAMESPACE_PREFIX -#undef MLD_CONFIG_API_NO_SUPERCOP -#undef MLD_CONFIG_API_CONSTANTS_ONLY -#undef MLD_CONFIG_API_QUALIFIER -#endif /* !MLD_API_LEGACY_CONFIG */ - -#undef MLD_API_CONCAT -#undef MLD_API_CONCAT_ -#undef MLD_API_CONCAT_UNDERSCORE -#undef MLD_API_NAMESPACE -#undef MLD_API_MUST_CHECK_RETURN_VALUE -#undef MLD_API_QUALIFIER -#undef MLD_API_LEGACY_CONFIG - -#endif /* MLD_CONFIG_API_NO_SUPERCOP */ -#endif /* !MLD_CONFIG_API_CONSTANTS_ONLY */ +#undef MLD_API_NAMESPACE_PREFIX +#endif /* !MLD_CONFIG_CONSTANTS_ONLY */ /***************************** Memory Usage **********************************/ @@ -941,7 +821,7 @@ int MLD_API_NAMESPACE(pk_from_sk)( * fixed-sized buffer and a simple allocator (e.g., bump allocator). */ /* check-magic: off */ -#if defined(MLD_API_LEGACY_CONFIG) || !defined(MLD_CONFIG_REDUCE_RAM) +#if !defined(MLD_CONFIG_REDUCE_RAM) #define MLD_TOTAL_ALLOC_44_KEYPAIR_NO_PCT 26912 #define MLD_TOTAL_ALLOC_44_KEYPAIR_PCT 48480 #define MLD_TOTAL_ALLOC_44_PK_FROM_SK 28480 @@ -957,7 +837,7 @@ int MLD_API_NAMESPACE(pk_from_sk)( #define MLD_TOTAL_ALLOC_87_PK_FROM_SK 78272 #define MLD_TOTAL_ALLOC_87_SIGN 108224 #define MLD_TOTAL_ALLOC_87_VERIFY 68800 -#else /* MLD_API_LEGACY_CONFIG || !MLD_CONFIG_REDUCE_RAM */ +#else /* !MLD_CONFIG_REDUCE_RAM */ #define MLD_TOTAL_ALLOC_44_KEYPAIR_NO_PCT 11584 #define MLD_TOTAL_ALLOC_44_KEYPAIR_PCT 16896 #define MLD_TOTAL_ALLOC_44_PK_FROM_SK 13152 @@ -973,15 +853,13 @@ int MLD_API_NAMESPACE(pk_from_sk)( #define MLD_TOTAL_ALLOC_87_PK_FROM_SK 21984 #define MLD_TOTAL_ALLOC_87_SIGN 21344 #define MLD_TOTAL_ALLOC_87_VERIFY 12512 -#endif /* !(MLD_API_LEGACY_CONFIG || !MLD_CONFIG_REDUCE_RAM) */ +#endif /* MLD_CONFIG_REDUCE_RAM */ /* check-magic: on */ /* * MLD_TOTAL_ALLOC_*_KEYPAIR adapts based on MLD_CONFIG_KEYGEN_PCT. - * For legacy config, we don't know which options are used, so assume - * the worst case (PCT enabled). */ -#if defined(MLD_API_LEGACY_CONFIG) || defined(MLD_CONFIG_KEYGEN_PCT) +#if defined(MLD_CONFIG_KEYGEN_PCT) #define MLD_TOTAL_ALLOC_44_KEYPAIR MLD_TOTAL_ALLOC_44_KEYPAIR_PCT #define MLD_TOTAL_ALLOC_65_KEYPAIR MLD_TOTAL_ALLOC_65_KEYPAIR_PCT #define MLD_TOTAL_ALLOC_87_KEYPAIR MLD_TOTAL_ALLOC_87_KEYPAIR_PCT diff --git a/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native_bcm.c b/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native_bcm.c index 69dd3e7e1b..3c1a77ce27 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native_bcm.c +++ b/crypto/fipsmodule/ml_dsa/mldsa/mldsa_native_bcm.c @@ -78,7 +78,8 @@ #endif /* MLD_SYS_AARCH64 */ #if defined(MLD_SYS_X86_64) #include "native/x86_64/src/consts.c" -#endif /* MLD_SYS_X86_64 */ +#include "native/x86_64/src/rej_uniform_table.c" +#endif #endif /* MLD_CONFIG_USE_NATIVE_BACKEND_ARITH */ @@ -97,9 +98,6 @@ * Undefine macros from MLD_CONFIG_PARAMETER_SET-specific files */ /* mldsa/mldsa_native.h */ -#undef CRYPTO_BYTES -#undef CRYPTO_PUBLICKEYBYTES -#undef CRYPTO_SECRETKEYBYTES #undef MLDSA44_BYTES #undef MLDSA44_CRHBYTES #undef MLDSA44_PUBLICKEYBYTES @@ -134,19 +132,15 @@ #undef MLD_API_CONCAT #undef MLD_API_CONCAT_ #undef MLD_API_CONCAT_UNDERSCORE -#undef MLD_API_LEGACY_CONFIG #undef MLD_API_MUST_CHECK_RETURN_VALUE #undef MLD_API_NAMESPACE +#undef MLD_API_NAMESPACE_PREFIX #undef MLD_API_QUALIFIER -#undef MLD_CONFIG_API_CONSTANTS_ONLY -#undef MLD_CONFIG_API_NAMESPACE_PREFIX -#undef MLD_CONFIG_API_NO_SUPERCOP -#undef MLD_CONFIG_API_PARAMETER_SET -#undef MLD_CONFIG_API_QUALIFIER #undef MLD_DOMAIN_SEPARATION_MAX_BYTES #undef MLD_ERR_FAIL #undef MLD_ERR_OUT_OF_MEMORY #undef MLD_ERR_RNG_FAIL +#undef MLD_ERR_SIGNING_PAUSED #undef MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED #undef MLD_H #undef MLD_MAX3_ @@ -185,11 +179,6 @@ #undef MLD_TOTAL_ALLOC_87_PK_FROM_SK #undef MLD_TOTAL_ALLOC_87_SIGN #undef MLD_TOTAL_ALLOC_87_VERIFY -#undef crypto_sign -#undef crypto_sign_keypair -#undef crypto_sign_open -#undef crypto_sign_signature -#undef crypto_sign_verify /* mldsa/src/common.h */ #undef MLD_ADD_PARAM_SET #undef MLD_ALLOC @@ -202,20 +191,11 @@ #undef MLD_COMMON_H #undef MLD_CONCAT #undef MLD_CONCAT_ -#undef MLD_CONTEXT_PARAMETERS_0 -#undef MLD_CONTEXT_PARAMETERS_1 -#undef MLD_CONTEXT_PARAMETERS_2 -#undef MLD_CONTEXT_PARAMETERS_3 -#undef MLD_CONTEXT_PARAMETERS_4 -#undef MLD_CONTEXT_PARAMETERS_5 -#undef MLD_CONTEXT_PARAMETERS_6 -#undef MLD_CONTEXT_PARAMETERS_7 -#undef MLD_CONTEXT_PARAMETERS_8 -#undef MLD_CONTEXT_PARAMETERS_9 #undef MLD_EMPTY_CU #undef MLD_ERR_FAIL #undef MLD_ERR_OUT_OF_MEMORY #undef MLD_ERR_RNG_FAIL +#undef MLD_ERR_SIGNING_PAUSED #undef MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED #undef MLD_EXTERNAL_API #undef MLD_FIPS202X4_HEADER_FILE @@ -297,6 +277,7 @@ #undef MLDSA_SK_TR_OFFSET #undef MLDSA_TAU #undef MLDSA_TRBYTES +#undef MLD_MAX_KAPPA #undef MLD_PARAMS_H /* mldsa/src/poly_kl.h */ #undef MLD_POLYETA_UNPACK_LOWER_BOUND @@ -409,10 +390,8 @@ #undef MLD_PREHASH_SHAKE_256 #undef MLD_SIGN_H #undef mld_prepare_domain_separation_prefix -#undef mld_sign #undef mld_sign_keypair #undef mld_sign_keypair_internal -#undef mld_sign_open #undef mld_sign_pk_from_sk #undef mld_sign_signature #undef mld_sign_signature_extmu @@ -429,6 +408,22 @@ /* * Undefine macros from MLD_CONFIG_PARAMETER_SET-generic files */ +/* mldsa/src/context.h */ +#undef MLD_CONTEXT_H +#undef MLD_CONTEXT_PARAMETERS_0 +#undef MLD_CONTEXT_PARAMETERS_1 +#undef MLD_CONTEXT_PARAMETERS_2 +#undef MLD_CONTEXT_PARAMETERS_3 +#undef MLD_CONTEXT_PARAMETERS_4 +#undef MLD_CONTEXT_PARAMETERS_5 +#undef MLD_CONTEXT_PARAMETERS_6 +#undef MLD_CONTEXT_PARAMETERS_7 +#undef MLD_CONTEXT_PARAMETERS_8 +#undef MLD_CONTEXT_PARAMETERS_9 +#undef MLD_CONTEXT_UNUSED +#undef mld_sign_attempt +#undef mld_sign_finish +#undef mld_sign_resume /* mldsa/src/ct.h */ #undef MLD_CT_H #undef MLD_USE_ASM_VALUE_BARRIER @@ -509,6 +504,8 @@ #undef MLD_MUST_CHECK_RETURN_VALUE #undef MLD_RESTRICT #undef MLD_STATIC_TESTABLE +#undef MLD_SYSV_ABI +#undef MLD_SYSV_ABI_SUPPORTED #undef MLD_SYS_AARCH64 #undef MLD_SYS_AARCH64_EB #undef MLD_SYS_APPLE @@ -633,16 +630,16 @@ #undef mld_pointwise_acc_l7_avx2_asm #undef mld_pointwise_avx2_asm #undef mld_poly_caddq_avx2_asm -#undef mld_poly_chknorm_avx2 -#undef mld_poly_decompose_32_avx2 -#undef mld_poly_decompose_88_avx2 -#undef mld_poly_use_hint_32_avx2 -#undef mld_poly_use_hint_88_avx2 -#undef mld_polyz_unpack_17_avx2 -#undef mld_polyz_unpack_19_avx2 -#undef mld_rej_uniform_avx2 -#undef mld_rej_uniform_eta2_avx2 -#undef mld_rej_uniform_eta4_avx2 +#undef mld_poly_chknorm_avx2_asm +#undef mld_poly_decompose_32_avx2_asm +#undef mld_poly_decompose_88_avx2_asm +#undef mld_poly_use_hint_32_avx2_asm +#undef mld_poly_use_hint_88_avx2_asm +#undef mld_polyz_unpack_17_avx2_asm +#undef mld_polyz_unpack_19_avx2_asm +#undef mld_rej_uniform_avx2_asm +#undef mld_rej_uniform_eta2_avx2_asm +#undef mld_rej_uniform_eta4_avx2_asm #undef mld_rej_uniform_table /* mldsa/src/native/x86_64/src/consts.h */ #undef MLD_AVX2_BACKEND_DATA_OFFSET_8XDIV diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/arith_native_aarch64.h b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/arith_native_aarch64.h index 94e113016b..81a39fdb76 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/arith_native_aarch64.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/arith_native_aarch64.h @@ -61,8 +61,8 @@ MLD_INTERNAL_DATA_DECLARATION const uint8_t mld_polyz_unpack_19_indices[64]; #define MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN (2 * 136) #define mld_ntt_aarch64_asm MLD_NAMESPACE(ntt_aarch64_asm) -void mld_ntt_aarch64_asm(int32_t *r, const int32_t *zetas_l123456, - const int32_t *zetas_l78) +void mld_ntt_aarch64_asm(int32_t r[MLDSA_N], const int32_t zetas_l123456[144], + const int32_t zetas_l78[384]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/ntt_aarch64_asm.ml */ __contract__( @@ -78,8 +78,8 @@ __contract__( ); #define mld_intt_aarch64_asm MLD_NAMESPACE(intt_aarch64_asm) -void mld_intt_aarch64_asm(int32_t *r, const int32_t *zetas_l78, - const int32_t *zetas_l123456) +void mld_intt_aarch64_asm(int32_t r[MLDSA_N], const int32_t zetas_l78[384], + const int32_t zetas_l123456[160]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/intt_aarch64_asm.ml */ __contract__( @@ -95,8 +95,8 @@ __contract__( #define mld_rej_uniform_aarch64_asm MLD_NAMESPACE(rej_uniform_aarch64_asm) MLD_MUST_CHECK_RETURN_VALUE -uint64_t mld_rej_uniform_aarch64_asm(int32_t *r, const uint8_t *buf, - unsigned buflen, const uint8_t *table) +uint64_t mld_rej_uniform_aarch64_asm(int32_t r[MLDSA_N], const uint8_t *buf, + unsigned buflen, const uint8_t table[256]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/rej_uniform_aarch64_asm.ml. */ __contract__( @@ -113,8 +113,9 @@ __contract__( #define mld_rej_uniform_eta2_aarch64_asm \ MLD_NAMESPACE(rej_uniform_eta2_aarch64_asm) MLD_MUST_CHECK_RETURN_VALUE -uint64_t mld_rej_uniform_eta2_aarch64_asm(int32_t *r, const uint8_t *buf, - unsigned buflen, const uint8_t *table) +uint64_t mld_rej_uniform_eta2_aarch64_asm(int32_t r[MLDSA_N], + const uint8_t *buf, unsigned buflen, + const uint8_t table[4096]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/rej_uniform_eta2_aarch64_asm.ml */ __contract__( @@ -132,8 +133,9 @@ __contract__( #define mld_rej_uniform_eta4_aarch64_asm \ MLD_NAMESPACE(rej_uniform_eta4_aarch64_asm) MLD_MUST_CHECK_RETURN_VALUE -uint64_t mld_rej_uniform_eta4_aarch64_asm(int32_t *r, const uint8_t *buf, - unsigned buflen, const uint8_t *table) +uint64_t mld_rej_uniform_eta4_aarch64_asm(int32_t r[MLDSA_N], + const uint8_t *buf, unsigned buflen, + const uint8_t table[4096]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/rej_uniform_eta4_aarch64_asm.ml */ __contract__( @@ -152,7 +154,7 @@ __contract__( #if !defined(MLD_CONFIG_NO_SIGN_API) #define mld_poly_decompose_32_aarch64_asm \ MLD_NAMESPACE(poly_decompose_32_aarch64_asm) -void mld_poly_decompose_32_aarch64_asm(int32_t *a1, int32_t *a0) +void mld_poly_decompose_32_aarch64_asm(int32_t a1[MLDSA_N], int32_t a0[MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/poly_decompose_32_aarch64_asm.ml */ __contract__( @@ -169,7 +171,7 @@ __contract__( #define mld_poly_decompose_88_aarch64_asm \ MLD_NAMESPACE(poly_decompose_88_aarch64_asm) -void mld_poly_decompose_88_aarch64_asm(int32_t *a1, int32_t *a0) +void mld_poly_decompose_88_aarch64_asm(int32_t a1[MLDSA_N], int32_t a0[MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/poly_decompose_88_aarch64_asm.ml */ __contract__( @@ -186,7 +188,7 @@ __contract__( #endif /* !MLD_CONFIG_NO_SIGN_API */ #define mld_poly_caddq_aarch64_asm MLD_NAMESPACE(poly_caddq_aarch64_asm) -void mld_poly_caddq_aarch64_asm(int32_t *a) +void mld_poly_caddq_aarch64_asm(int32_t a[MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/poly_caddq_aarch64_asm.ml */ __contract__( @@ -199,7 +201,8 @@ __contract__( #if !defined(MLD_CONFIG_NO_VERIFY_API) #define mld_poly_use_hint_32_aarch64_asm \ MLD_NAMESPACE(poly_use_hint_32_aarch64_asm) -void mld_poly_use_hint_32_aarch64_asm(int32_t *a, const int32_t *h) +void mld_poly_use_hint_32_aarch64_asm(int32_t a[MLDSA_N], + const int32_t h[MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/poly_use_hint_32_aarch64_asm.ml */ __contract__( @@ -213,7 +216,8 @@ __contract__( #define mld_poly_use_hint_88_aarch64_asm \ MLD_NAMESPACE(poly_use_hint_88_aarch64_asm) -void mld_poly_use_hint_88_aarch64_asm(int32_t *a, const int32_t *h) +void mld_poly_use_hint_88_aarch64_asm(int32_t a[MLDSA_N], + const int32_t h[MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/poly_use_hint_88_aarch64_asm.ml */ __contract__( @@ -228,7 +232,7 @@ __contract__( #define mld_poly_chknorm_aarch64_asm MLD_NAMESPACE(poly_chknorm_aarch64_asm) MLD_MUST_CHECK_RETURN_VALUE -int mld_poly_chknorm_aarch64_asm(const int32_t *a, int32_t B) +int mld_poly_chknorm_aarch64_asm(const int32_t a[MLDSA_N], int32_t B) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/poly_chknorm_aarch64_asm.ml */ __contract__( @@ -243,8 +247,8 @@ __contract__( #if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 #define mld_polyz_unpack_17_aarch64_asm \ MLD_NAMESPACE(polyz_unpack_17_aarch64_asm) -void mld_polyz_unpack_17_aarch64_asm(int32_t *r, const uint8_t *buf, - const uint8_t *indices) +void mld_polyz_unpack_17_aarch64_asm(int32_t r[MLDSA_N], const uint8_t buf[576], + const uint8_t indices[64]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/polyz_unpack_17_aarch64_asm.ml */ __contract__( @@ -261,8 +265,8 @@ __contract__( (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) #define mld_polyz_unpack_19_aarch64_asm \ MLD_NAMESPACE(polyz_unpack_19_aarch64_asm) -void mld_polyz_unpack_19_aarch64_asm(int32_t *r, const uint8_t *buf, - const uint8_t *indices) +void mld_polyz_unpack_19_aarch64_asm(int32_t r[MLDSA_N], const uint8_t buf[640], + const uint8_t indices[64]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/polyz_unpack_19_aarch64_asm.ml */ __contract__( @@ -280,7 +284,8 @@ __contract__( defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) #define mld_poly_pointwise_montgomery_aarch64_asm \ MLD_NAMESPACE(poly_pointwise_montgomery_aarch64_asm) -void mld_poly_pointwise_montgomery_aarch64_asm(int32_t *a, const int32_t *b) +void mld_poly_pointwise_montgomery_aarch64_asm(int32_t a[MLDSA_N], + const int32_t b[MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/aarch64/proofs/pointwise_montgomery_aarch64_asm.ml */ __contract__( @@ -301,7 +306,8 @@ __contract__( #define mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm \ MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l4_aarch64_asm) void mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm( - int32_t *r, const int32_t a[4][MLDSA_N], const int32_t b[4][MLDSA_N]) + int32_t r[MLDSA_N], const int32_t a[4][MLDSA_N], + const int32_t b[4][MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in * proofs/hol_light/aarch64/proofs/mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm.ml @@ -320,7 +326,8 @@ __contract__( #define mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm \ MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l5_aarch64_asm) void mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm( - int32_t *r, const int32_t a[5][MLDSA_N], const int32_t b[5][MLDSA_N]) + int32_t r[MLDSA_N], const int32_t a[5][MLDSA_N], + const int32_t b[5][MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in * proofs/hol_light/aarch64/proofs/mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm.ml @@ -339,7 +346,8 @@ __contract__( #define mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm \ MLD_NAMESPACE(polyvecl_pointwise_acc_montgomery_l7_aarch64_asm) void mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm( - int32_t *r, const int32_t a[7][MLDSA_N], const int32_t b[7][MLDSA_N]) + int32_t r[MLDSA_N], const int32_t a[7][MLDSA_N], + const int32_t b[7][MLDSA_N]) /* This must be kept in sync with the HOL-Light specification * in * proofs/hol_light/aarch64/proofs/mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm.ml diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/intt_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/intt_aarch64_asm.S index 886adfe238..7b3657a99c 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/intt_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/intt_aarch64_asm.S @@ -14,13 +14,45 @@ * Becker, Hwang, Kannwischer, Yang, Yang * https://eprint.iacr.org/2021/986 * + * - [NeonNTT_Autoformalised] + * Neon NTT - (Auto)formalised + * Hanno Becker + * https://eprint.iacr.org/2026/1223 + * * - [SLOTHY_Paper] * Fast and Clean: Auditable high-performance assembly via constraint solving * Abdulrahman, Becker, Kannwischer, Klein * https://eprint.iacr.org/2022/1303 */ -/* AArch64 ML-DSA inverse NTT following @[NeonNTT] and @[SLOTHY_Paper] */ +/* AArch64 ML-DSA inverse NTT following @[NeonNTT], @[SLOTHY_Paper], and @[NeonNTT_Autoformalised] */ + +/*yaml + Name: intt_aarch64_asm + Description: AArch64 ML-DSA inverse NTT + Signature: void mld_intt_aarch64_asm(int32_t r[256], const int32_t zetas_l78[384], const int32_t zetas_l123456[160]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t r[256] + description: Input/output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int32_t zetas_l78[384] + description: Twiddle factors for layers 7-8 (384 x int32_t) + x2: + type: buffer + size_bytes: 640 + permissions: read-only + c_parameter: const int32_t zetas_l123456[160] + description: Twiddle factors for layers 1-6 (160 x int32_t) +*/ #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm.S index 8252f91637..8991096030 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm.S @@ -2,6 +2,33 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l4_aarch64_asm + Description: AArch64 pointwise multiply-accumulate of length-4 polynomial vectors + Signature: void mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm(int32_t r[256], const int32_t a[4][256], const int32_t b[4][256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t a[4][256] + description: Input polynomial vector a (4 x 256 x int32_t) + x2: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t b[4][256] + description: Input polynomial vector b (4 x 256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm.S index db1e70a48b..82d597aac5 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm.S @@ -2,6 +2,33 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l5_aarch64_asm + Description: AArch64 pointwise multiply-accumulate of length-5 polynomial vectors + Signature: void mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm(int32_t r[256], const int32_t a[5][256], const int32_t b[5][256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t a[5][256] + description: Input polynomial vector a (5 x 256 x int32_t) + x2: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t b[5][256] + description: Input polynomial vector b (5 x 256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm.S index 821b5eadcb..735ccab5e5 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm.S @@ -2,6 +2,33 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: polyvecl_pointwise_acc_montgomery_l7_aarch64_asm + Description: AArch64 pointwise multiply-accumulate of length-7 polynomial vectors + Signature: void mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm(int32_t r[256], const int32_t a[7][256], const int32_t b[7][256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t a[7][256] + description: Input polynomial vector a (7 x 256 x int32_t) + x2: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t b[7][256] + description: Input polynomial vector b (7 x 256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/ntt_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/ntt_aarch64_asm.S index 9488daf695..b99a7a9276 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/ntt_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/ntt_aarch64_asm.S @@ -14,13 +14,45 @@ * Becker, Hwang, Kannwischer, Yang, Yang * https://eprint.iacr.org/2021/986 * + * - [NeonNTT_Autoformalised] + * Neon NTT - (Auto)formalised + * Hanno Becker + * https://eprint.iacr.org/2026/1223 + * * - [SLOTHY_Paper] * Fast and Clean: Auditable high-performance assembly via constraint solving * Abdulrahman, Becker, Kannwischer, Klein * https://eprint.iacr.org/2022/1303 */ -/* AArch64 ML-DSA forward NTT following @[NeonNTT] and @[SLOTHY_Paper] */ +/* AArch64 ML-DSA forward NTT following @[NeonNTT], @[SLOTHY_Paper], and @[NeonNTT_Autoformalised] */ + +/*yaml + Name: ntt_aarch64_asm + Description: AArch64 ML-DSA forward NTT + Signature: void mld_ntt_aarch64_asm(int32_t r[256], const int32_t zetas_l123456[144], const int32_t zetas_l78[384]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t r[256] + description: Input/output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 576 + permissions: read-only + c_parameter: const int32_t zetas_l123456[144] + description: Twiddle factors for layers 1-6 (144 x int32_t) + x2: + type: buffer + size_bytes: 1536 + permissions: read-only + c_parameter: const int32_t zetas_l78[384] + description: Twiddle factors for layers 7-8 (384 x int32_t) +*/ #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/pointwise_montgomery_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/pointwise_montgomery_aarch64_asm.S index 89908c8fc4..798cce48b7 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/pointwise_montgomery_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/pointwise_montgomery_aarch64_asm.S @@ -2,6 +2,27 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_pointwise_montgomery_aarch64_asm + Description: AArch64 pointwise Montgomery multiplication of two polynomials + Signature: void mld_poly_pointwise_montgomery_aarch64_asm(int32_t a[256], const int32_t b[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t b[256] + description: Input polynomial (256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_caddq_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_caddq_aarch64_asm.S index 58927648c4..d7aaa45453 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_caddq_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_caddq_aarch64_asm.S @@ -2,6 +2,21 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_caddq_aarch64_asm + Description: AArch64 conditional addition of q to each coefficient + Signature: void mld_poly_caddq_aarch64_asm(int32_t a[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial (256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_chknorm_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_chknorm_aarch64_asm.S index af2df2534b..a27a018a36 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_chknorm_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_chknorm_aarch64_asm.S @@ -2,6 +2,26 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_chknorm_aarch64_asm + Description: AArch64 infinity-norm bound check on polynomial coefficients + Signature: int mld_poly_chknorm_aarch64_asm(const int32_t a[256], int32_t B) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t a[256] + description: Input polynomial (256 x int32_t) + x1: + type: scalar + c_parameter: int32_t B + description: Norm bound + test_with: 131072 # representative non-negative bound (1 << 17) +*/ + #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_32_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_32_aarch64_asm.S index 24ad69dd40..30768f1f9a 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_32_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_32_aarch64_asm.S @@ -2,6 +2,27 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_decompose_32_aarch64_asm + Description: AArch64 coefficient decomposition (alpha = (Q-1)/32) + Signature: void mld_poly_decompose_32_aarch64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output high-part polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial / output low-part (256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_88_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_88_aarch64_asm.S index df79653850..983b245d4f 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_88_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_decompose_88_aarch64_asm.S @@ -2,6 +2,27 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_decompose_88_aarch64_asm + Description: AArch64 coefficient decomposition (alpha = (Q-1)/88) + Signature: void mld_poly_decompose_88_aarch64_asm(int32_t a1[256], int32_t a0[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t a1[256] + description: Output high-part polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a0[256] + description: Input polynomial / output low-part (256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_32_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_32_aarch64_asm.S index d20fcefa97..7e39eefa54 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_32_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_32_aarch64_asm.S @@ -2,6 +2,27 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_use_hint_32_aarch64_asm + Description: AArch64 hint application (alpha = (Q-1)/32) + Signature: void mld_poly_use_hint_32_aarch64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial (256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_88_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_88_aarch64_asm.S index 15161c7f6c..6828960b5f 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_88_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/poly_use_hint_88_aarch64_asm.S @@ -2,6 +2,27 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: poly_use_hint_88_aarch64_asm + Description: AArch64 hint application (alpha = (Q-1)/88) + Signature: void mld_poly_use_hint_88_aarch64_asm(int32_t a[256], const int32_t h[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t a[256] + description: Input/output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t h[256] + description: Hint polynomial (256 x int32_t) +*/ + #include "_internal_s2n_bignum_arm.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_17_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_17_aarch64_asm.S index 0775d4cb9a..72dc34488d 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_17_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_17_aarch64_asm.S @@ -4,6 +4,33 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: polyz_unpack_17_aarch64_asm + Description: AArch64 unpacking of 17-bit packed coefficients + Signature: void mld_polyz_unpack_17_aarch64_asm(int32_t r[256], const uint8_t buf[576], const uint8_t indices[64]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 576 + permissions: read-only + c_parameter: const uint8_t buf[576] + description: Packed input bytes + x2: + type: buffer + size_bytes: 64 + permissions: read-only + c_parameter: const uint8_t indices[64] + description: Permutation index table (64 x uint8_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_19_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_19_aarch64_asm.S index 267ad4e909..3fa0c91dc5 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_19_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/polyz_unpack_19_aarch64_asm.S @@ -4,6 +4,33 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: polyz_unpack_19_aarch64_asm + Description: AArch64 unpacking of 19-bit packed coefficients + Signature: void mld_polyz_unpack_19_aarch64_asm(int32_t r[256], const uint8_t buf[640], const uint8_t indices[64]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output polynomial (256 x int32_t) + x1: + type: buffer + size_bytes: 640 + permissions: read-only + c_parameter: const uint8_t buf[640] + description: Packed input bytes + x2: + type: buffer + size_bytes: 64 + permissions: read-only + c_parameter: const uint8_t indices[64] + description: Permutation index table (64 x uint8_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_aarch64_asm.S index 4e261ccdd0..36e50f4da5 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_aarch64_asm.S @@ -4,6 +4,38 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: rej_uniform_aarch64_asm + Description: AArch64 rejection sampling of uniform coefficients mod q + Signature: uint64_t mld_rej_uniform_aarch64_asm(int32_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[256]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output buffer (256 x int32_t) + x1: + type: buffer + size_bytes: x2 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + x2: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be a multiple of 24) + test_with: 840 # MLD_POLY_UNIFORM_NBLOCKS * SHAKE128_RATE = 5 * 168 + x3: + type: buffer + size_bytes: 256 + permissions: read-only + c_parameter: const uint8_t table[256] + description: Lookup table (256 x uint8_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta2_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta2_aarch64_asm.S index b79d4bc60c..1a62bd5abd 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta2_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta2_aarch64_asm.S @@ -4,6 +4,38 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: rej_uniform_eta2_aarch64_asm + Description: AArch64 rejection sampling of eta=2 secret coefficients + Signature: uint64_t mld_rej_uniform_eta2_aarch64_asm(int32_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output buffer (256 x int32_t) + x1: + type: buffer + size_bytes: x2 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + x2: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be a multiple of 8) + test_with: 136 # MLD_AARCH64_REJ_UNIFORM_ETA2_BUFLEN + x3: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t table[4096] + description: Lookup table (4096 x uint8_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta4_aarch64_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta4_aarch64_asm.S index bad80dc158..9d7488f382 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta4_aarch64_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/aarch64/src/rej_uniform_eta4_aarch64_asm.S @@ -4,6 +4,38 @@ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ +/*yaml + Name: rej_uniform_eta4_aarch64_asm + Description: AArch64 rejection sampling of eta=4 secret coefficients + Signature: uint64_t mld_rej_uniform_eta4_aarch64_asm(int32_t r[256], const uint8_t *buf, unsigned buflen, const uint8_t table[4096]) + ABI: + Architecture: aarch64 + CallingConvention: AAPCS64 + x0: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t r[256] + description: Output buffer (256 x int32_t) + x1: + type: buffer + size_bytes: x2 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input buffer + x2: + type: scalar + c_parameter: unsigned buflen + description: Length of input buffer (must be a multiple of 8) + test_with: 272 # MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN + x3: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const uint8_t table[4096] + description: Lookup table (4096 x uint8_t) +*/ + #include "_internal_s2n_bignum_arm.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/meta.h b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/meta.h new file mode 100644 index 0000000000..ead52f0aaf --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/meta.h @@ -0,0 +1,323 @@ +/* + * Copyright (c) The mlkem-native project authors + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +#ifndef MLD_NATIVE_X86_64_META_H +#define MLD_NATIVE_X86_64_META_H + +/* Identifier for this backend so that source and assembly files + * in the build can be appropriately guarded. */ +#define MLD_ARITH_BACKEND_X86_64_DEFAULT + +#define MLD_USE_NATIVE_NTT_CUSTOM_ORDER +#define MLD_USE_NATIVE_NTT +#define MLD_USE_NATIVE_INTT +#define MLD_USE_NATIVE_REJ_UNIFORM +#define MLD_USE_NATIVE_REJ_UNIFORM_ETA2 +#define MLD_USE_NATIVE_REJ_UNIFORM_ETA4 +#define MLD_USE_NATIVE_POLY_DECOMPOSE_32 +#define MLD_USE_NATIVE_POLY_DECOMPOSE_88 +#define MLD_USE_NATIVE_POLY_CADDQ +#define MLD_USE_NATIVE_POLY_USE_HINT_32 +#define MLD_USE_NATIVE_POLY_USE_HINT_88 +#define MLD_USE_NATIVE_POLY_CHKNORM +#define MLD_USE_NATIVE_POLYZ_UNPACK_17 +#define MLD_USE_NATIVE_POLYZ_UNPACK_19 +#define MLD_USE_NATIVE_POINTWISE_MONTGOMERY +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4 +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5 +#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7 + +#if !defined(__ASSEMBLER__) +#include +#include "../../common.h" +#include "../api.h" +#include "src/arith_native_x86_64.h" + +static MLD_INLINE void mld_poly_permute_bitrev_to_custom(int32_t data[MLDSA_N]) +{ + if (mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + mld_nttunpack_avx2_asm(data); + } +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + mld_ntt_avx2_asm(data, mld_qdata); + return MLD_NATIVE_FUNC_SUCCESS; +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_invntt_avx2_asm(data, mld_qdata); + return MLD_NATIVE_FUNC_SUCCESS; +} + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + /* AVX2 implementation assumes specific buffer lengths */ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2) || len != MLDSA_N || + buflen != MLD_AVX2_REJ_UNIFORM_BUFLEN) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + /* Safety: outlen is at most MLDSA_N and, hence, this cast is safe. */ + return (int)mld_rej_uniform_avx2_asm(r, buf, mld_rej_uniform_table); +} + +#if !defined(MLD_CONFIG_NO_KEYPAIR_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_eta2_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + unsigned int outlen; + /* AVX2 implementation assumes specific buffer lengths */ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2) || len != MLDSA_N || + buflen != MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + /* Constant time: Inputs and outputs to this function are secret. + * It is safe to leak which coefficients are accepted/rejected. + * The assembly implementation must not leak any other information about the + * accepted coefficients. Constant-time testing cannot cover this, and we + * hence have to manually verify the assembly. + * We declassify prior the input data and mark the outputs as secret. + */ + MLD_CT_TESTING_DECLASSIFY(buf, buflen); + outlen = mld_rej_uniform_eta2_avx2_asm(r, buf, mld_rej_uniform_table); + MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen); + /* Safety: outlen is at most MLDSA_N and, hence, this cast is safe. */ + return (int)outlen; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len, + const uint8_t *buf, + unsigned buflen) +{ + unsigned int outlen; + /* AVX2 implementation assumes specific buffer lengths */ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2) || len != MLDSA_N || + buflen != MLD_AVX2_REJ_UNIFORM_ETA4_BUFLEN) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + + /* Constant time: Inputs and outputs to this function are secret. + * It is safe to leak which coefficients are accepted/rejected. + * The assembly implementation must not leak any other information about the + * accepted coefficients. Constant-time testing cannot cover this, and we + * hence have to manually verify the assembly. + * We declassify prior the input data and mark the outputs as secret. + */ + MLD_CT_TESTING_DECLASSIFY(buf, buflen); + outlen = mld_rej_uniform_eta4_avx2_asm(r, buf, mld_rej_uniform_table); + MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen); + /* Safety: outlen is at most MLDSA_N and, hence, this cast is safe. */ + return (int)outlen; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4 */ +#endif /* !MLD_CONFIG_NO_KEYPAIR_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_poly_decompose_32_avx2_asm(a1, a0); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_poly_decompose_88_avx2_asm(a1, a0); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_SIGN_API */ + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_poly_caddq_avx2_asm(a); + return MLD_NATIVE_FUNC_SUCCESS; +} + +#if !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_poly_use_hint_32_avx2_asm(a, h); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_poly_use_hint_88_avx2_asm(a, h); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ +#endif /* !MLD_CONFIG_NO_VERIFY_API */ + +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + return mld_poly_chknorm_avx2_asm(a, B); +} + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *a) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_polyz_unpack_17_avx2_asm(r, a); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \ + */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *a) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_polyz_unpack_19_avx2_asm(r, a); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ + || MLD_CONFIG_PARAMETER_SET == 87 */ +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ + +#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \ + defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_poly_pointwise_montgomery_native( + int32_t a[MLDSA_N], const int32_t b[MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_pointwise_avx2_asm(a, b, mld_qdata); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \ + MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native( + int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], + const int32_t v[4][MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_pointwise_acc_l4_avx2_asm(w, u, v, mld_qdata); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native( + int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], + const int32_t v[5][MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_pointwise_acc_l5_avx2_asm(w, u, v, mld_qdata); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5 */ + +#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7 +MLD_MUST_CHECK_RETURN_VALUE +static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native( + int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], + const int32_t v[7][MLDSA_N]) +{ + if (!mld_sys_check_capability(MLD_SYS_CAP_X86_64_AVX2)) + { + return MLD_NATIVE_FUNC_FALLBACK; + } + mld_pointwise_acc_l7_avx2_asm(w, u, v, mld_qdata); + return MLD_NATIVE_FUNC_SUCCESS; +} +#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7 */ + +#endif /* !__ASSEMBLER__ */ + +#endif /* !MLD_NATIVE_X86_64_META_H */ diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/arith_native_x86_64.h b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/arith_native_x86_64.h index 6833ada9c0..d2b640d714 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/arith_native_x86_64.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/arith_native_x86_64.h @@ -32,6 +32,7 @@ MLD_INTERNAL_DATA_DECLARATION const uint8_t mld_rej_uniform_table[256][8]; #define mld_ntt_avx2_asm MLD_NAMESPACE(ntt_avx2_asm) +MLD_SYSV_ABI void mld_ntt_avx2_asm(int32_t *r, const int32_t *qdata) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/x86_64/proofs/ntt_avx2_asm.ml */ @@ -47,6 +48,7 @@ __contract__( ); #define mld_invntt_avx2_asm MLD_NAMESPACE(invntt_avx2_asm) +MLD_SYSV_ABI void mld_invntt_avx2_asm(int32_t *r, const int32_t *qdata) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/x86_64/proofs/intt_avx2_asm.ml */ @@ -61,6 +63,7 @@ __contract__( ); #define mld_nttunpack_avx2_asm MLD_NAMESPACE(nttunpack_avx2_asm) +MLD_SYSV_ABI void mld_nttunpack_avx2_asm(int32_t *r) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/x86_64/proofs/nttunpack_avx2_asm.ml */ @@ -74,32 +77,94 @@ __contract__( r[i] == old(*(int32_t (*)[MLDSA_N])r)[j]))) ); -#define mld_rej_uniform_avx2 MLD_NAMESPACE(mld_rej_uniform_avx2) -MLD_MUST_CHECK_RETURN_VALUE -unsigned mld_rej_uniform_avx2(int32_t *r, - const uint8_t buf[MLD_AVX2_REJ_UNIFORM_BUFLEN]); +#define mld_rej_uniform_avx2_asm MLD_NAMESPACE(rej_uniform_avx2_asm) +/* This contract must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/rej_uniform_avx2_asm.ml */ +MLD_MUST_CHECK_RETURN_VALUE MLD_SYSV_ABI +unsigned mld_rej_uniform_avx2_asm( + int32_t *r, const uint8_t buf[MLD_AVX2_REJ_UNIFORM_BUFLEN], + const uint8_t table[256][8]) +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, 840)) + requires(table == mld_rej_uniform_table) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(return_value <= MLDSA_N) + ensures(array_bound(r, 0, return_value, 0, MLDSA_Q)) +); #if !defined(MLD_CONFIG_NO_KEYPAIR_API) -#define mld_rej_uniform_eta2_avx2 MLD_NAMESPACE(mld_rej_uniform_eta2_avx2) -MLD_MUST_CHECK_RETURN_VALUE -unsigned mld_rej_uniform_eta2_avx2( - int32_t *r, const uint8_t buf[MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN]); - -#define mld_rej_uniform_eta4_avx2 MLD_NAMESPACE(mld_rej_uniform_eta4_avx2) -MLD_MUST_CHECK_RETURN_VALUE -unsigned mld_rej_uniform_eta4_avx2( - int32_t *r, const uint8_t buf[MLD_AVX2_REJ_UNIFORM_ETA4_BUFLEN]); +#define mld_rej_uniform_eta2_avx2_asm MLD_NAMESPACE(rej_uniform_eta2_avx2_asm) +/* This contract must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/rej_uniform_eta2_avx2_asm.ml */ +MLD_MUST_CHECK_RETURN_VALUE MLD_SYSV_ABI +unsigned mld_rej_uniform_eta2_avx2_asm( + int32_t *r, const uint8_t buf[MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN], + const uint8_t table[256][8]) +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN)) + requires(table == mld_rej_uniform_table) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(return_value <= MLDSA_N) + ensures(array_abs_bound(r, 0, return_value, 3)) +); + +#define mld_rej_uniform_eta4_avx2_asm MLD_NAMESPACE(rej_uniform_eta4_avx2_asm) +/* This contract must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/rej_uniform_eta4_avx2_asm.ml */ +MLD_MUST_CHECK_RETURN_VALUE MLD_SYSV_ABI +unsigned mld_rej_uniform_eta4_avx2_asm( + int32_t *r, const uint8_t buf[MLD_AVX2_REJ_UNIFORM_ETA4_BUFLEN], + const uint8_t table[256][8]) +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(buf, MLD_AVX2_REJ_UNIFORM_ETA4_BUFLEN)) + requires(table == mld_rej_uniform_table) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(return_value <= MLDSA_N) + ensures(array_abs_bound(r, 0, return_value, 5)) +); #endif /* !MLD_CONFIG_NO_KEYPAIR_API */ #if !defined(MLD_CONFIG_NO_SIGN_API) -#define mld_poly_decompose_32_avx2 MLD_NAMESPACE(mld_poly_decompose_32_avx2) -void mld_poly_decompose_32_avx2(int32_t *a1, int32_t *a0); +#define mld_poly_decompose_32_avx2_asm MLD_NAMESPACE(poly_decompose_32_avx2_asm) +MLD_SYSV_ABI +void mld_poly_decompose_32_avx2_asm(int32_t *a1, int32_t *a0) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/poly_decompose_32_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(a1, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a0, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a0, 0, MLDSA_N, 0, MLDSA_Q)) + assigns(memory_slice(a1, sizeof(int32_t) * MLDSA_N)) + assigns(memory_slice(a0, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 16 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 32)) */ + ensures(array_bound(a1, 0, MLDSA_N, 0, 16)) + /* check-magic: 261889 == (MLDSA_Q - 1) / 32 + 1 */ + ensures(array_abs_bound(a0, 0, MLDSA_N, 261889)) +); -#define mld_poly_decompose_88_avx2 MLD_NAMESPACE(mld_poly_decompose_88_avx2) -void mld_poly_decompose_88_avx2(int32_t *a1, int32_t *a0); +#define mld_poly_decompose_88_avx2_asm MLD_NAMESPACE(poly_decompose_88_avx2_asm) +MLD_SYSV_ABI +void mld_poly_decompose_88_avx2_asm(int32_t *a1, int32_t *a0) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/poly_decompose_88_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(a1, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a0, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a0, 0, MLDSA_N, 0, MLDSA_Q)) + assigns(memory_slice(a1, sizeof(int32_t) * MLDSA_N)) + assigns(memory_slice(a0, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 44 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 88)) */ + ensures(array_bound(a1, 0, MLDSA_N, 0, 44)) + /* check-magic: 95233 == (MLDSA_Q - 1) / 88 + 1 */ + ensures(array_abs_bound(a0, 0, MLDSA_N, 95233)) +); #endif /* !MLD_CONFIG_NO_SIGN_API */ #define mld_poly_caddq_avx2_asm MLD_NAMESPACE(poly_caddq_avx2_asm) +MLD_SYSV_ABI void mld_poly_caddq_avx2_asm(int32_t *r) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/x86_64/proofs/poly_caddq_avx2_asm.ml */ @@ -111,26 +176,80 @@ __contract__( ); #if !defined(MLD_CONFIG_NO_VERIFY_API) -#define mld_poly_use_hint_32_avx2 MLD_NAMESPACE(mld_poly_use_hint_32_avx2) -void mld_poly_use_hint_32_avx2(int32_t *a, const int32_t *h); +#define mld_poly_use_hint_32_avx2_asm MLD_NAMESPACE(poly_use_hint_32_avx2_asm) +MLD_SYSV_ABI +void mld_poly_use_hint_32_avx2_asm(int32_t *a, const int32_t *h) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/poly_use_hint_32_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(h, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) + requires(array_bound(h, 0, MLDSA_N, 0, 2)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 16 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 32)) */ + ensures(array_bound(a, 0, MLDSA_N, 0, 16)) +); -#define mld_poly_use_hint_88_avx2 MLD_NAMESPACE(mld_poly_use_hint_88_avx2) -void mld_poly_use_hint_88_avx2(int32_t *a, const int32_t *h); +#define mld_poly_use_hint_88_avx2_asm MLD_NAMESPACE(poly_use_hint_88_avx2_asm) +MLD_SYSV_ABI +void mld_poly_use_hint_88_avx2_asm(int32_t *a, const int32_t *h) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/poly_use_hint_88_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(h, sizeof(int32_t) * MLDSA_N)) + requires(array_bound(a, 0, MLDSA_N, 0, MLDSA_Q)) + requires(array_bound(h, 0, MLDSA_N, 0, 2)) + assigns(memory_slice(a, sizeof(int32_t) * MLDSA_N)) + /* check-magic: 44 == (MLDSA_Q - 1) / (2 * ((MLDSA_Q - 1) / 88)) */ + ensures(array_bound(a, 0, MLDSA_N, 0, 44)) +); #endif /* !MLD_CONFIG_NO_VERIFY_API */ -#define mld_poly_chknorm_avx2 MLD_NAMESPACE(mld_poly_chknorm_avx2) -MLD_MUST_CHECK_RETURN_VALUE -int mld_poly_chknorm_avx2(const int32_t *a, int32_t B); +#define mld_poly_chknorm_avx2_asm MLD_NAMESPACE(poly_chknorm_avx2_asm) +MLD_MUST_CHECK_RETURN_VALUE MLD_SYSV_ABI +int mld_poly_chknorm_avx2_asm(const int32_t *a, int32_t B) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/poly_chknorm_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(a, sizeof(int32_t) * MLDSA_N)) + /* HOL Light precondition: abs(ival(x i)) < 2^31, i.e., a[i] != INT32_MIN */ + requires(forall(k0, 0, MLDSA_N, a[k0] > INT32_MIN)) + /* HOL Light precondition: 0 <= ival bound (asm computes B-1 internally) */ + requires(B >= 0) + ensures(return_value == 0 || return_value == 1) + ensures((return_value == 0) == array_abs_bound(a, 0, MLDSA_N, B)) +); #if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) -#define mld_polyz_unpack_17_avx2 MLD_NAMESPACE(mld_polyz_unpack_17_avx2) -void mld_polyz_unpack_17_avx2(int32_t *r, const uint8_t *a); +#define mld_polyz_unpack_17_avx2_asm MLD_NAMESPACE(polyz_unpack_17_avx2_asm) +MLD_SYSV_ABI +void mld_polyz_unpack_17_avx2_asm(int32_t *r, const uint8_t *a) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/polyz_unpack_17_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a, 576)) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(r, 0, MLDSA_N, -((1 << 17) - 1), (1 << 17) + 1)) +); -#define mld_polyz_unpack_19_avx2 MLD_NAMESPACE(mld_polyz_unpack_19_avx2) -void mld_polyz_unpack_19_avx2(int32_t *r, const uint8_t *a); +#define mld_polyz_unpack_19_avx2_asm MLD_NAMESPACE(polyz_unpack_19_avx2_asm) +MLD_SYSV_ABI +void mld_polyz_unpack_19_avx2_asm(int32_t *r, const uint8_t *a) +/* This must be kept in sync with the HOL-Light specification + * in proofs/hol_light/x86_64/proofs/polyz_unpack_19_avx2_asm.ml */ +__contract__( + requires(memory_no_alias(r, sizeof(int32_t) * MLDSA_N)) + requires(memory_no_alias(a, 640)) + assigns(memory_slice(r, sizeof(int32_t) * MLDSA_N)) + ensures(array_bound(r, 0, MLDSA_N, -((1 << 19) - 1), (1 << 19) + 1)) +); #endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ #define mld_pointwise_avx2_asm MLD_NAMESPACE(pointwise_avx2_asm) +MLD_SYSV_ABI void mld_pointwise_avx2_asm(int32_t *a, const int32_t *b, const int32_t *qdata) /* This must be kept in sync with the HOL-Light specification * in proofs/hol_light/x86_64/proofs/pointwise_avx2_asm.ml */ @@ -149,6 +268,7 @@ __contract__( ); #define mld_pointwise_acc_l4_avx2_asm MLD_NAMESPACE(pointwise_acc_l4_avx2_asm) +MLD_SYSV_ABI void mld_pointwise_acc_l4_avx2_asm(int32_t c[MLDSA_N], const int32_t a[4][MLDSA_N], const int32_t b[4][MLDSA_N], @@ -168,6 +288,7 @@ __contract__( ); #define mld_pointwise_acc_l5_avx2_asm MLD_NAMESPACE(pointwise_acc_l5_avx2_asm) +MLD_SYSV_ABI void mld_pointwise_acc_l5_avx2_asm(int32_t c[MLDSA_N], const int32_t a[5][MLDSA_N], const int32_t b[5][MLDSA_N], @@ -187,6 +308,7 @@ __contract__( ); #define mld_pointwise_acc_l7_avx2_asm MLD_NAMESPACE(pointwise_acc_l7_avx2_asm) +MLD_SYSV_ABI void mld_pointwise_acc_l7_avx2_asm(int32_t c[MLDSA_N], const int32_t a[7][MLDSA_N], const int32_t b[7][MLDSA_N], diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/intt_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/intt_avx2_asm.S index 88a277a1e3..62676bf9a8 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/intt_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/intt_avx2_asm.S @@ -18,6 +18,28 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: invntt_avx2_asm + Description: x86_64 AVX2 inverse NTT + Signature: void mld_invntt_avx2_asm(int32_t *r, const int32_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *r + description: Input/output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 2496 + permissions: read-only + c_parameter: const int32_t *qdata + description: Precomputed constants (624 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/ntt_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/ntt_avx2_asm.S index 71d7cffc8c..6b921c87c8 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/ntt_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/ntt_avx2_asm.S @@ -18,6 +18,28 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: ntt_avx2_asm + Description: x86_64 AVX2 forward NTT + Signature: void mld_ntt_avx2_asm(int32_t *r, const int32_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *r + description: Input/output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 2496 + permissions: read-only + c_parameter: const int32_t *qdata + description: Precomputed constants (624 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/nttunpack_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/nttunpack_avx2_asm.S index 104282c499..57eeacc463 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/nttunpack_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/nttunpack_avx2_asm.S @@ -18,6 +18,22 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: nttunpack_avx2_asm + Description: x86_64 AVX2 NTT coefficient unpacking/permutation + Signature: void mld_nttunpack_avx2_asm(int32_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *r + description: Input/output polynomial (256 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l4_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l4_avx2_asm.S index 8be18af787..9232ac8ae7 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l4_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l4_avx2_asm.S @@ -17,6 +17,40 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: pointwise_acc_l4_avx2_asm + Description: x86_64 AVX2 pointwise multiply-accumulate of length-4 polynomial vectors + Signature: void mld_pointwise_acc_l4_avx2_asm(int32_t *c, const int32_t a[4][256], const int32_t b[4][256], const int32_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *c + description: Output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t a[4][256] + description: Input polynomial vector a (4 x 256 x int32_t) + rdx: + type: buffer + size_bytes: 4096 + permissions: read-only + c_parameter: const int32_t b[4][256] + description: Input polynomial vector b (4 x 256 x int32_t) + rcx: + type: buffer + size_bytes: 2496 + permissions: read-only + c_parameter: const int32_t *qdata + description: Precomputed constants (624 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l5_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l5_avx2_asm.S index a823f16088..f6f81be6a4 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l5_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l5_avx2_asm.S @@ -17,6 +17,40 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: pointwise_acc_l5_avx2_asm + Description: x86_64 AVX2 pointwise multiply-accumulate of length-5 polynomial vectors + Signature: void mld_pointwise_acc_l5_avx2_asm(int32_t *c, const int32_t a[5][256], const int32_t b[5][256], const int32_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *c + description: Output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t a[5][256] + description: Input polynomial vector a (5 x 256 x int32_t) + rdx: + type: buffer + size_bytes: 5120 + permissions: read-only + c_parameter: const int32_t b[5][256] + description: Input polynomial vector b (5 x 256 x int32_t) + rcx: + type: buffer + size_bytes: 2496 + permissions: read-only + c_parameter: const int32_t *qdata + description: Precomputed constants (624 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l7_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l7_avx2_asm.S index d4e37367f0..b0977622df 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l7_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_acc_l7_avx2_asm.S @@ -17,6 +17,40 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: pointwise_acc_l7_avx2_asm + Description: x86_64 AVX2 pointwise multiply-accumulate of length-7 polynomial vectors + Signature: void mld_pointwise_acc_l7_avx2_asm(int32_t *c, const int32_t a[7][256], const int32_t b[7][256], const int32_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *c + description: Output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t a[7][256] + description: Input polynomial vector a (7 x 256 x int32_t) + rdx: + type: buffer + size_bytes: 7168 + permissions: read-only + c_parameter: const int32_t b[7][256] + description: Input polynomial vector b (7 x 256 x int32_t) + rcx: + type: buffer + size_bytes: 2496 + permissions: read-only + c_parameter: const int32_t *qdata + description: Precomputed constants (624 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_avx2_asm.S index bcf3ee96de..1823dd9757 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/pointwise_avx2_asm.S @@ -17,6 +17,34 @@ * AVX2 Dilithium implementation @[REF_AVX2]. */ +/*yaml + Name: pointwise_avx2_asm + Description: x86_64 AVX2 pointwise Montgomery multiplication + Signature: void mld_pointwise_avx2_asm(int32_t *a, const int32_t *b, const int32_t *qdata) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *a + description: Input/output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t *b + description: Input polynomial (256 x int32_t) + rdx: + type: buffer + size_bytes: 2496 + permissions: read-only + c_parameter: const int32_t *qdata + description: Precomputed constants (624 x int32_t) +*/ + #include "_internal_s2n_bignum_x86_att.h" /* diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_caddq_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_caddq_avx2_asm.S index bcf74dfa2a..5f710008ec 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_caddq_avx2_asm.S +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_caddq_avx2_asm.S @@ -18,14 +18,23 @@ */ -/************************************************* - * Name: mld_poly_caddq_avx2_asm - * - * Description: For all coefficients of in/out polynomial add Q if - * coefficient is negative. - * - * Arguments: - int32_t *r: pointer to input/output polynomial - **************************************************/ +/*yaml + Name: poly_caddq_avx2_asm + Description: x86_64 AVX2 conditional addition of q to each coefficient. + For all coefficients of the in/out polynomial, add Q if the coefficient + is negative. + Signature: void mld_poly_caddq_avx2_asm(int32_t *r) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *r + description: Input/output polynomial (256 x int32_t) +*/ #include "_internal_s2n_bignum_x86_att.h" diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_chknorm_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_chknorm_avx2_asm.S new file mode 100644 index 0000000000..041e755015 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_chknorm_avx2_asm.S @@ -0,0 +1,173 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + + +/*yaml + Name: poly_chknorm_avx2_asm + Description: x86_64 AVX2 infinity-norm bound check on polynomial coefficients. + Check the infinity norm of the polynomial against the given bound B. + Returns 0 if the norm is strictly smaller than B; otherwise returns 1 + (i.e. returns 1 if any |coefficient| >= B, 0 otherwise). + Signature: int mld_poly_chknorm_avx2_asm(const int32_t *a, int32_t B) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t *a + description: Input polynomial (256 x int32_t) + rsi: + type: scalar + c_parameter: int32_t B + description: Norm bound (must be non-negative) + test_with: 131072 # representative non-negative bound (1 << 17) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/poly_chknorm_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_poly_chknorm_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_poly_chknorm_avx2_asm) +S2N_BN_SYMBOL(mldsa_poly_chknorm_avx2_asm): + + .cfi_startproc + subl $0x1, %esi + vpxor %xmm1, %xmm1, %xmm1 + vmovd %esi, %xmm2 + vpbroadcastd %xmm2, %ymm2 + vpabsd (%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x20(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x40(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x60(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x80(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0xa0(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0xc0(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0xe0(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x100(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x120(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x140(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x160(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x180(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x1a0(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x1c0(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x1e0(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x200(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x220(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x240(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x260(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x280(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x2a0(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x2c0(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x2e0(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x300(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x320(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x340(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x360(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + vpabsd 0x380(%rdi), %ymm0 + vpcmpgtd %ymm2, %ymm0, %ymm0 + vpor %ymm0, %ymm1, %ymm1 + vpabsd 0x3a0(%rdi), %ymm3 + vpcmpgtd %ymm2, %ymm3, %ymm3 + vpor %ymm3, %ymm1, %ymm1 + vpabsd 0x3c0(%rdi), %ymm4 + vpcmpgtd %ymm2, %ymm4, %ymm4 + vpor %ymm4, %ymm1, %ymm1 + vpabsd 0x3e0(%rdi), %ymm5 + vpcmpgtd %ymm2, %ymm5, %ymm5 + vpor %ymm5, %ymm1, %ymm1 + xorl %eax, %eax + vptest %ymm1, %ymm1 + setne %al + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_poly_chknorm_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_decompose_32_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_decompose_32_avx2_asm.S new file mode 100644 index 0000000000..24e60cbe11 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_decompose_32_avx2_asm.S @@ -0,0 +1,482 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + * + * The algorithm for Decompose(r) (more specifically the handling for the + * wrap-around cases) is modified. See the AVX2 intrinsics version + * (poly_decompose_32_avx2.c, predecessor of this file) for a more detailed + * comparison. + */ + + +/*yaml + Name: poly_decompose_32_avx2_asm + Description: x86_64 AVX2 coefficient decomposition (alpha = 2*(Q-1)/32). + For all coefficients c of the input polynomial, compute high and low bits + c0, c1 such that c = c1*ALPHA + c0 with -ALPHA/2 < c0 <= ALPHA/2, except if + c1 = (Q-1)/ALPHA where we set c1 = 0 and -ALPHA/2 <= c0 = c - Q < 0. + Assumes coefficients to be standard (unsigned canonical) representatives, + i.e. 0 <= c < Q. For ML-DSA-65 / ML-DSA-87 (gamma2 = (Q-1)/32, + alpha = 2*gamma2). + Signature: void mld_poly_decompose_32_avx2_asm(int32_t *a1, int32_t *a0) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *a1 + description: Output high-part polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *a0 + description: Input polynomial / output low-part (256 x int32_t) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/poly_decompose_32_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_poly_decompose_32_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_poly_decompose_32_avx2_asm) +S2N_BN_SYMBOL(mldsa_poly_decompose_32_avx2_asm): + + .cfi_startproc + movl $0x7f, %eax + vmovd %eax, %xmm10 + vpbroadcastd %xmm10, %ymm10 + movl $0x401, %eax # imm = 0x401 + vmovd %eax, %xmm11 + vpbroadcastd %xmm11, %ymm11 + movl $0x200, %eax # imm = 0x200 + vmovd %eax, %xmm12 + vpbroadcastd %xmm12, %ymm12 + movl $0x7be100, %eax # imm = 0x7BE100 + vmovd %eax, %xmm13 + vpbroadcastd %xmm13, %ymm13 + movl $0x7fe00, %eax # imm = 0x7FE00 + vmovd %eax, %xmm14 + vpbroadcastd %xmm14, %ymm14 + vmovdqa (%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, (%rdi) + vmovdqa %ymm2, (%rsi) + vmovdqa 0x20(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x20(%rdi) + vmovdqa %ymm2, 0x20(%rsi) + vmovdqa 0x40(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x40(%rdi) + vmovdqa %ymm2, 0x40(%rsi) + vmovdqa 0x60(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x60(%rdi) + vmovdqa %ymm2, 0x60(%rsi) + vmovdqa 0x80(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x80(%rdi) + vmovdqa %ymm2, 0x80(%rsi) + vmovdqa 0xa0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0xa0(%rdi) + vmovdqa %ymm2, 0xa0(%rsi) + vmovdqa 0xc0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0xc0(%rdi) + vmovdqa %ymm2, 0xc0(%rsi) + vmovdqa 0xe0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0xe0(%rdi) + vmovdqa %ymm2, 0xe0(%rsi) + vmovdqa 0x100(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x100(%rdi) + vmovdqa %ymm2, 0x100(%rsi) + vmovdqa 0x120(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x120(%rdi) + vmovdqa %ymm2, 0x120(%rsi) + vmovdqa 0x140(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x140(%rdi) + vmovdqa %ymm2, 0x140(%rsi) + vmovdqa 0x160(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x160(%rdi) + vmovdqa %ymm2, 0x160(%rsi) + vmovdqa 0x180(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x180(%rdi) + vmovdqa %ymm2, 0x180(%rsi) + vmovdqa 0x1a0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x1a0(%rdi) + vmovdqa %ymm2, 0x1a0(%rsi) + vmovdqa 0x1c0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x1c0(%rdi) + vmovdqa %ymm2, 0x1c0(%rsi) + vmovdqa 0x1e0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x1e0(%rdi) + vmovdqa %ymm2, 0x1e0(%rsi) + vmovdqa 0x200(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x200(%rdi) + vmovdqa %ymm2, 0x200(%rsi) + vmovdqa 0x220(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x220(%rdi) + vmovdqa %ymm2, 0x220(%rsi) + vmovdqa 0x240(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x240(%rdi) + vmovdqa %ymm2, 0x240(%rsi) + vmovdqa 0x260(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x260(%rdi) + vmovdqa %ymm2, 0x260(%rsi) + vmovdqa 0x280(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x280(%rdi) + vmovdqa %ymm2, 0x280(%rsi) + vmovdqa 0x2a0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x2a0(%rdi) + vmovdqa %ymm2, 0x2a0(%rsi) + vmovdqa 0x2c0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x2c0(%rdi) + vmovdqa %ymm2, 0x2c0(%rsi) + vmovdqa 0x2e0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x2e0(%rdi) + vmovdqa %ymm2, 0x2e0(%rsi) + vmovdqa 0x300(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x300(%rdi) + vmovdqa %ymm2, 0x300(%rsi) + vmovdqa 0x320(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x320(%rdi) + vmovdqa %ymm2, 0x320(%rsi) + vmovdqa 0x340(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x340(%rdi) + vmovdqa %ymm2, 0x340(%rsi) + vmovdqa 0x360(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x360(%rdi) + vmovdqa %ymm2, 0x360(%rsi) + vmovdqa 0x380(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x380(%rdi) + vmovdqa %ymm2, 0x380(%rsi) + vmovdqa 0x3a0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x3a0(%rdi) + vmovdqa %ymm2, 0x3a0(%rsi) + vmovdqa 0x3c0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x3c0(%rdi) + vmovdqa %ymm2, 0x3c0(%rsi) + vmovdqa 0x3e0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x3e0(%rdi) + vmovdqa %ymm2, 0x3e0(%rsi) + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_poly_decompose_32_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_decompose_88_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_decompose_88_avx2_asm.S new file mode 100644 index 0000000000..1517dee088 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_decompose_88_avx2_asm.S @@ -0,0 +1,481 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + * + * The algorithm for Decompose(r) (more specifically the handling for the + * wrap-around cases) is modified. See the AVX2 intrinsics version + * (poly_decompose_88_avx2.c, predecessor of this file) for a more detailed + * comparison. + */ + + +/*yaml + Name: poly_decompose_88_avx2_asm + Description: x86_64 AVX2 coefficient decomposition (alpha = 2*(Q-1)/88). + For all coefficients c of the input polynomial, compute high and low bits + c0, c1 such that c = c1*ALPHA + c0 with -ALPHA/2 < c0 <= ALPHA/2, except if + c1 = (Q-1)/ALPHA where we set c1 = 0 and -ALPHA/2 <= c0 = c - Q < 0. + Assumes coefficients to be standard (unsigned canonical) representatives, + i.e. 0 <= c < Q. For ML-DSA-44 (gamma2 = (Q-1)/88, alpha = 2*gamma2). + Signature: void mld_poly_decompose_88_avx2_asm(int32_t *a1, int32_t *a0) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *a1 + description: Output high-part polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *a0 + description: Input polynomial / output low-part (256 x int32_t) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/poly_decompose_88_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_poly_decompose_88_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_poly_decompose_88_avx2_asm) +S2N_BN_SYMBOL(mldsa_poly_decompose_88_avx2_asm): + + .cfi_startproc + movl $0x7f, %eax + vmovd %eax, %xmm10 + vpbroadcastd %xmm10, %ymm10 + movl $0x2c0b, %eax # imm = 0x2C0B + vmovd %eax, %xmm11 + vpbroadcastd %xmm11, %ymm11 + movl $0x80, %eax + vmovd %eax, %xmm12 + vpbroadcastd %xmm12, %ymm12 + movl $0x7e6c00, %eax # imm = 0x7E6C00 + vmovd %eax, %xmm13 + vpbroadcastd %xmm13, %ymm13 + movl $0x2e800, %eax # imm = 0x2E800 + vmovd %eax, %xmm14 + vpbroadcastd %xmm14, %ymm14 + vmovdqa (%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, (%rdi) + vmovdqa %ymm2, (%rsi) + vmovdqa 0x20(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x20(%rdi) + vmovdqa %ymm2, 0x20(%rsi) + vmovdqa 0x40(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x40(%rdi) + vmovdqa %ymm2, 0x40(%rsi) + vmovdqa 0x60(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x60(%rdi) + vmovdqa %ymm2, 0x60(%rsi) + vmovdqa 0x80(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x80(%rdi) + vmovdqa %ymm2, 0x80(%rsi) + vmovdqa 0xa0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0xa0(%rdi) + vmovdqa %ymm2, 0xa0(%rsi) + vmovdqa 0xc0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0xc0(%rdi) + vmovdqa %ymm2, 0xc0(%rsi) + vmovdqa 0xe0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0xe0(%rdi) + vmovdqa %ymm2, 0xe0(%rsi) + vmovdqa 0x100(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x100(%rdi) + vmovdqa %ymm2, 0x100(%rsi) + vmovdqa 0x120(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x120(%rdi) + vmovdqa %ymm2, 0x120(%rsi) + vmovdqa 0x140(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x140(%rdi) + vmovdqa %ymm2, 0x140(%rsi) + vmovdqa 0x160(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x160(%rdi) + vmovdqa %ymm2, 0x160(%rsi) + vmovdqa 0x180(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x180(%rdi) + vmovdqa %ymm2, 0x180(%rsi) + vmovdqa 0x1a0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x1a0(%rdi) + vmovdqa %ymm2, 0x1a0(%rsi) + vmovdqa 0x1c0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x1c0(%rdi) + vmovdqa %ymm2, 0x1c0(%rsi) + vmovdqa 0x1e0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x1e0(%rdi) + vmovdqa %ymm2, 0x1e0(%rsi) + vmovdqa 0x200(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x200(%rdi) + vmovdqa %ymm2, 0x200(%rsi) + vmovdqa 0x220(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x220(%rdi) + vmovdqa %ymm2, 0x220(%rsi) + vmovdqa 0x240(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x240(%rdi) + vmovdqa %ymm2, 0x240(%rsi) + vmovdqa 0x260(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x260(%rdi) + vmovdqa %ymm2, 0x260(%rsi) + vmovdqa 0x280(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x280(%rdi) + vmovdqa %ymm2, 0x280(%rsi) + vmovdqa 0x2a0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x2a0(%rdi) + vmovdqa %ymm2, 0x2a0(%rsi) + vmovdqa 0x2c0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x2c0(%rdi) + vmovdqa %ymm2, 0x2c0(%rsi) + vmovdqa 0x2e0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x2e0(%rdi) + vmovdqa %ymm2, 0x2e0(%rsi) + vmovdqa 0x300(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x300(%rdi) + vmovdqa %ymm2, 0x300(%rsi) + vmovdqa 0x320(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x320(%rdi) + vmovdqa %ymm2, 0x320(%rsi) + vmovdqa 0x340(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x340(%rdi) + vmovdqa %ymm2, 0x340(%rsi) + vmovdqa 0x360(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x360(%rdi) + vmovdqa %ymm2, 0x360(%rsi) + vmovdqa 0x380(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x380(%rdi) + vmovdqa %ymm2, 0x380(%rsi) + vmovdqa 0x3a0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x3a0(%rdi) + vmovdqa %ymm2, 0x3a0(%rsi) + vmovdqa 0x3c0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x3c0(%rdi) + vmovdqa %ymm2, 0x3c0(%rsi) + vmovdqa 0x3e0(%rsi), %ymm0 + vpaddd %ymm10, %ymm0, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm11, %ymm1, %ymm1 + vpmulhrsw %ymm12, %ymm1, %ymm1 + vpcmpgtd %ymm13, %ymm0, %ymm3 + vpmulld %ymm14, %ymm1, %ymm2 + vpsubd %ymm2, %ymm0, %ymm2 + vpandn %ymm1, %ymm3, %ymm1 + vpaddd %ymm3, %ymm2, %ymm2 + vmovdqa %ymm1, 0x3e0(%rdi) + vmovdqa %ymm2, 0x3e0(%rsi) + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_poly_decompose_88_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_use_hint_32_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_use_hint_32_avx2_asm.S new file mode 100644 index 0000000000..a80a753507 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_use_hint_32_avx2_asm.S @@ -0,0 +1,115 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + + +/*yaml + Name: poly_use_hint_32_avx2_asm + Description: x86_64 AVX2 hint application (alpha = (Q-1)/32). + Use the hint polynomial h to correct the high bits of the polynomial a, + in place. Variant for parameter sets ML-DSA-65 and ML-DSA-87 + (GAMMA2 = (Q-1)/32). + Signature: void mld_poly_use_hint_32_avx2_asm(int32_t *a, const int32_t *h) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *a + description: Input/output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t *h + description: Hint polynomial (256 x int32_t) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/poly_use_hint_32_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_poly_use_hint_32_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_poly_use_hint_32_avx2_asm) +S2N_BN_SYMBOL(mldsa_poly_use_hint_32_avx2_asm): + + .cfi_startproc + movl $0x7f, %ecx + movl $0x401, %r8d # imm = 0x401 + vmovd %r8d, %xmm8 + vpbroadcastd %xmm8, %ymm8 + xorl %eax, %eax + vpxor %xmm6, %xmm6, %xmm6 + vmovd %ecx, %xmm5 + movl $0x7be100, %ecx # imm = 0x7BE100 + movl $0x200, %r9d # imm = 0x200 + vmovd %r9d, %xmm7 + vpbroadcastd %xmm7, %ymm7 + vmovd %ecx, %xmm4 + movl $0xf, %ecx + vpbroadcastd %xmm5, %ymm5 + vmovd %ecx, %xmm3 + vpbroadcastd %xmm4, %ymm4 + vpbroadcastd %xmm3, %ymm3 + +Lmldsa_poly_use_hint_32_avx2_asm_loop: + vmovdqa (%rdi), %ymm0 + vmovdqa (%rsi), %ymm2 + vpaddd %ymm0, %ymm5, %ymm1 + vpsrld $0x7, %ymm1, %ymm1 + vpmulhuw %ymm8, %ymm1, %ymm1 + vpmulhrsw %ymm7, %ymm1, %ymm1 + vpcmpgtd %ymm4, %ymm0, %ymm11 + vpandn %ymm1, %ymm11, %ymm9 + vpslld $0xa, %ymm1, %ymm10 + vpsubd %ymm1, %ymm10, %ymm1 + vpslld $0x9, %ymm1, %ymm1 + vpsubd %ymm1, %ymm0, %ymm0 + vpaddd %ymm11, %ymm0, %ymm0 + vpcmpgtd %ymm6, %ymm0, %ymm0 + vpandn %ymm2, %ymm0, %ymm0 + vpslld $0x1, %ymm0, %ymm0 + vpsubd %ymm0, %ymm2, %ymm2 + vpaddd %ymm9, %ymm2, %ymm2 + vpand %ymm3, %ymm2, %ymm2 + vmovdqa %ymm2, (%rdi) + addq $0x20, %rdi + addq $0x20, %rsi + addq $0x20, %rax + cmpq $0x400, %rax # imm = 0x400 + jne Lmldsa_poly_use_hint_32_avx2_asm_loop + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_poly_use_hint_32_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_use_hint_88_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_use_hint_88_avx2_asm.S new file mode 100644 index 0000000000..8c4a5b638a --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/poly_use_hint_88_avx2_asm.S @@ -0,0 +1,117 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + + +/*yaml + Name: poly_use_hint_88_avx2_asm + Description: x86_64 AVX2 hint application (alpha = (Q-1)/88). + Use the hint polynomial h to correct the high bits of the polynomial a, + in place. Variant for parameter set ML-DSA-44 (GAMMA2 = (Q-1)/88). + Signature: void mld_poly_use_hint_88_avx2_asm(int32_t *a, const int32_t *h) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: read/write + c_parameter: int32_t *a + description: Input/output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 1024 + permissions: read-only + c_parameter: const int32_t *h + description: Hint polynomial (256 x int32_t) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/poly_use_hint_88_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_poly_use_hint_88_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_poly_use_hint_88_avx2_asm) +S2N_BN_SYMBOL(mldsa_poly_use_hint_88_avx2_asm): + + .cfi_startproc + movl $0x7f, %ecx + xorl %eax, %eax + vpxor %xmm5, %xmm5, %xmm5 + movl $0x2c0b, %r8d # imm = 0x2C0B + vmovd %r8d, %xmm8 + vpbroadcastd %xmm8, %ymm8 + vmovd %ecx, %xmm4 + movl $0x7e6c00, %ecx # imm = 0x7E6C00 + movl $0x80, %r9d + vmovd %r9d, %xmm7 + vpbroadcastd %xmm7, %ymm7 + movl $0x2b, %r10d + vmovd %r10d, %xmm6 + vpbroadcastd %xmm6, %ymm6 + vmovd %ecx, %xmm3 + vpbroadcastd %xmm4, %ymm4 + vpbroadcastd %xmm3, %ymm3 + +Lmldsa_poly_use_hint_88_avx2_asm_loop: + vmovdqa (%rdi), %ymm0 + vmovdqa (%rsi), %ymm1 + vpaddd %ymm0, %ymm4, %ymm10 + vpsrld $0x7, %ymm10, %ymm10 + vpmulhuw %ymm8, %ymm10, %ymm10 + vpmulhrsw %ymm7, %ymm10, %ymm10 + vpcmpgtd %ymm3, %ymm0, %ymm11 + vpandn %ymm10, %ymm11, %ymm9 + vpslld $0x1, %ymm10, %ymm12 + vpaddd %ymm10, %ymm12, %ymm12 + vpslld $0x5, %ymm12, %ymm10 + vpsubd %ymm12, %ymm10, %ymm10 + vpslld $0xb, %ymm10, %ymm10 + vpsubd %ymm10, %ymm0, %ymm0 + vpaddd %ymm11, %ymm0, %ymm0 + vpcmpgtd %ymm5, %ymm0, %ymm0 + vpandn %ymm1, %ymm0, %ymm0 + vpslld $0x1, %ymm0, %ymm0 + vpsubd %ymm0, %ymm1, %ymm0 + vpaddd %ymm9, %ymm0, %ymm0 + vpblendvb %ymm0, %ymm6, %ymm0, %ymm0 + vpcmpgtd %ymm6, %ymm0, %ymm1 + vpandn %ymm0, %ymm1, %ymm0 + vmovdqa %ymm0, (%rdi) + addq $0x20, %rdi + addq $0x20, %rsi + addq $0x20, %rax + cmpq $0x400, %rax # imm = 0x400 + jne Lmldsa_poly_use_hint_88_avx2_asm_loop + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_poly_use_hint_88_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/polyz_unpack_17_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/polyz_unpack_17_avx2_asm.S new file mode 100644 index 0000000000..bc37861fb9 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/polyz_unpack_17_avx2_asm.S @@ -0,0 +1,346 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + + +/*yaml + Name: polyz_unpack_17_avx2_asm + Description: x86_64 AVX2 unpacking of 17-bit packed coefficients. + Unpack polynomial z with 18-bit packed coefficients (GAMMA1 = 2^17), + mapping packed [0, 2^18-1] to signed [-(2^17-1), 2^17] via GAMMA1 - x. + Signature: void mld_polyz_unpack_17_avx2_asm(int32_t *r, const uint8_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 576 + permissions: read-only + c_parameter: const uint8_t *a + description: Packed input bytes +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/polyz_unpack_17_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_polyz_unpack_17_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_polyz_unpack_17_avx2_asm) +S2N_BN_SYMBOL(mldsa_polyz_unpack_17_avx2_asm): + + .cfi_startproc + movabsq $-0xfbfcfd00fdff00, %rax # imm = 0xFF040302FF020100 + vmovq %rax, %xmm1 + movabsq $-0xf7f8f900f9fafc, %rax # imm = 0xFF080706FF060504 + vpinsrq $0x1, %rax, %xmm1, %xmm1 + movabsq $-0xe4e5e600e6e7e9, %rax # imm = 0xFF1B1A19FF191817 + vmovq %rax, %xmm5 + movabsq $-0xe0e1e200e2e3e5, %rax # imm = 0xFF1F1E1DFF1D1C1B + vpinsrq $0x1, %rax, %xmm5, %xmm5 + vinserti128 $0x1, %xmm5, %ymm1, %ymm1 + movabsq $0x200000000, %rax # imm = 0x200000000 + vmovq %rax, %xmm2 + movabsq $0x600000004, %rax # imm = 0x600000004 + vpinsrq $0x1, %rax, %xmm2, %xmm2 + vinserti128 $0x1, %xmm2, %ymm2, %ymm2 + movl $0x3ffff, %eax # imm = 0x3FFFF + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x20000, %eax # imm = 0x20000 + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + vmovdqu (%rsi), %xmm0 + vmovdqu 0x2(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, (%rdi) + vmovdqu 0x12(%rsi), %xmm0 + vmovdqu 0x14(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x20(%rdi) + vmovdqu 0x24(%rsi), %xmm0 + vmovdqu 0x26(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x40(%rdi) + vmovdqu 0x36(%rsi), %xmm0 + vmovdqu 0x38(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x60(%rdi) + vmovdqu 0x48(%rsi), %xmm0 + vmovdqu 0x4a(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x80(%rdi) + vmovdqu 0x5a(%rsi), %xmm0 + vmovdqu 0x5c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0xa0(%rdi) + vmovdqu 0x6c(%rsi), %xmm0 + vmovdqu 0x6e(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0xc0(%rdi) + vmovdqu 0x7e(%rsi), %xmm0 + vmovdqu 0x80(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0xe0(%rdi) + vmovdqu 0x90(%rsi), %xmm0 + vmovdqu 0x92(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x100(%rdi) + vmovdqu 0xa2(%rsi), %xmm0 + vmovdqu 0xa4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x120(%rdi) + vmovdqu 0xb4(%rsi), %xmm0 + vmovdqu 0xb6(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x140(%rdi) + vmovdqu 0xc6(%rsi), %xmm0 + vmovdqu 0xc8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x160(%rdi) + vmovdqu 0xd8(%rsi), %xmm0 + vmovdqu 0xda(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x180(%rdi) + vmovdqu 0xea(%rsi), %xmm0 + vmovdqu 0xec(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x1a0(%rdi) + vmovdqu 0xfc(%rsi), %xmm0 + vmovdqu 0xfe(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x1c0(%rdi) + vmovdqu 0x10e(%rsi), %xmm0 + vmovdqu 0x110(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x1e0(%rdi) + vmovdqu 0x120(%rsi), %xmm0 + vmovdqu 0x122(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x200(%rdi) + vmovdqu 0x132(%rsi), %xmm0 + vmovdqu 0x134(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x220(%rdi) + vmovdqu 0x144(%rsi), %xmm0 + vmovdqu 0x146(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x240(%rdi) + vmovdqu 0x156(%rsi), %xmm0 + vmovdqu 0x158(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x260(%rdi) + vmovdqu 0x168(%rsi), %xmm0 + vmovdqu 0x16a(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x280(%rdi) + vmovdqu 0x17a(%rsi), %xmm0 + vmovdqu 0x17c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x2a0(%rdi) + vmovdqu 0x18c(%rsi), %xmm0 + vmovdqu 0x18e(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x2c0(%rdi) + vmovdqu 0x19e(%rsi), %xmm0 + vmovdqu 0x1a0(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x2e0(%rdi) + vmovdqu 0x1b0(%rsi), %xmm0 + vmovdqu 0x1b2(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x300(%rdi) + vmovdqu 0x1c2(%rsi), %xmm0 + vmovdqu 0x1c4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x320(%rdi) + vmovdqu 0x1d4(%rsi), %xmm0 + vmovdqu 0x1d6(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x340(%rdi) + vmovdqu 0x1e6(%rsi), %xmm0 + vmovdqu 0x1e8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x360(%rdi) + vmovdqu 0x1f8(%rsi), %xmm0 + vmovdqu 0x1fa(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x380(%rdi) + vmovdqu 0x20a(%rsi), %xmm0 + vmovdqu 0x20c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x3a0(%rdi) + vmovdqu 0x21c(%rsi), %xmm0 + vmovdqu 0x21e(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x3c0(%rdi) + vmovdqu 0x22e(%rsi), %xmm0 + vmovdqu 0x230(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x3e0(%rdi) + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_polyz_unpack_17_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/polyz_unpack_19_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/polyz_unpack_19_avx2_asm.S new file mode 100644 index 0000000000..c41e6b1786 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/polyz_unpack_19_avx2_asm.S @@ -0,0 +1,346 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + + +/*yaml + Name: polyz_unpack_19_avx2_asm + Description: x86_64 AVX2 unpacking of 19-bit packed coefficients. + Unpack polynomial z with 20-bit packed coefficients (GAMMA1 = 2^19), + mapping packed [0, 2^20-1] to signed [-(2^19-1), 2^19] via GAMMA1 - x. + Signature: void mld_polyz_unpack_19_avx2_asm(int32_t *r, const uint8_t *a) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output polynomial (256 x int32_t) + rsi: + type: buffer + size_bytes: 640 + permissions: read-only + c_parameter: const uint8_t *a + description: Packed input bytes +*/ + +#include "_internal_s2n_bignum_x86_att.h" + + + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/polyz_unpack_19_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_polyz_unpack_19_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_polyz_unpack_19_avx2_asm) +S2N_BN_SYMBOL(mldsa_polyz_unpack_19_avx2_asm): + + .cfi_startproc + movabsq $-0xfbfcfd00fdff00, %rax # imm = 0xFF040302FF020100 + vmovq %rax, %xmm1 + movabsq $-0xf6f7f800f8f9fb, %rax # imm = 0xFF090807FF070605 + vpinsrq $0x1, %rax, %xmm1, %xmm1 + movabsq $-0xe5e6e700e7e8ea, %rax # imm = 0xFF1A1918FF181716 + vmovq %rax, %xmm5 + movabsq $-0xe0e1e200e2e3e5, %rax # imm = 0xFF1F1E1DFF1D1C1B + vpinsrq $0x1, %rax, %xmm5, %xmm5 + vinserti128 $0x1, %xmm5, %ymm1, %ymm1 + movabsq $0x400000000, %rax # imm = 0x400000000 + vmovq %rax, %xmm2 + movabsq $0x400000000, %rax # imm = 0x400000000 + vpinsrq $0x1, %rax, %xmm2, %xmm2 + vinserti128 $0x1, %xmm2, %ymm2, %ymm2 + movl $0xfffff, %eax # imm = 0xFFFFF + vmovd %eax, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x80000, %eax # imm = 0x80000 + vmovd %eax, %xmm4 + vpbroadcastd %xmm4, %ymm4 + vmovdqu (%rsi), %xmm0 + vmovdqu 0x4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, (%rdi) + vmovdqu 0x14(%rsi), %xmm0 + vmovdqu 0x18(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x20(%rdi) + vmovdqu 0x28(%rsi), %xmm0 + vmovdqu 0x2c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x40(%rdi) + vmovdqu 0x3c(%rsi), %xmm0 + vmovdqu 0x40(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x60(%rdi) + vmovdqu 0x50(%rsi), %xmm0 + vmovdqu 0x54(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x80(%rdi) + vmovdqu 0x64(%rsi), %xmm0 + vmovdqu 0x68(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0xa0(%rdi) + vmovdqu 0x78(%rsi), %xmm0 + vmovdqu 0x7c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0xc0(%rdi) + vmovdqu 0x8c(%rsi), %xmm0 + vmovdqu 0x90(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0xe0(%rdi) + vmovdqu 0xa0(%rsi), %xmm0 + vmovdqu 0xa4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x100(%rdi) + vmovdqu 0xb4(%rsi), %xmm0 + vmovdqu 0xb8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x120(%rdi) + vmovdqu 0xc8(%rsi), %xmm0 + vmovdqu 0xcc(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x140(%rdi) + vmovdqu 0xdc(%rsi), %xmm0 + vmovdqu 0xe0(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x160(%rdi) + vmovdqu 0xf0(%rsi), %xmm0 + vmovdqu 0xf4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x180(%rdi) + vmovdqu 0x104(%rsi), %xmm0 + vmovdqu 0x108(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x1a0(%rdi) + vmovdqu 0x118(%rsi), %xmm0 + vmovdqu 0x11c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x1c0(%rdi) + vmovdqu 0x12c(%rsi), %xmm0 + vmovdqu 0x130(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x1e0(%rdi) + vmovdqu 0x140(%rsi), %xmm0 + vmovdqu 0x144(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x200(%rdi) + vmovdqu 0x154(%rsi), %xmm0 + vmovdqu 0x158(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x220(%rdi) + vmovdqu 0x168(%rsi), %xmm0 + vmovdqu 0x16c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x240(%rdi) + vmovdqu 0x17c(%rsi), %xmm0 + vmovdqu 0x180(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x260(%rdi) + vmovdqu 0x190(%rsi), %xmm0 + vmovdqu 0x194(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x280(%rdi) + vmovdqu 0x1a4(%rsi), %xmm0 + vmovdqu 0x1a8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x2a0(%rdi) + vmovdqu 0x1b8(%rsi), %xmm0 + vmovdqu 0x1bc(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x2c0(%rdi) + vmovdqu 0x1cc(%rsi), %xmm0 + vmovdqu 0x1d0(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x2e0(%rdi) + vmovdqu 0x1e0(%rsi), %xmm0 + vmovdqu 0x1e4(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x300(%rdi) + vmovdqu 0x1f4(%rsi), %xmm0 + vmovdqu 0x1f8(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x320(%rdi) + vmovdqu 0x208(%rsi), %xmm0 + vmovdqu 0x20c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x340(%rdi) + vmovdqu 0x21c(%rsi), %xmm0 + vmovdqu 0x220(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x360(%rdi) + vmovdqu 0x230(%rsi), %xmm0 + vmovdqu 0x234(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x380(%rdi) + vmovdqu 0x244(%rsi), %xmm0 + vmovdqu 0x248(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x3a0(%rdi) + vmovdqu 0x258(%rsi), %xmm0 + vmovdqu 0x25c(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x3c0(%rdi) + vmovdqu 0x26c(%rsi), %xmm0 + vmovdqu 0x270(%rsi), %xmm5 + vinserti128 $0x1, %xmm5, %ymm0, %ymm0 + vpshufb %ymm1, %ymm0, %ymm0 + vpsrlvd %ymm2, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubd %ymm0, %ymm4, %ymm0 + vmovdqa %ymm0, 0x3e0(%rdi) + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_polyz_unpack_19_avx2_asm) + + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_avx2_asm.S new file mode 100644 index 0000000000..fd6d2024ee --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_avx2_asm.S @@ -0,0 +1,129 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + +/*yaml + Name: rej_uniform_avx2_asm + Description: x86_64 AVX2 rejection sampling of uniform coefficients mod q. + Extract 23-bit values from the input byte buffer and accept those that are + < MLDSA_Q, writing accepted coefficients to the output buffer. Returns the + number of valid coefficients written (at most 256). + Signature: unsigned mld_rej_uniform_avx2_asm(int32_t *r, const uint8_t buf[840], const uint8_t table[256][8]) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output buffer (256 x int32_t) + rsi: + type: buffer + size_bytes: 840 + permissions: read-only + c_parameter: const uint8_t buf[840] + description: Input buffer (MLD_POLY_UNIFORM_NBLOCKS * SHAKE128_RATE = 5 * 168) + rdx: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const uint8_t table[256][8] + description: Lookup table (256 x 8 x uint8_t) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/rej_uniform_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_rej_uniform_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_rej_uniform_avx2_asm) +S2N_BN_SYMBOL(mldsa_rej_uniform_avx2_asm): + + .cfi_startproc + movabsq $-0xfafbfc00fdff00, %r10 # imm = 0xFF050403FF020100 + vmovq %r10, %xmm0 + movabsq $-0xf4f5f600f7f8fa, %r10 # imm = 0xFF0B0A09FF080706 + vpinsrq $0x1, %r10, %xmm0, %xmm0 + movabsq $-0xf6f7f800f9fafc, %r10 # imm = 0xFF090807FF060504 + vmovq %r10, %xmm3 + movabsq $-0xf0f1f200f3f4f6, %r10 # imm = 0xFF0F0E0DFF0C0B0A + vpinsrq $0x1, %r10, %xmm3, %xmm3 + vinserti128 $0x1, %xmm3, %ymm0, %ymm0 + movl $0x7fffff, %r8d # imm = 0x7FFFFF + vmovd %r8d, %xmm1 + vpbroadcastd %xmm1, %ymm1 + movl $0x7fe001, %r8d # imm = 0x7FE001 + vmovd %r8d, %xmm2 + vpbroadcastd %xmm2, %ymm2 + xorl %eax, %eax + xorl %ecx, %ecx + +Lmldsa_rej_uniform_avx2_asm_loop: + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_avx2_asm_scalar + cmpl $0x328, %ecx # imm = 0x328 + ja Lmldsa_rej_uniform_avx2_asm_scalar + vmovdqu (%rsi,%rcx), %ymm3 + addl $0x18, %ecx + vpermq $0x94, %ymm3, %ymm3 # ymm3 = ymm3[0,1,1,2] + vpshufb %ymm0, %ymm3, %ymm3 + vpand %ymm1, %ymm3, %ymm3 + vpsubd %ymm2, %ymm3, %ymm4 + vmovmskps %ymm4, %r8d + popcntl %r8d, %r9d + vmovq (%rdx,%r8,8), %xmm4 + vpmovzxbd %xmm4, %ymm4 # ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero + vpermd %ymm3, %ymm4, %ymm3 + vmovdqu %ymm3, (%rdi,%rax,4) + addl %r9d, %eax + jmp Lmldsa_rej_uniform_avx2_asm_loop + +Lmldsa_rej_uniform_avx2_asm_scalar: + cmpl $0x100, %eax # imm = 0x100 + jae Lmldsa_rej_uniform_avx2_asm_done + cmpl $0x345, %ecx # imm = 0x345 + ja Lmldsa_rej_uniform_avx2_asm_done + movzwl (%rsi,%rcx), %r8d + movzbl 0x2(%rsi,%rcx), %r9d + shll $0x10, %r9d + orl %r9d, %r8d + andl $0x7fffff, %r8d # imm = 0x7FFFFF + addl $0x3, %ecx + cmpl $0x7fe001, %r8d # imm = 0x7FE001 + jae Lmldsa_rej_uniform_avx2_asm_scalar + movl %r8d, (%rdi,%rax,4) + addl $0x1, %eax + jmp Lmldsa_rej_uniform_avx2_asm_scalar + +Lmldsa_rej_uniform_avx2_asm_done: + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_rej_uniform_avx2_asm) + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_eta2_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_eta2_avx2_asm.S new file mode 100644 index 0000000000..0a26a934c7 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_eta2_avx2_asm.S @@ -0,0 +1,203 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + +/*yaml + Name: rej_uniform_eta2_avx2_asm + Description: x86_64 AVX2 rejection sampling of eta=2 secret coefficients. + Extracts 4-bit nibbles from the input byte buffer, accepts those < 15, + and maps an accepted nibble n to the coefficient 2 - (n mod 5) via a + centered modulo-5 reduction, producing values in [-2, 2]. + Signature: unsigned mld_rej_uniform_eta2_avx2_asm(int32_t *r, const uint8_t *buf, const uint8_t table[256][8]) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output coefficient buffer (256 x int32_t) + rsi: + type: buffer + size_bytes: 136 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input byte buffer (MLD_AVX2_REJ_UNIFORM_ETA2_BUFLEN = 136) + rdx: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const uint8_t *table + description: Lookup table (256 x 8 uint8_t = mld_rej_uniform_table) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/rej_uniform_eta2_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_rej_uniform_eta2_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_rej_uniform_eta2_avx2_asm) +S2N_BN_SYMBOL(mldsa_rej_uniform_eta2_avx2_asm): + + .cfi_startproc + movl $0xf0f0f0f, %r8d # imm = 0xF0F0F0F + vmovd %r8d, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x2020202, %r8d # imm = 0x2020202 + vmovd %r8d, %xmm4 + vpbroadcastd %xmm4, %ymm4 + movl $0xf0f0f0f, %r8d # imm = 0xF0F0F0F + vmovd %r8d, %xmm5 + vpbroadcastd %xmm5, %ymm5 + movl $0xffffe660, %r8d # imm = 0xFFFFE660 + vpinsrw $0x0, %r8d, %xmm6, %xmm6 + vpbroadcastw %xmm6, %ymm6 + movl $0x5, %r8d + vpinsrw $0x0, %r8d, %xmm7, %xmm7 + vpbroadcastw %xmm7, %ymm7 + xorl %eax, %eax + xorl %ecx, %ecx + +Lmldsa_rej_uniform_eta2_avx2_asm_loop: + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta2_avx2_asm_scalar + cmpl $0x78, %ecx + ja Lmldsa_rej_uniform_eta2_avx2_asm_scalar + vpmovzxbw (%rsi,%rcx), %ymm0 + vpsllw $0x4, %ymm0, %ymm1 + vpor %ymm1, %ymm0, %ymm0 + vpand %ymm3, %ymm0, %ymm0 + vpsubb %ymm5, %ymm0, %ymm1 + vpsubb %ymm0, %ymm4, %ymm0 + vpmovmskb %ymm1, %r8d + vextracti128 $0x0, %ymm0, %xmm8 + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm9 + vpshufb %xmm9, %xmm8, %xmm9 + vpmovsxbd %xmm9, %ymm1 + vpmulhrsw %ymm6, %ymm1, %ymm2 + vpmullw %ymm7, %ymm2, %ymm2 + vpaddd %ymm2, %ymm1, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + shrl $0x8, %r8d + addl $0x4, %ecx + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta2_avx2_asm_scalar + vpsrldq $0x8, %xmm8, %xmm8 # xmm8 = xmm8[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm9 + vpshufb %xmm9, %xmm8, %xmm9 + vpmovsxbd %xmm9, %ymm1 + vpmulhrsw %ymm6, %ymm1, %ymm2 + vpmullw %ymm7, %ymm2, %ymm2 + vpaddd %ymm2, %ymm1, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + shrl $0x8, %r8d + addl $0x4, %ecx + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta2_avx2_asm_scalar + vextracti128 $0x1, %ymm0, %xmm8 + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm9 + vpshufb %xmm9, %xmm8, %xmm9 + vpmovsxbd %xmm9, %ymm1 + vpmulhrsw %ymm6, %ymm1, %ymm2 + vpmullw %ymm7, %ymm2, %ymm2 + vpaddd %ymm2, %ymm1, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + shrl $0x8, %r8d + addl $0x4, %ecx + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta2_avx2_asm_scalar + vpsrldq $0x8, %xmm8, %xmm8 # xmm8 = xmm8[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm9 + vpshufb %xmm9, %xmm8, %xmm9 + vpmovsxbd %xmm9, %ymm1 + vpmulhrsw %ymm6, %ymm1, %ymm2 + vpmullw %ymm7, %ymm2, %ymm2 + vpaddd %ymm2, %ymm1, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + addl $0x4, %ecx + jmp Lmldsa_rej_uniform_eta2_avx2_asm_loop + +Lmldsa_rej_uniform_eta2_avx2_asm_scalar: + cmpl $0x100, %eax # imm = 0x100 + jae Lmldsa_rej_uniform_eta2_avx2_asm_done + cmpl $0x88, %ecx + jae Lmldsa_rej_uniform_eta2_avx2_asm_done + movzbl (%rsi,%rcx), %r11d + incl %ecx + movl %r11d, %r10d + andl $0xf, %r10d + cmpl $0xf, %r10d + jae Lmldsa_rej_uniform_eta2_avx2_asm_high_nibble + movl %r10d, %r11d + imull $0xcd, %r11d, %r11d + shrl $0xa, %r11d + imull $0x5, %r11d, %r11d + subl %r11d, %r10d + movl $0x2, %r11d + subl %r10d, %r11d + movl %r11d, (%rdi,%rax,4) + incl %eax + cmpl $0x100, %eax # imm = 0x100 + jae Lmldsa_rej_uniform_eta2_avx2_asm_done + +Lmldsa_rej_uniform_eta2_avx2_asm_high_nibble: + movzbl -0x1(%rsi,%rcx), %r11d + shrl $0x4, %r11d + andl $0xf, %r11d + cmpl $0xf, %r11d + jae Lmldsa_rej_uniform_eta2_avx2_asm_scalar + movl %r11d, %r10d + imull $0xcd, %r10d, %r10d + shrl $0xa, %r10d + imull $0x5, %r10d, %r10d + subl %r10d, %r11d + movl $0x2, %r10d + subl %r11d, %r10d + movl %r10d, (%rdi,%rax,4) + incl %eax + jmp Lmldsa_rej_uniform_eta2_avx2_asm_scalar + +Lmldsa_rej_uniform_eta2_avx2_asm_done: + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_rej_uniform_eta2_avx2_asm) + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_eta4_avx2_asm.S b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_eta4_avx2_asm.S new file mode 100644 index 0000000000..22888026bd --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_eta4_avx2_asm.S @@ -0,0 +1,174 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* References + * ========== + * + * - [REF_AVX2] + * CRYSTALS-Dilithium optimized AVX2 implementation + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé + * https://github.com/pq-crystals/dilithium/tree/master/avx2 + */ + +/* + * This file is derived from the public domain + * AVX2 Dilithium implementation @[REF_AVX2]. + */ + +/*yaml + Name: rej_uniform_eta4_avx2_asm + Description: x86_64 AVX2 rejection sampling of eta=4 secret coefficients. + Extracts 4-bit nibbles from the input byte buffer, accepts those < 9, + and maps an accepted nibble n to the coefficient 4 - n, producing + values in [-4, 4]. + Signature: unsigned mld_rej_uniform_eta4_avx2_asm(int32_t *r, const uint8_t *buf, const uint8_t table[256][8]) + ABI: + Architecture: x86_64 + CallingConvention: SysV + Features: [AVX2] + rdi: + type: buffer + size_bytes: 1024 + permissions: write-only + c_parameter: int32_t *r + description: Output coefficient buffer (256 x int32_t) + rsi: + type: buffer + size_bytes: 272 + permissions: read-only + c_parameter: const uint8_t *buf + description: Input byte buffer (MLD_AVX2_REJ_UNIFORM_ETA4_BUFLEN = 272) + rdx: + type: buffer + size_bytes: 2048 + permissions: read-only + c_parameter: const uint8_t *table + description: Lookup table (256 x 8 uint8_t = mld_rej_uniform_table) +*/ + +#include "_internal_s2n_bignum_x86_att.h" + +/* + * WARNING: This file is auto-derived from the mldsa-native source file + * dev/x86_64/src/rej_uniform_eta4_avx2_asm.S using scripts/simpasm. Do not modify it directly. + */ + +.text +.balign 4 + S2N_BN_SYM_VISIBILITY_DIRECTIVE(mldsa_rej_uniform_eta4_avx2_asm) + S2N_BN_SYM_PRIVACY_DIRECTIVE(mldsa_rej_uniform_eta4_avx2_asm) +S2N_BN_SYMBOL(mldsa_rej_uniform_eta4_avx2_asm): + + .cfi_startproc + movl $0xf0f0f0f, %r8d # imm = 0xF0F0F0F + vmovd %r8d, %xmm2 + vpbroadcastd %xmm2, %ymm2 + movl $0x4040404, %r8d # imm = 0x4040404 + vmovd %r8d, %xmm3 + vpbroadcastd %xmm3, %ymm3 + movl $0x9090909, %r8d # imm = 0x9090909 + vmovd %r8d, %xmm4 + vpbroadcastd %xmm4, %ymm4 + xorl %eax, %eax + xorl %ecx, %ecx + +Lmldsa_rej_uniform_eta4_avx2_asm_loop: + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta4_avx2_asm_scalar + cmpl $0x100, %ecx # imm = 0x100 + ja Lmldsa_rej_uniform_eta4_avx2_asm_scalar + vpmovzxbw (%rsi,%rcx), %ymm0 + vpsllw $0x4, %ymm0, %ymm1 + vpor %ymm1, %ymm0, %ymm0 + vpand %ymm2, %ymm0, %ymm0 + vpsubb %ymm4, %ymm0, %ymm1 + vpsubb %ymm0, %ymm3, %ymm0 + vpmovmskb %ymm1, %r8d + vextracti128 $0x0, %ymm0, %xmm5 + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm6 + vpshufb %xmm6, %xmm5, %xmm6 + vpmovsxbd %xmm6, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + shrl $0x8, %r8d + addl $0x4, %ecx + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta4_avx2_asm_scalar + vpsrldq $0x8, %xmm5, %xmm5 # xmm5 = xmm5[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm6 + vpshufb %xmm6, %xmm5, %xmm6 + vpmovsxbd %xmm6, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + shrl $0x8, %r8d + addl $0x4, %ecx + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta4_avx2_asm_scalar + vextracti128 $0x1, %ymm0, %xmm5 + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm6 + vpshufb %xmm6, %xmm5, %xmm6 + vpmovsxbd %xmm6, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + shrl $0x8, %r8d + addl $0x4, %ecx + cmpl $0xf8, %eax + ja Lmldsa_rej_uniform_eta4_avx2_asm_scalar + vpsrldq $0x8, %xmm5, %xmm5 # xmm5 = xmm5[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero + movzbl %r8b, %r10d + vmovq (%rdx,%r10,8), %xmm6 + vpshufb %xmm6, %xmm5, %xmm6 + vpmovsxbd %xmm6, %ymm1 + vmovdqu %ymm1, (%rdi,%rax,4) + popcntl %r10d, %r9d + addl %r9d, %eax + addl $0x4, %ecx + jmp Lmldsa_rej_uniform_eta4_avx2_asm_loop + +Lmldsa_rej_uniform_eta4_avx2_asm_scalar: + cmpl $0x100, %eax # imm = 0x100 + jae Lmldsa_rej_uniform_eta4_avx2_asm_done + cmpl $0x110, %ecx # imm = 0x110 + jae Lmldsa_rej_uniform_eta4_avx2_asm_done + movzbl (%rsi,%rcx), %r11d + incl %ecx + movl %r11d, %r10d + andl $0xf, %r10d + cmpl $0x9, %r10d + jae Lmldsa_rej_uniform_eta4_avx2_asm_high_nibble + movl $0x4, %r9d + subl %r10d, %r9d + movl %r9d, (%rdi,%rax,4) + incl %eax + cmpl $0x100, %eax # imm = 0x100 + jae Lmldsa_rej_uniform_eta4_avx2_asm_done + +Lmldsa_rej_uniform_eta4_avx2_asm_high_nibble: + shrl $0x4, %r11d + andl $0xf, %r11d + cmpl $0x9, %r11d + jae Lmldsa_rej_uniform_eta4_avx2_asm_scalar + movl $0x4, %r10d + subl %r11d, %r10d + movl %r10d, (%rdi,%rax,4) + incl %eax + jmp Lmldsa_rej_uniform_eta4_avx2_asm_scalar + +Lmldsa_rej_uniform_eta4_avx2_asm_done: + retq + .cfi_endproc + +S2N_BN_SIZE_DIRECTIVE(mldsa_rej_uniform_eta4_avx2_asm) + + +#if defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_table.c b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_table.c new file mode 100644 index 0000000000..c36da44254 --- /dev/null +++ b/crypto/fipsmodule/ml_dsa/mldsa/native/x86_64/src/rej_uniform_table.c @@ -0,0 +1,161 @@ +/* + * Copyright (c) The mldsa-native project authors + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT + */ + +/* + * WARNING: This file is auto-generated from scripts/autogen + * in the mldsa-native repository. + * Do not modify it directly. + */ + +#include "../../../common.h" + +#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) && \ + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) + +#include "arith_native_x86_64.h" + +/* + * Lookup table used by rejection sampling. + * See autogen for details. + */ +MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint8_t + mld_rej_uniform_table[256][8] = { + {0, 0, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0, 0, 0}, + {1, 0, 0, 0, 0, 0, 0, 0}, {0, 1, 0, 0, 0, 0, 0, 0}, + {2, 0, 0, 0, 0, 0, 0, 0}, {0, 2, 0, 0, 0, 0, 0, 0}, + {1, 2, 0, 0, 0, 0, 0, 0}, {0, 1, 2, 0, 0, 0, 0, 0}, + {3, 0, 0, 0, 0, 0, 0, 0}, {0, 3, 0, 0, 0, 0, 0, 0}, + {1, 3, 0, 0, 0, 0, 0, 0}, {0, 1, 3, 0, 0, 0, 0, 0}, + {2, 3, 0, 0, 0, 0, 0, 0}, {0, 2, 3, 0, 0, 0, 0, 0}, + {1, 2, 3, 0, 0, 0, 0, 0}, {0, 1, 2, 3, 0, 0, 0, 0}, + {4, 0, 0, 0, 0, 0, 0, 0}, {0, 4, 0, 0, 0, 0, 0, 0}, + {1, 4, 0, 0, 0, 0, 0, 0}, {0, 1, 4, 0, 0, 0, 0, 0}, + {2, 4, 0, 0, 0, 0, 0, 0}, {0, 2, 4, 0, 0, 0, 0, 0}, + {1, 2, 4, 0, 0, 0, 0, 0}, {0, 1, 2, 4, 0, 0, 0, 0}, + {3, 4, 0, 0, 0, 0, 0, 0}, {0, 3, 4, 0, 0, 0, 0, 0}, + {1, 3, 4, 0, 0, 0, 0, 0}, {0, 1, 3, 4, 0, 0, 0, 0}, + {2, 3, 4, 0, 0, 0, 0, 0}, {0, 2, 3, 4, 0, 0, 0, 0}, + {1, 2, 3, 4, 0, 0, 0, 0}, {0, 1, 2, 3, 4, 0, 0, 0}, + {5, 0, 0, 0, 0, 0, 0, 0}, {0, 5, 0, 0, 0, 0, 0, 0}, + {1, 5, 0, 0, 0, 0, 0, 0}, {0, 1, 5, 0, 0, 0, 0, 0}, + {2, 5, 0, 0, 0, 0, 0, 0}, {0, 2, 5, 0, 0, 0, 0, 0}, + {1, 2, 5, 0, 0, 0, 0, 0}, {0, 1, 2, 5, 0, 0, 0, 0}, + {3, 5, 0, 0, 0, 0, 0, 0}, {0, 3, 5, 0, 0, 0, 0, 0}, + {1, 3, 5, 0, 0, 0, 0, 0}, {0, 1, 3, 5, 0, 0, 0, 0}, + {2, 3, 5, 0, 0, 0, 0, 0}, {0, 2, 3, 5, 0, 0, 0, 0}, + {1, 2, 3, 5, 0, 0, 0, 0}, {0, 1, 2, 3, 5, 0, 0, 0}, + {4, 5, 0, 0, 0, 0, 0, 0}, {0, 4, 5, 0, 0, 0, 0, 0}, + {1, 4, 5, 0, 0, 0, 0, 0}, {0, 1, 4, 5, 0, 0, 0, 0}, + {2, 4, 5, 0, 0, 0, 0, 0}, {0, 2, 4, 5, 0, 0, 0, 0}, + {1, 2, 4, 5, 0, 0, 0, 0}, {0, 1, 2, 4, 5, 0, 0, 0}, + {3, 4, 5, 0, 0, 0, 0, 0}, {0, 3, 4, 5, 0, 0, 0, 0}, + {1, 3, 4, 5, 0, 0, 0, 0}, {0, 1, 3, 4, 5, 0, 0, 0}, + {2, 3, 4, 5, 0, 0, 0, 0}, {0, 2, 3, 4, 5, 0, 0, 0}, + {1, 2, 3, 4, 5, 0, 0, 0}, {0, 1, 2, 3, 4, 5, 0, 0}, + {6, 0, 0, 0, 0, 0, 0, 0}, {0, 6, 0, 0, 0, 0, 0, 0}, + {1, 6, 0, 0, 0, 0, 0, 0}, {0, 1, 6, 0, 0, 0, 0, 0}, + {2, 6, 0, 0, 0, 0, 0, 0}, {0, 2, 6, 0, 0, 0, 0, 0}, + {1, 2, 6, 0, 0, 0, 0, 0}, {0, 1, 2, 6, 0, 0, 0, 0}, + {3, 6, 0, 0, 0, 0, 0, 0}, {0, 3, 6, 0, 0, 0, 0, 0}, + {1, 3, 6, 0, 0, 0, 0, 0}, {0, 1, 3, 6, 0, 0, 0, 0}, + {2, 3, 6, 0, 0, 0, 0, 0}, {0, 2, 3, 6, 0, 0, 0, 0}, + {1, 2, 3, 6, 0, 0, 0, 0}, {0, 1, 2, 3, 6, 0, 0, 0}, + {4, 6, 0, 0, 0, 0, 0, 0}, {0, 4, 6, 0, 0, 0, 0, 0}, + {1, 4, 6, 0, 0, 0, 0, 0}, {0, 1, 4, 6, 0, 0, 0, 0}, + {2, 4, 6, 0, 0, 0, 0, 0}, {0, 2, 4, 6, 0, 0, 0, 0}, + {1, 2, 4, 6, 0, 0, 0, 0}, {0, 1, 2, 4, 6, 0, 0, 0}, + {3, 4, 6, 0, 0, 0, 0, 0}, {0, 3, 4, 6, 0, 0, 0, 0}, + {1, 3, 4, 6, 0, 0, 0, 0}, {0, 1, 3, 4, 6, 0, 0, 0}, + {2, 3, 4, 6, 0, 0, 0, 0}, {0, 2, 3, 4, 6, 0, 0, 0}, + {1, 2, 3, 4, 6, 0, 0, 0}, {0, 1, 2, 3, 4, 6, 0, 0}, + {5, 6, 0, 0, 0, 0, 0, 0}, {0, 5, 6, 0, 0, 0, 0, 0}, + {1, 5, 6, 0, 0, 0, 0, 0}, {0, 1, 5, 6, 0, 0, 0, 0}, + {2, 5, 6, 0, 0, 0, 0, 0}, {0, 2, 5, 6, 0, 0, 0, 0}, + {1, 2, 5, 6, 0, 0, 0, 0}, {0, 1, 2, 5, 6, 0, 0, 0}, + {3, 5, 6, 0, 0, 0, 0, 0}, {0, 3, 5, 6, 0, 0, 0, 0}, + {1, 3, 5, 6, 0, 0, 0, 0}, {0, 1, 3, 5, 6, 0, 0, 0}, + {2, 3, 5, 6, 0, 0, 0, 0}, {0, 2, 3, 5, 6, 0, 0, 0}, + {1, 2, 3, 5, 6, 0, 0, 0}, {0, 1, 2, 3, 5, 6, 0, 0}, + {4, 5, 6, 0, 0, 0, 0, 0}, {0, 4, 5, 6, 0, 0, 0, 0}, + {1, 4, 5, 6, 0, 0, 0, 0}, {0, 1, 4, 5, 6, 0, 0, 0}, + {2, 4, 5, 6, 0, 0, 0, 0}, {0, 2, 4, 5, 6, 0, 0, 0}, + {1, 2, 4, 5, 6, 0, 0, 0}, {0, 1, 2, 4, 5, 6, 0, 0}, + {3, 4, 5, 6, 0, 0, 0, 0}, {0, 3, 4, 5, 6, 0, 0, 0}, + {1, 3, 4, 5, 6, 0, 0, 0}, {0, 1, 3, 4, 5, 6, 0, 0}, + {2, 3, 4, 5, 6, 0, 0, 0}, {0, 2, 3, 4, 5, 6, 0, 0}, + {1, 2, 3, 4, 5, 6, 0, 0}, {0, 1, 2, 3, 4, 5, 6, 0}, + {7, 0, 0, 0, 0, 0, 0, 0}, {0, 7, 0, 0, 0, 0, 0, 0}, + {1, 7, 0, 0, 0, 0, 0, 0}, {0, 1, 7, 0, 0, 0, 0, 0}, + {2, 7, 0, 0, 0, 0, 0, 0}, {0, 2, 7, 0, 0, 0, 0, 0}, + {1, 2, 7, 0, 0, 0, 0, 0}, {0, 1, 2, 7, 0, 0, 0, 0}, + {3, 7, 0, 0, 0, 0, 0, 0}, {0, 3, 7, 0, 0, 0, 0, 0}, + {1, 3, 7, 0, 0, 0, 0, 0}, {0, 1, 3, 7, 0, 0, 0, 0}, + {2, 3, 7, 0, 0, 0, 0, 0}, {0, 2, 3, 7, 0, 0, 0, 0}, + {1, 2, 3, 7, 0, 0, 0, 0}, {0, 1, 2, 3, 7, 0, 0, 0}, + {4, 7, 0, 0, 0, 0, 0, 0}, {0, 4, 7, 0, 0, 0, 0, 0}, + {1, 4, 7, 0, 0, 0, 0, 0}, {0, 1, 4, 7, 0, 0, 0, 0}, + {2, 4, 7, 0, 0, 0, 0, 0}, {0, 2, 4, 7, 0, 0, 0, 0}, + {1, 2, 4, 7, 0, 0, 0, 0}, {0, 1, 2, 4, 7, 0, 0, 0}, + {3, 4, 7, 0, 0, 0, 0, 0}, {0, 3, 4, 7, 0, 0, 0, 0}, + {1, 3, 4, 7, 0, 0, 0, 0}, {0, 1, 3, 4, 7, 0, 0, 0}, + {2, 3, 4, 7, 0, 0, 0, 0}, {0, 2, 3, 4, 7, 0, 0, 0}, + {1, 2, 3, 4, 7, 0, 0, 0}, {0, 1, 2, 3, 4, 7, 0, 0}, + {5, 7, 0, 0, 0, 0, 0, 0}, {0, 5, 7, 0, 0, 0, 0, 0}, + {1, 5, 7, 0, 0, 0, 0, 0}, {0, 1, 5, 7, 0, 0, 0, 0}, + {2, 5, 7, 0, 0, 0, 0, 0}, {0, 2, 5, 7, 0, 0, 0, 0}, + {1, 2, 5, 7, 0, 0, 0, 0}, {0, 1, 2, 5, 7, 0, 0, 0}, + {3, 5, 7, 0, 0, 0, 0, 0}, {0, 3, 5, 7, 0, 0, 0, 0}, + {1, 3, 5, 7, 0, 0, 0, 0}, {0, 1, 3, 5, 7, 0, 0, 0}, + {2, 3, 5, 7, 0, 0, 0, 0}, {0, 2, 3, 5, 7, 0, 0, 0}, + {1, 2, 3, 5, 7, 0, 0, 0}, {0, 1, 2, 3, 5, 7, 0, 0}, + {4, 5, 7, 0, 0, 0, 0, 0}, {0, 4, 5, 7, 0, 0, 0, 0}, + {1, 4, 5, 7, 0, 0, 0, 0}, {0, 1, 4, 5, 7, 0, 0, 0}, + {2, 4, 5, 7, 0, 0, 0, 0}, {0, 2, 4, 5, 7, 0, 0, 0}, + {1, 2, 4, 5, 7, 0, 0, 0}, {0, 1, 2, 4, 5, 7, 0, 0}, + {3, 4, 5, 7, 0, 0, 0, 0}, {0, 3, 4, 5, 7, 0, 0, 0}, + {1, 3, 4, 5, 7, 0, 0, 0}, {0, 1, 3, 4, 5, 7, 0, 0}, + {2, 3, 4, 5, 7, 0, 0, 0}, {0, 2, 3, 4, 5, 7, 0, 0}, + {1, 2, 3, 4, 5, 7, 0, 0}, {0, 1, 2, 3, 4, 5, 7, 0}, + {6, 7, 0, 0, 0, 0, 0, 0}, {0, 6, 7, 0, 0, 0, 0, 0}, + {1, 6, 7, 0, 0, 0, 0, 0}, {0, 1, 6, 7, 0, 0, 0, 0}, + {2, 6, 7, 0, 0, 0, 0, 0}, {0, 2, 6, 7, 0, 0, 0, 0}, + {1, 2, 6, 7, 0, 0, 0, 0}, {0, 1, 2, 6, 7, 0, 0, 0}, + {3, 6, 7, 0, 0, 0, 0, 0}, {0, 3, 6, 7, 0, 0, 0, 0}, + {1, 3, 6, 7, 0, 0, 0, 0}, {0, 1, 3, 6, 7, 0, 0, 0}, + {2, 3, 6, 7, 0, 0, 0, 0}, {0, 2, 3, 6, 7, 0, 0, 0}, + {1, 2, 3, 6, 7, 0, 0, 0}, {0, 1, 2, 3, 6, 7, 0, 0}, + {4, 6, 7, 0, 0, 0, 0, 0}, {0, 4, 6, 7, 0, 0, 0, 0}, + {1, 4, 6, 7, 0, 0, 0, 0}, {0, 1, 4, 6, 7, 0, 0, 0}, + {2, 4, 6, 7, 0, 0, 0, 0}, {0, 2, 4, 6, 7, 0, 0, 0}, + {1, 2, 4, 6, 7, 0, 0, 0}, {0, 1, 2, 4, 6, 7, 0, 0}, + {3, 4, 6, 7, 0, 0, 0, 0}, {0, 3, 4, 6, 7, 0, 0, 0}, + {1, 3, 4, 6, 7, 0, 0, 0}, {0, 1, 3, 4, 6, 7, 0, 0}, + {2, 3, 4, 6, 7, 0, 0, 0}, {0, 2, 3, 4, 6, 7, 0, 0}, + {1, 2, 3, 4, 6, 7, 0, 0}, {0, 1, 2, 3, 4, 6, 7, 0}, + {5, 6, 7, 0, 0, 0, 0, 0}, {0, 5, 6, 7, 0, 0, 0, 0}, + {1, 5, 6, 7, 0, 0, 0, 0}, {0, 1, 5, 6, 7, 0, 0, 0}, + {2, 5, 6, 7, 0, 0, 0, 0}, {0, 2, 5, 6, 7, 0, 0, 0}, + {1, 2, 5, 6, 7, 0, 0, 0}, {0, 1, 2, 5, 6, 7, 0, 0}, + {3, 5, 6, 7, 0, 0, 0, 0}, {0, 3, 5, 6, 7, 0, 0, 0}, + {1, 3, 5, 6, 7, 0, 0, 0}, {0, 1, 3, 5, 6, 7, 0, 0}, + {2, 3, 5, 6, 7, 0, 0, 0}, {0, 2, 3, 5, 6, 7, 0, 0}, + {1, 2, 3, 5, 6, 7, 0, 0}, {0, 1, 2, 3, 5, 6, 7, 0}, + {4, 5, 6, 7, 0, 0, 0, 0}, {0, 4, 5, 6, 7, 0, 0, 0}, + {1, 4, 5, 6, 7, 0, 0, 0}, {0, 1, 4, 5, 6, 7, 0, 0}, + {2, 4, 5, 6, 7, 0, 0, 0}, {0, 2, 4, 5, 6, 7, 0, 0}, + {1, 2, 4, 5, 6, 7, 0, 0}, {0, 1, 2, 4, 5, 6, 7, 0}, + {3, 4, 5, 6, 7, 0, 0, 0}, {0, 3, 4, 5, 6, 7, 0, 0}, + {1, 3, 4, 5, 6, 7, 0, 0}, {0, 1, 3, 4, 5, 6, 7, 0}, + {2, 3, 4, 5, 6, 7, 0, 0}, {0, 2, 3, 4, 5, 6, 7, 0}, + {1, 2, 3, 4, 5, 6, 7, 0}, {0, 1, 2, 3, 4, 5, 6, 7}, +}; + +#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ + */ + +MLD_EMPTY_CU(avx2_rej_uniform_table) + +#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT && \ + !MLD_CONFIG_MULTILEVEL_NO_SHARED) */ diff --git a/crypto/fipsmodule/ml_dsa/mldsa/packing.h b/crypto/fipsmodule/ml_dsa/mldsa/packing.h index a039db99dd..ec5755433f 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/packing.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/packing.h @@ -2,6 +2,16 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + #ifndef MLD_PACKING_H #define MLD_PACKING_H @@ -13,6 +23,9 @@ /** * Bit-pack the s1 component into the secret key. * + * @spec{Partially implements @[FIPS204, Algorithm 24, skEncode] (s1 + * component).} + * * @param[out] sk Output byte array. * @param[in] s1 Pointer to vector s1. */ @@ -34,6 +47,9 @@ __contract__( * s1 must already be packed via mld_pack_sk_s1, and t0 via * mld_compute_pack_t0_t1. * + * @spec{Partially implements @[FIPS204, Algorithm 24, skEncode] (rho, key, tr, + * s2 components).} + * * @param[out] sk Output byte array. * @param[in] rho Byte array containing rho. * @param[in] tr Byte array containing tr. @@ -64,6 +80,9 @@ __contract__( /** * Bit-pack challenge c into sig = (c, z, h). * + * @spec{Partially implements @[FIPS204, Algorithm 26, sigEncode] (c + * component).} + * * @param[out] sig Output byte array. * @param[in] c Pointer to challenge hash. */ @@ -80,6 +99,14 @@ __contract__( /** * Compute hints from (w0, w1) and pack them into the hint section of sig. * + * @spec{Combines the hint computation @[FIPS204, Algorithm 39, MakeHint] with + * the packing @[FIPS204, Algorithm 20, HintBitPack] (the h component of + * @[FIPS204, Algorithm 26, sigEncode]): it computes the hint vector h from + * (w0, w1) and packs it, rather than receiving a ready-made h as HintBitPack + * does. The hints are computed via mld_make_hint (rounding.h), a specialized + * MakeHint valid only for the values arising during signing; see the block + * comment in mld_attempt_signature_generation (sign.c).} + * * @param[in,out] sig Byte array containing signature. * @param[in] w0 Pointer to low part of input vector. * @param[in] w1 Pointer to high part of input vector. @@ -109,6 +136,9 @@ __contract__( * The c and h components are packed separately using mld_pack_sig_c and * mld_pack_sig_h. * + * @spec{Partially implements @[FIPS204, Algorithm 26, sigEncode] (one + * polynomial of the z component).} + * * @param[in,out] sig Output byte array. * @param[in] zi Pointer to a single polynomial in z. * @param i Index of zi in vector z. @@ -131,6 +161,9 @@ __contract__( * Unpack a single polynomial of the t1 component of a public key * pk = (rho, t1). * + * @spec{Partially implements @[FIPS204, Algorithm 23, pkDecode] (one polynomial + * of t1).} + * * @param[out] t1 Pointer to output polynomial t1[i]. * @param[in] pk Byte array containing bit-packed pk. * @param i Row index, must be < MLDSA_K. @@ -155,6 +188,8 @@ __contract__( * * NOTE: In REDUCE_RAM mode, s1/s2/t0 borrow from sk rather than copying. * + * @spec{Implements @[FIPS204, Algorithm 25, skDecode].} + * * @param[out] rho Output byte array for rho. * @param[out] tr Output byte array for tr. * @param[out] key Output byte array for key. @@ -215,6 +250,9 @@ __contract__( * call returns MLD_ERR_FAIL the encoding is malformed and the signature must * be rejected. * + * @spec{Implements @[FIPS204, Algorithm 21, HintBitUnpack] (one row; part of + * @[FIPS204, Algorithm 27, sigDecode]).} + * * @param[out] h Pointer to output polynomial h[i]. * @param[in] sig Signature buffer. * @param i Row index, must be < MLDSA_K. diff --git a/crypto/fipsmodule/ml_dsa/mldsa/params.h b/crypto/fipsmodule/ml_dsa/mldsa/params.h index acc9a1a0e9..db3bf6c539 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/params.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/params.h @@ -26,7 +26,7 @@ #define MLDSA_ETA 2 #define MLDSA_TAU 39 #define MLDSA_BETA 78 -#define MLDSA_GAMMA1 (1 << 17) +#define MLDSA_GAMMA1 ((int32_t)1 << 17) #define MLDSA_GAMMA2 MLDSA_GAMMA2_88 #define MLDSA_OMEGA 80 #define MLDSA_CTILDEBYTES 32 @@ -41,7 +41,7 @@ #define MLDSA_ETA 4 #define MLDSA_TAU 49 #define MLDSA_BETA 196 -#define MLDSA_GAMMA1 (1 << 19) +#define MLDSA_GAMMA1 ((int32_t)1 << 19) #define MLDSA_GAMMA2 MLDSA_GAMMA2_32 #define MLDSA_OMEGA 55 #define MLDSA_CTILDEBYTES 48 @@ -56,7 +56,7 @@ #define MLDSA_ETA 2 #define MLDSA_TAU 60 #define MLDSA_BETA 120 -#define MLDSA_GAMMA1 (1 << 19) +#define MLDSA_GAMMA1 ((int32_t)1 << 19) #define MLDSA_GAMMA2 MLDSA_GAMMA2_32 #define MLDSA_OMEGA 75 #define MLDSA_CTILDEBYTES 64 @@ -70,6 +70,12 @@ #define MLDSA_POLYT0_PACKEDBYTES 416 #define MLDSA_POLYVECH_PACKEDBYTES (MLDSA_OMEGA + MLDSA_K) +/* Sampling y from counter kappa uses nonces kappa, ..., kappa+L-1, which fit in + * uint16_t iff kappa <= UINT16_MAX - MLDSA_L. With kappa = attempt*MLDSA_L this + * bounds the number of signing attempts by MLD_MAX_KAPPA / MLDSA_L; see + * MLD_MAX_SIGNING_ATTEMPTS in sign.c. */ +#define MLD_MAX_KAPPA (UINT16_MAX - MLDSA_L) + /* Layout of the packed public key pk[MLDSA_CRYPTO_PUBLICKEYBYTES] = (rho, t1): * * +-------------+--------------------------+ diff --git a/crypto/fipsmodule/ml_dsa/mldsa/poly.c b/crypto/fipsmodule/ml_dsa/mldsa/poly.c index 12ed2cf8d7..4d74306841 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/poly.c +++ b/crypto/fipsmodule/ml_dsa/mldsa/poly.c @@ -217,7 +217,7 @@ static MLD_INLINE void mld_ntt_butterfly_block(int32_t r[MLDSA_N], const int32_t zeta, const unsigned start, const unsigned len, - const unsigned bound) + const uint32_t bound) __contract__( requires(start < MLDSA_N) requires(1 <= len && len <= MLDSA_N / 2 && start + 2 * len <= MLDSA_N) diff --git a/crypto/fipsmodule/ml_dsa/mldsa/poly.h b/crypto/fipsmodule/ml_dsa/mldsa/poly.h index e7d4871b14..2467f23798 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/poly.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/poly.h @@ -2,6 +2,16 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + #ifndef MLD_POLY_H #define MLD_POLY_H @@ -64,6 +74,9 @@ __contract__( /** * Add polynomials. No modular reduction is performed. * + * @spec{Implements @[FIPS204, Algorithm 44, AddNTT] (coefficientwise + * polynomial addition; also used for addition in the normal domain).} + * * @param[in,out] r Pointer to input-output polynomial to be added to. * @param[in] b Pointer to input polynomial that should be added to r. * Must be disjoint from r. @@ -136,6 +149,8 @@ __contract__( * In-place forward NTT. Output coefficients are bounded by MLD_NTT_BOUND in * absolute value. * + * @spec{Implements @[FIPS204, Algorithm 41, NTT].} + * * @param[in,out] a Pointer to input/output polynomial. */ MLD_INTERNAL_API @@ -150,11 +165,16 @@ __contract__( #define mld_poly_invntt_tomont MLD_NAMESPACE(poly_invntt_tomont) /** - * In-place inverse NTT and multiplication by 2^{32}. + * In-place inverse NTT. * * Input coefficients need to be less than MLDSA_Q in absolute value and * output coefficients are bounded by MLD_INTT_BOUND. * + * @spec{Implements @[FIPS204, Algorithm 42, NTT^{-1}] up to scaling: + * The input is scaled by 2^{-32} as a result of the Montgomery base + * multiplication. The output is in normal domain. In other words, this + * function implements `NTT^{-1} o mult(2^32)`.} + * * @param[in,out] a Pointer to input/output polynomial. */ MLD_INTERNAL_API @@ -170,9 +190,12 @@ __contract__( defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) #define mld_poly_pointwise_montgomery MLD_NAMESPACE(poly_pointwise_montgomery) /** - * Pointwise multiplication of polynomials in NTT domain representation and - * multiplication of resulting polynomial by 2^{-32}. Destructive in the first - * argument. + * Pointwise multiplication of polynomials. Destructive in the first argument. + * + * @spec{Implements @[FIPS204, Algorithm 45, MultiplyNTT], up to scaling: The + * input is in normal domain, the output is scaled by 2^{-32} as a result of + * the use of Montgomery multiplication. In other words, this function + * implements `mult(2^{-32}) o MultiplyNTT`.} * * @param[in,out] a Pointer to first input/output polynomial. On entry, holds * the first multiplicand; on exit, holds the product @@ -225,6 +248,8 @@ __contract__( * Sample polynomial with uniformly random coefficients in [0, MLDSA_Q-1] by * performing rejection sampling on the output stream of SHAKE128(seed|nonce). * + * @spec{Implements @[FIPS204, Algorithm 30, RejNTTPoly].} + * * @param[out] a Pointer to output polynomial. * @param[in] seed Byte array with seed of length MLDSA_SEEDBYTES and the * packed 2-byte nonce. @@ -245,6 +270,8 @@ __contract__( * Generate four polynomials using rejection sampling on (pseudo-)uniformly * random bytes sampled from a seed. * + * @spec{Implements @[FIPS204, Algorithm 30, RejNTTPoly] (four-way batched).} + * * @param[out] vec0 Pointer to first polynomial to be sampled. * @param[out] vec1 Pointer to second polynomial to be sampled. * @param[out] vec2 Pointer to third polynomial to be sampled. @@ -280,6 +307,8 @@ __contract__( * Bit-pack polynomial t1 with coefficients fitting in 10 bits. Input * coefficients are assumed to be standard representatives. * + * @spec{Implements @[FIPS204, Algorithm 16, SimpleBitPack].} + * * @param[out] r Pointer to output byte array with at least * MLDSA_POLYT1_PACKEDBYTES bytes. * @param[in] a Pointer to input polynomial. @@ -300,6 +329,8 @@ __contract__( * Unpack polynomial t1 with 10-bit coefficients. Output coefficients are * standard representatives. * + * @spec{Implements @[FIPS204, Algorithm 18, SimpleBitUnpack].} + * * @param[out] r Pointer to output polynomial. * @param[in] a Byte array with bit-packed polynomial. */ @@ -318,6 +349,8 @@ __contract__( /** * Bit-pack polynomial t0 with coefficients in ]-2^{MLDSA_D-1}, 2^{MLDSA_D-1}]. * + * @spec{Implements @[FIPS204, Algorithm 17, BitPack].} + * * @param[out] r Pointer to output byte array with at least * MLDSA_POLYT0_PACKEDBYTES bytes. * @param[in] a Pointer to input polynomial. @@ -337,6 +370,8 @@ __contract__( /** * Unpack polynomial t0 with coefficients in ]-2^{MLDSA_D-1}, 2^{MLDSA_D-1}]. * + * @spec{Implements @[FIPS204, Algorithm 19, BitUnpack].} + * * @param[out] r Pointer to output polynomial. * @param[in] a Byte array with bit-packed polynomial. */ @@ -355,11 +390,12 @@ __contract__( * Check infinity norm of polynomial against given bound. Assumes input * coefficients were reduced by mld_reduce32(). * - * @spec{The definition in FIPS-204 requires signed canonical reduction prior - * to applying the bounds check. However, `-B < (a mod± MLDSA_Q) < B` is - * equivalent to `-B < a < B` under the assumption that - * `B <= MLDSA_Q - MLD_REDUCE32_RANGE_MAX` (cf. the assertion in the code). - * Hence, the present spec and implementation are correct without reduction.} + * @spec{@[FIPS204] defines the infinity norm via signed canonical reduction + * (mod± MLDSA_Q) prior to applying the bounds check. However, + * `-B < (a mod± MLDSA_Q) < B` is equivalent to `-B < a < B` under the + * assumption that `B <= MLDSA_Q - MLD_REDUCE32_RANGE_MAX` (cf. the assertion in + * the code). Hence, this contract and implementation are correct without + * reduction.} * * @param[in] a Pointer to polynomial. * @param B Norm bound. diff --git a/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.c b/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.c index 5be537d468..befe58a8b6 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.c +++ b/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.c @@ -905,5 +905,4 @@ void mld_polyz_unpack(mld_poly *r, const uint8_t a[MLDSA_POLYZ_PACKEDBYTES]) #undef mld_poly_use_hint_c #undef mld_polyz_unpack_c #undef MLD_POLY_UNIFORM_ETA_NBLOCKS -#undef MLD_POLY_UNIFORM_ETA_NBLOCKS #undef MLD_POLY_UNIFORM_GAMMA1_NBLOCKS diff --git a/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.h b/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.h index 341cd987a4..2dca8f17cd 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/poly_kl.h @@ -2,6 +2,16 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + #ifndef MLD_POLY_KL_H #define MLD_POLY_KL_H @@ -68,6 +78,9 @@ __contract__( * [-MLDSA_ETA, MLDSA_ETA] by performing rejection sampling on the output * stream from SHAKE256(seed|nonce_i). * + * @spec{Implements @[FIPS204, Algorithm 31, RejBoundedPoly] (four-way + * batched).} + * * @param[out] r0 Pointer to first output polynomial. * @param[out] r1 Pointer to second output polynomial. * @param[out] r2 Pointer to third output polynomial. @@ -107,6 +120,8 @@ __contract__( * [-MLDSA_ETA, MLDSA_ETA] by performing rejection sampling on the output * stream from SHAKE256(seed|nonce). * + * @spec{Implements @[FIPS204, Algorithm 31, RejBoundedPoly].} + * * @param[out] r Pointer to output polynomial. * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. * @param nonce Nonce. @@ -133,6 +148,9 @@ __contract__( * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1] by unpacking output stream of * SHAKE256(seed|nonce). * + * @spec{Partially implements @[FIPS204, Algorithm 34, ExpandMask] (one + * polynomial, i.e. the loop body of lines 3-5).} + * * @param[out] a Pointer to output polynomial. * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. * @param nonce 16-bit nonce. @@ -157,6 +175,9 @@ __contract__( * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1] by unpacking output streams of * SHAKE256(seed|nonce_i). * + * @spec{Partially implements @[FIPS204, Algorithm 34, ExpandMask] (four-way + * batched, i.e. four iterations of the loop body of lines 3-5).} + * * @param[out] r0 Pointer to first output polynomial. * @param[out] r1 Pointer to second output polynomial. * @param[out] r2 Pointer to third output polynomial. @@ -195,8 +216,10 @@ __contract__( #if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) #define mld_poly_challenge MLD_NAMESPACE_KL(poly_challenge) /** - * Implementation of H. Samples polynomial with MLDSA_TAU nonzero coefficients - * in {-1, 1} using the output stream of SHAKE256(seed). + * Samples polynomial with MLDSA_TAU nonzero coefficients in {-1, 1} using the + * output stream of SHAKE256(seed). + * + * @spec{Implements @[FIPS204, Algorithm 29, SampleInBall].} * * @param[out] c Pointer to output polynomial. * @param[in] seed Byte array containing seed of length MLDSA_CTILDEBYTES. @@ -217,6 +240,8 @@ __contract__( /** * Bit-pack polynomial with coefficients in [-MLDSA_ETA, MLDSA_ETA]. * + * @spec{Implements @[FIPS204, Algorithm 17, BitPack].} + * * @param[out] r Pointer to output byte array with at least * MLDSA_POLYETA_PACKEDBYTES bytes. * @param[in] a Pointer to input polynomial. @@ -252,6 +277,8 @@ __contract__( /** * Unpack polynomial with coefficients in [-MLDSA_ETA, MLDSA_ETA]. * + * @spec{Implements @[FIPS204, Algorithm 19, BitUnpack].} + * * @param[out] r Pointer to output polynomial. * @param[in] a Byte array with bit-packed polynomial. */ @@ -271,6 +298,8 @@ __contract__( * Bit-pack polynomial with coefficients in * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1]. * + * @spec{Implements @[FIPS204, Algorithm 17, BitPack].} + * * @param[out] r Pointer to output byte array with at least * MLDSA_POLYZ_PACKEDBYTES bytes. * @param[in] a Pointer to input polynomial. @@ -291,6 +320,8 @@ __contract__( * Unpack polynomial z with coefficients in * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1]. * + * @spec{Implements @[FIPS204, Algorithm 19, BitUnpack].} + * * @param[out] r Pointer to output polynomial. * @param[in] a Byte array with bit-packed polynomial. */ @@ -310,6 +341,8 @@ __contract__( * for ML-DSA-65/87. Dispatches to the value-specialized variant for the * selected parameter set. * + * @spec{Implements @[FIPS204, Algorithm 16, SimpleBitPack].} + * * @param[out] r Pointer to output byte array with at least * MLDSA_POLYW1_PACKEDBYTES bytes. * @param[in] a Pointer to input polynomial. diff --git a/crypto/fipsmodule/ml_dsa/mldsa/polyvec.c b/crypto/fipsmodule/ml_dsa/mldsa/polyvec.c index 58571973df..8d427a8f85 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/polyvec.c +++ b/crypto/fipsmodule/ml_dsa/mldsa/polyvec.c @@ -30,39 +30,38 @@ MLD_INTERNAL_API void mld_polyvecl_uniform_gamma1(mld_polyvecl *v, const uint8_t seed[MLDSA_CRHBYTES], - uint16_t nonce) + uint16_t kappa) { #if defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) int i; #endif - /* Safety: nonce is at most ((UINT16_MAX - MLDSA_L) / MLDSA_L), and, hence, - * this cast is safe. See MLD_NONCE_UB comment in sign.c. */ - nonce = (uint16_t)(MLDSA_L * nonce); - /* Now, nonce <= UINT16_MAX - (MLDSA_L - 1), so the casts below are safe. */ + /* The caller passes the base counter kappa; component i is sampled from + * kappa + i. Safety: kappa <= MLD_MAX_KAPPA and i < MLDSA_L, so the + * casts below are safe. See MLD_MAX_KAPPA comment in params.h. */ #if defined(MLD_CONFIG_SERIAL_FIPS202_ONLY) for (i = 0; i < MLDSA_L; i++) { - mld_poly_uniform_gamma1(&v->vec[i], seed, (uint16_t)(nonce + i)); + mld_poly_uniform_gamma1(&v->vec[i], seed, (uint16_t)(kappa + i)); } #else /* MLD_CONFIG_SERIAL_FIPS202_ONLY */ #if MLDSA_L == 4 mld_poly_uniform_gamma1_4x(&v->vec[0], &v->vec[1], &v->vec[2], &v->vec[3], - seed, nonce, (uint16_t)(nonce + 1), - (uint16_t)(nonce + 2), (uint16_t)(nonce + 3)); + seed, kappa, (uint16_t)(kappa + 1), + (uint16_t)(kappa + 2), (uint16_t)(kappa + 3)); #elif MLDSA_L == 5 mld_poly_uniform_gamma1_4x(&v->vec[0], &v->vec[1], &v->vec[2], &v->vec[3], - seed, nonce, (uint16_t)(nonce + 1), - (uint16_t)(nonce + 2), (uint16_t)(nonce + 3)); - mld_poly_uniform_gamma1(&v->vec[4], seed, (uint16_t)(nonce + 4)); + seed, kappa, (uint16_t)(kappa + 1), + (uint16_t)(kappa + 2), (uint16_t)(kappa + 3)); + mld_poly_uniform_gamma1(&v->vec[4], seed, (uint16_t)(kappa + 4)); #elif MLDSA_L == 7 mld_poly_uniform_gamma1_4x(&v->vec[0], &v->vec[1], &v->vec[2], - &v->vec[3 /* irrelevant */], seed, nonce, - (uint16_t)(nonce + 1), (uint16_t)(nonce + 2), + &v->vec[3 /* irrelevant */], seed, kappa, + (uint16_t)(kappa + 1), (uint16_t)(kappa + 2), 0xFF /* irrelevant */); mld_poly_uniform_gamma1_4x(&v->vec[3], &v->vec[4], &v->vec[5], &v->vec[6], - seed, (uint16_t)(nonce + 3), (uint16_t)(nonce + 4), - (uint16_t)(nonce + 5), (uint16_t)(nonce + 6)); + seed, (uint16_t)(kappa + 3), (uint16_t)(kappa + 4), + (uint16_t)(kappa + 5), (uint16_t)(kappa + 6)); #endif /* MLDSA_L == 7 */ #endif /* !MLD_CONFIG_SERIAL_FIPS202_ONLY */ diff --git a/crypto/fipsmodule/ml_dsa/mldsa/polyvec.h b/crypto/fipsmodule/ml_dsa/mldsa/polyvec.h index 7ad4617126..8a09c66711 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/polyvec.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/polyvec.h @@ -2,6 +2,16 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + #ifndef MLD_POLYVEC_H #define MLD_POLYVEC_H @@ -31,20 +41,22 @@ typedef struct /** * Sample vector of polynomials with uniformly random coefficients in * [-(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1] by unpacking output stream of - * SHAKE256(seed|nonce). + * SHAKE256(seed|kappa+i) for component i. + * + * @spec{Implements @[FIPS204, Algorithm 34, ExpandMask].} * * @param[out] v Pointer to output vector. * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. - * @param nonce 16-bit nonce. + * @param kappa Base counter; component i uses kappa + i. */ MLD_INTERNAL_API void mld_polyvecl_uniform_gamma1(mld_polyvecl *v, const uint8_t seed[MLDSA_CRHBYTES], - uint16_t nonce) + uint16_t kappa) __contract__( requires(memory_no_alias(v, sizeof(mld_polyvecl))) requires(memory_no_alias(seed, MLDSA_CRHBYTES)) - requires(nonce <= (UINT16_MAX - MLDSA_L) / MLDSA_L) + requires(kappa <= MLD_MAX_KAPPA) assigns(memory_slice(v, sizeof(mld_polyvecl))) ensures(forall(k0, 0, MLDSA_L, array_bound(v->vec[k0].coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1))) @@ -89,6 +101,9 @@ __contract__( * The second input "v" is assumed to be output of an NTT, and hence must have * coefficients bounded by [-(MLD_NTT_BOUND-1), MLD_NTT_BOUND-1] inclusive. * + * @spec{Partially implements @[FIPS204, Algorithm 48, MatrixVectorNTT] + * (one output polynomial; multiply-accumulate of two NTT-domain vectors).} + * * @param[out] w Output polynomial. * @param[in] u Pointer to first input vector. * @param[in] v Pointer to second input vector. @@ -290,6 +305,8 @@ __contract__( * Bit-pack polynomial vector w1 with coefficients in [0, 15] or [0, 43]. Input * coefficients are assumed to be standard representatives. * + * @spec{Implements @[FIPS204, Algorithm 28, w1Encode].} + * * @param[out] r Pointer to output byte array with at least * MLDSA_K * MLDSA_POLYW1_PACKEDBYTES bytes. * @param[in] w1 Pointer to input polynomial vector. diff --git a/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.c b/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.c index 6e0971fb61..d85761ea03 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.c +++ b/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.c @@ -29,6 +29,9 @@ * of seed_ext. This function writes the domain-separation bytes * seed_ext[SEEDBYTES..+2] = {l, k} before sampling. * + * @spec{Partially implements @[FIPS204, Algorithm 32, ExpandA] (samples one + * matrix entry via @[FIPS204, Algorithm 30, RejNTTPoly]).} + * * @param[out] p Pointer to output polynomial. * @param[in,out] seed_ext Seed buffer pre-filled with rho in the first * MLDSA_SEEDBYTES; the final two bytes are diff --git a/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.h b/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.h index 8149220bc6..baecdb8226 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/polyvec_lazy.h @@ -127,11 +127,11 @@ typedef struct mld_polyvecl vec; /**< Masking vector y. */ } mld_yvec_eager; -/** Lazy yvec: store seed and nonce, regenerate y[i] on demand. */ +/** Lazy yvec: store seed and base counter kappa, regenerate y[i] on demand. */ typedef struct { const uint8_t *rhoprime; /**< Pointer to seed used to derive y. */ - uint16_t nonce; /**< Base nonce; component i uses MLDSA_L*nonce + i. */ + uint16_t kappa; /**< Base counter; component i uses kappa + i. */ } mld_yvec_lazy; #if !defined(MLD_CONFIG_NO_KEYPAIR_API) || !defined(MLD_CONFIG_NO_SIGN_API) @@ -341,17 +341,17 @@ __contract__( #if !defined(MLD_CONFIG_NO_SIGN_API) && \ (!defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) static MLD_INLINE void mld_yvec_init_eager( - mld_yvec_eager *y, const uint8_t rhoprime[MLDSA_CRHBYTES], uint16_t nonce) + mld_yvec_eager *y, const uint8_t rhoprime[MLDSA_CRHBYTES], uint16_t kappa) __contract__( requires(memory_no_alias(y, sizeof(mld_yvec_eager))) requires(memory_no_alias(rhoprime, MLDSA_CRHBYTES)) - requires(nonce <= (UINT16_MAX - MLDSA_L) / MLDSA_L) + requires(kappa <= MLD_MAX_KAPPA) assigns(memory_slice(y, sizeof(mld_yvec_eager))) ensures(forall(k1, 0, MLDSA_L, array_bound(y->vec.vec[k1].coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1))) ) { - mld_polyvecl_uniform_gamma1(&y->vec, rhoprime, nonce); + mld_polyvecl_uniform_gamma1(&y->vec, rhoprime, kappa); } static MLD_INLINE void mld_yvec_get_poly_eager(mld_poly *buf, @@ -370,16 +370,16 @@ __contract__( #if !defined(MLD_CONFIG_NO_SIGN_API) && \ (defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)) static MLD_INLINE void mld_yvec_init_lazy( - mld_yvec_lazy *y, const uint8_t rhoprime[MLDSA_CRHBYTES], uint16_t nonce) + mld_yvec_lazy *y, const uint8_t rhoprime[MLDSA_CRHBYTES], uint16_t kappa) __contract__( requires(memory_no_alias(y, sizeof(mld_yvec_lazy))) assigns(memory_slice(y, sizeof(mld_yvec_lazy))) ensures(y->rhoprime == old(rhoprime)) - ensures(y->nonce == old(nonce)) + ensures(y->kappa == old(kappa)) ) { y->rhoprime = rhoprime; - y->nonce = nonce; + y->kappa = kappa; } static MLD_INLINE void mld_yvec_get_poly_lazy(mld_poly *buf, @@ -390,16 +390,14 @@ __contract__( requires(memory_no_alias(y, sizeof(mld_yvec_lazy))) requires(i < MLDSA_L) requires(memory_no_alias(y->rhoprime, MLDSA_CRHBYTES)) - requires(y->nonce <= ((UINT16_MAX - MLDSA_L) / MLDSA_L)) + requires(y->kappa <= MLD_MAX_KAPPA) assigns(memory_slice(buf, sizeof(mld_poly))) ensures(array_bound(buf->coeffs, 0, MLDSA_N, -(MLDSA_GAMMA1 - 1), MLDSA_GAMMA1 + 1)) ) { - /* Safety: y->nonce is at most ((UINT16_MAX - MLDSA_L) / MLDSA_L) and - * i < MLDSA_L, so MLDSA_L * y->nonce + i fits in uint16_t. See - * MLD_NONCE_UB comment in sign.c. */ - mld_poly_uniform_gamma1(buf, y->rhoprime, - (uint16_t)(MLDSA_L * y->nonce + (int)i)); + /* Safety: y->kappa <= MLD_MAX_KAPPA and i < MLDSA_L, so y->kappa + i + * fits in uint16_t. See MLD_MAX_KAPPA comment in params.h. */ + mld_poly_uniform_gamma1(buf, y->rhoprime, (uint16_t)(y->kappa + i)); } #endif /* !MLD_CONFIG_NO_SIGN_API && (MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST) \ */ @@ -441,9 +439,10 @@ __contract__( #if !defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST) /** - * Implementation of ExpandA. Generates matrix A with uniformly random - * coefficients a_{i,j} by performing rejection sampling on the output stream - * of SHAKE128(rho|j|i). + * Generates matrix A with uniformly random coefficients a_{i,j} by performing + * rejection sampling on the output stream of SHAKE128(rho|j|i). + * + * @spec{Implements @[FIPS204, Algorithm 32, ExpandA].} * * @param[out] mat Pointer to output matrix. * @param[in] rho Byte array containing seed rho. @@ -579,7 +578,7 @@ __contract__( * * @param[out] w Pointer to output vector. * @param[in,out] mat Pointer to input matrix. - * @param[in] y Pointer to y seed/nonce. + * @param[in] y Pointer to y seed/kappa. * @param[out] scratch Scratch (only &scratch->vec[0] used). */ MLD_INTERNAL_API @@ -593,7 +592,7 @@ __contract__( requires(memory_no_alias(y, sizeof(mld_yvec_lazy))) requires(memory_no_alias(scratch, sizeof(mld_polyvecl))) requires(memory_no_alias(y->rhoprime, MLDSA_CRHBYTES)) - requires(y->nonce <= ((UINT16_MAX - MLDSA_L) / MLDSA_L)) + requires(y->kappa <= MLD_MAX_KAPPA) assigns(memory_slice(w, sizeof(mld_polyveck))) assigns(memory_slice(mat, sizeof(mld_polymat_lazy))) assigns(memory_slice(scratch, sizeof(mld_polyvecl))) diff --git a/crypto/fipsmodule/ml_dsa/mldsa/reduce.h b/crypto/fipsmodule/ml_dsa/mldsa/reduce.h index bb86d07056..199fe222af 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/reduce.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/reduce.h @@ -2,6 +2,16 @@ * Copyright (c) The mldsa-native project authors * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT */ + +/* References + * ========== + * + * - [FIPS204] + * FIPS 204 Module-Lattice-Based Digital Signature Standard + * National Institute of Standards and Technology + * https://csrc.nist.gov/pubs/fips/204/final + */ + #ifndef MLD_REDUCE_H #define MLD_REDUCE_H @@ -11,10 +21,10 @@ #include "debug.h" /* check-magic: -4186625 == pow(2,32,MLDSA_Q) */ -#define MLD_MONT -4186625 +#define MLD_MONT (-4186625) /* Upper bound for domain of mld_reduce32() */ -#define MLD_REDUCE32_DOMAIN_MAX (INT32_MAX - (1 << 22)) +#define MLD_REDUCE32_DOMAIN_MAX (INT32_MAX - ((int32_t)1 << 22)) /* Absolute bound for range of mld_reduce32() */ /* check-magic: 6283009 == (MLD_REDUCE32_DOMAIN_MAX - 255 * MLDSA_Q + 1) */ @@ -24,6 +34,8 @@ * Generic Montgomery reduction; given a 64-bit integer a, computes a 32-bit * integer congruent to a * R^-1 mod MLDSA_Q, where R=2^32. * + * @spec{Implements @[FIPS204, Algorithm 49, MontgomeryReduce].} + * * @param a Input integer to be reduced, of absolute value smaller or equal * to INT64_MAX - 2^31 * MLDSA_Q. * @@ -102,7 +114,7 @@ __contract__( { int32_t t; - t = (a + (1 << 22)) >> 23; + t = (a + ((int32_t)1 << 22)) >> 23; t = a - t * MLDSA_Q; mld_assert((t - a) % MLDSA_Q == 0); return t; diff --git a/crypto/fipsmodule/ml_dsa/mldsa/rounding.h b/crypto/fipsmodule/ml_dsa/mldsa/rounding.h index 21fae3c989..97f70e2013 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/rounding.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/rounding.h @@ -38,6 +38,8 @@ * -2^{MLDSA_D-1} < a0 <= 2^{MLDSA_D-1}. Assumes a to be standard * representative. * + * @spec{Implements @[FIPS204, Algorithm 35, Power2Round].} + * * @reference{In the reference implementation, a1 is passed as a return value * instead.} * @@ -69,6 +71,8 @@ __contract__( * -MLDSA_GAMMA2 <= a0 = a mod^+ MLDSA_Q - MLDSA_Q < 0. Assumes a to be * standard representative. * + * @spec{Implements @[FIPS204, Algorithm 36, Decompose].} + * * @reference{In the reference implementation, a1 is passed as a return value * instead.} * @@ -83,8 +87,9 @@ __contract__( requires(a >= 0 && a < MLDSA_Q) assigns(memory_slice(a0, sizeof(int32_t))) assigns(memory_slice(a1, sizeof(int32_t))) - /* a0 = -MLDSA_GAMMA2 can only occur when (q-1) = a - (a mod MLDSA_GAMMA2), - * then a1=1; and a0 = a - (a mod MLDSA_GAMMA2) - 1 (@[FIPS204, Algorithm 36 (Decompose)]) */ + /* a0 = -MLDSA_GAMMA2 occurs exactly when a = MLDSA_Q - MLDSA_GAMMA2: the + * border case of Decompose where a1 = (MLDSA_Q-1)/(2*MLDSA_GAMMA2) is + * wrapped to 0 and a0 = a - MLDSA_Q (@[FIPS204, Algorithm 36, Decompose]) */ ensures(*a0 >= -MLDSA_GAMMA2 && *a0 <= MLDSA_GAMMA2) ensures(*a1 >= 0 && *a1 < (MLDSA_Q-1)/(2*MLDSA_GAMMA2)) ensures((*a1 * 2 * MLDSA_GAMMA2 + *a0 - a) % MLDSA_Q == 0) @@ -141,7 +146,7 @@ __contract__( * * See proofs/isabelle/compress for a formalization of the above argument. */ - *a1 = (*a1 * 11275 + (1 << 23)) >> 24; + *a1 = (*a1 * 11275 + ((int32_t)1 << 23)) >> 24; mld_assert(*a1 >= 0 && *a1 <= 44); *a1 = mld_ct_sel_int32(0, *a1, mld_ct_cmask_neg_i32(43 - *a1)); @@ -157,7 +162,7 @@ __contract__( * eps = 1 / 4290772992 ≈ 2^(-31.99) < 2^(-31), therefore f1' * eps < * 2^16 * 2^(-31) = 1 / 2^15 < 1 / 2^12 < 1 / B. */ - *a1 = (*a1 * 1025 + (1 << 21)) >> 22; + *a1 = (*a1 * 1025 + ((int32_t)1 << 21)) >> 22; mld_assert(*a1 >= 0 && *a1 <= 16); *a1 &= 15; @@ -171,8 +176,21 @@ __contract__( } /** - * Compute hint bit indicating whether the low bits of the input element - * overflow into the high bits. + * Decide a single hint bit from the low part a0 and high part a1 of a + * coefficient: return 1 unless a0 lies in the range (-GAMMA2, GAMMA2] that + * LowBits would produce, with the boundary value -GAMMA2 also admitted when + * a1 == 0 (the Decompose border case). + * + * @note This is not a line-for-line implementation of FIPS 204's MakeHint(z, r) + * (@[FIPS204, Algorithm 39, MakeHint]), which takes two ring elements and + * returns [[HighBits(r) != HighBits(r + z)]]. Instead, it takes the already + * decomposed low/high parts (a0, a1) of a coefficient and decides the hint bit + * from them directly. As explained in the block comment of + * mld_attempt_signature_generation (sign.c), for the specific values that arise + * during signing -- a0 = w0 - cs2 + ct0 and a1 = w1 = HighBits(w) -- this is + * equivalent to the spec's MakeHint(-ct0, w - cs2 + ct0) coefficient-wise. + * Because it consumes (a0, a1) rather than (z, r), it relies on the caller + * having computed a compatible decomposition. * * @param a0 Low bits of input element. * @param a1 High bits of input element. @@ -197,6 +215,8 @@ __contract__( /** * Correct high bits according to hint. * + * @spec{Implements @[FIPS204, Algorithm 40, UseHint].} + * * @param a Input element. * @param hint Hint bit. * diff --git a/crypto/fipsmodule/ml_dsa/mldsa/sign.c b/crypto/fipsmodule/ml_dsa/mldsa/sign.c index f5e1d3f08d..ce9773adc0 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/sign.c +++ b/crypto/fipsmodule/ml_dsa/mldsa/sign.c @@ -90,7 +90,6 @@ static int mld_check_pct(uint8_t const pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) { MLD_ALIGN uint8_t message[1] = {0}; - size_t siglen; int ret; MLD_ALLOC(signature, uint8_t, MLDSA_CRYPTO_BYTES, context); MLD_ALLOC(pk_test, uint8_t, MLDSA_CRYPTO_PUBLICKEYBYTES, context); @@ -105,8 +104,8 @@ static int mld_check_pct(uint8_t const pk[MLDSA_CRYPTO_PUBLICKEYBYTES], mld_memcpy(pk_test, pk, MLDSA_CRYPTO_PUBLICKEYBYTES); /* Sign a test message using the original secret key */ - ret = mld_sign_signature(signature, &siglen, message, sizeof(message), NULL, - 0, sk, context); + ret = mld_sign_signature(signature, message, sizeof(message), NULL, 0, sk, + context); if (ret != 0) { goto cleanup; @@ -121,8 +120,8 @@ static int mld_check_pct(uint8_t const pk[MLDSA_CRYPTO_PUBLICKEYBYTES], #endif /* MLD_CONFIG_KEYGEN_PCT_BREAKAGE_TEST */ /* Verify the signature using the (potentially corrupted) public key */ - ret = mld_sign_verify(signature, siglen, message, sizeof(message), NULL, 0, - pk_test, context); + ret = mld_sign_verify(signature, message, sizeof(message), NULL, 0, pk_test, + context); cleanup: /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ @@ -131,7 +130,7 @@ static int mld_check_pct(uint8_t const pk[MLDSA_CRYPTO_PUBLICKEYBYTES], return ret; } -#else /* MLD_CONFIG_KEYGEN_PCT */ +#else /* MLD_CONFIG_KEYGEN_PCT */ static int mld_check_pct(uint8_t const pk[MLDSA_CRYPTO_PUBLICKEYBYTES], uint8_t const sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) @@ -139,13 +138,21 @@ static int mld_check_pct(uint8_t const pk[MLDSA_CRYPTO_PUBLICKEYBYTES], /* Skip PCT */ ((void)pk); ((void)sk); -#if defined(MLD_CONFIG_CONTEXT_PARAMETER) - ((void)context); -#endif + MLD_CONTEXT_UNUSED(context); return 0; } #endif /* !MLD_CONFIG_KEYGEN_PCT */ +/** + * Sample the short secret vectors s1 (length MLDSA_L) and s2 (length MLDSA_K) + * with coefficients in [-MLDSA_ETA, MLDSA_ETA] from the seed. + * + * @spec{Implements @[FIPS204, Algorithm 33, ExpandS].} + * + * @param[out] s1 Output vector s1. + * @param[out] s2 Output vector s2. + * @param[in] seed Byte array with seed of length MLDSA_CRHBYTES. + */ static void mld_sample_s1_s2(mld_polyvecl *s1, mld_polyveck *s2, const uint8_t seed[MLDSA_CRHBYTES]) __contract__( @@ -204,6 +211,9 @@ __contract__( * t1[k] via power2round, and bit-pack t1[k] into pk_t1 and t0[k] into the * t0_packed buffer. Used by both keygen and pk_from_sk. * + * @spec{Partially implements @[FIPS204, Algorithm 22, pkEncode] (t1) and + * @[FIPS204, Algorithm 24, skEncode] (t0).} + * * @param[out] pk_t1 Output buffer for packed t1 (size * MLDSA_K * MLDSA_POLYT1_PACKEDBYTES; i.e. the t1 * region of pk). @@ -477,13 +487,8 @@ __contract__( #endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */ #if !defined(MLD_CONFIG_NO_SIGN_API) -/* Reference: The reference implementation does not explicitly check the - * maximum nonce value, but instead loops indefinitely (even when the nonce - * would overflow). Internally, sampling of y uses - * (nonce*L), (nonce*L+1), ..., (nonce*L + L - 1). - * Hence, there are no overflows if nonce < (UINT16_MAX - L)/L. - * Explicitly checking for this explicitly allows us to prove type-safety. */ -#define MLD_NONCE_UB ((UINT16_MAX - MLDSA_L) / MLDSA_L) +/* MLD_MAX_KAPPA (see params.h) bounds the rejection-sampling counter kappa; + * MLD_MAX_SIGNING_ATTEMPTS below turns that into a bound on attempts. */ /** * Compute z = y + s1*c, check that z has coefficients smaller than @@ -525,7 +530,7 @@ __contract__( MLD_IF_REDUCE_RAM( requires(memory_no_alias(s1hat->packed, MLDSA_L * MLDSA_POLYETA_PACKEDBYTES)) requires(memory_no_alias(y->rhoprime, MLDSA_CRHBYTES)) - requires(y->nonce <= MLD_NONCE_UB) + requires(y->kappa <= MLD_MAX_KAPPA) ) assigns(memory_slice(sig, MLDSA_CRYPTO_BYTES)) assigns(memory_slice(z, sizeof(mld_poly))) @@ -577,12 +582,11 @@ __contract__( return 0; } -/* User-facing bound on signing attempts. See MLD_CONFIG_MAX_SIGNING_ATTEMPTS - * in mldsa_native_config.h. Default is chosen so that failure probability - * is < 2^{-256}, that is, signatures will practically always succeed. */ -#ifndef MLD_CONFIG_MAX_SIGNING_ATTEMPTS -#define MLD_CONFIG_MAX_SIGNING_ATTEMPTS MLD_NONCE_UB -#endif +/* Effective bound on signing attempts: the configured bound + * MLD_CONFIG_MAX_SIGNING_ATTEMPTS (see mldsa_native_config.h) if set, otherwise + * the hard type-safety bound MLD_MAX_KAPPA / MLDSA_L (see MLD_MAX_KAPPA in + * params.h). */ +#if defined(MLD_CONFIG_MAX_SIGNING_ATTEMPTS) #if !defined(MLD_ALLOW_NONCOMPLIANT_SIGNING_BOUND) && \ MLD_CONFIG_MAX_SIGNING_ATTEMPTS < 814 @@ -593,25 +597,39 @@ __contract__( #error Bad configuration: MLD_CONFIG_MAX_SIGNING_ATTEMPTS must be >= 1 #endif -#if MLD_CONFIG_MAX_SIGNING_ATTEMPTS > MLD_NONCE_UB +#if MLD_CONFIG_MAX_SIGNING_ATTEMPTS > MLD_MAX_KAPPA / MLDSA_L #error Bad configuration: MLD_CONFIG_MAX_SIGNING_ATTEMPTS exceeds the maximum allowed value. #endif +#define MLD_MAX_SIGNING_ATTEMPTS MLD_CONFIG_MAX_SIGNING_ATTEMPTS +#else /* MLD_CONFIG_MAX_SIGNING_ATTEMPTS */ +#define MLD_MAX_SIGNING_ATTEMPTS (MLD_MAX_KAPPA / MLDSA_L) +#endif /* !MLD_CONFIG_MAX_SIGNING_ATTEMPTS */ + MLD_MUST_CHECK_RETURN_VALUE static MLD_INLINE uint16_t mld_get_max_signing_attempts(void) __contract__( ensures(return_value >= 1) - ensures(return_value <= MLD_NONCE_UB) + ensures(return_value <= MLD_MAX_KAPPA / MLDSA_L) ) { /* cassert(0) ensures CBMC uses the contract rather than inlining the body, * keeping proofs agnostic of the configured value. */ cassert(0); - return MLD_CONFIG_MAX_SIGNING_ATTEMPTS; + return MLD_MAX_SIGNING_ATTEMPTS; } /** - * Attempt to generate a single signature. + * Attempt to generate a single signature: one iteration of the + * ML-DSA.Sign_internal rejection-sampling loop. + * + * @spec{Implements one iteration of the rejection-sampling loop body of + * @[FIPS204, Algorithm 7, ML-DSA.Sign_internal] (lines 11-30) plus, on success, + * the sigEncode step (line 33). The per-signature setup (Algorithm 7 lines 1-7: + * skDecode, NTT of s1/s2/t0, ExpandA, and computation of mu and rhoprime) and + * the loop itself (lines 8-10, 31-32) live in the caller + * mld_sign_signature_internal; kappa is this iteration's counter, used to + * sample y.} * * @reference{This code differs from the reference implementation in that it * factors out the core signature generation step into a distinct function @@ -621,7 +639,7 @@ __contract__( * @param[in] mu Pointer to message or hash of exactly MLDSA_CRHBYTES * bytes. * @param[in] rhoprime Pointer to randomness seed. - * @param nonce Current nonce value. + * @param kappa Counter for this iteration (= attempt*MLDSA_L). * @param[in] mat Expanded matrix. * @param[in] s1hat Secret vector s1 in NTT domain. * @param[in] s2hat Secret vector s2 in NTT domain. @@ -636,9 +654,10 @@ __contract__( * an allocation via MLD_CUSTOM_ALLOC returned NULL. */ MLD_MUST_CHECK_RETURN_VALUE +/* NOLINTNEXTLINE(readability-function-cognitive-complexity) */ static int mld_attempt_signature_generation( uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *mu, - const uint8_t rhoprime[MLDSA_CRHBYTES], uint16_t nonce, mld_polymat *mat, + const uint8_t rhoprime[MLDSA_CRHBYTES], uint16_t kappa, mld_polymat *mat, const mld_sk_s1hat *s1hat, const mld_sk_s2hat *s2hat, const mld_sk_t0hat *t0hat, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( @@ -649,7 +668,7 @@ __contract__( requires(memory_no_alias(s1hat, sizeof(mld_sk_s1hat))) requires(memory_no_alias(s2hat, sizeof(mld_sk_s2hat))) requires(memory_no_alias(t0hat, sizeof(mld_sk_t0hat))) - requires(nonce <= MLD_NONCE_UB) + requires(kappa <= MLD_MAX_KAPPA) MLD_IF_NOT_REDUCE_RAM( requires(forall(k1, 0, MLDSA_K, forall(l1, 0, MLDSA_L, array_bound(mat->vec[k1].vec[l1].coeffs, 0, MLDSA_N, 0, MLDSA_Q)))) @@ -699,15 +718,24 @@ __contract__( w1 = &w1tmp->w1; tmp = &w1tmp->tmp; - /* Sample/initialize intermediate vector y */ - mld_yvec_init(y, rhoprime, nonce); + /* @[FIPS204, Algorithm 7, line 11] y <- ExpandMask(rhoprime, kappa). */ + mld_yvec_init(y, rhoprime, kappa); - /* Matrix-vector multiplication, fused with y sampling in REDUCE_RAM mode */ + /* @[FIPS204, Algorithm 7, line 12] w <- invNTT(A_hat o NTT(y)). This call + * performs the whole line: it NTTs y, accumulates the pointwise product with + * A_hat, and applies the inverse NTT. In REDUCE_RAM mode the y sampling is + * fused into the same pass. */ mld_polyvec_matrix_pointwise_montgomery_yvec(w0, mat, y, tmp); - /* Decompose w and call the random oracle */ + /* @[FIPS204, Algorithm 7, line 13] w1 <- HighBits(w), here together with the + * low part: Decompose yields w = 2*GAMMA2*w1 + w0, keeping both w1 and w0 + * (w0 is reused below in the line-21/26 alternative, see further down). */ mld_polyveck_caddq(w0); mld_polyveck_decompose(w1, w0); + + /* @[FIPS204, Algorithm 7, line 15] ctilde <- H(mu || w1Encode(w1), lambda/4). + * w1Encode(w1) is packed into the w1 region of sig (mld_polyveck_pack_w1), + * then absorbed by H together with mu. */ mld_polyveck_pack_w1(sig, w1); mld_H(challenge_bytes, MLDSA_CTILDEBYTES, mu, MLDSA_CRHBYTES, sig, @@ -717,18 +745,56 @@ __contract__( * This also applies to challenges for rejected signatures. * See Section 5.5 of @[Round3_Spec]. */ MLD_CT_TESTING_DECLASSIFY(challenge_bytes, MLDSA_CTILDEBYTES); + /* @[FIPS204, Algorithm 7, line 16] c <- SampleInBall(ctilde) and + * @[FIPS204, Algorithm 7, line 17] c_hat <- NTT(c). */ mld_poly_challenge(cp, challenge_bytes); mld_poly_ntt(cp); - /* Compute z, reject if it reveals secret */ + /* @[FIPS204, Algorithm 7, lines 18+20] cs1 <- invNTT(c_hat o s1_hat) and + * z <- y + cs1, followed by the line-23 norm check ||z||_inf >= GAMMA1 - + * BETA. mld_compute_pack_z fuses all three per polynomial and, on success, + * packs z into sig; it returns MLD_ERR_FAIL if the norm check rejects z. */ ret = mld_compute_pack_z(sig, cp, s1hat, y, t, z); if (ret != 0) { goto cleanup; } - /* Compute w0 - cs2 + ct0 per-component, checking norms incrementally. - * This avoids allocating a full polyveck for h. */ + /* The remaining steps realize @[FIPS204, Algorithm 7, lines 21-28] (the + * low-bits norm check and the hint h) via the faster alternative formulation + * of @[Round3_Spec, Section 5.1]. @[FIPS204] explicitly permits this: the + * note accompanying Algorithm 7 states that the validity checks on z and the + * computation of h may instead be implemented "as described in Section 5.1 of + * [6]", and that reference is @[Round3_Spec, Section 5.1]. + * + * The loop below builds w0 - cs2 + ct0 in place in w0; w1 is unmodified, and + * is HighBits(w) from line 13. Those are the inputs to the streamlined + * computation of MakeHint explained below. + * + * Low-bits norm check: + * @[FIPS204, Algorithm 7, line 21] computes r0 = LowBits(w - cs2) and line + * 23 rejects when ||r0||_inf >= GAMMA2 - BETA. By @[Round3_Spec, Section + * 5.1] (Lemma 3), this line-23 check on r0 = LowBits(w - cs2) is implied by + * ||w0 - cs2||_inf < GAMMA2 - BETA, where w0 is the low part of w. In our + * context, w0 already holds the low part of w from the line-13 Decompose; + * after subtracting cs2 from it in place, the mld_poly_chknorm(w0, GAMMA2 - + * BETA) call below is exactly that check. + * + * Hint: + * @[FIPS204, Algorithm 7, line 26] sets h = MakeHint(-ct0, w - cs2 + ct0), + * and line 28 rejects when ||ct0||_inf >= GAMMA2 or h has more than OMEGA + * nonzero coefficients. @[Round3_Spec, Section 5.1] provides the following + * alternative description for MakeHint(-ct0, w - cs2 + ct0): a hint bit is + * zero exactly when the coefficient of w0 - cs2 + ct0 lies in + * (-GAMMA2, GAMMA2], or equals -GAMMA2 while the matching w1 coefficient is + * zero (the Decompose border case), and is set otherwise. This equivalence + * is precisely what mld_pack_sig_h -> mld_make_hint compute from w0 + * (= w0 - cs2 + ct0) and w1. The line-28 ||ct0||_inf >= GAMMA2 check is the + * mld_poly_chknorm(z, GAMMA2) call on ct0 below; the weight bound is + * enforced by mld_pack_sig_h. + * + * Building w0 per-component and checking norms incrementally also avoids + * allocating a full polyveck for h. */ for (k = 0; k < MLDSA_K; k++) __loop__( assigns(k, @@ -740,7 +806,8 @@ __contract__( decreases(MLDSA_K - k) ) { - /* Compute cs2[k] and subtract from w0[k] */ + /* @[FIPS204, Algorithm 7, line 19] cs2[k] <- invNTT(c_hat o s2_hat)[k], + * then subtract from w0[k] to form (w0 - cs2)[k]. */ mld_sk_s2hat_get_poly(z, s2hat, k); mld_poly_pointwise_montgomery(z, cp); mld_poly_invntt_tomont(z); @@ -748,8 +815,8 @@ __contract__( mld_poly_sub(&w0->vec[k], z); mld_poly_reduce(&w0->vec[k]); - /* Check that subtracting cs2 does not change high bits of w and low bits - * do not reveal secret information */ + /* Low-bits norm check (see block comment above): the line-23 check on + * r0 = LowBits(w - cs2) holds via ||w0 - cs2||_inf < GAMMA2 - BETA. */ w0_invalid = mld_poly_chknorm(&w0->vec[k], MLDSA_GAMMA2 - MLDSA_BETA); /* Constant time: w0_invalid may be leaked - see comment for z_invalid. */ MLD_CT_TESTING_DECLASSIFY(&w0_invalid, sizeof(uint32_t)); @@ -759,12 +826,14 @@ __contract__( goto cleanup; } - /* Compute ct0[k], check norm, and add to w0[k] */ + /* @[FIPS204, Algorithm 7, line 25] ct0[k] <- invNTT(c_hat o t0_hat)[k]. */ mld_sk_t0hat_get_poly(z, t0hat, k); mld_poly_pointwise_montgomery(z, cp); mld_poly_invntt_tomont(z); mld_poly_reduce(z); + /* @[FIPS204, Algorithm 7, line 28] reject when ||ct0||_inf >= GAMMA2 (the + * second part, the OMEGA weight bound, is enforced by mld_pack_sig_h). */ h_invalid = mld_poly_chknorm(z, MLDSA_GAMMA2); /* Constant time: h_invalid may be leaked - see comment for z_invalid. */ MLD_CT_TESTING_DECLASSIFY(&h_invalid, sizeof(uint32_t)); @@ -774,6 +843,8 @@ __contract__( goto cleanup; } + /* Add ct0[k] to (w0 - cs2)[k], leaving (w0 - cs2 + ct0)[k] in w0[k] -- the + * MakeHint input prepared for mld_pack_sig_h (see block comment above). */ mld_poly_add(&w0->vec[k], z); } @@ -790,9 +861,16 @@ __contract__( MLD_CT_TESTING_DECLASSIFY(w0, sizeof(*w0)); MLD_CT_TESTING_DECLASSIFY(w1, sizeof(*w1)); - /* Pack challenge bytes and hints. */ + /* @[FIPS204, Algorithm 7, line 33] sigEncode(ctilde, z mod+/- q, h) is split + * across three calls: z was already packed by mld_compute_pack_z, this call + * packs ctilde, and mld_pack_sig_h below packs the hint h. */ mld_pack_sig_c(sig, challenge_bytes); + /* @[FIPS204, Algorithm 7, line 26] h <- MakeHint(-ct0, w - cs2 + ct0), + * computed from (w0 = w0 - cs2 + ct0, w1) as described in the block comment + * above, and packed as the h component of the line-33 sigEncode. Returns + * MLD_ERR_FAIL if h would exceed OMEGA nonzero coefficients (the remaining + * part of the line-28 check), in which case we reject. */ ret = mld_pack_sig_h(sig, w0, w1); if (ret != 0) { @@ -818,7 +896,7 @@ __contract__( } MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, +int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *pre, size_t prelen, const uint8_t rnd[MLDSA_RNDBYTES], @@ -828,8 +906,8 @@ int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, { int ret; uint8_t *rho, *tr, *key, *mu, *rhoprime; - uint16_t nonce = 0; - const uint16_t nonce_limit = mld_get_max_signing_attempts(); + uint16_t attempt; + const uint16_t max_signing_attempts = mld_get_max_signing_attempts(); MLD_ALLOC(seedbuf, uint8_t, 2 * MLDSA_SEEDBYTES + MLDSA_TRBYTES + 2 * MLDSA_CRHBYTES, context); MLD_ALLOC(mat, mld_polymat, 1, context); @@ -844,46 +922,67 @@ int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, goto cleanup; } + /* If a resume hook is configured via MLD_CONFIG_SIGN_HOOK_RESUME, it provides + * the attempt to resume from after an earlier pause. Otherwise, we start at + * 0. Clamp to max_signing_attempts. */ + attempt = mld_sign_resume(context); + if (attempt > max_signing_attempts) + { + attempt = max_signing_attempts; + } + rho = seedbuf; tr = rho + MLDSA_SEEDBYTES; key = tr + MLDSA_TRBYTES; mu = key + MLDSA_SEEDBYTES; rhoprime = mu + MLDSA_CRHBYTES; + /* @[FIPS204, Algorithm 7, line 1] (rho, K, tr, s1, s2, t0) <- skDecode(sk) + * and @[FIPS204, Algorithm 7, lines 2-4] s1_hat/s2_hat/t0_hat <- NTT(...): + * mld_unpack_sk returns s1hat, s2hat, t0hat already in NTT domain. The spec's + * private random seed K is held in the local variable key. */ mld_unpack_sk(rho, tr, key, t0hat, s1hat, s2hat, sk); if (!externalmu) { - /* Compute mu = CRH(tr, pre, msg) */ + /* @[FIPS204, Algorithm 7, line 6] mu <- H(BytesToBits(tr) || M', 64). */ mld_H(mu, MLDSA_CRHBYTES, tr, MLDSA_TRBYTES, pre, prelen, m, mlen); } else { - /* mu has been provided directly */ + /* mu has been provided directly (external-mu variant; line 6 done by the + * caller in a separate cryptographic module). */ mld_memcpy(mu, m, MLDSA_CRHBYTES); } - /* Compute rhoprime = CRH(key, rnd, mu) */ + /* @[FIPS204, Algorithm 7, line 7] rhoprime <- H(K || rnd || mu, 64). */ mld_H(rhoprime, MLDSA_CRHBYTES, key, MLDSA_SEEDBYTES, rnd, MLDSA_RNDBYTES, mu, MLDSA_CRHBYTES); /* Constant time: rho is part of the public key and, hence, public. */ MLD_CT_TESTING_DECLASSIFY(rho, MLDSA_SEEDBYTES); - /* Expand matrix and transform vectors */ + /* @[FIPS204, Algorithm 7, line 5] A_hat <- ExpandA(rho). */ mld_polyvec_matrix_expand(mat, rho); - /* Reference: This code is re-structured using a while(1), */ - /* with explicit "continue" statements (rather than "goto") */ - /* to implement rejection of invalid signatures. */ - while (1) + /* @[FIPS204, Algorithm 7, lines 8-10 and 31-32] the rejection-sampling loop, + * tracked by attempt (kappa = attempt*MLDSA_L). Each iteration's body (lines + * 11-30) plus, on success, the line-33 sigEncode are performed by + * mld_attempt_signature_generation. */ + + /* Reference: the reference loops unboundedly; we instead iterate over the + * bounded range [0, max_signing_attempts) for predictable termination. + * A success or fatal error exits via goto cleanup; running to completion + * means every attempt was rejected; with a FIPS compliant choice of + * MLD_CONFIG_MAX_SIGNING_ATTEMPTS, this should never happen. */ + for (; attempt < max_signing_attempts; attempt++) __loop__( MLD_IF_NOT_REDUCE_RAM( - assigns(nonce, ret, object_whole(siglen), memory_slice(sig, MLDSA_CRYPTO_BYTES)) + assigns(attempt, ret, memory_slice(sig, MLDSA_CRYPTO_BYTES)) ) MLD_IF_REDUCE_RAM( - assigns(nonce, ret, object_whole(siglen), memory_slice(sig, MLDSA_CRYPTO_BYTES), + assigns(attempt, ret, memory_slice(sig, MLDSA_CRYPTO_BYTES), memory_slice(mat, sizeof(mld_polymat))) ) - invariant(nonce <= nonce_limit) + invariant(attempt <= max_signing_attempts) /* t0, s1, s2, and mat are initialized above and are NOT changed by this */ /* loop. We can therefore re-assert their bounds here as part of the */ @@ -895,42 +994,55 @@ int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, invariant(forall(k3, 0, MLDSA_L, array_abs_bound(s1hat->vec.vec[k3].coeffs, 0, MLDSA_N, MLD_NTT_BOUND))) invariant(forall(k4, 0, MLDSA_K, array_abs_bound(s2hat->vec.vec[k4].coeffs, 0, MLDSA_N, MLD_NTT_BOUND))) ) - decreases(nonce_limit - nonce) + decreases(max_signing_attempts - attempt) ) { - /* Reference: this code explicitly checks for exhaustion of signing */ - /* attempts to provide predictable termination and results in that */ - /* case. Checking here also means that incrementing nonce below can */ - /* be proven to be type-safe. */ - if (nonce == nonce_limit) + /* Safety: attempt < max_signing_attempts <= MLD_MAX_KAPPA / MLDSA_L, so + * kappa <= MLD_MAX_KAPPA and the cast is safe. */ + const uint16_t kappa = (uint16_t)(attempt * MLDSA_L); + + /* Query configurable signing hook whether signing should be paused. + * This is skipped by default and only used if the user sets the + * configuration option MLD_CONFIG_SIGN_HOOK_ATTEMPT. */ + if (mld_sign_attempt(attempt, context) != 0) { - ret = MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED; - break; + ret = MLD_ERR_SIGNING_PAUSED; + goto cleanup; } - ret = mld_attempt_signature_generation(sig, mu, rhoprime, nonce, mat, s1hat, + ret = mld_attempt_signature_generation(sig, mu, rhoprime, kappa, mat, s1hat, s2hat, t0hat, context); - nonce++; + + /* Decide whether to keep trying based on the return value: + * - ret == 0: a valid signature was produced; we are done. + * - ret == MLD_ERR_FAIL: this candidate was rejected by one of the norm + * or hint checks. We continue the loop and try again with the next + * nonce. + * - any other value (e.g. MLD_ERR_OUT_OF_MEMORY): an unrecoverable error + * occurred, so we propagate it to the caller. */ if (ret == 0) { - *siglen = MLDSA_CRYPTO_BYTES; - break; + /* Signing succeeded: record the attempt that succeeded. No-op in the + * default build. */ + mld_sign_finish(attempt, context); + goto cleanup; } - else if (ret != MLD_ERR_FAIL) + if (ret != MLD_ERR_FAIL) { - /* For failures such as out-of-memory, propagate and exit immediately. */ - break; + goto cleanup; } - - /* Otherwise, try again. */ } + /* Loop ran to completion: all attempts rejected, budget exhausted. + * This should never happen with a FIPS compliant choice of + * MLD_CONFIG_MAX_SIGNING_ATTEMPTS. */ + ret = MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED; + cleanup: if (ret != 0) { /* To be on the safe-side, we zeroize the signature buffer. */ - *siglen = 0; mld_memset(sig, 0, MLDSA_CRYPTO_BYTES); } @@ -948,9 +1060,8 @@ int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, #if !defined(MLD_CONFIG_NO_RANDOMIZED_API) MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_signature(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, - const uint8_t *m, size_t mlen, const uint8_t *ctx, - size_t ctxlen, +int mld_sign_signature(uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, + size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) { @@ -983,19 +1094,18 @@ int mld_sign_signature(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, } MLD_CT_TESTING_SECRET(rnd, MLDSA_RNDBYTES); - ret = mld_sign_signature_internal(sig, siglen, m, mlen, pre, pre_len, rnd, sk, - 0, context); + ret = mld_sign_signature_internal(sig, m, mlen, pre, pre_len, rnd, sk, 0, + context); cleanup: if (ret != 0) { - /* To be on the safe-side, make sure *siglen and sig have a well-defined - * value, even in the case of error. + /* To be on the safe-side, make sure sig has a well-defined value, even in + * the case of error. * - * If we come from mld_sign_signature_internal, both are redundant, - * but the error case should not be the norm, and the added cost of the - * memset insignificant. */ - *siglen = 0; + * If we come from mld_sign_signature_internal, this is redundant, but the + * error case should not be the norm, and the added cost of the memset + * insignificant. */ mld_memset(sig, 0, MLDSA_CRYPTO_BYTES); } @@ -1010,7 +1120,7 @@ int mld_sign_signature(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, #if !defined(MLD_CONFIG_NO_RANDOMIZED_API) MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, +int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t mu[MLDSA_CRHBYTES], const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) @@ -1020,7 +1130,6 @@ int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, if (rnd == NULL) { - *siglen = 0; ret = MLD_ERR_OUT_OF_MEMORY; goto cleanup; } @@ -1029,14 +1138,13 @@ int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, * call mld_sign_signature_internal directly with all-zero rnd. */ if (mld_randombytes(rnd, MLDSA_RNDBYTES) != 0) { - *siglen = 0; ret = MLD_ERR_RNG_FAIL; goto cleanup; } MLD_CT_TESTING_SECRET(rnd, MLDSA_RNDBYTES); - ret = mld_sign_signature_internal(sig, siglen, mu, MLDSA_CRHBYTES, NULL, 0, - rnd, sk, 1, context); + ret = mld_sign_signature_internal(sig, mu, MLDSA_CRHBYTES, NULL, 0, rnd, sk, + 1, context); cleanup: /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ @@ -1045,43 +1153,13 @@ int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, return ret; } #endif /* !MLD_CONFIG_NO_RANDOMIZED_API */ - -#if !defined(MLD_CONFIG_NO_RANDOMIZED_API) -MLD_MUST_CHECK_RETURN_VALUE -MLD_EXTERNAL_API -int mld_sign(uint8_t *sm, size_t *smlen, const uint8_t *m, size_t mlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], - MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) -{ - int ret; - size_t i; - - for (i = 0; i < mlen; ++i) - __loop__( - assigns(i, object_whole(sm)) - invariant(i <= mlen) - decreases(mlen - i) - ) - { - sm[MLDSA_CRYPTO_BYTES + mlen - 1 - i] = m[mlen - 1 - i]; - } - ret = mld_sign_signature(sm, smlen, sm + MLDSA_CRYPTO_BYTES, mlen, ctx, - ctxlen, sk, context); - if (ret == 0) - { - *smlen += mlen; - } - return ret; -} -#endif /* !MLD_CONFIG_NO_RANDOMIZED_API */ #endif /* !MLD_CONFIG_CORE_API_ONLY */ #endif /* !MLD_CONFIG_NO_SIGN_API */ #if !defined(MLD_CONFIG_NO_VERIFY_API) MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_verify_internal(const uint8_t *sig, size_t siglen, +int mld_sign_verify_internal(const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *pre, size_t prelen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], @@ -1108,12 +1186,6 @@ int mld_sign_verify_internal(const uint8_t *sig, size_t siglen, goto cleanup; } - if (siglen != MLDSA_CRYPTO_BYTES) - { - ret = MLD_ERR_FAIL; - goto cleanup; - } - mld_memcpy(c, sig, MLDSA_CTILDEBYTES); mld_polyvecl_unpack_z(z, sig + MLDSA_SIG_Z_OFFSET); @@ -1215,7 +1287,7 @@ int mld_sign_verify_internal(const uint8_t *sig, size_t siglen, #if !defined(MLD_CONFIG_CORE_API_ONLY) MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_verify(const uint8_t *sig, size_t siglen, const uint8_t *m, +int mld_sign_verify(const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) @@ -1232,8 +1304,7 @@ int mld_sign_verify(const uint8_t *sig, size_t siglen, const uint8_t *m, goto cleanup; } - ret = mld_sign_verify_internal(sig, siglen, m, mlen, pre, pre_len, pk, 0, - context); + ret = mld_sign_verify_internal(sig, m, mlen, pre, pre_len, pk, 0, context); cleanup: /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ @@ -1244,58 +1315,13 @@ int mld_sign_verify(const uint8_t *sig, size_t siglen, const uint8_t *m, MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_verify_extmu(const uint8_t *sig, size_t siglen, +int mld_sign_verify_extmu(const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t mu[MLDSA_CRHBYTES], const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) { - return mld_sign_verify_internal(sig, siglen, mu, MLDSA_CRHBYTES, NULL, 0, pk, - 1, context); -} - -MLD_MUST_CHECK_RETURN_VALUE -MLD_EXTERNAL_API -int mld_sign_open(uint8_t *m, size_t *mlen, const uint8_t *sm, size_t smlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], - MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) -{ - int ret; - size_t i; - - if (smlen < MLDSA_CRYPTO_BYTES) - { - ret = MLD_ERR_FAIL; - goto cleanup; - } - - *mlen = smlen - MLDSA_CRYPTO_BYTES; - ret = mld_sign_verify(sm, MLDSA_CRYPTO_BYTES, sm + MLDSA_CRYPTO_BYTES, *mlen, - ctx, ctxlen, pk, context); - if (ret == 0) - { - /* All good, copy msg, return 0 */ - for (i = 0; i < *mlen; ++i) - __loop__( - assigns(i, memory_slice(m, *mlen)) - invariant(i <= *mlen) - decreases(*mlen - i) - ) - { - m[i] = sm[MLDSA_CRYPTO_BYTES + i]; - } - } - -cleanup: - - if (ret != 0) - { - /* To be on the safe-side, we zeroize the message buffer. */ - *mlen = 0; - mld_memset(m, 0, smlen); - } - - return ret; + return mld_sign_verify_internal(sig, mu, MLDSA_CRHBYTES, NULL, 0, pk, 1, + context); } #endif /* !MLD_CONFIG_CORE_API_ONLY */ #endif /* !MLD_CONFIG_NO_VERIFY_API */ @@ -1305,9 +1331,8 @@ int mld_sign_open(uint8_t *m, size_t *mlen, const uint8_t *sm, size_t smlen, MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_signature_pre_hash_internal( - uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, const uint8_t *ph, - size_t phlen, const uint8_t *ctx, size_t ctxlen, - const uint8_t rnd[MLDSA_RNDBYTES], + uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *ph, size_t phlen, + const uint8_t *ctx, size_t ctxlen, const uint8_t rnd[MLDSA_RNDBYTES], const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], int hashalg, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) { @@ -1315,6 +1340,12 @@ int mld_sign_signature_pre_hash_internal( size_t pre_len; int ret; + if (hashalg == MLD_PREHASH_NONE) + { + ret = MLD_ERR_FAIL; + goto cleanup; + } + pre_len = mld_prepare_domain_separation_prefix(pre, ph, phlen, ctx, ctxlen, hashalg); if (pre_len == 0) @@ -1323,18 +1354,17 @@ int mld_sign_signature_pre_hash_internal( goto cleanup; } - ret = mld_sign_signature_internal(sig, siglen, pre, pre_len, NULL, 0, rnd, sk, - 0, context); + ret = mld_sign_signature_internal(sig, pre, pre_len, NULL, 0, rnd, sk, 0, + context); cleanup: if (ret != 0) { - /* To be on the safe-side, make sure *siglen and sig have a well-defined - * value, even in the case of error. + /* To be on the safe-side, make sure sig has a well-defined value, even in + * the case of error. * - * If we come from mld_sign_signature_internal, both are redundant, - * but the error case should not be the norm, and the added cost of the - * memset insignificant. */ - *siglen = 0; + * If we come from mld_sign_signature_internal, this is redundant, but the + * error case should not be the norm, and the added cost of the memset + * insignificant. */ mld_memset(sig, 0, MLDSA_CRYPTO_BYTES); } @@ -1348,7 +1378,7 @@ int mld_sign_signature_pre_hash_internal( MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_verify_pre_hash_internal( - const uint8_t *sig, size_t siglen, const uint8_t *ph, size_t phlen, + const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *ph, size_t phlen, const uint8_t *ctx, size_t ctxlen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], int hashalg, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) @@ -1357,6 +1387,12 @@ int mld_sign_verify_pre_hash_internal( size_t pre_len; int ret; + if (hashalg == MLD_PREHASH_NONE) + { + ret = MLD_ERR_FAIL; + goto cleanup; + } + pre_len = mld_prepare_domain_separation_prefix(pre, ph, phlen, ctx, ctxlen, hashalg); if (pre_len == 0) @@ -1365,8 +1401,7 @@ int mld_sign_verify_pre_hash_internal( goto cleanup; } - ret = mld_sign_verify_internal(sig, siglen, pre, pre_len, NULL, 0, pk, 0, - context); + ret = mld_sign_verify_internal(sig, pre, pre_len, NULL, 0, pk, 0, context); cleanup: /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ @@ -1379,18 +1414,17 @@ int mld_sign_verify_pre_hash_internal( MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_signature_pre_hash_shake256( - uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, const uint8_t *m, - size_t mlen, const uint8_t *ctx, size_t ctxlen, - const uint8_t rnd[MLDSA_RNDBYTES], + uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, + const uint8_t *ctx, size_t ctxlen, const uint8_t rnd[MLDSA_RNDBYTES], const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) { MLD_ALIGN uint8_t ph[64]; int ret; mld_shake256(ph, sizeof(ph), m, mlen); - ret = mld_sign_signature_pre_hash_internal(sig, siglen, ph, sizeof(ph), ctx, - ctxlen, rnd, sk, - MLD_PREHASH_SHAKE_256, context); + ret = mld_sign_signature_pre_hash_internal(sig, ph, sizeof(ph), ctx, ctxlen, + rnd, sk, MLD_PREHASH_SHAKE_256, + context); /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ mld_zeroize(ph, sizeof(ph)); return ret; @@ -1401,7 +1435,7 @@ int mld_sign_signature_pre_hash_shake256( MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_verify_pre_hash_shake256( - const uint8_t *sig, size_t siglen, const uint8_t *m, size_t mlen, + const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) @@ -1409,9 +1443,8 @@ int mld_sign_verify_pre_hash_shake256( MLD_ALIGN uint8_t ph[64]; int ret; mld_shake256(ph, sizeof(ph), m, mlen); - ret = mld_sign_verify_pre_hash_internal(sig, siglen, ph, sizeof(ph), ctx, - ctxlen, pk, MLD_PREHASH_SHAKE_256, - context); + ret = mld_sign_verify_pre_hash_internal(sig, ph, sizeof(ph), ctx, ctxlen, pk, + MLD_PREHASH_SHAKE_256, context); /* @[FIPS204, Section 3.6.3] Destruction of intermediate values. */ mld_zeroize(ph, sizeof(ph)); return ret; @@ -1502,8 +1535,9 @@ static int mld_validate_hash_length(int hashalg, size_t len) return (len == 256 / 8) ? 0 : -1; case MLD_PREHASH_SHAKE_256: return (len == 512 / 8) ? 0 : -1; + default: + return -1; } - return -1; } size_t mld_prepare_domain_separation_prefix( @@ -1643,6 +1677,5 @@ int mld_sign_pk_from_sk(uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], #undef mld_attempt_signature_generation #undef mld_compute_pack_t0_t1 #undef mld_get_max_signing_attempts -#undef MLD_NONCE_UB -#undef MLD_CONFIG_MAX_SIGNING_ATTEMPTS +#undef MLD_MAX_SIGNING_ATTEMPTS #undef MLD_PRE_HASH_OID_LEN diff --git a/crypto/fipsmodule/ml_dsa/mldsa/sign.h b/crypto/fipsmodule/ml_dsa/mldsa/sign.h index 68135380e2..9573e0413b 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/sign.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/sign.h @@ -38,7 +38,7 @@ #endif #if MLDSA_CRYPTO_BYTES != MLDSA_BYTES(MLD_CONFIG_PARAMETER_SET) -#error Mismatch for CRYPTO_BYTES between sign.h and mldsa_native.h +#error Mismatch for BYTES between sign.h and mldsa_native.h #endif #endif /* MLD_CHECK_APIS */ @@ -47,25 +47,23 @@ MLD_NAMESPACE_KL(keypair_internal) MLD_CONTEXT_PARAMETERS_3 #define mld_sign_keypair MLD_NAMESPACE_KL(keypair) MLD_CONTEXT_PARAMETERS_2 #define mld_sign_signature_internal \ - MLD_NAMESPACE_KL(signature_internal) MLD_CONTEXT_PARAMETERS_9 -#define mld_sign_signature MLD_NAMESPACE_KL(signature) MLD_CONTEXT_PARAMETERS_7 + MLD_NAMESPACE_KL(signature_internal) MLD_CONTEXT_PARAMETERS_8 +#define mld_sign_signature MLD_NAMESPACE_KL(signature) MLD_CONTEXT_PARAMETERS_6 #define mld_sign_signature_extmu \ - MLD_NAMESPACE_KL(signature_extmu) MLD_CONTEXT_PARAMETERS_4 -#define mld_sign MLD_NAMESPACE_KL(sign) MLD_CONTEXT_PARAMETERS_7 + MLD_NAMESPACE_KL(signature_extmu) MLD_CONTEXT_PARAMETERS_3 #define mld_sign_verify_internal \ - MLD_NAMESPACE_KL(verify_internal) MLD_CONTEXT_PARAMETERS_8 -#define mld_sign_verify MLD_NAMESPACE_KL(verify) MLD_CONTEXT_PARAMETERS_7 + MLD_NAMESPACE_KL(verify_internal) MLD_CONTEXT_PARAMETERS_7 +#define mld_sign_verify MLD_NAMESPACE_KL(verify) MLD_CONTEXT_PARAMETERS_6 #define mld_sign_verify_extmu \ - MLD_NAMESPACE_KL(verify_extmu) MLD_CONTEXT_PARAMETERS_4 -#define mld_sign_open MLD_NAMESPACE_KL(open) MLD_CONTEXT_PARAMETERS_7 + MLD_NAMESPACE_KL(verify_extmu) MLD_CONTEXT_PARAMETERS_3 #define mld_sign_signature_pre_hash_internal \ - MLD_NAMESPACE_KL(signature_pre_hash_internal) MLD_CONTEXT_PARAMETERS_9 + MLD_NAMESPACE_KL(signature_pre_hash_internal) MLD_CONTEXT_PARAMETERS_8 #define mld_sign_verify_pre_hash_internal \ - MLD_NAMESPACE_KL(verify_pre_hash_internal) MLD_CONTEXT_PARAMETERS_8 + MLD_NAMESPACE_KL(verify_pre_hash_internal) MLD_CONTEXT_PARAMETERS_7 #define mld_sign_signature_pre_hash_shake256 \ - MLD_NAMESPACE_KL(signature_pre_hash_shake256) MLD_CONTEXT_PARAMETERS_8 + MLD_NAMESPACE_KL(signature_pre_hash_shake256) MLD_CONTEXT_PARAMETERS_7 #define mld_sign_verify_pre_hash_shake256 \ - MLD_NAMESPACE_KL(verify_pre_hash_shake256) MLD_CONTEXT_PARAMETERS_7 + MLD_NAMESPACE_KL(verify_pre_hash_shake256) MLD_CONTEXT_PARAMETERS_6 #define mld_prepare_domain_separation_prefix \ MLD_NAMESPACE_KL(prepare_domain_separation_prefix) #define mld_sign_pk_from_sk \ @@ -93,7 +91,7 @@ * When MLD_CONFIG_KEYGEN_PCT is set, performs a Pairwise Consistency Test * (PCT) as required by FIPS 140-3 IG. * - * @spec{Implements @[FIPS204 Algorithm 6 (ML-DSA.KeyGen_internal)].} + * @spec{Implements @[FIPS204, Algorithm 6, ML-DSA.KeyGen_internal].} * * @param[out] pk Output public key. * @param[out] sk Output private key. @@ -137,7 +135,7 @@ __contract__( * When MLD_CONFIG_KEYGEN_PCT is set, performs a Pairwise Consistency Test * (PCT) as required by FIPS 140-3 IG. * - * @spec{Implements @[FIPS204 Algorithm 1 (ML-DSA.KeyGen)].} + * @spec{Implements @[FIPS204, Algorithm 1, ML-DSA.KeyGen].} * * @param[out] pk Output public key. * @param[out] sk Output private key. @@ -177,11 +175,16 @@ __contract__( /** * Compute signature using internal randomness. * - * If the returned value is non-zero, then the values of *sig and *siglen - * should not be referenced. + * On error (non-zero return value), the signature buffer sig is zeroized. * - * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. + * @spec{Implements @[FIPS204, Algorithm 7, ML-DSA.Sign_internal].} + * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use mld_sign_pk_from_sk + * to validate them before signing. + * + * @param[out] sig Pointer to buffer to hold the generated signature of + * MLDSA_CRYPTO_BYTES bytes. * @param[in] m Pointer to message to be signed (when * externalmu == 0), or to a precomputed * message representative mu (when externalmu != 0). @@ -192,7 +195,7 @@ __contract__( * @param prelen Length of prefix string. Ignored when * externalmu != 0. * @param[in] rnd Random seed. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param externalmu 0: m/mlen is the raw message; mu = H(tr, pre, m) is * computed internally. * non-zero: m points to a precomputed mu of @@ -208,11 +211,13 @@ __contract__( * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS * iterations. + * @retval MLD_ERR_SIGNING_PAUSED A MLD_CONFIG_SIGN_HOOK_ATTEMPT hook + * paused signing; re-invoke to resume. * @retval MLD_ERR_FAIL Other kinds of failure. */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, +int mld_sign_signature_internal(uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *pre, size_t prelen, const uint8_t rnd[MLDSA_RNDBYTES], @@ -223,20 +228,16 @@ __contract__( requires(mlen <= MLD_MAX_BUFFER_SIZE) requires(prelen <= MLD_MAX_BUFFER_SIZE) requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) - requires(memory_no_alias(siglen, sizeof(size_t))) requires(memory_no_alias(m, mlen)) requires(memory_no_alias(rnd, MLDSA_RNDBYTES)) requires(memory_no_alias(sk, MLDSA_CRYPTO_SECRETKEYBYTES)) requires((externalmu == 0) ==> ((prelen == 0) || memory_no_alias(pre, prelen))) requires((externalmu != 0) ==> (mlen == MLDSA_CRHBYTES)) assigns(memory_slice(sig, MLDSA_CRYPTO_BYTES)) - assigns(object_whole(siglen)) ensures(return_value == 0 || return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY || - return_value == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED) - ensures(return_value == 0 ==> *siglen == MLDSA_CRYPTO_BYTES) - ensures(return_value != 0 ==> *siglen == 0) -); + return_value == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED || + return_value == MLD_ERR_SIGNING_PAUSED)); #if !defined(MLD_CONFIG_CORE_API_ONLY) /** @@ -244,15 +245,18 @@ __contract__( * ML-DSA. If you require the deterministic variant, use * mld_sign_signature_internal directly. * - * @spec{Implements @[FIPS204 Algorithm 2 (ML-DSA.Sign)].} + * @spec{Implements @[FIPS204, Algorithm 2, ML-DSA.Sign].} + * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use mld_sign_pk_from_sk + * to validate them before signing. * * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. * @param[in] m Pointer to message to be signed. * @param mlen Length of message. * @param[in] ctx Pointer to context string. May be NULL if ctxlen == 0. * @param ctxlen Length of context string. Should be <= 255. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. @@ -269,23 +273,19 @@ __contract__( */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_signature(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, - const uint8_t *m, size_t mlen, const uint8_t *ctx, - size_t ctxlen, +int mld_sign_signature(uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, + size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(mlen <= MLD_MAX_BUFFER_SIZE) requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) - requires(memory_no_alias(siglen, sizeof(size_t))) requires(memory_no_alias(m, mlen)) requires(ctxlen <= MLD_MAX_BUFFER_SIZE) requires(ctxlen == 0 || memory_no_alias(ctx, ctxlen)) requires(memory_no_alias(sk, MLDSA_CRYPTO_SECRETKEYBYTES)) assigns(memory_slice(sig, MLDSA_CRYPTO_BYTES)) - assigns(object_whole(siglen)) - ensures((return_value == 0 && *siglen == MLDSA_CRYPTO_BYTES) || - (MLD_ANY_ERROR(return_value) && *siglen == 0)) + ensures(return_value == 0 || MLD_ANY_ERROR(return_value)) ); /** @@ -296,12 +296,15 @@ __contract__( * variant; for the deterministic variant, use mld_sign_signature_internal * directly with externalmu set to non-zero and an all-zero rnd. * - * @spec{Implements @[FIPS204 Algorithm 2 (ML-DSA.Sign external mu variant)].} + * @spec{Implements @[FIPS204, Algorithm 2, ML-DSA.Sign external mu variant].} + * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use mld_sign_pk_from_sk + * to validate them before signing. * * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. * @param[in] mu Precomputed message representative. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. @@ -318,64 +321,18 @@ __contract__( */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, +int mld_sign_signature_extmu(uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t mu[MLDSA_CRHBYTES], const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) - requires(memory_no_alias(siglen, sizeof(size_t))) requires(memory_no_alias(mu, MLDSA_CRHBYTES)) requires(memory_no_alias(sk, MLDSA_CRYPTO_SECRETKEYBYTES)) assigns(memory_slice(sig, MLDSA_CRYPTO_BYTES)) - assigns(object_whole(siglen)) - ensures((return_value == 0 && *siglen == MLDSA_CRYPTO_BYTES) || - (MLD_ANY_ERROR(return_value) && *siglen == 0)) + ensures(return_value == 0 || MLD_ANY_ERROR(return_value)) ); -/** - * Compute signed message. - * - * @param[out] sm Pointer to output signed message (allocated array with - * MLDSA_CRYPTO_BYTES + mlen bytes); can be equal to m. - * @param[out] smlen Pointer to output length of signed message. - * @param[in] m Pointer to message to be signed. - * @param mlen Length of message. - * @param[in] ctx Pointer to context string. - * @param ctxlen Length of context string. - * @param[in] sk Bit-packed secret key. - * @param context Application context. Only present when - * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by - * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. - * - * @retval 0 Success. - * @retval MLD_ERR_OUT_OF_MEMORY MLD_CONFIG_CUSTOM_ALLOC_FREE was - * used and an allocation via - * MLD_CUSTOM_ALLOC returned NULL. - * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded - * MLD_CONFIG_MAX_SIGNING_ATTEMPTS - * iterations. - * @retval MLD_ERR_FAIL Other kinds of failure. - */ -MLD_MUST_CHECK_RETURN_VALUE -MLD_EXTERNAL_API -int mld_sign(uint8_t *sm, size_t *smlen, const uint8_t *m, size_t mlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], - MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) -__contract__( - requires(mlen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(sm, MLDSA_CRYPTO_BYTES + mlen)) - requires(memory_no_alias(smlen, sizeof(size_t))) - requires(m == sm || memory_no_alias(m, mlen)) - requires(ctxlen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(ctx, ctxlen)) - requires(memory_no_alias(sk, MLDSA_CRYPTO_SECRETKEYBYTES)) - assigns(memory_slice(sm, MLDSA_CRYPTO_BYTES + mlen)) - assigns(object_whole(smlen)) - ensures((return_value == 0 && *smlen == MLDSA_CRYPTO_BYTES + mlen) || - (MLD_ANY_ERROR(return_value) && *smlen == 0)) -); #endif /* !MLD_CONFIG_CORE_API_ONLY */ #endif /* !MLD_CONFIG_NO_SIGN_API */ @@ -383,10 +340,10 @@ __contract__( /** * Verify signature. * - * @spec{Implements @[FIPS204 Algorithm 8 (ML-DSA.Verify_internal)].} + * @spec{Implements @[FIPS204, Algorithm 8, ML-DSA.Verify_internal].} * - * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. + * @param[in] sig Pointer to input signature of + * MLDSA_CRYPTO_BYTES bytes. * @param[in] m Pointer to message (when externalmu == 0), or to a * precomputed message representative mu (when * externalmu != 0). @@ -410,7 +367,7 @@ __contract__( */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_verify_internal(const uint8_t *sig, size_t siglen, +int mld_sign_verify_internal(const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *pre, size_t prelen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], @@ -419,8 +376,7 @@ int mld_sign_verify_internal(const uint8_t *sig, size_t siglen, __contract__( requires(prelen <= MLD_MAX_BUFFER_SIZE) requires(mlen <= MLD_MAX_BUFFER_SIZE) - requires(siglen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(sig, siglen)) + requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) requires(memory_no_alias(m, mlen)) requires((externalmu == 0) ==> ((prelen == 0) || memory_no_alias(pre, prelen))) requires((externalmu != 0) ==> (mlen == MLDSA_CRHBYTES)) @@ -432,10 +388,9 @@ __contract__( /** * Verify signature. * - * @spec{Implements @[FIPS204 Algorithm 3 (ML-DSA.Verify)].} + * @spec{Implements @[FIPS204, Algorithm 3, ML-DSA.Verify].} * * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. * @param[in] m Pointer to message. * @param mlen Length of message. * @param[in] ctx Pointer to context string. May be NULL if ctxlen == 0. @@ -452,15 +407,14 @@ __contract__( */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_verify(const uint8_t *sig, size_t siglen, const uint8_t *m, +int mld_sign_verify(const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(mlen <= MLD_MAX_BUFFER_SIZE) - requires(siglen <= MLD_MAX_BUFFER_SIZE) requires(ctxlen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(sig, siglen)) + requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) requires(memory_no_alias(m, mlen)) requires(ctxlen == 0 || memory_no_alias(ctx, ctxlen)) requires(memory_no_alias(pk, MLDSA_CRYPTO_PUBLICKEYBYTES)) @@ -474,10 +428,9 @@ __contract__( * 0x00 || ctxlen || ctx || msg for pure ML-DSA). The same mu must have been * used at signing time. * - * @spec{Implements @[FIPS204 Algorithm 3 (ML-DSA.Verify external mu variant)].} + * @spec{Implements @[FIPS204, Algorithm 3, ML-DSA.Verify external mu variant].} * * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. * @param[in] mu Precomputed message representative. * @param[in] pk Bit-packed public key. * @param context Application context. Only present when @@ -491,64 +444,25 @@ __contract__( */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API -int mld_sign_verify_extmu(const uint8_t *sig, size_t siglen, +int mld_sign_verify_extmu(const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t mu[MLDSA_CRHBYTES], const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) -__contract__( - requires(siglen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(sig, siglen)) +__contract__( requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) requires(memory_no_alias(mu, MLDSA_CRHBYTES)) requires(memory_no_alias(pk, MLDSA_CRYPTO_PUBLICKEYBYTES)) ensures(return_value == 0 || return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY) ); -/** - * Verify signed message. - * - * @param[out] m Pointer to output message (allocated array with smlen - * bytes); can be equal to sm. - * @param[out] mlen Pointer to output length of message. - * @param[in] sm Pointer to signed message. - * @param smlen Length of signed message. - * @param[in] ctx Pointer to context string. - * @param ctxlen Length of context string. - * @param[in] pk Bit-packed public key. - * @param context Application context. Only present when - * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by - * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. - * - * @retval 0 Success. - * @retval MLD_ERR_OUT_OF_MEMORY MLD_CONFIG_CUSTOM_ALLOC_FREE was used and an - * allocation via MLD_CUSTOM_ALLOC returned NULL. - * @retval MLD_ERR_FAIL Signature verification failed. - */ -MLD_MUST_CHECK_RETURN_VALUE -MLD_EXTERNAL_API -int mld_sign_open(uint8_t *m, size_t *mlen, const uint8_t *sm, size_t smlen, - const uint8_t *ctx, size_t ctxlen, - const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], - MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) -__contract__( - requires(smlen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(m, smlen)) - requires(memory_no_alias(mlen, sizeof(size_t))) - requires(m == sm || memory_no_alias(sm, smlen)) - requires(ctxlen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(ctx, ctxlen)) - requires(memory_no_alias(pk, MLDSA_CRYPTO_PUBLICKEYBYTES)) - assigns(memory_slice(m, smlen)) - assigns(memory_slice(mlen, sizeof(size_t))) - ensures(return_value == 0 || return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY) -); #endif /* !MLD_CONFIG_CORE_API_ONLY */ #endif /* !MLD_CONFIG_NO_VERIFY_API */ #if !defined(MLD_CONFIG_CORE_API_ONLY) #if !defined(MLD_CONFIG_NO_SIGN_API) /** - * FIPS 204: Algorithm 4 HashML-DSA.Sign. Compute signature with pre-hashed - * message. + * Compute signature with pre-hashed message. + * + * @spec{Implements @[FIPS204, Algorithm 4, HashML-DSA.Sign].} * * Supported hash algorithm constants: * MLD_PREHASH_SHA2_224, MLD_PREHASH_SHA2_256, MLD_PREHASH_SHA2_384, @@ -556,17 +470,22 @@ __contract__( * MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384, * MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256. * + * MLD_PREHASH_NONE is rejected by this API. + * * @warning This is an unstable API that may change in the future. If you need * a stable API use mld_sign_signature_pre_hash_shake256. * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use mld_sign_pk_from_sk + * to validate them before signing. + * * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. * @param[in] ph Pointer to pre-hashed message. * @param phlen Length of pre-hashed message. * @param[in] ctx Pointer to context string. * @param ctxlen Length of context string. * @param[in] rnd Random seed. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param hashalg Hash algorithm constant (one of MLD_PREHASH_*). * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by @@ -579,36 +498,35 @@ __contract__( * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS * iterations. + * @retval MLD_ERR_SIGNING_PAUSED A MLD_CONFIG_SIGN_HOOK_ATTEMPT hook + * paused signing; re-invoke to resume. * @retval MLD_ERR_FAIL Other kinds of failure. */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_signature_pre_hash_internal( - uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, const uint8_t *ph, - size_t phlen, const uint8_t *ctx, size_t ctxlen, - const uint8_t rnd[MLDSA_RNDBYTES], + uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *ph, size_t phlen, + const uint8_t *ctx, size_t ctxlen, const uint8_t rnd[MLDSA_RNDBYTES], const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], int hashalg, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(ctxlen <= MLD_MAX_BUFFER_SIZE) requires(phlen <= MLD_MAX_BUFFER_SIZE) requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) - requires(memory_no_alias(siglen, sizeof(size_t))) requires(memory_no_alias(ph, phlen)) requires(ctxlen == 0 || memory_no_alias(ctx, ctxlen)) requires(memory_no_alias(rnd, MLDSA_RNDBYTES)) requires(memory_no_alias(sk, MLDSA_CRYPTO_SECRETKEYBYTES)) assigns(memory_slice(sig, MLDSA_CRYPTO_BYTES)) - assigns(object_whole(siglen)) - ensures((return_value == 0 && *siglen == MLDSA_CRYPTO_BYTES) || - ((return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY || return_value == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED) && *siglen == 0)) + ensures(return_value == 0 || return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY || return_value == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED || return_value == MLD_ERR_SIGNING_PAUSED) ); #endif /* !MLD_CONFIG_NO_SIGN_API */ #if !defined(MLD_CONFIG_NO_VERIFY_API) /** - * FIPS 204: Algorithm 5 HashML-DSA.Verify. Verify signature with pre-hashed - * message. + * Verify signature with pre-hashed message. + * + * @spec{Implements @[FIPS204, Algorithm 5, HashML-DSA.Verify].} * * Supported hash algorithm constants: * MLD_PREHASH_SHA2_224, MLD_PREHASH_SHA2_256, MLD_PREHASH_SHA2_384, @@ -616,11 +534,12 @@ __contract__( * MLD_PREHASH_SHA3_224, MLD_PREHASH_SHA3_256, MLD_PREHASH_SHA3_384, * MLD_PREHASH_SHA3_512, MLD_PREHASH_SHAKE_128, MLD_PREHASH_SHAKE_256. * + * MLD_PREHASH_NONE is rejected by this API. + * * @warning This is an unstable API that may change in the future. If you need * a stable API use mld_sign_verify_pre_hash_shake256. * * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. * @param[in] ph Pointer to pre-hashed message. * @param phlen Length of pre-hashed message. * @param[in] ctx Pointer to context string. @@ -639,15 +558,14 @@ __contract__( MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_verify_pre_hash_internal( - const uint8_t *sig, size_t siglen, const uint8_t *ph, size_t phlen, + const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *ph, size_t phlen, const uint8_t *ctx, size_t ctxlen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], int hashalg, MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(phlen <= MLD_MAX_BUFFER_SIZE) requires(ctxlen <= MLD_MAX_BUFFER_SIZE - 77) - requires(siglen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(sig, siglen)) + requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) requires(memory_no_alias(ph, phlen)) requires(ctxlen == 0 || memory_no_alias(ctx, ctxlen)) requires(memory_no_alias(pk, MLDSA_CRYPTO_PUBLICKEYBYTES)) @@ -657,19 +575,23 @@ __contract__( #if !defined(MLD_CONFIG_NO_SIGN_API) /** - * FIPS 204: Algorithm 4 HashML-DSA.Sign with SHAKE256. - * * Compute signature with pre-hashed message using SHAKE256. This function * computes the SHAKE256 hash of the message internally. * + * @spec{Implements @[FIPS204, Algorithm 4, HashML-DSA.Sign] with SHAKE256 as + * the pre-hash.} + * + * @warning This function does not perform secret key validation. + * Callers importing serialized keys can use mld_sign_pk_from_sk + * to validate them before signing. + * * @param[out] sig Output signature. - * @param[out] siglen Pointer to output length of signature. * @param[in] m Pointer to message to be hashed and signed. * @param mlen Length of message. * @param[in] ctx Pointer to context string. * @param ctxlen Length of context string. * @param[in] rnd Random seed. - * @param[in] sk Bit-packed secret key. + * @param[in] sk Bit-packed secret key; assumed to be valid. * @param context Application context. Only present when * MLD_CONFIG_CONTEXT_PARAMETER is defined; type set by * MLD_CONFIG_CONTEXT_PARAMETER_TYPE. @@ -681,41 +603,39 @@ __contract__( * @retval MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED The rejection-sampling loop exceeded * MLD_CONFIG_MAX_SIGNING_ATTEMPTS * iterations. + * @retval MLD_ERR_SIGNING_PAUSED A MLD_CONFIG_SIGN_HOOK_ATTEMPT hook + * paused signing; re-invoke to resume. * @retval MLD_ERR_FAIL Other kinds of failure. */ MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_signature_pre_hash_shake256( - uint8_t sig[MLDSA_CRYPTO_BYTES], size_t *siglen, const uint8_t *m, - size_t mlen, const uint8_t *ctx, size_t ctxlen, - const uint8_t rnd[MLDSA_RNDBYTES], + uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, + const uint8_t *ctx, size_t ctxlen, const uint8_t rnd[MLDSA_RNDBYTES], const uint8_t sk[MLDSA_CRYPTO_SECRETKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(mlen <= MLD_MAX_BUFFER_SIZE) requires(ctxlen <= MLD_MAX_BUFFER_SIZE) requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) - requires(memory_no_alias(siglen, sizeof(size_t))) requires(memory_no_alias(m, mlen)) requires(ctxlen == 0 || memory_no_alias(ctx, ctxlen)) requires(memory_no_alias(rnd, MLDSA_RNDBYTES)) requires(memory_no_alias(sk, MLDSA_CRYPTO_SECRETKEYBYTES)) assigns(memory_slice(sig, MLDSA_CRYPTO_BYTES)) - assigns(object_whole(siglen)) - ensures((return_value == 0 && *siglen == MLDSA_CRYPTO_BYTES) || - ((return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY || return_value == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED) && *siglen == 0)) + ensures(return_value == 0 || return_value == MLD_ERR_FAIL || return_value == MLD_ERR_OUT_OF_MEMORY || return_value == MLD_ERR_SIGN_ATTEMPTS_EXHAUSTED || return_value == MLD_ERR_SIGNING_PAUSED) ); #endif /* !MLD_CONFIG_NO_SIGN_API */ #if !defined(MLD_CONFIG_NO_VERIFY_API) /** - * FIPS 204: Algorithm 5 HashML-DSA.Verify with SHAKE256. - * * Verify signature with pre-hashed message using SHAKE256. This function * computes the SHAKE256 hash of the message internally. * + * @spec{Implements @[FIPS204, Algorithm 5, HashML-DSA.Verify] with SHAKE256 as + * the pre-hash.} + * * @param[in] sig Pointer to input signature. - * @param siglen Length of signature. * @param[in] m Pointer to message to be hashed and verified. * @param mlen Length of message. * @param[in] ctx Pointer to context string. @@ -733,15 +653,14 @@ __contract__( MLD_MUST_CHECK_RETURN_VALUE MLD_EXTERNAL_API int mld_sign_verify_pre_hash_shake256( - const uint8_t *sig, size_t siglen, const uint8_t *m, size_t mlen, + const uint8_t sig[MLDSA_CRYPTO_BYTES], const uint8_t *m, size_t mlen, const uint8_t *ctx, size_t ctxlen, const uint8_t pk[MLDSA_CRYPTO_PUBLICKEYBYTES], MLD_CONFIG_CONTEXT_PARAMETER_TYPE context) __contract__( requires(mlen <= MLD_MAX_BUFFER_SIZE) requires(ctxlen <= MLD_MAX_BUFFER_SIZE - 77) - requires(siglen <= MLD_MAX_BUFFER_SIZE) - requires(memory_no_alias(sig, siglen)) + requires(memory_no_alias(sig, MLDSA_CRYPTO_BYTES)) requires(memory_no_alias(m, mlen)) requires(ctxlen == 0 || memory_no_alias(ctx, ctxlen)) requires(memory_no_alias(pk, MLDSA_CRYPTO_PUBLICKEYBYTES)) @@ -767,15 +686,15 @@ __contract__( * This function is useful for building incremental signing APIs. * * @spec{For HashML-DSA (hashalg != MLD_PREHASH_NONE), implements - * @[FIPS204, Algorithm 4, L23]. For Pure ML-DSA (hashalg == MLD_PREHASH_NONE), - * implements + * @[FIPS204, Algorithm 4, line 23]. For Pure ML-DSA + * (hashalg == MLD_PREHASH_NONE), implements * ``` * M' <- BytesToBits(IntegerToBytes(0, 1) * || IntegerToBytes(|ctx|, 1) * || ctx * ``` - * which is part of @[FIPS204, Algorithm 2 (ML-DSA.Sign), L10] and - * @[FIPS204, Algorithm 3 (ML-DSA.Verify), L5].} + * which is part of @[FIPS204, Algorithm 2, ML-DSA.Sign, line 10] and + * @[FIPS204, Algorithm 3, ML-DSA.Verify, line 5].} * * @param[out] prefix Output domain separation prefix buffer. * @param[in] ph Pointer to pre-hashed message (ignored for pure @@ -809,7 +728,7 @@ __contract__( * Perform basic validity checks on secret key, and derive public key. * * Referring to the decoding of the secret key `sk=(rho, K, tr, s1, s2, t0)` - * (cf. @[FIPS204, Algorithm 25 skDecode]), the following checks are + * (cf. @[FIPS204, Algorithm 25, skDecode]), the following checks are * performed: * - Check that s1 and s2 have coefficients in [-MLDSA_ETA, MLDSA_ETA]. * - Check that t0 and tr stored in sk match recomputed values. diff --git a/crypto/fipsmodule/ml_dsa/mldsa/sys.h b/crypto/fipsmodule/ml_dsa/mldsa/sys.h index 27834690a9..a96ba759ae 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa/sys.h +++ b/crypto/fipsmodule/ml_dsa/mldsa/sys.h @@ -49,12 +49,13 @@ #define MLD_SYS_ARMV81M_MVE #endif -#if defined(__x86_64__) +/* Check if we're running on an x86_64 system. */ +#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) #define MLD_SYS_X86_64 #if defined(__AVX2__) #define MLD_SYS_X86_64_AVX2 #endif -#endif /* __x86_64__ */ +#endif /* __x86_64__ || _M_X64 || _M_AMD64 */ #if defined(MLD_SYS_LITTLE_ENDIAN) && defined(__powerpc64__) #define MLD_SYS_PPC64LE @@ -233,15 +234,41 @@ #define MLD_MUST_CHECK_RETURN_VALUE #endif +/* The x86_64 assembly backend uses the SysV calling convention. On Windows, + * where the Microsoft x64 calling convention is the default, it can still be + * used with compilers that allow choosing the calling convention per + * function: GCC and Clang support __attribute__((sysv_abi)), which makes + * calls to the annotated function follow the SysV calling convention. + * + * MLD_SYSV_ABI_SUPPORTED signals that the toolchain can call SysV assembly + * routines; the x86_64 assembly backend is only enabled if it is defined. + * MLD_SYSV_ABI is the attribute carried by declarations of x86_64 assembly + * routines. Both macros can be set externally for toolchains offering an + * equivalent mechanism that is not recognized here. */ +#if defined(MLD_SYS_X86_64) && !defined(MLD_SYSV_ABI_SUPPORTED) +#if !defined(MLD_SYS_WINDOWS) || defined(__GNUC__) || defined(__clang__) +#define MLD_SYSV_ABI_SUPPORTED +#endif +#endif + +#if !defined(MLD_SYSV_ABI) +#if defined(MLD_SYS_WINDOWS) && defined(MLD_SYSV_ABI_SUPPORTED) +#define MLD_SYSV_ABI __attribute__((sysv_abi)) +#else +#define MLD_SYSV_ABI +#endif +#endif /* !MLD_SYSV_ABI */ #if !defined(__ASSEMBLER__) /* System capability enumeration */ typedef enum { /* x86_64 */ - MLD_SYS_CAP_AVX2, + MLD_SYS_CAP_X86_64_AVX2, /* AArch64 */ - MLD_SYS_CAP_SHA3 + MLD_SYS_CAP_AARCH64_SHA3, + /* Armv8.1-M */ + MLD_SYS_CAP_ARMV81M_MVE } mld_sys_cap; #if !defined(MLD_CONFIG_CUSTOM_CAPABILITY_FUNC) diff --git a/crypto/fipsmodule/ml_dsa/mldsa_native_backend.h b/crypto/fipsmodule/ml_dsa/mldsa_native_backend.h index 06cd3bd434..98a9ded903 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa_native_backend.h +++ b/crypto/fipsmodule/ml_dsa/mldsa_native_backend.h @@ -14,7 +14,7 @@ // NEON capability gate; falls back to the C reference impl when NEON is absent. #include "mldsa_aarch64_meta.h" #elif defined(OPENSSL_X86_64) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX) -#include "mldsa_x86_64_meta.h" +#include "mldsa/native/x86_64/meta.h" #endif #endif diff --git a/crypto/fipsmodule/ml_dsa/mldsa_native_config.h b/crypto/fipsmodule/ml_dsa/mldsa_native_config.h index a5f540f64a..bf35b9a057 100644 --- a/crypto/fipsmodule/ml_dsa/mldsa_native_config.h +++ b/crypto/fipsmodule/ml_dsa/mldsa_native_config.h @@ -48,7 +48,7 @@ static MLD_INLINE int mld_sys_check_capability(int cap) { #if defined(MLD_SYS_X86_64) - if (cap == MLD_SYS_CAP_AVX2) + if (cap == MLD_SYS_CAP_X86_64_AVX2) { return CRYPTO_is_AVX2_capable(); } diff --git a/crypto/fipsmodule/ml_dsa/mldsa_x86_64_meta.h b/crypto/fipsmodule/ml_dsa/mldsa_x86_64_meta.h deleted file mode 100644 index 3b905ff78e..0000000000 --- a/crypto/fipsmodule/ml_dsa/mldsa_x86_64_meta.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 OR ISC - -/* - * Custom x86_64 backend header for the mldsa-native import. - * - * mldsa-native's upstream meta.h declares native implementations for both - * assembly-backed operations (NTT, INTT, pointwise multiplication, caddq) and - * AVX2 C-intrinsic operations (rej_uniform, decompose, use_hint, chknorm, - * polyz_unpack). AWS-LC only imports the assembly-backed operations, so we - * replace the upstream meta.h with this trimmed-down version that declares - * only the subset we actually provide. - * - * Kept outside the imported `mldsa/` tree so that `importer.sh` does not - * need to modify upstream sources. - */ - -#ifndef MLD_NATIVE_X86_64_META_H -#define MLD_NATIVE_X86_64_META_H - -/* Identifier for this backend so that source and assembly files - * in the build can be appropriately guarded. */ -#define MLD_ARITH_BACKEND_X86_64_DEFAULT - -#define MLD_USE_NATIVE_NTT_CUSTOM_ORDER -#define MLD_USE_NATIVE_NTT -#define MLD_USE_NATIVE_INTT -#define MLD_USE_NATIVE_POLY_CADDQ -#define MLD_USE_NATIVE_POINTWISE_MONTGOMERY -#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4 -#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5 -#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7 - -#if !defined(__ASSEMBLER__) -#include "mldsa/native/api.h" -#include "mldsa/native/x86_64/src/arith_native_x86_64.h" - -static MLD_INLINE void mld_poly_permute_bitrev_to_custom(int32_t data[MLDSA_N]) -{ - if (mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - mld_nttunpack_avx2_asm(data); - } -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_ntt_avx2_asm(data, mld_qdata); - return MLD_NATIVE_FUNC_SUCCESS; -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_invntt_avx2_asm(data, mld_qdata); - return MLD_NATIVE_FUNC_SUCCESS; -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_poly_caddq_avx2_asm(a); - return MLD_NATIVE_FUNC_SUCCESS; -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_poly_pointwise_montgomery_native( - int32_t a[MLDSA_N], const int32_t b[MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_pointwise_avx2_asm(a, b, mld_qdata); - return MLD_NATIVE_FUNC_SUCCESS; -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native( - int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N], - const int32_t v[4][MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_pointwise_acc_l4_avx2_asm(w, u, v, mld_qdata); - return MLD_NATIVE_FUNC_SUCCESS; -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native( - int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N], - const int32_t v[5][MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_pointwise_acc_l5_avx2_asm(w, u, v, mld_qdata); - return MLD_NATIVE_FUNC_SUCCESS; -} - -MLD_MUST_CHECK_RETURN_VALUE -static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native( - int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N], - const int32_t v[7][MLDSA_N]) -{ - if (!mld_sys_check_capability(MLD_SYS_CAP_AVX2)) - { - return MLD_NATIVE_FUNC_FALLBACK; - } - mld_pointwise_acc_l7_avx2_asm(w, u, v, mld_qdata); - return MLD_NATIVE_FUNC_SUCCESS; -} - -#endif /* !__ASSEMBLER__ */ - -#endif /* !MLD_NATIVE_X86_64_META_H */