From b6f4c68532afbb3de6fcc8c2fd3d04165bdf4a21 Mon Sep 17 00:00:00 2001 From: "valery.osheter@coinbase.com" Date: Fri, 10 Jul 2026 15:11:14 +0300 Subject: [PATCH 1/2] seck256k1: ASM implementation of Field64 on AMD64 using MULX/ADX --- dcrec/secp256k1/field64.go | 14 -- dcrec/secp256k1/field64_amd64.go | 72 ++++++++++ dcrec/secp256k1/field64_amd64.s | 220 +++++++++++++++++++++++++++++ dcrec/secp256k1/field64_generic.go | 21 +++ 4 files changed, 313 insertions(+), 14 deletions(-) create mode 100644 dcrec/secp256k1/field64_amd64.go create mode 100644 dcrec/secp256k1/field64_amd64.s create mode 100644 dcrec/secp256k1/field64_generic.go diff --git a/dcrec/secp256k1/field64.go b/dcrec/secp256k1/field64.go index 53df4f1ae..6d2a06b9f 100644 --- a/dcrec/secp256k1/field64.go +++ b/dcrec/secp256k1/field64.go @@ -929,20 +929,6 @@ func field64Reduce512(r *[4]uint64, x *[8]uint64) { r[3] = constantTimeSelect64(borrow, t3, s3) } -// field64Mul sets r = a * b (mod p). -func field64Mul(r *[4]uint64, a, b *[4]uint64) { - var product [8]uint64 - field64Mul512(&product, a, b) - field64Reduce512(r, &product) -} - -// field64Square sets r = a^2 (mod p). -func field64Square(r *[4]uint64, a *[4]uint64) { - var product [8]uint64 - field64Square512(&product, a) - field64Reduce512(r, &product) -} - // Inverse finds the modular multiplicative inverse of the field value in // constant time. The existing field value is modified. // diff --git a/dcrec/secp256k1/field64_amd64.go b/dcrec/secp256k1/field64_amd64.go new file mode 100644 index 000000000..4b2cf6897 --- /dev/null +++ b/dcrec/secp256k1/field64_amd64.go @@ -0,0 +1,72 @@ +// Copyright (c) 2026 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +//go:build amd64 && !purego + +package secp256k1 + +// field64UseADX reports whether the CPU supports both BMI2 (MULX) and ADX (ADCX/ADOX) +var field64UseADX = func() bool { + const ( + eaxMax = 0 + eaxExt = 7 + bmi2Bit = 8 + adxBit = 19 + ) + + // Leaf 0x07 is only valid when the CPU reports it within the max input. + eax, _, _, _ := field64CPUID(eaxMax, 0) + if eax < eaxExt { + return false + } + + _, ebx, _, _ := field64CPUID(eaxExt, 0) + hasBMI2 := ebx>>bmi2Bit&1 == 1 + hasADX := ebx>>adxBit&1 == 1 + return hasBMI2 && hasADX +}() + +//go:noescape +func field64MulADX(r *[4]uint64, a, b *[4]uint64) + +//go:noescape +func field64SquareADX(r *[4]uint64, a *[4]uint64) + +// field64CPUID provides access to the CPUID opcode. +// +//go:noescape +func field64CPUID(eaxIn, ecxIn uint32) (eax, ebx, ecx, edx uint32) + +var ( + field64Mul = pickField64Mul() + field64Square = pickField64Square() +) + +func pickField64Mul() func(r *[4]uint64, a, b *[4]uint64) { + if field64UseADX { + return field64MulADX + } + return field64MulGeneric +} + +func pickField64Square() func(r *[4]uint64, a *[4]uint64) { + if field64UseADX { + return field64SquareADX + } + return field64SquareGeneric +} + +// field64MulGeneric sets r = a * b (mod p). +func field64MulGeneric(r *[4]uint64, a, b *[4]uint64) { + var product [8]uint64 + field64Mul512(&product, a, b) + field64Reduce512(r, &product) +} + +// field64SquareGeneric sets r = a^2 (mod p) +func field64SquareGeneric(r *[4]uint64, a *[4]uint64) { + var product [8]uint64 + field64Square512(&product, a) + field64Reduce512(r, &product) +} diff --git a/dcrec/secp256k1/field64_amd64.s b/dcrec/secp256k1/field64_amd64.s new file mode 100644 index 000000000..347e43e16 --- /dev/null +++ b/dcrec/secp256k1/field64_amd64.s @@ -0,0 +1,220 @@ +// Copyright (c) 2026 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +//go:build amd64 && !purego + +#include "textflag.h" + +#define REDUCE() \ + MOVQ $0x1000003D1, DX \ + XORQ CX, CX \ + \ + \ // First fold: t = (p0..p3) + c*(p4..p7) as 5 limbs in R12..R15,SI. + MULXQ R12, R12, AX \ + MULXQ R13, R13, BX \ + ADCXQ AX, R13 \ + MULXQ R14, R14, AX \ + ADCXQ BX, R14 \ + MULXQ R15, R15, SI \ + ADCXQ AX, R15 \ + ADCXQ CX, SI \ + \ + ADOXQ R8, R12 \ + ADOXQ R9, R13 \ + ADOXQ R10, R14 \ + ADOXQ R11, R15 \ + ADOXQ CX, SI \ + \ + \ // Second fold: t4*c back into the low limbs, carry kept in SI. + MULXQ SI, AX, CX \ + ADDQ AX, R12 \ + ADCQ CX, R13 \ + ADCQ $0, R14 \ + ADCQ $0, R15 \ + MOVQ $0, SI \ + ADCQ $0, SI \ + \ + \ // Constant-time conditional subtract of p. + MOVQ $0xFFFFFFFEFFFFFC2F, AX \ + MOVQ R12, R8 \ + SUBQ AX, R8 \ + MOVQ R13, R9 \ + SBBQ $-1, R9 \ + MOVQ R14, R10 \ + SBBQ $-1, R10 \ + MOVQ R15, R11 \ + SBBQ $-1, R11 \ + MOVQ SI, AX \ + SBBQ $0, AX \ + CMOVQCC R8, R12 \ + CMOVQCC R9, R13 \ + CMOVQCC R10, R14 \ + CMOVQCC R11, R15 \ + \ + MOVQ r+0(FP), AX \ + MOVQ R12, 0(AX) \ + MOVQ R13, 8(AX) \ + MOVQ R14, 16(AX) \ + MOVQ R15, 24(AX) + +// func field64MulADX(r *[4]uint64, a, b *[4]uint64) +TEXT ·field64MulADX(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), SI + MOVQ b+16(FP), DI + MOVOU 0(DI), X0 + MOVOU 16(DI), X1 + XORQ BX, BX + + // Row 0: p0..p4 = a0*b. + MOVQ 0(SI), DX + MOVQ X0, AX + PEXTRQ $1, X0, R13 + MOVQ X1, R14 + PEXTRQ $1, X1, R15 + MULXQ AX, R8, CX + MULXQ R13, R9, DI + ADCXQ CX, R9 + MULXQ R14, R10, CX + ADCXQ DI, R10 + MULXQ R15, R11, R12 + ADCXQ CX, R11 + ADCXQ BX, R12 + + // Row 1: p1..p4 += a1*b; new top p5 in R13. + MOVQ 8(SI), DX + MOVQ X0, R13 + PEXTRQ $1, X0, R14 + PEXTRQ $1, X1, R15 + MULXQ R13, AX, CX + ADOXQ AX, R9 + MOVQ X1, R13 + MULXQ R14, AX, DI + ADCXQ CX, AX + ADOXQ AX, R10 + MULXQ R13, AX, CX + ADCXQ DI, AX + ADOXQ AX, R11 + MULXQ R15, AX, R13 + ADCXQ CX, AX + ADOXQ AX, R12 + ADCXQ BX, R13 + ADOXQ BX, R13 + + // Row 2: p2..p5 += a2*b; new top p6 in R14. + MOVQ 16(SI), DX + MOVQ X0, R14 + PEXTRQ $1, X0, R15 + MULXQ R14, AX, CX + ADOXQ AX, R10 + MOVQ X1, R14 + MULXQ R15, AX, DI + ADCXQ CX, AX + ADOXQ AX, R11 + PEXTRQ $1, X1, R15 + MULXQ R14, AX, CX + ADCXQ DI, AX + ADOXQ AX, R12 + MULXQ R15, AX, R14 + ADCXQ CX, AX + ADOXQ AX, R13 + ADCXQ BX, R14 + ADOXQ BX, R14 + + // Row 3: p3..p6 += a3*b; new top p7 in R15. + MOVQ 24(SI), DX + MOVQ X0, AX + PEXTRQ $1, X0, R15 + MOVQ X1, SI + MULXQ AX, AX, CX + ADOXQ AX, R11 + MULXQ R15, AX, DI + ADCXQ CX, AX + ADOXQ AX, R12 + PEXTRQ $1, X1, R15 + MULXQ SI, AX, CX + ADCXQ DI, AX + ADOXQ AX, R13 + MULXQ R15, AX, R15 + ADCXQ CX, AX + ADOXQ AX, R14 + ADCXQ BX, R15 + ADOXQ BX, R15 + + REDUCE() + RET + +// func field64SquareADX(r *[4]uint64, a *[4]uint64) +TEXT ·field64SquareADX(SB), NOSPLIT, $0-16 + MOVQ a+8(FP), SI + MOVOU 0(SI), X0 + MOVOU 16(SI), X1 + XORQ R13, R13 + XORQ R15, R15 + XORQ CX, CX + + // Off-diagonal upper-triangle products into p1..p6 (a1->SI, a3->DI reused). + MOVQ X0, DX + PEXTRQ $1, X1, DI + MOVQ X1, R8 + PEXTRQ $1, X0, SI + MULXQ SI, R9, R10 + MULXQ R8, BX, R11 + MULXQ DI, AX, R12 + ADCXQ BX, R10 + ADCXQ AX, R11 + ADCXQ CX, R12 + + MOVQ SI, DX + MULXQ R8, BX, AX + ADOXQ BX, R11 + ADOXQ AX, R12 + ADOXQ CX, R13 + MULXQ DI, BX, AX + ADDQ BX, R12 + ADCXQ AX, R13 + + MOVQ R8, DX + MULXQ DI, BX, R14 + ADDQ BX, R13 + ADCXQ CX, R14 + + // Double p1..p6, capturing the top carry into p7. + ADDQ R9, R9 + ADCXQ R10, R10 + ADCXQ R11, R11 + ADCXQ R12, R12 + ADCXQ R13, R13 + ADCXQ R14, R14 + ADCXQ R15, R15 + + // Add the diagonal squares a[i]^2 at columns 0,2,4,6. + MOVQ X0, DX + MULXQ DX, R8, BX + ADDQ BX, R9 + MOVQ SI, DX + MULXQ DX, BX, AX + ADCXQ BX, R10 + ADCXQ AX, R11 + MOVQ X1, DX + MULXQ DX, BX, AX + ADCXQ BX, R12 + ADCXQ AX, R13 + MOVQ DI, DX + MULXQ DX, BX, AX + ADCXQ BX, R14 + ADCXQ AX, R15 + + REDUCE() + RET + +// func field64CPUID(eaxIn, ecxIn uint32) (eax, ebx, ecx, edx uint32) +TEXT ·field64CPUID(SB), NOSPLIT, $0-24 + MOVL eaxIn+0(FP), AX + MOVL ecxIn+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET diff --git a/dcrec/secp256k1/field64_generic.go b/dcrec/secp256k1/field64_generic.go new file mode 100644 index 000000000..bea665bc6 --- /dev/null +++ b/dcrec/secp256k1/field64_generic.go @@ -0,0 +1,21 @@ +// Copyright (c) 2026 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +//go:build !amd64 || purego + +package secp256k1 + +// field64Mul sets r = a * b (mod p). +func field64Mul(r *[4]uint64, a, b *[4]uint64) { + var product [8]uint64 + field64Mul512(&product, a, b) + field64Reduce512(r, &product) +} + +// field64Square sets r = a^2 (mod p). +func field64Square(r *[4]uint64, a *[4]uint64) { + var product [8]uint64 + field64Square512(&product, a) + field64Reduce512(r, &product) +} From a5eba071167f4680c2b56fbf0fa29bfd893388d5 Mon Sep 17 00:00:00 2001 From: "valery.osheter@coinbase.com" Date: Wed, 15 Jul 2026 18:38:02 +0300 Subject: [PATCH 2/2] secp256k1: Fix field64 amd64 dispatch and add comparative benchmarks. --- dcrec/secp256k1/field64_amd64.go | 25 ++++--- dcrec/secp256k1/field64_amd64.s | 79 ++++++++------------- dcrec/secp256k1/field64_bench_amd64_test.go | 60 ++++++++++++++++ 3 files changed, 102 insertions(+), 62 deletions(-) create mode 100644 dcrec/secp256k1/field64_bench_amd64_test.go diff --git a/dcrec/secp256k1/field64_amd64.go b/dcrec/secp256k1/field64_amd64.go index 4b2cf6897..bf068120f 100644 --- a/dcrec/secp256k1/field64_amd64.go +++ b/dcrec/secp256k1/field64_amd64.go @@ -6,7 +6,7 @@ package secp256k1 -// field64UseADX reports whether the CPU supports both BMI2 (MULX) and ADX (ADCX/ADOX) +// field64UseADX reports whether the CPU supports both BMI2 (MULX) and ADX (ADCX/ADOX). var field64UseADX = func() bool { const ( eaxMax = 0 @@ -38,26 +38,25 @@ func field64SquareADX(r *[4]uint64, a *[4]uint64) //go:noescape func field64CPUID(eaxIn, ecxIn uint32) (eax, ebx, ecx, edx uint32) -var ( - field64Mul = pickField64Mul() - field64Square = pickField64Square() -) - -func pickField64Mul() func(r *[4]uint64, a, b *[4]uint64) { +// field64Mul sets r = a * b (mod p) +func field64Mul(r *[4]uint64, a, b *[4]uint64) { if field64UseADX { - return field64MulADX + field64MulADX(r, a, b) + return } - return field64MulGeneric + field64MulGeneric(r, a, b) } -func pickField64Square() func(r *[4]uint64, a *[4]uint64) { +// field64Square sets r = a^2 (mod p) +func field64Square(r *[4]uint64, a *[4]uint64) { if field64UseADX { - return field64SquareADX + field64SquareADX(r, a) + return } - return field64SquareGeneric + field64SquareGeneric(r, a) } -// field64MulGeneric sets r = a * b (mod p). +// field64MulGeneric sets r = a * b (mod p) func field64MulGeneric(r *[4]uint64, a, b *[4]uint64) { var product [8]uint64 field64Mul512(&product, a, b) diff --git a/dcrec/secp256k1/field64_amd64.s b/dcrec/secp256k1/field64_amd64.s index 347e43e16..aeb851ae5 100644 --- a/dcrec/secp256k1/field64_amd64.s +++ b/dcrec/secp256k1/field64_amd64.s @@ -62,40 +62,30 @@ TEXT ·field64MulADX(SB), NOSPLIT, $0-24 MOVQ a+8(FP), SI MOVQ b+16(FP), DI - MOVOU 0(DI), X0 - MOVOU 16(DI), X1 XORQ BX, BX // Row 0: p0..p4 = a0*b. MOVQ 0(SI), DX - MOVQ X0, AX - PEXTRQ $1, X0, R13 - MOVQ X1, R14 - PEXTRQ $1, X1, R15 - MULXQ AX, R8, CX - MULXQ R13, R9, DI + MULXQ 0(DI), R8, CX + MULXQ 8(DI), R9, R15 ADCXQ CX, R9 - MULXQ R14, R10, CX - ADCXQ DI, R10 - MULXQ R15, R11, R12 + MULXQ 16(DI), R10, CX + ADCXQ R15, R10 + MULXQ 24(DI), R11, R12 ADCXQ CX, R11 ADCXQ BX, R12 // Row 1: p1..p4 += a1*b; new top p5 in R13. MOVQ 8(SI), DX - MOVQ X0, R13 - PEXTRQ $1, X0, R14 - PEXTRQ $1, X1, R15 - MULXQ R13, AX, CX + MULXQ 0(DI), AX, CX ADOXQ AX, R9 - MOVQ X1, R13 - MULXQ R14, AX, DI + MULXQ 8(DI), AX, R15 ADCXQ CX, AX ADOXQ AX, R10 - MULXQ R13, AX, CX - ADCXQ DI, AX + MULXQ 16(DI), AX, CX + ADCXQ R15, AX ADOXQ AX, R11 - MULXQ R15, AX, R13 + MULXQ 24(DI), AX, R13 ADCXQ CX, AX ADOXQ AX, R12 ADCXQ BX, R13 @@ -103,19 +93,15 @@ TEXT ·field64MulADX(SB), NOSPLIT, $0-24 // Row 2: p2..p5 += a2*b; new top p6 in R14. MOVQ 16(SI), DX - MOVQ X0, R14 - PEXTRQ $1, X0, R15 - MULXQ R14, AX, CX + MULXQ 0(DI), AX, CX ADOXQ AX, R10 - MOVQ X1, R14 - MULXQ R15, AX, DI + MULXQ 8(DI), AX, R15 ADCXQ CX, AX ADOXQ AX, R11 - PEXTRQ $1, X1, R15 - MULXQ R14, AX, CX - ADCXQ DI, AX + MULXQ 16(DI), AX, CX + ADCXQ R15, AX ADOXQ AX, R12 - MULXQ R15, AX, R14 + MULXQ 24(DI), AX, R14 ADCXQ CX, AX ADOXQ AX, R13 ADCXQ BX, R14 @@ -123,19 +109,15 @@ TEXT ·field64MulADX(SB), NOSPLIT, $0-24 // Row 3: p3..p6 += a3*b; new top p7 in R15. MOVQ 24(SI), DX - MOVQ X0, AX - PEXTRQ $1, X0, R15 - MOVQ X1, SI - MULXQ AX, AX, CX + MULXQ 0(DI), AX, CX ADOXQ AX, R11 - MULXQ R15, AX, DI + MULXQ 8(DI), AX, R15 ADCXQ CX, AX ADOXQ AX, R12 - PEXTRQ $1, X1, R15 - MULXQ SI, AX, CX - ADCXQ DI, AX + MULXQ 16(DI), AX, CX + ADCXQ R15, AX ADOXQ AX, R13 - MULXQ R15, AX, R15 + MULXQ 24(DI), AX, R15 ADCXQ CX, AX ADOXQ AX, R14 ADCXQ BX, R15 @@ -146,18 +128,16 @@ TEXT ·field64MulADX(SB), NOSPLIT, $0-24 // func field64SquareADX(r *[4]uint64, a *[4]uint64) TEXT ·field64SquareADX(SB), NOSPLIT, $0-16 - MOVQ a+8(FP), SI - MOVOU 0(SI), X0 - MOVOU 16(SI), X1 + MOVQ a+8(FP), AX + MOVQ 0(AX), DX // a0 + MOVQ 8(AX), SI // a1 + MOVQ 16(AX), R8 // a2 + MOVQ 24(AX), DI // a3 XORQ R13, R13 XORQ R15, R15 XORQ CX, CX - // Off-diagonal upper-triangle products into p1..p6 (a1->SI, a3->DI reused). - MOVQ X0, DX - PEXTRQ $1, X1, DI - MOVQ X1, R8 - PEXTRQ $1, X0, SI + // Off-diagonal upper-triangle products into p1..p6 MULXQ SI, R9, R10 MULXQ R8, BX, R11 MULXQ DI, AX, R12 @@ -179,7 +159,7 @@ TEXT ·field64SquareADX(SB), NOSPLIT, $0-16 ADDQ BX, R13 ADCXQ CX, R14 - // Double p1..p6, capturing the top carry into p7. + // Double p1..p6, capturing the top carry into p7 ADDQ R9, R9 ADCXQ R10, R10 ADCXQ R11, R11 @@ -189,14 +169,15 @@ TEXT ·field64SquareADX(SB), NOSPLIT, $0-16 ADCXQ R15, R15 // Add the diagonal squares a[i]^2 at columns 0,2,4,6. - MOVQ X0, DX + MOVQ a+8(FP), CX + MOVQ 0(CX), DX MULXQ DX, R8, BX ADDQ BX, R9 MOVQ SI, DX MULXQ DX, BX, AX ADCXQ BX, R10 ADCXQ AX, R11 - MOVQ X1, DX + MOVQ 16(CX), DX MULXQ DX, BX, AX ADCXQ BX, R12 ADCXQ AX, R13 diff --git a/dcrec/secp256k1/field64_bench_amd64_test.go b/dcrec/secp256k1/field64_bench_amd64_test.go new file mode 100644 index 000000000..adb4c5225 --- /dev/null +++ b/dcrec/secp256k1/field64_bench_amd64_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2026 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +//go:build amd64 && !purego + +package secp256k1 + +import ( + "testing" +) + +func BenchmarkField64MulAMD64(b *testing.B) { + a := mustFieldVal64("d2e670a19c6d753d1a6d8b20bd045df8a08fb162cf508956c31268c6d81ffdab").n + c := mustFieldVal64("16fb970147a9acc73654d4be233cc48b875ce20a2122d24f073d29bd28805aca").n + var r [4]uint64 + + b.Run("Generic", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + field64MulGeneric(&r, &a, &c) + } + }) + if field64UseADX { + b.Run("MULX/ADX", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + field64MulADX(&r, &a, &c) + } + }) + } else { + b.Log("Skipping MULX/ADX bench (disabled or no instruction set support)") + } +} + +func BenchmarkField64SquareAMD64(b *testing.B) { + a := mustFieldVal64("16fb970147a9acc73654d4be233cc48b875ce20a2122d24f073d29bd28805aca").n + var r [4]uint64 + + b.Run("Generic", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + field64SquareGeneric(&r, &a) + } + }) + if field64UseADX { + b.Run("MULX/ADX", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + field64SquareADX(&r, &a) + } + }) + } else { + b.Log("Skipping MULX/ADX bench (disabled or no instruction set support)") + } +}