-
Notifications
You must be signed in to change notification settings - Fork 319
secp256k1: ASM implementation of Field64 on AMD64 using MULX/ADX #3734
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
valery-osheter-cb
wants to merge
3
commits into
decred:master
Choose a base branch
from
valery-osheter-cb:field64_amd_asm
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+353
−14
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
b6f4c68
seck256k1: ASM implementation of Field64 on AMD64 using MULX/ADX
valery-osheter-cb 8836a33
Merge remote-tracking branch 'origin/master' into field64_amd_asm
valery-osheter-cb a5eba07
secp256k1: Fix field64 amd64 dispatch and add comparative benchmarks.
valery-osheter-cb File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| // Copyright (c) 2026 The Decred developers | ||
| // Use of this source code is governed by an ISC | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| //go:build amd64 && !purego | ||
|
|
||
| package secp256k1 | ||
|
|
||
| // field64UseADX reports whether the CPU supports both BMI2 (MULX) and ADX (ADCX/ADOX). | ||
| var field64UseADX = func() bool { | ||
| const ( | ||
| eaxMax = 0 | ||
| eaxExt = 7 | ||
| bmi2Bit = 8 | ||
| adxBit = 19 | ||
| ) | ||
|
|
||
| // Leaf 0x07 is only valid when the CPU reports it within the max input. | ||
| eax, _, _, _ := field64CPUID(eaxMax, 0) | ||
| if eax < eaxExt { | ||
| return false | ||
| } | ||
|
|
||
| _, ebx, _, _ := field64CPUID(eaxExt, 0) | ||
| hasBMI2 := ebx>>bmi2Bit&1 == 1 | ||
| hasADX := ebx>>adxBit&1 == 1 | ||
| return hasBMI2 && hasADX | ||
| }() | ||
|
|
||
| //go:noescape | ||
| func field64MulADX(r *[4]uint64, a, b *[4]uint64) | ||
|
|
||
| //go:noescape | ||
| func field64SquareADX(r *[4]uint64, a *[4]uint64) | ||
|
|
||
| // field64CPUID provides access to the CPUID opcode. | ||
| // | ||
| //go:noescape | ||
| func field64CPUID(eaxIn, ecxIn uint32) (eax, ebx, ecx, edx uint32) | ||
|
|
||
| // field64Mul sets r = a * b (mod p) | ||
| func field64Mul(r *[4]uint64, a, b *[4]uint64) { | ||
| if field64UseADX { | ||
| field64MulADX(r, a, b) | ||
| return | ||
| } | ||
| field64MulGeneric(r, a, b) | ||
| } | ||
|
|
||
| // field64Square sets r = a^2 (mod p) | ||
| func field64Square(r *[4]uint64, a *[4]uint64) { | ||
| if field64UseADX { | ||
| field64SquareADX(r, a) | ||
| return | ||
| } | ||
| field64SquareGeneric(r, a) | ||
| } | ||
|
|
||
| // field64MulGeneric sets r = a * b (mod p) | ||
| func field64MulGeneric(r *[4]uint64, a, b *[4]uint64) { | ||
| var product [8]uint64 | ||
| field64Mul512(&product, a, b) | ||
| field64Reduce512(r, &product) | ||
| } | ||
|
|
||
| // field64SquareGeneric sets r = a^2 (mod p) | ||
| func field64SquareGeneric(r *[4]uint64, a *[4]uint64) { | ||
| var product [8]uint64 | ||
| field64Square512(&product, a) | ||
| field64Reduce512(r, &product) | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,201 @@ | ||
| // Copyright (c) 2026 The Decred developers | ||
| // Use of this source code is governed by an ISC | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| //go:build amd64 && !purego | ||
|
|
||
| #include "textflag.h" | ||
|
|
||
| #define REDUCE() \ | ||
| MOVQ $0x1000003D1, DX \ | ||
| XORQ CX, CX \ | ||
| \ | ||
| \ // First fold: t = (p0..p3) + c*(p4..p7) as 5 limbs in R12..R15,SI. | ||
| MULXQ R12, R12, AX \ | ||
| MULXQ R13, R13, BX \ | ||
| ADCXQ AX, R13 \ | ||
| MULXQ R14, R14, AX \ | ||
| ADCXQ BX, R14 \ | ||
| MULXQ R15, R15, SI \ | ||
| ADCXQ AX, R15 \ | ||
| ADCXQ CX, SI \ | ||
| \ | ||
| ADOXQ R8, R12 \ | ||
| ADOXQ R9, R13 \ | ||
| ADOXQ R10, R14 \ | ||
| ADOXQ R11, R15 \ | ||
| ADOXQ CX, SI \ | ||
| \ | ||
| \ // Second fold: t4*c back into the low limbs, carry kept in SI. | ||
| MULXQ SI, AX, CX \ | ||
| ADDQ AX, R12 \ | ||
| ADCQ CX, R13 \ | ||
| ADCQ $0, R14 \ | ||
| ADCQ $0, R15 \ | ||
| MOVQ $0, SI \ | ||
| ADCQ $0, SI \ | ||
| \ | ||
| \ // Constant-time conditional subtract of p. | ||
| MOVQ $0xFFFFFFFEFFFFFC2F, AX \ | ||
| MOVQ R12, R8 \ | ||
| SUBQ AX, R8 \ | ||
| MOVQ R13, R9 \ | ||
| SBBQ $-1, R9 \ | ||
| MOVQ R14, R10 \ | ||
| SBBQ $-1, R10 \ | ||
| MOVQ R15, R11 \ | ||
| SBBQ $-1, R11 \ | ||
| MOVQ SI, AX \ | ||
| SBBQ $0, AX \ | ||
| CMOVQCC R8, R12 \ | ||
| CMOVQCC R9, R13 \ | ||
| CMOVQCC R10, R14 \ | ||
| CMOVQCC R11, R15 \ | ||
| \ | ||
| MOVQ r+0(FP), AX \ | ||
| MOVQ R12, 0(AX) \ | ||
| MOVQ R13, 8(AX) \ | ||
| MOVQ R14, 16(AX) \ | ||
| MOVQ R15, 24(AX) | ||
|
|
||
| // func field64MulADX(r *[4]uint64, a, b *[4]uint64) | ||
| TEXT ·field64MulADX(SB), NOSPLIT, $0-24 | ||
| MOVQ a+8(FP), SI | ||
| MOVQ b+16(FP), DI | ||
| XORQ BX, BX | ||
|
|
||
| // Row 0: p0..p4 = a0*b. | ||
| MOVQ 0(SI), DX | ||
| MULXQ 0(DI), R8, CX | ||
| MULXQ 8(DI), R9, R15 | ||
| ADCXQ CX, R9 | ||
| MULXQ 16(DI), R10, CX | ||
| ADCXQ R15, R10 | ||
| MULXQ 24(DI), R11, R12 | ||
| ADCXQ CX, R11 | ||
| ADCXQ BX, R12 | ||
|
|
||
| // Row 1: p1..p4 += a1*b; new top p5 in R13. | ||
| MOVQ 8(SI), DX | ||
| MULXQ 0(DI), AX, CX | ||
| ADOXQ AX, R9 | ||
| MULXQ 8(DI), AX, R15 | ||
| ADCXQ CX, AX | ||
| ADOXQ AX, R10 | ||
| MULXQ 16(DI), AX, CX | ||
| ADCXQ R15, AX | ||
| ADOXQ AX, R11 | ||
| MULXQ 24(DI), AX, R13 | ||
| ADCXQ CX, AX | ||
| ADOXQ AX, R12 | ||
| ADCXQ BX, R13 | ||
| ADOXQ BX, R13 | ||
|
|
||
| // Row 2: p2..p5 += a2*b; new top p6 in R14. | ||
| MOVQ 16(SI), DX | ||
| MULXQ 0(DI), AX, CX | ||
| ADOXQ AX, R10 | ||
| MULXQ 8(DI), AX, R15 | ||
| ADCXQ CX, AX | ||
| ADOXQ AX, R11 | ||
| MULXQ 16(DI), AX, CX | ||
| ADCXQ R15, AX | ||
| ADOXQ AX, R12 | ||
| MULXQ 24(DI), AX, R14 | ||
| ADCXQ CX, AX | ||
| ADOXQ AX, R13 | ||
| ADCXQ BX, R14 | ||
| ADOXQ BX, R14 | ||
|
|
||
| // Row 3: p3..p6 += a3*b; new top p7 in R15. | ||
| MOVQ 24(SI), DX | ||
| MULXQ 0(DI), AX, CX | ||
| ADOXQ AX, R11 | ||
| MULXQ 8(DI), AX, R15 | ||
| ADCXQ CX, AX | ||
| ADOXQ AX, R12 | ||
| MULXQ 16(DI), AX, CX | ||
| ADCXQ R15, AX | ||
| ADOXQ AX, R13 | ||
| MULXQ 24(DI), AX, R15 | ||
| ADCXQ CX, AX | ||
| ADOXQ AX, R14 | ||
| ADCXQ BX, R15 | ||
| ADOXQ BX, R15 | ||
|
|
||
| REDUCE() | ||
| RET | ||
|
|
||
| // func field64SquareADX(r *[4]uint64, a *[4]uint64) | ||
| TEXT ·field64SquareADX(SB), NOSPLIT, $0-16 | ||
| MOVQ a+8(FP), AX | ||
| MOVQ 0(AX), DX // a0 | ||
| MOVQ 8(AX), SI // a1 | ||
| MOVQ 16(AX), R8 // a2 | ||
| MOVQ 24(AX), DI // a3 | ||
| XORQ R13, R13 | ||
| XORQ R15, R15 | ||
| XORQ CX, CX | ||
|
|
||
| // Off-diagonal upper-triangle products into p1..p6 | ||
| MULXQ SI, R9, R10 | ||
| MULXQ R8, BX, R11 | ||
| MULXQ DI, AX, R12 | ||
| ADCXQ BX, R10 | ||
| ADCXQ AX, R11 | ||
| ADCXQ CX, R12 | ||
|
|
||
| MOVQ SI, DX | ||
| MULXQ R8, BX, AX | ||
| ADOXQ BX, R11 | ||
| ADOXQ AX, R12 | ||
| ADOXQ CX, R13 | ||
| MULXQ DI, BX, AX | ||
| ADDQ BX, R12 | ||
| ADCXQ AX, R13 | ||
|
|
||
| MOVQ R8, DX | ||
| MULXQ DI, BX, R14 | ||
| ADDQ BX, R13 | ||
| ADCXQ CX, R14 | ||
|
|
||
| // Double p1..p6, capturing the top carry into p7 | ||
| ADDQ R9, R9 | ||
| ADCXQ R10, R10 | ||
| ADCXQ R11, R11 | ||
| ADCXQ R12, R12 | ||
| ADCXQ R13, R13 | ||
| ADCXQ R14, R14 | ||
| ADCXQ R15, R15 | ||
|
|
||
| // Add the diagonal squares a[i]^2 at columns 0,2,4,6. | ||
| MOVQ a+8(FP), CX | ||
| MOVQ 0(CX), DX | ||
| MULXQ DX, R8, BX | ||
| ADDQ BX, R9 | ||
| MOVQ SI, DX | ||
| MULXQ DX, BX, AX | ||
| ADCXQ BX, R10 | ||
| ADCXQ AX, R11 | ||
| MOVQ 16(CX), DX | ||
| MULXQ DX, BX, AX | ||
| ADCXQ BX, R12 | ||
| ADCXQ AX, R13 | ||
| MOVQ DI, DX | ||
| MULXQ DX, BX, AX | ||
| ADCXQ BX, R14 | ||
| ADCXQ AX, R15 | ||
|
|
||
| REDUCE() | ||
| RET | ||
|
|
||
| // func field64CPUID(eaxIn, ecxIn uint32) (eax, ebx, ecx, edx uint32) | ||
| TEXT ·field64CPUID(SB), NOSPLIT, $0-24 | ||
| MOVL eaxIn+0(FP), AX | ||
| MOVL ecxIn+4(FP), CX | ||
| CPUID | ||
| MOVL AX, eax+8(FP) | ||
| MOVL BX, ebx+12(FP) | ||
| MOVL CX, ecx+16(FP) | ||
| MOVL DX, edx+20(FP) | ||
| RET | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| // Copyright (c) 2026 The Decred developers | ||
| // Use of this source code is governed by an ISC | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| //go:build amd64 && !purego | ||
|
|
||
| package secp256k1 | ||
|
|
||
| import ( | ||
| "testing" | ||
| ) | ||
|
|
||
| func BenchmarkField64MulAMD64(b *testing.B) { | ||
| a := mustFieldVal64("d2e670a19c6d753d1a6d8b20bd045df8a08fb162cf508956c31268c6d81ffdab").n | ||
| c := mustFieldVal64("16fb970147a9acc73654d4be233cc48b875ce20a2122d24f073d29bd28805aca").n | ||
| var r [4]uint64 | ||
|
|
||
| b.Run("Generic", func(b *testing.B) { | ||
| b.ReportAllocs() | ||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| field64MulGeneric(&r, &a, &c) | ||
| } | ||
| }) | ||
| if field64UseADX { | ||
| b.Run("MULX/ADX", func(b *testing.B) { | ||
| b.ReportAllocs() | ||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| field64MulADX(&r, &a, &c) | ||
| } | ||
| }) | ||
| } else { | ||
| b.Log("Skipping MULX/ADX bench (disabled or no instruction set support)") | ||
| } | ||
| } | ||
|
|
||
| func BenchmarkField64SquareAMD64(b *testing.B) { | ||
| a := mustFieldVal64("16fb970147a9acc73654d4be233cc48b875ce20a2122d24f073d29bd28805aca").n | ||
| var r [4]uint64 | ||
|
|
||
| b.Run("Generic", func(b *testing.B) { | ||
| b.ReportAllocs() | ||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| field64SquareGeneric(&r, &a) | ||
| } | ||
| }) | ||
| if field64UseADX { | ||
| b.Run("MULX/ADX", func(b *testing.B) { | ||
| b.ReportAllocs() | ||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| field64SquareADX(&r, &a) | ||
| } | ||
| }) | ||
| } else { | ||
| b.Log("Skipping MULX/ADX bench (disabled or no instruction set support)") | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| // Copyright (c) 2026 The Decred developers | ||
| // Use of this source code is governed by an ISC | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| //go:build !amd64 || purego | ||
|
|
||
| package secp256k1 | ||
|
|
||
| // field64Mul sets r = a * b (mod p). | ||
| func field64Mul(r *[4]uint64, a, b *[4]uint64) { | ||
| var product [8]uint64 | ||
| field64Mul512(&product, a, b) | ||
| field64Reduce512(r, &product) | ||
| } | ||
|
|
||
| // field64Square sets r = a^2 (mod p). | ||
| func field64Square(r *[4]uint64, a *[4]uint64) { | ||
| var product [8]uint64 | ||
| field64Square512(&product, a) | ||
| field64Reduce512(r, &product) | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Commenting for future reviewers that this is safe because the
XORQ BX, BXabove unconditionally clearsCF(andOFfor theADOXcall below) per the Intel Software Developer's Manual (and AMD's) despite many other sources claiming that it only affectsZFandSFand leavesCFandOFunmodified.