Arm64: [PAC-RET] NativeAOT changes

src/coreclr/inc/cfi.h

@@ -10,6 +10,7 @@ enum CFI_OPCODE
    CFI_ADJUST_CFA_OFFSET,    // Offset is adjusted relative to the current one.
    CFI_DEF_CFA_REGISTER,     // New register is used to compute CFA
    CFI_REL_OFFSET,           // Register is saved at offset from the current CFA
+   CFI_DEF_CFA,              // Take address from register and add offset to it
    CFI_NEGATE_RA_STATE,      // Sign the return address in lr with paciasp
 };
 

src/coreclr/inc/clrnt.h

@@ -333,7 +333,7 @@ RtlpGetFunctionEndAddress (
     else
     {
         // Get from the xdata record.
-        FunctionLength = *(PTR_ULONG64)(ImageBase + FunctionLength) & 0x3ffff;
+        FunctionLength = *(PTR_uint64_t)(ImageBase + FunctionLength) & 0x3ffff;
     }
 
     return FunctionEntry->BeginAddress + 4 * FunctionLength;
@@ -422,7 +422,7 @@ RtlpGetFunctionEndAddress (
     if ((FunctionLength & 3) != 0) {
         FunctionLength = (FunctionLength >> 2) & 0x7ff;
     } else {
-        FunctionLength = *(PTR_ULONG64)(ImageBase + FunctionLength) & 0x3ffff;
+        FunctionLength = *(PTR_uint64_t)(ImageBase + FunctionLength) & 0x3ffff;
     }
 
     return FunctionEntry->BeginAddress + 4 * FunctionLength;
@@ -478,7 +478,7 @@ RtlpGetFunctionEndAddress (
     if ((FunctionLength & 3) != 0) {
         FunctionLength = (FunctionLength >> 2) & 0x7ff;
     } else {
-        FunctionLength = *(PTR_ULONG64)(ImageBase + FunctionLength) & 0x3ffff;
+        FunctionLength = *(PTR_uint64_t)(ImageBase + FunctionLength) & 0x3ffff;
     }
 
     return FunctionEntry->BeginAddress + 2 * FunctionLength;

src/coreclr/inc/daccess.h

@@ -2372,23 +2372,23 @@ typedef DPTR(TADDR)        PTR_TADDR;
 
 #ifndef NATIVEAOT
 typedef ArrayDPTR(BYTE)    PTR_BYTE;
+typedef DPTR(WORD)         PTR_WORD;
+typedef DPTR(DWORD)        PTR_DWORD;
+typedef DPTR(ULONG64)      PTR_ULONG64;
+typedef DPTR(UINT64)       PTR_UINT64;
 typedef DPTR(PTR_BYTE) PTR_PTR_BYTE;
 typedef DPTR(PTR_PTR_BYTE) PTR_PTR_PTR_BYTE;
 typedef ArrayDPTR(signed char) PTR_SBYTE;
 typedef ArrayDPTR(const BYTE) PTR_CBYTE;
 typedef DPTR(INT8)    PTR_INT8;
 typedef DPTR(INT16)   PTR_INT16;
 typedef DPTR(UINT16)  PTR_UINT16;
-typedef DPTR(WORD)    PTR_WORD;
 typedef DPTR(USHORT)  PTR_USHORT;
-typedef DPTR(DWORD)   PTR_DWORD;
 typedef DPTR(LONG)    PTR_LONG;
 typedef DPTR(ULONG)   PTR_ULONG;
 typedef DPTR(INT32)   PTR_INT32;
 typedef DPTR(UINT32)  PTR_UINT32;
-typedef DPTR(ULONG64) PTR_ULONG64;
 typedef DPTR(INT64)   PTR_INT64;
-typedef DPTR(UINT64)  PTR_UINT64;
 typedef DPTR(SIZE_T)  PTR_SIZE_T;
 typedef DPTR(int)     PTR_int;
 typedef DPTR(BOOL)    PTR_BOOL;

src/coreclr/nativeaot/Runtime/AsmOffsets.h

@@ -54,7 +54,12 @@ ASM_OFFSET(   30,    48, Thread, m_pTransitionFrame)
 ASM_OFFSET(   34,    50, Thread, m_pDeferredTransitionFrame)
 ASM_OFFSET(   40,    68, Thread, m_ppvHijackedReturnAddressLocation)
 ASM_OFFSET(   44,    70, Thread, m_pvHijackedReturnAddress)
+#if defined(TARGET_ARM64)
+ASM_OFFSET(   48,    78, Thread, m_pSpForPacSign)
+ASM_OFFSET(   4c,    80, Thread, m_pExInfoStackHead)
+#else
 ASM_OFFSET(   48,    78, Thread, m_pExInfoStackHead)
+#endif
 #ifdef TARGET_X86
 ASM_OFFSET(   4c,    FF, Thread, m_uHijackedReturnValueFlags)
 #endif

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -132,6 +132,15 @@ if (WIN32)
 
   list(APPEND FULL_RUNTIME_SOURCES windows/CoffNativeCodeManager.cpp)
 
+  if(CLR_CMAKE_TARGET_ARCH_ARM64)
+    list(APPEND FULL_RUNTIME_SOURCES
+      ../../unwinder/arm64/unwinder.cpp
+    )
+    include_directories(../../inc)
+    include_directories(../../unwinder)
+    include_directories(../../unwinder/arm64)
+  endif()
+
   set(ASM_SUFFIX asm)
 else()
 

src/coreclr/nativeaot/Runtime/ICodeManager.h

@@ -165,7 +165,8 @@ class ICodeManager
 
     virtual bool GetReturnAddressHijackInfo(MethodInfo *    pMethodInfo,
                                             REGDISPLAY *    pRegisterSet,          // in
-                                            PTR_PTR_VOID *  ppvRetAddrLocation     // out
+                                            PTR_PTR_VOID *  ppvRetAddrLocation,    // out
+                                            uintptr_t *     pSpForArm64PacSign     // out
                                             ) PURE_VIRTUAL
 
 #ifdef TARGET_X86

src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp

@@ -64,6 +64,18 @@ EXTERN_C CODE_LOCATION RhpRethrow2;
 #define FAILFAST_OR_DAC_FAIL_UNCONDITIONALLY(msg) { ASSERT_UNCONDITIONALLY(msg); RhFailFast(); }
 #endif
 
+#if defined(TARGET_ARM64)
+extern "C" void* PacStripPtr(void* ptr);
+#endif // TARGET_ARM64
+
+static TADDR ReturnAddressToCanonicalPC(TADDR returnAddress)
+{
+#if defined(TARGET_ARM64)
+    returnAddress = (TADDR)PacStripPtr((void*)returnAddress);
+#endif // TARGET_ARM64
+    return PCODEToPINSTR(dac_cast<PCODE>(returnAddress));
+}
+
 StackFrameIterator::StackFrameIterator(Thread * pThreadToWalk, PInvokeTransitionFrame* pInitialTransitionFrame)
 {
     STRESS_LOG0(LF_STACKWALK, LL_INFO10000, "----Init---- [ GC ]\n");
@@ -164,7 +176,7 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PInvokeTransitionF
 
 #if !defined(FEATURE_PORTABLE_HELPERS) // @TODO: no portable version of regdisplay
     memset(&m_RegDisplay, 0, sizeof(m_RegDisplay));
-    m_RegDisplay.SetIP((PCODE)PCODEToPINSTR((PCODE)pFrame->m_RIP));
+    m_RegDisplay.SetIP(ReturnAddressToCanonicalPC(dac_cast<TADDR>(pFrame->m_RIP)));
     SetControlPC(dac_cast<PTR_VOID>(m_RegDisplay.GetIP()));
 
     PTR_uintptr_t pPreservedRegsCursor = (PTR_uintptr_t)PTR_HOST_MEMBER_TADDR(PInvokeTransitionFrame, pFrame, m_PreservedRegs);
@@ -413,14 +425,15 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PTR_PAL_LIMITED_CO
 
     // This codepath is used by the hijack stackwalk and we can get arbitrary ControlPCs from there.  If this
     // context has a non-managed control PC, then we're done.
-    if (!m_pInstance->IsManaged(dac_cast<PTR_VOID>(pCtx->GetIp())))
+    TADDR controlPC = ReturnAddressToCanonicalPC(pCtx->GetIp());
+    if (!m_pInstance->IsManaged(dac_cast<PTR_VOID>(controlPC)))
         return;
 
     //
     // control state
     //
     m_RegDisplay.SP   = pCtx->GetSp();
-    m_RegDisplay.IP   = PCODEToPINSTR(pCtx->GetIp());
+    m_RegDisplay.IP   = controlPC;
     SetControlPC(dac_cast<PTR_VOID>(m_RegDisplay.GetIP()));
 
 #ifdef TARGET_ARM
@@ -633,14 +646,15 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, NATIVE_CONTEXT* pC
 
     // This codepath is used by the hijack stackwalk. The IP must be in managed code
     // or in a conservatively reported assembly thunk.
-    ASSERT(IsValidReturnAddress((void*)pCtx->GetIp()));
+    TADDR controlPC = ReturnAddressToCanonicalPC(pCtx->GetIp());
+    ASSERT(IsValidReturnAddress(dac_cast<PTR_VOID>(controlPC)));
 
     //
     // control state
     //
-    SetControlPC(dac_cast<PTR_VOID>(pCtx->GetIp()));
+    SetControlPC(dac_cast<PTR_VOID>(controlPC));
     m_RegDisplay.SP   = pCtx->GetSp();
-    m_RegDisplay.IP   = pCtx->GetIp();
+    m_RegDisplay.IP   = controlPC;
 
 #ifdef TARGET_UNIX
 #define PTR_TO_REG(ptr, reg) (&((ptr)->reg()))
@@ -1223,7 +1237,7 @@ void StackFrameIterator::UnwindFuncletInvokeThunk()
 
     m_RegDisplay.pFP  = SP++;
 
-    m_RegDisplay.SetIP(*SP++);
+    m_RegDisplay.SetIP(ReturnAddressToCanonicalPC(*SP++));
 
     m_RegDisplay.pX19 = SP++;
     m_RegDisplay.pX20 = SP++;
@@ -1636,7 +1650,7 @@ void StackFrameIterator::UnwindUniversalTransitionThunk()
     stackFrame->UnwindVolatileArgRegisters(&m_RegDisplay);
 
     PTR_uintptr_t addressOfPushedCallerIP = stackFrame->get_AddressOfPushedCallerIP();
-    m_RegDisplay.SetIP(PCODEToPINSTR(*addressOfPushedCallerIP));
+    m_RegDisplay.SetIP(ReturnAddressToCanonicalPC(*addressOfPushedCallerIP));
     m_RegDisplay.SetSP((uintptr_t)dac_cast<TADDR>(stackFrame->get_CallerSP()));
     SetControlPC(dac_cast<PTR_VOID>(m_RegDisplay.GetIP()));
 #if defined(TARGET_AMD64) && defined(TARGET_WINDOWS)
@@ -1767,7 +1781,7 @@ void StackFrameIterator::UnwindThrowSiteThunk()
     ASSERT_UNCONDITIONALLY("NYI for this arch");
 #endif
 
-    m_RegDisplay.SetIP(PCODEToPINSTR(pContext->IP));
+    m_RegDisplay.SetIP(ReturnAddressToCanonicalPC(pContext->IP));
     m_RegDisplay.SetSP(pContext->GetSp());
     SetControlPC(dac_cast<PTR_VOID>(m_RegDisplay.GetIP()));
 
@@ -1862,7 +1876,8 @@ void StackFrameIterator::NextInternal()
         // if the thread is safe to walk, it better not have a hijack in place.
         ASSERT(!m_pThread->IsHijacked());
 
-        SetControlPC(dac_cast<PTR_VOID>(PCODEToPINSTR(m_RegDisplay.GetIP())));
+        m_RegDisplay.SetIP(ReturnAddressToCanonicalPC(m_RegDisplay.GetIP()));
+        SetControlPC(dac_cast<PTR_VOID>(m_RegDisplay.GetIP()));
 
         PTR_VOID collapsingTargetFrame = NULL;
 

src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h

@@ -224,6 +224,7 @@ TrashRegister32Bit SETS "w":CC:("$TrashRegister32Bit":RIGHT:((:LEN:TrashRegister
         str         $trashReg1, [$trashReg2]
         str         xzr, [$threadReg, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation]
         str         xzr, [$threadReg, #OFFSETOF__Thread__m_pvHijackedReturnAddress]
+        str         xzr, [$threadReg, #OFFSETOF__Thread__m_pSpForPacSign]
 0
     MEND
 

src/coreclr/nativeaot/Runtime/arm64/ExceptionHandling.S

@@ -331,6 +331,7 @@ LOCAL_LABEL(ClearThreadState):
         // clear the Thread's hijack state
         str         xzr, [x2, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation]
         str         xzr, [x2, #OFFSETOF__Thread__m_pvHijackedReturnAddress]
+        str         xzr, [x2, #OFFSETOF__Thread__m_pSpForPacSign]
 
 LOCAL_LABEL(NotHijacked):
 

src/coreclr/nativeaot/Runtime/arm64/ExceptionHandling.asm

@@ -333,6 +333,7 @@ ClearThreadState
         ;; clear the Thread's hijack state
         str         xzr, [x2, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation]
         str         xzr, [x2, #OFFSETOF__Thread__m_pvHijackedReturnAddress]
+        str         xzr, [x2, #OFFSETOF__Thread__m_pSpForPacSign]
 
 NotHijacked
 

src/coreclr/nativeaot/Runtime/arm64/GcProbe.S

@@ -4,6 +4,8 @@
 #include <unixasmmacros.inc>
 #include "AsmOffsets.inc"
 
+.arch_extension pauth
+
 #define PROBE_FRAME_SIZE 0x140  // 4 * 8  for fixed part of PInvokeTransitionFrame (fp, lr, m_pThread, m_Flags) +
                                 // 10 * 8 for callee saved registers +
                                 // 1 * 8  for caller SP +
@@ -103,7 +105,7 @@
 //  x9: thread pointer
 //  x0-x7, q0-q7 preserved, x10 trashed
 //
-.macro FixupHijackedCallstack
+.macro FixupHijackedCallstack clearHijackState
 
     // x9 <- GetThread()
 
@@ -150,22 +152,27 @@
 #endif
 
     //
-    // Fix the stack by restoring the original return address
+    // Fix the stack by restoring and authenticating the original return address.
     //
     ldr         lr, [x9, #OFFSETOF__Thread__m_pvHijackedReturnAddress]
+    ldr         x16, [x9, #OFFSETOF__Thread__m_pSpForPacSign]
+    cbz         x16, \clearHijackState
+    autia       lr, x16
 
+\clearHijackState:
     //
     // Clear hijack state
     //
-    // Clear m_ppvHijackedReturnAddressLocation and m_pvHijackedReturnAddress
+    // Clear m_ppvHijackedReturnAddressLocation, m_pvHijackedReturnAddress, and m_pSpForPacSign
     stp         xzr, xzr, [x9, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation]
+    str         xzr, [x9, #OFFSETOF__Thread__m_pSpForPacSign]
 .endm
 
 //
 // GC Probe Hijack target
 //
 NESTED_ENTRY RhpGcProbeHijack, _TEXT, NoHandler
-    FixupHijackedCallstack
+    FixupHijackedCallstack LOCAL_LABEL(RhpGcProbeClearHijackState)
 
     PREPARE_EXTERNAL_VAR_INDIRECT_W RhpTrapThreads, 10
     tbnz        x10, #TrapThreadsFlags_TrapThreads_Bit, LOCAL_LABEL(WaitForGC)

src/coreclr/nativeaot/Runtime/arm64/GcProbe.asm

@@ -112,25 +112,30 @@ PROBE_FRAME_SIZE    field 0
 ;;
 ;; Register state on exit:
 ;;  x9: thread pointer
-;;  x0-x7 preserved, x10 trashed
+;;  x0-x7 preserved, x10 and x16 trashed
 ;;
     MACRO
-        FixupHijackedCallstack
+        FixupHijackedCallstack $ClearHijackStateLabel
 
         ;; x9 <- GetThread(), TRASHES x10
         INLINE_GETTHREAD x9, x10
 
         ;;
-        ;; Fix the stack by restoring the original return address
+        ;; Fix the stack by restoring and authenticating the original return address.
         ;;
         ldr         lr, [x9, #OFFSETOF__Thread__m_pvHijackedReturnAddress]
+        ldr         x16, [x9, #OFFSETOF__Thread__m_pSpForPacSign]
+        cbz         x16, $ClearHijackStateLabel
+        DCD         0xDAC1161E  ;; autib lr, x16 instruction in binary to avoid requiring PAC-enabled assemblers
 
+$ClearHijackStateLabel
         ;;
         ;; Clear hijack state
         ;;
         ASSERT OFFSETOF__Thread__m_pvHijackedReturnAddress == (OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation + 8)
-        ;; Clear m_ppvHijackedReturnAddressLocation and m_pvHijackedReturnAddress
+        ;; Clear m_ppvHijackedReturnAddressLocation, m_pvHijackedReturnAddress, and m_pSpForPacSign
         stp         xzr, xzr, [x9, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation]
+        str         xzr, [x9, #OFFSETOF__Thread__m_pSpForPacSign]
     MEND
 
     MACRO
@@ -154,7 +159,7 @@ PROBE_FRAME_SIZE    field 0
         HijackTargetFakeProlog
 
     LABELED_RETURN_ADDRESS RhpGcProbeHijack
-        FixupHijackedCallstack
+        FixupHijackedCallstack RhpGcProbeClearHijackState
 
         ldr         x10, =RhpTrapThreads
         ldr         w10, [x10]
@@ -201,7 +206,7 @@ WaitForGC
 ;;
 ;;
     LEAF_ENTRY RhpGcStressHijack
-        FixupHijackedCallstack
+        FixupHijackedCallstack RhpGcStressClearHijackState
         mov         x12, #(DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_X0 + PTFF_SAVE_X1 + PTFF_SAVE_X2 + PTFF_SAVE_X3 + PTFF_SAVE_X4 + PTFF_SAVE_X5 + PTFF_SAVE_X6 + PTFF_SAVE_X7)
         b           RhpGcStressProbe
     LEAF_END RhpGcStressHijack

src/coreclr/nativeaot/Runtime/arm64/MiscStubs.S

@@ -3,3 +3,34 @@
 
 #include <unixasmmacros.inc>
 #include "AsmOffsets.inc"
+
+// void* PacStripPtr(void *);
+// This function strips the pointer of PAC info that is passed as an argument.
+// We prefer to strip a pointer where it's not going to be used to branch execution to.
+.arch_extension pauth
+    LEAF_ENTRY PacStripPtr, _TEXT
+        xpaci x0
+        ret
+    LEAF_END PacStripPtr, _TEXT
+
+// void* PacSignPtr(void *, void *);
+// This function signs the input pointer using x1 as salt. It is a no-op on non-PAC enabled machines.
+.arch_extension pauth
+    LEAF_ENTRY PacSignPtr, _TEXT
+        mov x17, x0
+        mov x16, x1
+        pacia1716
+        mov x0, x17
+        ret
+    LEAF_END PacSignPtr, _TEXT
+
+// void* PacAuthPtr(void *, void *);
+// This function authenticates the input signed-pointer using x1 as salt. It is a no-op on non-PAC enabled machines.
+.arch_extension pauth
+    LEAF_ENTRY PacAuthPtr, _TEXT
+        mov x17, x0
+        mov x16, x1
+        autia1716
+        mov x0, x17
+        ret
+    LEAF_END PacAuthPtr, _TEXT

src/coreclr/nativeaot/Runtime/arm64/MiscStubs.asm

@@ -5,4 +5,32 @@
 
     TEXTAREA
 
+; void* PacStripPtr(void *);
+; This function strips the pointer of PAC info that is passed as an argument.
+; We prefer to strip a pointer where it's not going to be used to branch execution to.
+    LEAF_ENTRY PacStripPtr
+        DCD     0xDAC143E0  ; xpaci x0 instruction in binary to avoid requiring PAC-enabled assemblers
+        ret
+    LEAF_END PacStripPtr
+
+; void* PacSignPtr(void *, void *);
+; This function signs the input pointer using x1 as salt. It is a no-op on non-PAC enabled machines.
+    LEAF_ENTRY PacSignPtr
+        mov x17, x0
+        mov x16, x1
+        DCD 0xD503215F  ; pacib1716 instruction in binary to avoid error while compiling with non-PAC enabled compilers
+        mov x0, x17
+        ret
+    LEAF_END PacSignPtr
+
+; void* PacAuthPtr(void *, void *);
+; This function authenticates the input signed-pointer using x1 as salt. It is a no-op on non-PAC enabled machines.
+    LEAF_ENTRY PacAuthPtr
+        mov x17, x0
+        mov x16, x1
+        DCD 0xD50321DF  ; autib1716 instruction in binary to avoid error while compiling with non-PAC enabled compilers
+        mov x0, x17
+        ret
+    LEAF_END PacAuthPtr
+
     end