#include "common.h"

#include "jitinterface.h"

#include "corjit.h"

#include "jithost.h"

#include "eetwain.h"

#include "eeconfig.h"

#include "excep.h"

#include "appdomain.hpp"

#include "codeman.h"

#include "nibblemapmacros.h"

#include "generics.h"

#include "dynamicmethod.h"

#include "eventtrace.h"

#include "threadsuspend.h"

#include "daccess.h"

#include "exceptionhandling.h"

#include "rtlfunctions.h"

#include "debuginfostore.h"

#include "strsafe.h"

#include "configuration.h"

#if !defined(DACCESS_COMPILE) && !defined(TARGET_WASM)

#include "../debug/ee/executioncontrol.h"

#endif // !DACCESS_COMPILE && !TARGET_WASM

#include <minipal/cpufeatures.h>

#include <minipal/cpuid.h>

#ifdef HOST_64BIT

#define CHECK_DUPLICATED_STRUCT_LAYOUTS

#include "../debug/daccess/fntableaccess.h"

#endif // HOST_64BIT

#ifdef FEATURE_PERFMAP

#include "perfmap.h"

#endif

#define DEFAULT_JUMPSTUBS_PER_BLOCK 32

SPTR_IMPL(EECodeManager, ExecutionManager, m_pDefaultCodeMan);

SPTR_IMPL(EEJitManager, ExecutionManager, m_pEEJitManager);

#ifdef FEATURE_READYTORUN

SPTR_IMPL(ReadyToRunJitManager, ExecutionManager, m_pReadyToRunJitManager);

#endif

#ifdef FEATURE_INTERPRETER

SPTR_IMPL(InterpreterJitManager, ExecutionManager, m_pInterpreterJitManager);

SPTR_IMPL(InterpreterCodeManager, ExecutionManager, m_pInterpreterCodeMan);

#endif

SVAL_IMPL(RangeSectionMapData, ExecutionManager, g_codeRangeMap);

VOLATILE_SVAL_IMPL_INIT(LONG, ExecutionManager, m_dwReaderCount, 0);

VOLATILE_SVAL_IMPL_INIT(LONG, ExecutionManager, m_dwWriterLock, 0);

#ifndef DACCESS_COMPILE

CrstStatic ExecutionManager::m_JumpStubCrst;

unsigned ExecutionManager::m_normal_JumpStubLookup;

unsigned ExecutionManager::m_normal_JumpStubUnique;

unsigned ExecutionManager::m_normal_JumpStubBlockAllocCount;

unsigned ExecutionManager::m_normal_JumpStubBlockFullCount;

unsigned ExecutionManager::m_LCG_JumpStubLookup;

unsigned ExecutionManager::m_LCG_JumpStubUnique;

unsigned ExecutionManager::m_LCG_JumpStubBlockAllocCount;

unsigned ExecutionManager::m_LCG_JumpStubBlockFullCount;

#endif // DACCESS_COMPILE

#if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) && !defined(DACCESS_COMPILE)

typedef NTSTATUS (WINAPI* RtlAddGrowableFunctionTableFnPtr) (

PVOID *DynamicTable, PRUNTIME_FUNCTION FunctionTable, ULONG EntryCount,

ULONG MaximumEntryCount, ULONG_PTR rangeStart, ULONG_PTR rangeEnd);

typedef VOID (WINAPI* RtlGrowFunctionTableFnPtr) (PVOID DynamicTable, ULONG NewEntryCount);

typedef VOID (WINAPI* RtlDeleteGrowableFunctionTableFnPtr) (PVOID DynamicTable);

static RtlAddGrowableFunctionTableFnPtr pRtlAddGrowableFunctionTable;

static RtlGrowFunctionTableFnPtr pRtlGrowFunctionTable;

static RtlDeleteGrowableFunctionTableFnPtr pRtlDeleteGrowableFunctionTable;

static bool s_publishingActive; // Publishing to ETW is turned on

static Crst* s_pUnwindInfoTablePublishLock; // Protects main table, OS registration, and lazy init

static Crst* s_pUnwindInfoTablePendingLock; // Protects pending buffer only

bool InitUnwindFtns()

}

CONTRACTL_END;

HINSTANCE hNtdll = GetModuleHandle(W("ntdll.dll"));

if (hNtdll != NULL)

{

void* growFunctionTable = GetProcAddress(hNtdll, "RtlGrowFunctionTable");

void* deleteGrowableFunctionTable = GetProcAddress(hNtdll, "RtlDeleteGrowableFunctionTable");

void* addGrowableFunctionTable = GetProcAddress(hNtdll, "RtlAddGrowableFunctionTable");

// All or nothing AddGroableFunctionTable is last (marker)

if (growFunctionTable != NULL &&

deleteGrowableFunctionTable != NULL &&

addGrowableFunctionTable != NULL)

{

pRtlGrowFunctionTable = (RtlGrowFunctionTableFnPtr) growFunctionTable;

pRtlDeleteGrowableFunctionTable = (RtlDeleteGrowableFunctionTableFnPtr) deleteGrowableFunctionTable;

pRtlAddGrowableFunctionTable = (RtlAddGrowableFunctionTableFnPtr) addGrowableFunctionTable;

}

// Don't call FreeLibrary(hNtdll) because GetModuleHandle did *NOT* increment the reference count!

}

return (pRtlAddGrowableFunctionTable != NULL);

}

UnwindInfoTable::UnwindInfoTable(ULONG_PTR rangeStart, ULONG_PTR rangeEnd)

}

UnwindInfoTable::~UnwindInfoTable()

{

CONTRACTL {

NOTHROW;

GC_NOTRIGGER;

} CONTRACTL_END;

_ASSERTE(s_publishingActive);

// We do this lock free to because too many places still want no-trigger. It should be OK

// It would be cleaner if we could take the lock (we did not have to be GC_NOTRIGGER)

UnRegister();

delete[] pTable;

}

void UnwindInfoTable::Register()

}

void UnwindInfoTable::UnRegister()

}

void UnwindInfoTable::AddToUnwindInfoTable(PT_RUNTIME_FUNCTION data, int count)

}

CONTRACTL_END;

_ASSERTE(s_publishingActive);

for (int i = 0; i < count; )

{

{

CrstHolder pendingLock(s_pUnwindInfoTablePendingLock);

while (i < count && cPendingCount < cPendingMaxCount)

{

_ASSERTE(data[i].BeginAddress <= RUNTIME_FUNCTION__EndAddress(&data[i], iRangeStart));

_ASSERTE(RUNTIME_FUNCTION__EndAddress(&data[i], iRangeStart) <= (iRangeEnd - iRangeStart));

pendingTable[cPendingCount++] = data[i];

STRESS_LOG5(LF_JIT, LL_INFO1000, "AddToUnwindTable Handle: %p [%p, %p] BUFFERED 0x%x, pending 0x%x\n",

hHandle, iRangeStart, iRangeEnd,

data[i].BeginAddress, cPendingCount);

i++;

}

}

// Flush any pending entries if we run out of space, or when we are at the end

// of the batch so the OS can unwind this method immediately.

FlushPendingEntries();

}

}

void UnwindInfoTable::FlushPendingEntries()

}

CONTRACTL_END;

// Free the old table outside the lock

NewArrayHolder<T_RUNTIME_FUNCTION> oldPTable;

CrstHolder publishLock(s_pUnwindInfoTablePublishLock);

if (hHandle == NULL)

{

// If hHandle is null, it means Register() failed. Skip flushing to avoid calling

// RtlGrowFunctionTable with a null handle.

CrstHolder pendingLock(s_pUnwindInfoTablePendingLock);

cPendingCount = 0;

return;

}

// Grab the pending entries under the pending lock, then release it so

// other threads can keep accumulating new entries while we publish.

T_RUNTIME_FUNCTION localPending[cPendingMaxCount];

ULONG localPendingCount;

{

CrstHolder pendingLock(s_pUnwindInfoTablePendingLock);

localPendingCount = cPendingCount;

memcpy(localPending, pendingTable, cPendingCount * sizeof(T_RUNTIME_FUNCTION));

cPendingCount = 0;

INDEBUG( memset(pendingTable, 0xcc, sizeof(pendingTable)); )

}

if (localPendingCount == 0)

return;

// Sort the pending entries by BeginAddress.

// Use a simple insertion sort since cPendingMaxCount is small (32).

static_assert(cPendingMaxCount == 32,

"cPendingMaxCount was updated and might be too large for insertion sort, consider using a better algorithm");

for (ULONG i = 1; i < localPendingCount; i++)

{

T_RUNTIME_FUNCTION key = localPending[i];

ULONG j = i;

while (j > 0 && localPending[j - 1].BeginAddress > key.BeginAddress)

{

localPending[j] = localPending[j - 1];

j--;

}

localPending[j] = key;

}

// Fast path: if all pending entries can be appended in order with room to spare,

// we can just append and call RtlGrowFunctionTable.

if (cTableCurCount + localPendingCount <= cTableMaxCount

&& (cTableCurCount == 0 || pTable[cTableCurCount - 1].BeginAddress < localPending[0].BeginAddress))

{

memcpy(&pTable[cTableCurCount], localPending, localPendingCount * sizeof(T_RUNTIME_FUNCTION));

cTableCurCount += localPendingCount;

pRtlGrowFunctionTable(hHandle, cTableCurCount);

STRESS_LOG5(LF_JIT, LL_INFO1000, "FlushPendingEntries Handle: %p [%p, %p] APPENDED 0x%x entries, now 0x%x\n",

hHandle, iRangeStart, iRangeEnd, localPendingCount, cTableCurCount);

return;

}

// Merge main table and pending entries.

// Calculate the new table size: live entries from main table + all pending entries

ULONG liveCount = cTableCurCount - cDeletedEntries;

ULONG newCount = liveCount + localPendingCount;

ULONG desiredSpace = newCount * 5 / 4 + 1; // Increase by 20%

STRESS_LOG7(LF_JIT, LL_INFO100, "FlushPendingEntries Handle: %p [%p, %p] Merging 0x%x live + 0x%x pending into 0x%x max, from 0x%x\n",

hHandle, iRangeStart, iRangeEnd, liveCount, localPendingCount, desiredSpace, cTableMaxCount);

NewArrayHolder<T_RUNTIME_FUNCTION> newPTable(new T_RUNTIME_FUNCTION[desiredSpace]);

// Merge-sort the main table and pending buffer into newPTable.

ULONG mainIdx = 0;

ULONG pendIdx = 0;

ULONG toIdx = 0;

while (mainIdx < cTableCurCount && pendIdx < localPendingCount)

{

// Skip deleted entries in main table

if (pTable[mainIdx].UnwindData == 0)

{

mainIdx++;

continue;

}

if (localPending[pendIdx].BeginAddress < pTable[mainIdx].BeginAddress)

{

newPTable[toIdx++] = localPending[pendIdx++];

}

else

{

newPTable[toIdx++] = pTable[mainIdx++];

}

}

while (mainIdx < cTableCurCount)

{

if (pTable[mainIdx].UnwindData != 0)

newPTable[toIdx++] = pTable[mainIdx];

mainIdx++;

}

while (pendIdx < localPendingCount)

{

newPTable[toIdx++] = localPending[pendIdx++];

}

_ASSERTE(toIdx == newCount);

_ASSERTE(toIdx <= desiredSpace);

oldPTable = pTable;

// The OS growable function table API (RtlGrowFunctionTable) only supports

// appending sorted entries, it cannot shrink, reorder, or remove entries.

// We have to tear down the old OS registration and create a new one

// combining the old and pending entries while skipping the deleted ones.

// We should keep the gap between UnRegister and Register as short as possible,

// as OS stack walks will have no unwind info for this range during that

// window. The new table is fully built before UnRegister to minimize this gap.

UnRegister();

pTable = newPTable.Extract();

cTableCurCount = toIdx;

cTableMaxCount = desiredSpace;

cDeletedEntries = 0;

Register();

}

/* static */ void UnwindInfoTable::RemoveFromUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, TADDR baseAddress, TADDR entryPoint)

}

CONTRACTL_END;

_ASSERTE(unwindInfoPtr != NULL);

if (!s_publishingActive)

return;

UnwindInfoTable* unwindInfo = VolatileLoad(unwindInfoPtr);

if (unwindInfo == NULL)

return;

DWORD relativeEntryPoint = (DWORD)(entryPoint - baseAddress);

STRESS_LOG3(LF_JIT, LL_INFO100, "RemoveFromUnwindInfoTable Removing %p BaseAddress %p rel %x\n",

entryPoint, baseAddress, relativeEntryPoint);

// Check the main (published) table under the publish lock.

// We don't need to check the pending buffer because the method should have already been published

// before it can be removed.

{

CrstHolder publishLock(s_pUnwindInfoTablePublishLock);

for (ULONG i = 0; i < unwindInfo->cTableCurCount; i++)

{

if (unwindInfo->pTable[i].BeginAddress <= relativeEntryPoint &&

relativeEntryPoint < RUNTIME_FUNCTION__EndAddress(&unwindInfo->pTable[i], unwindInfo->iRangeStart))

{

if (unwindInfo->pTable[i].UnwindData != 0)

unwindInfo->cDeletedEntries++;

unwindInfo->pTable[i].UnwindData = 0; // Mark the entry for deletion

STRESS_LOG1(LF_JIT, LL_INFO100, "RemoveFromUnwindInfoTable Removed entry 0x%x\n", i);

return;

}

}

}

STRESS_LOG2(LF_JIT, LL_WARNING, "RemoveFromUnwindInfoTable COULD NOT FIND %p BaseAddress %p\n",

entryPoint, baseAddress);

}

/* static */ void UnwindInfoTable::PublishUnwindInfoForMethod(TADDR baseAddress, PT_RUNTIME_FUNCTION methodUnwindData, int methodUnwindDataCount)

}

/* static */ void UnwindInfoTable::UnpublishUnwindInfoForMethod(TADDR entryPoint)

}

CONTRACTL_END;

if (!s_publishingActive)

return;

RangeSection * pRS = ExecutionManager::FindCodeRange(entryPoint, ExecutionManager::GetScanFlags());

_ASSERTE(pRS != NULL);

if (pRS != NULL)

{

_ASSERTE(pRS->_pjit->GetCodeType() == (miManaged | miIL) || pRS->_pjit->GetCodeType() == (miManaged | miIL | miOPTIL));

if (pRS->_pjit->GetCodeType() == (miManaged | miIL)) // Do this only for Jitted code, not for interpreted code

{

// This cast is justified because only EEJitManager's have the code type above.

EEJitManager* pJitMgr = (EEJitManager*)(pRS->_pjit);

CodeHeader * pHeader = pJitMgr->GetCodeHeaderFromStartAddress(entryPoint);

for(ULONG i = 0; i < pHeader->GetNumberOfUnwindInfos(); i++)

RemoveFromUnwindInfoTable(&pRS->_pUnwindInfoTable, pRS->_range.RangeStart(), pRS->_range.RangeStart() + pHeader->GetUnwindInfo(i)->BeginAddress);

}

}

}

/* static */ void UnwindInfoTable::Initialize()

}

CONTRACTL_END;

_ASSERTE(!s_publishingActive);

// If we don't have the APIs we need, give up

if (!InitUnwindFtns())

return;

// Create the locks

s_pUnwindInfoTablePublishLock = new Crst(CrstUnwindInfoTablePublishLock);

s_pUnwindInfoTablePendingLock = new Crst(CrstUnwindInfoTablePendingLock);

s_publishingActive = true;

}

#else

/* static */ void UnwindInfoTable::PublishUnwindInfoForMethod(TADDR baseAddress, T_RUNTIME_FUNCTION* methodUnwindData, int methodUnwindDataCount)

{

LIMITED_METHOD_CONTRACT;

}

/* static */ void UnwindInfoTable::UnpublishUnwindInfoForMethod(TADDR entryPoint)

{

LIMITED_METHOD_CONTRACT;

}

/* static */ void UnwindInfoTable::Initialize()

{

LIMITED_METHOD_CONTRACT;

}

#endif // defined(TARGET_AMD64) && defined(TARGET_WINDOWS) && !defined(DACCESS_COMPILE)

#if !defined(DACCESS_COMPILE)

CodeHeapIterator::CodeHeapIterator(EECodeGenManager* manager, HeapList* heapList, LoaderAllocator* pLoaderAllocatorFilter)

}

CONTRACTL_END;

// Iterator through the heap list collecting the current state of the heaps.

HeapList* current = heapList;

while (current)

{

HeapListState* state = m_Heaps.AppendThrowing();

state->Heap = current;

state->MapBase = (void*)current->mapBase;

state->HdrMap = current->pHdrMap;

state->MaxCodeHeapSize = current->maxCodeHeapSize;

current = current->GetNext();

}

// Move to the first method section.

(void)NextMethodSectionIterator();

}

CodeHeapIterator::EECodeGenManagerReleaseIteratorHolder::EECodeGenManagerReleaseIteratorHolder(EECodeGenManager* manager) : m_manager(manager)

{

CONTRACTL

{

NOTHROW;

GC_NOTRIGGER;

}

CONTRACTL_END;

manager->AddRefIterator();

}

CodeHeapIterator::EECodeGenManagerReleaseIteratorHolder::~EECodeGenManagerReleaseIteratorHolder()

{

CONTRACTL

{

NOTHROW;

GC_NOTRIGGER;

}

CONTRACTL_END;

if (m_manager)

{

m_manager->ReleaseIterator();

m_manager = NULL;

}

}

CodeHeapIterator::EECodeGenManagerReleaseIteratorHolder& CodeHeapIterator::EECodeGenManagerReleaseIteratorHolder::operator=(EECodeGenManagerReleaseIteratorHolder&& other)

{

CONTRACTL

{

NOTHROW;

GC_NOTRIGGER;

}

CONTRACTL_END;

if (this != &other)

{

if (m_manager)

{

m_manager->ReleaseIterator();

}

m_manager = other.m_manager;

other.m_manager = NULL;

}

return *this;

}

bool CodeHeapIterator::Next()

}

CONTRACTL_END;

while (true)

{

if (!m_Iterator.Next())

{

if (!NextMethodSectionIterator())

return false;

}

else

{

BYTE* code = m_Iterator.GetMethodCode();

#ifdef FEATURE_INTERPRETER

if (m_codeType == (miManaged | miIL | miOPTIL))

{

// Interpreter case

InterpreterCodeHeader* pHdr = (InterpreterCodeHeader*)(code - sizeof(InterpreterCodeHeader));

m_pCurrent = pHdr->GetMethodDesc();

}

else

#endif

{

CodeHeader* pHdr = (CodeHeader*)(code - sizeof(CodeHeader));

m_pCurrent = !pHdr->IsStubCodeBlock() ? pHdr->GetMethodDesc() : NULL;

}

// LoaderAllocator filter

if (m_pLoaderAllocatorFilter && m_pCurrent)

{

LoaderAllocator *pCurrentLoaderAllocator = m_pCurrent->GetLoaderAllocator();

if (pCurrentLoaderAllocator != m_pLoaderAllocatorFilter)

continue;

}

return true;

}

}

}

bool CodeHeapIterator::NextMethodSectionIterator()

}

CONTRACTL_END;

if (m_HeapsIndexNext >= m_Heaps.Count())

{

m_Iterator = {};

return false;

}

HeapListState& curr = m_Heaps.Table()[m_HeapsIndexNext++];

m_Iterator = MethodSectionIterator{

curr.MapBase,

(COUNT_T)curr.MaxCodeHeapSize,

curr.HdrMap,

(COUNT_T)HEAP2MAPSIZE(ROUND_UP_TO_PAGE(curr.MaxCodeHeapSize))};

return true;

}

#endif // !DACCESS_COMPILE

#ifndef DACCESS_COMPILE

ExecutionManager::ReaderLockHolder::ReaderLockHolder()

{

CONTRACTL {

NOTHROW;

GC_NOTRIGGER;

CAN_TAKE_LOCK;

} CONTRACTL_END;

IncCantAllocCount();

InterlockedIncrement(&m_dwReaderCount);

EE_LOCK_TAKEN(GetPtrForLockContract());

if (VolatileLoad(&m_dwWriterLock) != 0)

{

YIELD_WHILE ((VolatileLoad(&m_dwWriterLock) != 0));

}

}

ExecutionManager::ReaderLockHolder::~ReaderLockHolder()

{

CONTRACTL

{

NOTHROW;

GC_NOTRIGGER;

MODE_ANY;

}

CONTRACTL_END;

InterlockedDecrement(&m_dwReaderCount);

DecCantAllocCount();

EE_LOCK_RELEASED(GetPtrForLockContract());

}

BOOL ExecutionManager::ReaderLockHolder::Acquired()

{

LIMITED_METHOD_CONTRACT;

return VolatileLoad(&m_dwWriterLock) == 0;

}

ExecutionManager::WriterLockHolder::WriterLockHolder()

{

CONTRACTL {

NOTHROW;

GC_NOTRIGGER;

CAN_TAKE_LOCK;

} CONTRACTL_END;

_ASSERTE(m_dwWriterLock == 0);

// Signal to a debugger that this thread cannot stop now

IncCantStopCount();

IncCantAllocCount();

DWORD dwSwitchCount = 0;

while (TRUE)

{

// While this thread holds the writer lock, we must not try to suspend it

// or allow a profiler to walk its stack

Thread::IncForbidSuspendThread();

InterlockedIncrement(&m_dwWriterLock);

if (m_dwReaderCount == 0)

break;

InterlockedDecrement(&m_dwWriterLock);

// Before we loop and retry, it's safe to suspend or hijack and inspect

// this thread

Thread::DecForbidSuspendThread();

__SwitchToThread(0, ++dwSwitchCount);

}

EE_LOCK_TAKEN(GetPtrForLockContract());

}

ExecutionManager::WriterLockHolder::~WriterLockHolder()

{

LIMITED_METHOD_CONTRACT;

InterlockedDecrement(&m_dwWriterLock);

// Writer lock released, so it's safe again for this thread to be

// suspended or have its stack walked by a profiler

Thread::DecForbidSuspendThread();

DecCantAllocCount();

// Signal to a debugger that it's again safe to stop this thread

DecCantStopCount();

EE_LOCK_RELEASED(GetPtrForLockContract());

}

#else

ExecutionManager::ReaderLockHolder::ReaderLockHolder()

{

SUPPORTS_DAC;

if (m_dwWriterLock != 0)

{

ThrowHR(CORDBG_E_PROCESS_NOT_SYNCHRONIZED);

}

}

ExecutionManager::ReaderLockHolder::~ReaderLockHolder()

{

}

#endif // DACCESS_COMPILE

#if defined(TARGET_ARM) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)

#define EXCEPTION_DATA_SUPPORTS_FUNCTION_FRAGMENTS

#endif

#if defined(EXCEPTION_DATA_SUPPORTS_FUNCTION_FRAGMENTS)

BOOL IsFunctionFragment(TADDR baseAddress, PTR_RUNTIME_FUNCTION pFunctionEntry)

}

#endif // EXCEPTION_DATA_SUPPORTS_FUNCTION_FRAGMENTS

#ifndef DACCESS_COMPILE

IJitManager::IJitManager()

}

#endif // #ifndef DACCESS_COMPILE

ExecutionManager::ScanFlag ExecutionManager::GetScanFlags(Thread *pThread)

} CONTRACTL_END;

#if !defined(DACCESS_COMPILE)

if (!pThread)

{