diff --git a/ARMeilleure/Common/Counter.cs b/ARMeilleure/Common/Counter.cs
new file mode 100644
index 000000000..defb5abaa
--- /dev/null
+++ b/ARMeilleure/Common/Counter.cs
@@ -0,0 +1,99 @@
+using System;
+
+namespace ARMeilleure.Common
+{
+ ///
+ /// Represents a numeric counter which can be used for instrumentation of compiled code.
+ ///
+ /// Type of the counter
+ class Counter : IDisposable where T : unmanaged
+ {
+ private bool _disposed;
+ private readonly int _index;
+ private readonly EntryTable _countTable;
+
+ ///
+ /// Initializes a new instance of the class from the specified
+ /// instance and index.
+ ///
+ /// instance
+ /// Index in the
+ /// is
+ /// is unsupported
+ public Counter(EntryTable countTable)
+ {
+ if (typeof(T) != typeof(byte) && typeof(T) != typeof(sbyte) &&
+ typeof(T) != typeof(short) && typeof(T) != typeof(ushort) &&
+ typeof(T) != typeof(int) && typeof(T) != typeof(uint) &&
+ typeof(T) != typeof(long) && typeof(T) != typeof(ulong) &&
+ typeof(T) != typeof(nint) && typeof(T) != typeof(nuint) &&
+ typeof(T) != typeof(float) && typeof(T) != typeof(double))
+ {
+ throw new ArgumentException("Counter does not support the specified type.");
+ }
+
+ _countTable = countTable ?? throw new ArgumentNullException(nameof(countTable));
+ _index = countTable.Allocate();
+ }
+
+ ///
+ /// Gets a reference to the value of the counter.
+ ///
+ /// instance was disposed
+ ///
+ /// This can refer to freed memory if the owning is disposed.
+ ///
+ public ref T Value
+ {
+ get
+ {
+ if (_disposed)
+ {
+ throw new ObjectDisposedException(null);
+ }
+
+ return ref _countTable.GetValue(_index);
+ }
+ }
+
+ ///
+ /// Releases all resources used by the instance.
+ ///
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ ///
+ /// Releases all unmanaged and optionally managed resources used by the instance.
+ ///
+ /// to dispose managed resources also; otherwise just unmanaged resouces
+ protected virtual void Dispose(bool disposing)
+ {
+ if (!_disposed)
+ {
+ try
+ {
+ // The index into the EntryTable is essentially an unmanaged resource since we allocate and free the
+ // resource ourselves.
+ _countTable.Free(_index);
+ }
+ catch (ObjectDisposedException)
+ {
+ // Can happen because _countTable may be disposed before the Counter instance.
+ }
+
+ _disposed = true;
+ }
+ }
+
+ ///
+ /// Frees resources used by the instance.
+ ///
+ ~Counter()
+ {
+ Dispose(false);
+ }
+ }
+}
diff --git a/ARMeilleure/Common/EntryTable.cs b/ARMeilleure/Common/EntryTable.cs
new file mode 100644
index 000000000..a0ed7c8e5
--- /dev/null
+++ b/ARMeilleure/Common/EntryTable.cs
@@ -0,0 +1,196 @@
+using System;
+using System.Collections.Generic;
+using System.Numerics;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Common
+{
+ ///
+ /// Represents an expandable table of the type , whose entries will remain at the same
+ /// address through out the table's lifetime.
+ ///
+ /// Type of the entry in the table
+ class EntryTable : IDisposable where TEntry : unmanaged
+ {
+ private bool _disposed;
+ private int _freeHint;
+ private readonly int _pageCapacity; // Number of entries per page.
+ private readonly int _pageLogCapacity;
+ private readonly Dictionary _pages;
+ private readonly BitMap _allocated;
+
+ ///
+ /// Initializes a new instance of the class with the desired page size in
+ /// bytes.
+ ///
+ /// Desired page size in bytes
+ /// is less than 0
+ /// 's size is zero
+ ///
+ /// The actual page size may be smaller or larger depending on the size of .
+ ///
+ public unsafe EntryTable(int pageSize = 4096)
+ {
+ if (pageSize < 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(pageSize), "Page size cannot be negative.");
+ }
+
+ if (sizeof(TEntry) == 0)
+ {
+ throw new ArgumentException("Size of TEntry cannot be zero.");
+ }
+
+ _allocated = new BitMap();
+ _pages = new Dictionary();
+ _pageLogCapacity = BitOperations.Log2((uint)(pageSize / sizeof(TEntry)));
+ _pageCapacity = 1 << _pageLogCapacity;
+ }
+
+ ///
+ /// Allocates an entry in the .
+ ///
+ /// Index of entry allocated in the table
+ /// instance was disposed
+ public int Allocate()
+ {
+ if (_disposed)
+ {
+ throw new ObjectDisposedException(null);
+ }
+
+ lock (_allocated)
+ {
+ if (_allocated.IsSet(_freeHint))
+ {
+ _freeHint = _allocated.FindFirstUnset();
+ }
+
+ int index = _freeHint++;
+ var page = GetPage(index);
+
+ _allocated.Set(index);
+
+ GetValue(page, index) = default;
+
+ return index;
+ }
+ }
+
+ ///
+ /// Frees the entry at the specified .
+ ///
+ /// Index of entry to free
+ /// instance was disposed
+ public void Free(int index)
+ {
+ if (_disposed)
+ {
+ throw new ObjectDisposedException(null);
+ }
+
+ lock (_allocated)
+ {
+ if (_allocated.IsSet(index))
+ {
+ _allocated.Clear(index);
+
+ _freeHint = index;
+ }
+ }
+ }
+
+ ///
+ /// Gets a reference to the entry at the specified allocated .
+ ///
+ /// Index of the entry
+ /// Reference to the entry at the specified
+ /// instance was disposed
+ /// Entry at is not allocated
+ public ref TEntry GetValue(int index)
+ {
+ if (_disposed)
+ {
+ throw new ObjectDisposedException(null);
+ }
+
+ lock (_allocated)
+ {
+ if (!_allocated.IsSet(index))
+ {
+ throw new ArgumentException("Entry at the specified index was not allocated", nameof(index));
+ }
+
+ var page = GetPage(index);
+
+ return ref GetValue(page, index);
+ }
+ }
+
+ ///
+ /// Gets a reference to the entry at using the specified from the specified
+ /// .
+ ///
+ /// Page to use
+ /// Index to use
+ /// Reference to the entry
+ private ref TEntry GetValue(Span page, int index)
+ {
+ return ref page[index & (_pageCapacity - 1)];
+ }
+
+ ///
+ /// Gets the page for the specified .
+ ///
+ /// Index to use
+ /// Page for the specified
+ private unsafe Span GetPage(int index)
+ {
+ var pageIndex = (int)((uint)(index & ~(_pageCapacity - 1)) >> _pageLogCapacity);
+
+ if (!_pages.TryGetValue(pageIndex, out IntPtr page))
+ {
+ page = Marshal.AllocHGlobal(sizeof(TEntry) * _pageCapacity);
+
+ _pages.Add(pageIndex, page);
+ }
+
+ return new Span((void*)page, _pageCapacity);
+ }
+
+ ///
+ /// Releases all resources used by the instance.
+ ///
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ ///
+ /// Releases all unmanaged and optionally managed resources used by the
+ /// instance.
+ ///
+ /// to dispose managed resources also; otherwise just unmanaged resouces
+ protected virtual void Dispose(bool disposing)
+ {
+ if (!_disposed)
+ {
+ foreach (var page in _pages.Values)
+ {
+ Marshal.FreeHGlobal(page);
+ }
+
+ _disposed = true;
+ }
+ }
+
+ ///
+ /// Frees resources used by the instance.
+ ///
+ ~EntryTable()
+ {
+ Dispose(false);
+ }
+ }
+}
diff --git a/ARMeilleure/Decoders/Block.cs b/ARMeilleure/Decoders/Block.cs
index 9d380bca6..f296d299d 100644
--- a/ARMeilleure/Decoders/Block.cs
+++ b/ARMeilleure/Decoders/Block.cs
@@ -11,8 +11,7 @@ namespace ARMeilleure.Decoders
public Block Next { get; set; }
public Block Branch { get; set; }
- public bool TailCall { get; set; }
- public bool Exit { get; set; }
+ public bool Exit { get; set; }
public List OpCodes { get; }
diff --git a/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs b/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs
index e64f9a54f..17c17812d 100644
--- a/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs
+++ b/ARMeilleure/Decoders/Optimizations/TailCallRemover.cs
@@ -58,15 +58,14 @@ namespace ARMeilleure.Decoders.Optimizations
return blocks.ToArray(); // Nothing to do here.
}
- // Mark branches outside of contiguous region as exit blocks.
+ // Mark branches whose target is outside of the contiguous region as an exit block.
for (int i = startBlockIndex; i <= endBlockIndex; i++)
{
Block block = blocks[i];
if (block.Branch != null && (block.Branch.Address > endBlock.EndAddress || block.Branch.EndAddress < startBlock.Address))
{
- block.Branch.Exit = true;
- block.Branch.TailCall = true;
+ block.Branch.Exit = true;
}
}
diff --git a/ARMeilleure/Instructions/InstEmitFlowHelper.cs b/ARMeilleure/Instructions/InstEmitFlowHelper.cs
index 296e20a5e..f995ffa1f 100644
--- a/ARMeilleure/Instructions/InstEmitFlowHelper.cs
+++ b/ARMeilleure/Instructions/InstEmitFlowHelper.cs
@@ -200,7 +200,7 @@ namespace ARMeilleure.Instructions
}
}
- public static void EmitTailContinue(ArmEmitterContext context, Operand address, bool allowRejit)
+ public static void EmitTailContinue(ArmEmitterContext context, Operand address)
{
// Left option here as it may be useful if we need to return to managed rather than tail call in future.
// (eg. for debug)
@@ -218,9 +218,7 @@ namespace ARMeilleure.Instructions
{
context.StoreToContext();
- Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(allowRejit
- ? nameof(NativeInterface.GetFunctionAddress)
- : nameof(NativeInterface.GetFunctionAddressWithoutRejit)), address);
+ Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
EmitNativeCall(context, fallbackAddr, isJump: true);
}
diff --git a/ARMeilleure/Instructions/NativeInterface.cs b/ARMeilleure/Instructions/NativeInterface.cs
index b8b7ff0e8..fa17d3349 100644
--- a/ARMeilleure/Instructions/NativeInterface.cs
+++ b/ARMeilleure/Instructions/NativeInterface.cs
@@ -220,6 +220,11 @@ namespace ARMeilleure.Instructions
}
#endregion
+ public static void EnqueueForRejit(ulong address)
+ {
+ Context.Translator.EnqueueForRejit(address, GetContext().ExecutionMode);
+ }
+
public static void SignalMemoryTracking(ulong address, ulong size, bool write)
{
GetMemoryManager().SignalMemoryTracking(address, size, write);
@@ -232,24 +237,14 @@ namespace ARMeilleure.Instructions
public static ulong GetFunctionAddress(ulong address)
{
- return GetFunctionAddressWithHint(address, true);
- }
-
- public static ulong GetFunctionAddressWithoutRejit(ulong address)
- {
- return GetFunctionAddressWithHint(address, false);
- }
-
- private static ulong GetFunctionAddressWithHint(ulong address, bool hintRejit)
- {
- TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit);
+ TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
return (ulong)function.FuncPtr.ToInt64();
}
public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress)
{
- TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit: true);
+ TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
ulong ptr = (ulong)function.FuncPtr.ToInt64();
diff --git a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs
index 26d664783..1b748f6a2 100644
--- a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs
+++ b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs
@@ -1,4 +1,5 @@
using ARMeilleure.Common;
+using System.Runtime.CompilerServices;
namespace ARMeilleure.IntermediateRepresentation
{
@@ -34,6 +35,11 @@ namespace ARMeilleure.IntermediateRepresentation
return Operand().With(value);
}
+ public static unsafe Operand Const(ref T reference, int? index = null)
+ {
+ return Operand().With((long)Unsafe.AsPointer(ref reference), index != null, index);
+ }
+
public static Operand ConstF(float value)
{
return Operand().With(value);
diff --git a/ARMeilleure/Translation/ArmEmitterContext.cs b/ARMeilleure/Translation/ArmEmitterContext.cs
index 8f1531922..ad44b0cfb 100644
--- a/ARMeilleure/Translation/ArmEmitterContext.cs
+++ b/ARMeilleure/Translation/ArmEmitterContext.cs
@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
using ARMeilleure.Decoders;
using ARMeilleure.Instructions;
using ARMeilleure.IntermediateRepresentation;
@@ -41,18 +42,26 @@ namespace ARMeilleure.Translation
public IMemoryManager Memory { get; }
public JumpTable JumpTable { get; }
+ public EntryTable CountTable { get; }
public ulong EntryAddress { get; }
public bool HighCq { get; }
public Aarch32Mode Mode { get; }
- public ArmEmitterContext(IMemoryManager memory, JumpTable jumpTable, ulong entryAddress, bool highCq, Aarch32Mode mode)
+ public ArmEmitterContext(
+ IMemoryManager memory,
+ JumpTable jumpTable,
+ EntryTable countTable,
+ ulong entryAddress,
+ bool highCq,
+ Aarch32Mode mode)
{
- Memory = memory;
- JumpTable = jumpTable;
+ Memory = memory;
+ JumpTable = jumpTable;
+ CountTable = countTable;
EntryAddress = entryAddress;
- HighCq = highCq;
- Mode = mode;
+ HighCq = highCq;
+ Mode = mode;
_labels = new Dictionary();
}
diff --git a/ARMeilleure/Translation/Delegates.cs b/ARMeilleure/Translation/Delegates.cs
index 5ad718724..a561d2653 100644
--- a/ARMeilleure/Translation/Delegates.cs
+++ b/ARMeilleure/Translation/Delegates.cs
@@ -103,6 +103,7 @@ namespace ARMeilleure.Translation
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Break)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
+ SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntfrqEl0)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntvctEl0)));
@@ -113,7 +114,6 @@ namespace ARMeilleure.Translation
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only.
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)));
- SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddressWithoutRejit)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only.
diff --git a/ARMeilleure/Translation/PTC/Ptc.cs b/ARMeilleure/Translation/PTC/Ptc.cs
index 32e0e7e8c..55a0f4d06 100644
--- a/ARMeilleure/Translation/PTC/Ptc.cs
+++ b/ARMeilleure/Translation/PTC/Ptc.cs
@@ -1,6 +1,7 @@
using ARMeilleure.CodeGen;
using ARMeilleure.CodeGen.Unwinding;
using ARMeilleure.CodeGen.X86;
+using ARMeilleure.Common;
using ARMeilleure.Memory;
using ARMeilleure.Translation.Cache;
using Ryujinx.Common;
@@ -27,7 +28,7 @@ namespace ARMeilleure.Translation.PTC
private const string OuterHeaderMagicString = "PTCohd\0\0";
private const string InnerHeaderMagicString = "PTCihd\0\0";
- private const uint InternalVersion = 2169; //! To be incremented manually for each change to the ARMeilleure project.
+ private const uint InternalVersion = 2190; //! To be incremented manually for each change to the ARMeilleure project.
private const string ActualDir = "0";
private const string BackupDir = "1";
@@ -38,6 +39,7 @@ namespace ARMeilleure.Translation.PTC
internal const int PageTablePointerIndex = -1; // Must be a negative value.
internal const int JumpPointerIndex = -2; // Must be a negative value.
internal const int DynamicPointerIndex = -3; // Must be a negative value.
+ internal const int CountTableIndex = -4; // Must be a negative value.
private const byte FillingByte = 0x00;
private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
@@ -538,7 +540,11 @@ namespace ARMeilleure.Translation.PTC
}
}
- internal static void LoadTranslations(ConcurrentDictionary funcs, IMemoryManager memory, JumpTable jumpTable)
+ internal static void LoadTranslations(
+ ConcurrentDictionary funcs,
+ IMemoryManager memory,
+ JumpTable jumpTable,
+ EntryTable countTable)
{
if (AreCarriersEmpty())
{
@@ -567,16 +573,18 @@ namespace ARMeilleure.Translation.PTC
{
byte[] code = ReadCode(index, infoEntry.CodeLength);
+ Counter callCounter = null;
+
if (infoEntry.RelocEntriesCount != 0)
{
RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
- PatchCode(code.AsSpan(), relocEntries, memory.PageTablePointer, jumpTable);
+ PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter);
}
UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
- TranslatedFunction func = FastTranslate(code, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
+ TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func);
@@ -670,8 +678,16 @@ namespace ARMeilleure.Translation.PTC
return relocEntries;
}
- private static void PatchCode(Span code, RelocEntry[] relocEntries, IntPtr pageTablePointer, JumpTable jumpTable)
+ private static void PatchCode(
+ Span code,
+ RelocEntry[] relocEntries,
+ IntPtr pageTablePointer,
+ JumpTable jumpTable,
+ EntryTable countTable,
+ out Counter callCounter)
{
+ callCounter = null;
+
foreach (RelocEntry relocEntry in relocEntries)
{
ulong imm;
@@ -688,6 +704,12 @@ namespace ARMeilleure.Translation.PTC
{
imm = (ulong)jumpTable.DynamicPointer.ToInt64();
}
+ else if (relocEntry.Index == CountTableIndex)
+ {
+ callCounter = new Counter(countTable);
+
+ unsafe { imm = (ulong)Unsafe.AsPointer(ref callCounter.Value); }
+ }
else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr))
{
imm = (ulong)funcPtr.ToInt64();
@@ -722,7 +744,12 @@ namespace ARMeilleure.Translation.PTC
return new UnwindInfo(pushEntries, prologueSize);
}
- private static TranslatedFunction FastTranslate(byte[] code, ulong guestSize, UnwindInfo unwindInfo, bool highCq)
+ private static TranslatedFunction FastTranslate(
+ byte[] code,
+ Counter callCounter,
+ ulong guestSize,
+ UnwindInfo unwindInfo,
+ bool highCq)
{
CompiledFunction cFunc = new CompiledFunction(code, unwindInfo);
@@ -730,7 +757,7 @@ namespace ARMeilleure.Translation.PTC
GuestFunction gFunc = Marshal.GetDelegateForFunctionPointer(codePtr);
- TranslatedFunction tFunc = new TranslatedFunction(gFunc, guestSize, highCq);
+ TranslatedFunction tFunc = new TranslatedFunction(gFunc, callCounter, guestSize, highCq);
return tFunc;
}
@@ -771,7 +798,11 @@ namespace ARMeilleure.Translation.PTC
}
}
- internal static void MakeAndSaveTranslations(ConcurrentDictionary funcs, IMemoryManager memory, JumpTable jumpTable)
+ internal static void MakeAndSaveTranslations(
+ ConcurrentDictionary funcs,
+ IMemoryManager memory,
+ JumpTable jumpTable,
+ EntryTable countTable)
{
var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs);
@@ -813,7 +844,7 @@ namespace ARMeilleure.Translation.PTC
Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address));
- TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.mode, item.highCq);
+ TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.mode, item.highCq);
bool isAddressUnique = funcs.TryAdd(address, func);
@@ -1058,4 +1089,4 @@ namespace ARMeilleure.Translation.PTC
}
}
}
-}
\ No newline at end of file
+}
diff --git a/ARMeilleure/Translation/TranslatedFunction.cs b/ARMeilleure/Translation/TranslatedFunction.cs
index 8b0759289..04dd769c1 100644
--- a/ARMeilleure/Translation/TranslatedFunction.cs
+++ b/ARMeilleure/Translation/TranslatedFunction.cs
@@ -1,24 +1,22 @@
+using ARMeilleure.Common;
using System;
using System.Runtime.InteropServices;
-using System.Threading;
namespace ARMeilleure.Translation
{
class TranslatedFunction
{
- private const int MinCallsForRejit = 100;
-
private readonly GuestFunction _func; // Ensure that this delegate will not be garbage collected.
- private int _callCount;
-
+ public Counter CallCounter { get; }
public ulong GuestSize { get; }
public bool HighCq { get; }
public IntPtr FuncPtr { get; }
- public TranslatedFunction(GuestFunction func, ulong guestSize, bool highCq)
+ public TranslatedFunction(GuestFunction func, Counter callCounter, ulong guestSize, bool highCq)
{
_func = func;
+ CallCounter = callCounter;
GuestSize = guestSize;
HighCq = highCq;
FuncPtr = Marshal.GetFunctionPointerForDelegate(func);
@@ -28,15 +26,5 @@ namespace ARMeilleure.Translation
{
return _func(context.NativeContextPtr);
}
-
- public bool ShouldRejit()
- {
- return !HighCq && Interlocked.Increment(ref _callCount) == MinCallsForRejit;
- }
-
- public void ResetCallCount()
- {
- Interlocked.Exchange(ref _callCount, 0);
- }
}
}
\ No newline at end of file
diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs
index 73a321fd2..9f88f17a9 100644
--- a/ARMeilleure/Translation/Translator.cs
+++ b/ARMeilleure/Translation/Translator.cs
@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
using ARMeilleure.Decoders;
using ARMeilleure.Diagnostics;
using ARMeilleure.Instructions;
@@ -10,7 +11,6 @@ using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
-using System.Linq;
using System.Runtime;
using System.Threading;
@@ -22,23 +22,27 @@ namespace ARMeilleure.Translation
{
public class Translator
{
+ private const int CountTableCapacity = 4 * 1024 * 1024;
+
private readonly IJitMemoryAllocator _allocator;
private readonly IMemoryManager _memory;
private readonly ConcurrentDictionary _funcs;
- private readonly ConcurrentQueue> _oldFuncs;
+ private readonly ConcurrentQueue> _oldFuncs;
+ private readonly ConcurrentDictionary _backgroundSet;
private readonly ConcurrentStack _backgroundStack;
private readonly AutoResetEvent _backgroundTranslatorEvent;
private readonly ReaderWriterLock _backgroundTranslatorLock;
private JumpTable _jumpTable;
internal JumpTable JumpTable => _jumpTable;
+ internal EntryTable CountTable { get; }
private volatile int _threadCount;
// FIXME: Remove this once the init logic of the emulator will be redone.
- public static ManualResetEvent IsReadyForTranslation = new ManualResetEvent(false);
+ public static readonly ManualResetEvent IsReadyForTranslation = new(false);
public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
{
@@ -46,12 +50,15 @@ namespace ARMeilleure.Translation
_memory = memory;
_funcs = new ConcurrentDictionary();
- _oldFuncs = new ConcurrentQueue>();
+ _oldFuncs = new ConcurrentQueue>();
+ _backgroundSet = new ConcurrentDictionary();
_backgroundStack = new ConcurrentStack();
_backgroundTranslatorEvent = new AutoResetEvent(false);
_backgroundTranslatorLock = new ReaderWriterLock();
+ CountTable = new EntryTable();
+
JitCache.Initialize(allocator);
DirectCallStubs.InitializeStubs();
@@ -63,9 +70,16 @@ namespace ARMeilleure.Translation
{
_backgroundTranslatorLock.AcquireReaderLock(Timeout.Infinite);
- if (_backgroundStack.TryPop(out RejitRequest request))
+ if (_backgroundStack.TryPop(out RejitRequest request) &&
+ _backgroundSet.TryRemove(request.Address, out _))
{
- TranslatedFunction func = Translate(_memory, _jumpTable, request.Address, request.Mode, highCq: true);
+ TranslatedFunction func = Translate(
+ _memory,
+ _jumpTable,
+ CountTable,
+ request.Address,
+ request.Mode,
+ highCq: true);
_funcs.AddOrUpdate(request.Address, func, (key, oldFunc) =>
{
@@ -89,7 +103,8 @@ namespace ARMeilleure.Translation
}
}
- _backgroundTranslatorEvent.Set(); // Wake up any other background translator threads, to encourage them to exit.
+ // Wake up any other background translator threads, to encourage them to exit.
+ _backgroundTranslatorEvent.Set();
}
public void Execute(State.ExecutionContext context, ulong address)
@@ -104,18 +119,21 @@ namespace ARMeilleure.Translation
if (Ptc.State == PtcState.Enabled)
{
Debug.Assert(_funcs.Count == 0);
- Ptc.LoadTranslations(_funcs, _memory, _jumpTable);
- Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable);
+ Ptc.LoadTranslations(_funcs, _memory, _jumpTable, CountTable);
+ Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable);
}
PtcProfiler.Start();
Ptc.Disable();
- // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core+ht etc).
- // All threads are normal priority except from the last, which just fills as much of the last core as the os lets it with a low priority.
- // If we only have one rejit thread, it should be normal priority as highCq code is performance critical.
- // TODO: Use physical cores rather than logical. This only really makes sense for processors with hyperthreading. Requires OS specific code.
+ // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core + ht
+ // etc). All threads are normal priority except from the last, which just fills as much of the last core
+ // as the os lets it with a low priority. If we only have one rejit thread, it should be normal priority
+ // as highCq code is performance critical.
+ //
+ // TODO: Use physical cores rather than logical. This only really makes sense for processors with
+ // hyperthreading. Requires OS specific code.
int unboundedThreadCount = Math.Max(1, (Environment.ProcessorCount - 6) / 3);
int threadCount = Math.Min(4, unboundedThreadCount);
@@ -156,6 +174,8 @@ namespace ARMeilleure.Translation
_jumpTable.Dispose();
_jumpTable = null;
+ CountTable.Dispose();
+
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
}
}
@@ -173,11 +193,11 @@ namespace ARMeilleure.Translation
return nextAddr;
}
- internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode, bool hintRejit = false)
+ internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode)
{
if (!_funcs.TryGetValue(address, out TranslatedFunction func))
{
- func = Translate(_memory, _jumpTable, address, mode, highCq: false);
+ func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false);
TranslatedFunction getFunc = _funcs.GetOrAdd(address, func);
@@ -193,18 +213,18 @@ namespace ARMeilleure.Translation
}
}
- if (hintRejit && func.ShouldRejit())
- {
- _backgroundStack.Push(new RejitRequest(address, mode));
- _backgroundTranslatorEvent.Set();
- }
-
return func;
}
- internal static TranslatedFunction Translate(IMemoryManager memory, JumpTable jumpTable, ulong address, ExecutionMode mode, bool highCq)
+ internal static TranslatedFunction Translate(
+ IMemoryManager memory,
+ JumpTable jumpTable,
+ EntryTable countTable,
+ ulong address,
+ ExecutionMode mode,
+ bool highCq)
{
- ArmEmitterContext context = new ArmEmitterContext(memory, jumpTable, address, highCq, Aarch32Mode.User);
+ var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User);
Logger.StartPass(PassName.Decoding);
@@ -216,6 +236,13 @@ namespace ARMeilleure.Translation
Logger.StartPass(PassName.Translation);
+ Counter counter = null;
+
+ if (!context.HighCq)
+ {
+ EmitRejitCheck(context, out counter);
+ }
+
EmitSynchronization(context);
if (blocks[0].Address != address)
@@ -258,7 +285,7 @@ namespace ARMeilleure.Translation
Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, highCq, ptcInfo);
}
- return new TranslatedFunction(func, funcSize, highCq);
+ return new TranslatedFunction(func, counter, funcSize, highCq);
}
internal static void PreparePool(int groupId = 0)
@@ -320,7 +347,7 @@ namespace ARMeilleure.Translation
if (block.Exit)
{
- InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address), block.TailCall);
+ InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address));
}
else
{
@@ -368,29 +395,43 @@ namespace ARMeilleure.Translation
return context.GetControlFlowGraph();
}
+ internal static void EmitRejitCheck(ArmEmitterContext context, out Counter counter)
+ {
+ const int MinsCallForRejit = 100;
+
+ counter = new Counter(context.CountTable);
+
+ Operand lblEnd = Label();
+
+ Operand address = Const(ref counter.Value, Ptc.CountTableIndex);
+ Operand curCount = context.Load(OperandType.I32, address);
+ Operand count = context.Add(curCount, Const(1));
+ context.Store(address, count);
+ context.BranchIf(lblEnd, curCount, Const(MinsCallForRejit), Comparison.NotEqual, BasicBlockFrequency.Cold);
+
+ context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)), Const(context.EntryAddress));
+
+ context.MarkLabel(lblEnd);
+ }
+
internal static void EmitSynchronization(EmitterContext context)
{
long countOffs = NativeContext.GetCounterOffset();
- Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
-
- Operand count = context.Load(OperandType.I32, countAddr);
-
Operand lblNonZero = Label();
- Operand lblExit = Label();
+ Operand lblExit = Label();
+ Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
+ Operand count = context.Load(OperandType.I32, countAddr);
context.BranchIfTrue(lblNonZero, count, BasicBlockFrequency.Cold);
Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
-
context.BranchIfTrue(lblExit, running, BasicBlockFrequency.Cold);
context.Return(Const(0L));
context.MarkLabel(lblNonZero);
-
count = context.Subtract(count, Const(1));
-
context.Store(countAddr, count);
context.MarkLabel(lblExit);
@@ -404,9 +445,18 @@ namespace ARMeilleure.Translation
// TODO: Completely remove functions overlapping the specified range from the cache.
}
+ internal void EnqueueForRejit(ulong guestAddress, ExecutionMode mode)
+ {
+ if (_backgroundSet.TryAdd(guestAddress, null))
+ {
+ _backgroundStack.Push(new RejitRequest(guestAddress, mode));
+ _backgroundTranslatorEvent.Set();
+ }
+ }
+
private void EnqueueForDeletion(ulong guestAddress, TranslatedFunction func)
{
- _oldFuncs.Enqueue(new KeyValuePair(guestAddress, func.FuncPtr));
+ _oldFuncs.Enqueue(new(guestAddress, func));
}
private void ClearJitCache()
@@ -414,16 +464,20 @@ namespace ARMeilleure.Translation
// Ensure no attempt will be made to compile new functions due to rejit.
ClearRejitQueue(allowRequeue: false);
- foreach (var kv in _funcs)
+ foreach (var func in _funcs.Values)
{
- JitCache.Unmap(kv.Value.FuncPtr);
+ JitCache.Unmap(func.FuncPtr);
+
+ func.CallCounter?.Dispose();
}
_funcs.Clear();
while (_oldFuncs.TryDequeue(out var kv))
{
- JitCache.Unmap(kv.Value);
+ JitCache.Unmap(kv.Value.FuncPtr);
+
+ kv.Value.CallCounter?.Dispose();
}
}
@@ -435,10 +489,12 @@ namespace ARMeilleure.Translation
{
while (_backgroundStack.TryPop(out var request))
{
- if (_funcs.TryGetValue(request.Address, out var func))
+ if (_funcs.TryGetValue(request.Address, out var func) && func.CallCounter != null)
{
- func.ResetCallCount();
+ Volatile.Write(ref func.CallCounter.Value, 0);
}
+
+ _backgroundSet.TryRemove(request.Address, out _);
}
}
else