diff --git a/ARMeilleure/CodeGen/X86/Assembler.cs b/ARMeilleure/CodeGen/X86/Assembler.cs index bab4c4530..39aeb7c9d 100644 --- a/ARMeilleure/CodeGen/X86/Assembler.cs +++ b/ARMeilleure/CodeGen/X86/Assembler.cs @@ -963,8 +963,6 @@ namespace ARMeilleure.CodeGen.X86 } else if (dest?.Kind == OperandKind.Register && info.OpRImm64 != BadOp) { - int? index = source.PtcIndex; - int rexPrefix = GetRexPrefix(dest, source, type, rrm: false); if (rexPrefix != 0) @@ -974,9 +972,9 @@ namespace ARMeilleure.CodeGen.X86 WriteByte((byte)(info.OpRImm64 + (dest.GetRegister().Index & 0b111))); - if (_ptcInfo != null && index != null) + if (_ptcInfo != null && source.Relocatable) { - _ptcInfo.WriteRelocEntry(new RelocEntry((int)_stream.Position, (int)index)); + _ptcInfo.WriteRelocEntry(new RelocEntry((int)_stream.Position, source.Symbol)); } WriteUInt64(imm); diff --git a/ARMeilleure/Common/AddressTable.cs b/ARMeilleure/Common/AddressTable.cs new file mode 100644 index 000000000..4af1dc3a0 --- /dev/null +++ b/ARMeilleure/Common/AddressTable.cs @@ -0,0 +1,261 @@ +using ARMeilleure.Diagnostics.EventSources; +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Common +{ + /// + /// Represents a table of guest address to a value. + /// + /// Type of the value + unsafe class AddressTable : IDisposable where TEntry : unmanaged + { + /// + /// Represents a level in an . + /// + public readonly struct Level + { + /// + /// Gets the index of the in the guest address. + /// + public int Index { get; } + + /// + /// Gets the length of the in the guest address. + /// + public int Length { get; } + + /// + /// Gets the mask which masks the bits used by the . + /// + public ulong Mask => ((1ul << Length) - 1) << Index; + + /// + /// Initializes a new instance of the structure with the specified + /// and . + /// + /// Index of the + /// Length of the + public Level(int index, int length) + { + (Index, Length) = (index, length); + } + + /// + /// Gets the value of the from the specified guest . + /// + /// Guest address + /// Value of the from the specified guest + public int GetValue(ulong address) + { + return (int)((address & Mask) >> Index); + } + } + + private bool _disposed; + private TEntry** _table; + private readonly List _pages; + + /// + /// Gets the bits used by the of the instance. + /// + public ulong Mask { get; } + + /// + /// Gets the s used by the instance. + /// + public Level[] Levels { get; } + + /// + /// Gets or sets the default fill value of newly created leaf pages. + /// + public TEntry Fill { get; set; } + + /// + /// Gets the base address of the . + /// + /// instance was disposed + public IntPtr Base + { + get + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + lock (_pages) + { + return (IntPtr)GetRootPage(); + } + } + } + + /// + /// Constructs a new instance of the class with the specified list of + /// . + /// + /// is null + /// Length of is less than 2 + public AddressTable(Level[] levels) + { + if (levels == null) + { + throw new ArgumentNullException(nameof(levels)); + } + + if (levels.Length < 2) + { + throw new ArgumentException("Table must be at least 2 levels deep.", nameof(levels)); + } + + _pages = new List(capacity: 16); + + Levels = levels; + Mask = 0; + + foreach (var level in Levels) + { + Mask |= level.Mask; + } + } + + /// + /// Determines if the specified is in the range of the + /// . + /// + /// Guest address + /// if is valid; otherwise + public bool IsValid(ulong address) + { + return (address & ~Mask) == 0; + } + + /// + /// Gets a reference to the value at the specified guest . + /// + /// Guest address + /// Reference to the value at the specified guest + /// instance was disposed + /// is not mapped + public ref TEntry GetValue(ulong address) + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + if (!IsValid(address)) + { + throw new ArgumentException($"Address 0x{address:X} is not mapped onto the table.", nameof(address)); + } + + lock (_pages) + { + return ref GetPage(address)[Levels[^1].GetValue(address)]; + } + } + + /// + /// Gets the leaf page for the specified guest . + /// + /// Guest address + /// Leaf page for the specified guest + private TEntry* GetPage(ulong address) + { + TEntry** page = GetRootPage(); + + for (int i = 0; i < Levels.Length - 1; i++) + { + ref Level level = ref Levels[i]; + ref TEntry* nextPage = ref page[level.GetValue(address)]; + + if (nextPage == null) + { + ref Level nextLevel = ref Levels[i + 1]; + + nextPage = i == Levels.Length - 2 ? + (TEntry*)Allocate(1 << nextLevel.Length, Fill, leaf: true) : + (TEntry*)Allocate(1 << nextLevel.Length, IntPtr.Zero, leaf: false); + } + + page = (TEntry**)nextPage; + } + + return (TEntry*)page; + } + + /// + /// Lazily initialize and get the root page of the . + /// + /// Root page of the + private TEntry** GetRootPage() + { + if (_table == null) + { + _table = (TEntry**)Allocate(1 << Levels[0].Length, fill: IntPtr.Zero, leaf: false); + } + + return _table; + } + + /// + /// Allocates a block of memory of the specified type and length. + /// + /// Type of elements + /// Number of elements + /// Fill value + /// if leaf; otherwise + /// Allocated block + private IntPtr Allocate(int length, T fill, bool leaf) where T : unmanaged + { + var size = sizeof(T) * length; + var page = Marshal.AllocHGlobal(size); + var span = new Span((void*)page, length); + + span.Fill(fill); + + _pages.Add(page); + + AddressTableEventSource.Log.Allocated(size, leaf); + + return page; + } + + /// + /// Releases all resources used by the instance. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases all unmanaged and optionally managed resources used by the + /// instance. + /// + /// to dispose managed resources also; otherwise just unmanaged resouces + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + foreach (var page in _pages) + { + Marshal.FreeHGlobal(page); + } + + _disposed = true; + } + } + + /// + /// Frees resources used by the instance. + /// + ~AddressTable() + { + Dispose(false); + } + } +} diff --git a/ARMeilleure/Common/EntryTable.cs b/ARMeilleure/Common/EntryTable.cs index a0ed7c8e5..b61af8f81 100644 --- a/ARMeilleure/Common/EntryTable.cs +++ b/ARMeilleure/Common/EntryTable.cs @@ -168,7 +168,7 @@ namespace ARMeilleure.Common } /// - /// Releases all unmanaged and optionally managed resources used by the + /// Releases all unmanaged and optionally managed resources used by the /// instance. /// /// to dispose managed resources also; otherwise just unmanaged resouces diff --git a/ARMeilleure/Diagnostics/EventSources/AddressTableEventSource.cs b/ARMeilleure/Diagnostics/EventSources/AddressTableEventSource.cs new file mode 100644 index 000000000..201a25623 --- /dev/null +++ b/ARMeilleure/Diagnostics/EventSources/AddressTableEventSource.cs @@ -0,0 +1,51 @@ +using System.Diagnostics.Tracing; + +namespace ARMeilleure.Diagnostics.EventSources +{ + [EventSource(Name = "ARMeilleure")] + class AddressTableEventSource : EventSource + { + public static readonly AddressTableEventSource Log = new(); + + private ulong _size; + private ulong _leafSize; + private PollingCounter _sizeCounter; + private PollingCounter _leafSizeCounter; + + public AddressTableEventSource() + { + _sizeCounter = new PollingCounter("addr-tab-alloc", this, () => _size / 1024d / 1024d) + { + DisplayName = "AddressTable Total Bytes Allocated", + DisplayUnits = "MB" + }; + + _leafSizeCounter = new PollingCounter("addr-tab-leaf-alloc", this, () => _leafSize / 1024d / 1024d) + { + DisplayName = "AddressTable Total Leaf Bytes Allocated", + DisplayUnits = "MB" + }; + } + + public void Allocated(int bytes, bool leaf) + { + _size += (uint)bytes; + + if (leaf) + { + _leafSize += (uint)bytes; + } + } + + protected override void Dispose(bool disposing) + { + _leafSizeCounter.Dispose(); + _leafSizeCounter = null; + + _sizeCounter.Dispose(); + _sizeCounter = null; + + base.Dispose(disposing); + } + } +} diff --git a/ARMeilleure/Instructions/InstEmitFlowHelper.cs b/ARMeilleure/Instructions/InstEmitFlowHelper.cs index e1309a4e3..808d15c80 100644 --- a/ARMeilleure/Instructions/InstEmitFlowHelper.cs +++ b/ARMeilleure/Instructions/InstEmitFlowHelper.cs @@ -150,21 +150,70 @@ namespace ARMeilleure.Instructions } else { - EmitJumpTableBranch(context, Const(immediate), isJump: false); + EmitTableBranch(context, Const(immediate), isJump: false); } } - private static void EmitNativeCall(ArmEmitterContext context, Operand nativeContextPtr, Operand funcAddr, bool isJump) + public static void EmitVirtualCall(ArmEmitterContext context, Operand target) { + EmitTableBranch(context, target, isJump: false); + } + + public static void EmitVirtualJump(ArmEmitterContext context, Operand target, bool isReturn) + { + if (isReturn) + { + context.Return(target); + } + else + { + EmitTableBranch(context, target, isJump: true); + } + } + + private static void EmitTableBranch(ArmEmitterContext context, Operand guestAddress, bool isJump) + { + context.StoreToContext(); + + if (guestAddress.Type == OperandType.I32) + { + guestAddress = context.ZeroExtend32(OperandType.I64, guestAddress); + } + + // Store the target guest address into the native context. The stubs uses this address to dispatch into the + // next translation. + Operand nativeContext = context.LoadArgument(OperandType.I64, 0); + Operand dispAddressAddr = context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset())); + context.Store(dispAddressAddr, guestAddress); + + Operand hostAddress; + + // If address is mapped onto the function table, we can skip the table walk. Otherwise we fallback + // onto the dispatch stub. + if (guestAddress.Kind == OperandKind.Constant && context.FunctionTable.IsValid(guestAddress.Value)) + { + Operand hostAddressAddr = !context.HasPtc ? + Const(ref context.FunctionTable.GetValue(guestAddress.Value)) : + Const(ref context.FunctionTable.GetValue(guestAddress.Value), new Symbol(SymbolType.FunctionTable, guestAddress.Value)); + + hostAddress = context.Load(OperandType.I64, hostAddressAddr); + } + else + { + hostAddress = !context.HasPtc ? + Const((long)context.Stubs.DispatchStub) : + Const((long)context.Stubs.DispatchStub, Ptc.DispatchStubSymbol); + } + if (isJump) { - context.Tailcall(funcAddr, nativeContextPtr); + context.Tailcall(hostAddress, nativeContext); } else { OpCode op = context.CurrOp; - Operand returnAddress = context.Call(funcAddr, OperandType.I64, nativeContextPtr); + Operand returnAddress = context.Call(hostAddress, OperandType.I64, nativeContext); context.LoadFromContext(); @@ -177,203 +226,10 @@ namespace ARMeilleure.Instructions // If the return address isn't to our next instruction, we need to return so the JIT can figure out // what to do. Operand lblContinue = context.GetLabel(nextAddr.Value); - - // We need to clear out the call flag for the return address before comparing it. context.BranchIf(lblContinue, returnAddress, nextAddr, Comparison.Equal, BasicBlockFrequency.Cold); context.Return(returnAddress); } } - - private static void EmitNativeCall(ArmEmitterContext context, Operand funcAddr, bool isJump) - { - EmitNativeCall(context, context.LoadArgument(OperandType.I64, 0), funcAddr, isJump); - } - - public static void EmitVirtualCall(ArmEmitterContext context, Operand target) - { - EmitJumpTableBranch(context, target, isJump: false); - } - - public static void EmitVirtualJump(ArmEmitterContext context, Operand target, bool isReturn) - { - if (isReturn) - { - context.Return(target); - } - else - { - EmitJumpTableBranch(context, target, isJump: true); - } - } - - public static void EmitTailContinue(ArmEmitterContext context, Operand address) - { - // Left option here as it may be useful if we need to return to managed rather than tail call in future. - // (eg. for debug) - bool useTailContinue = true; - - if (useTailContinue) - { - if (context.HighCq) - { - // If we're doing a tail continue in HighCq, reserve a space in the jump table to avoid calling back - // to the translator. This will always try to get a HighCq version of our continue target as well. - EmitJumpTableBranch(context, address, isJump: true); - } - else - { - context.StoreToContext(); - - Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address); - - EmitNativeCall(context, fallbackAddr, isJump: true); - } - } - else - { - context.Return(address); - } - } - - private static void EmitNativeCallWithGuestAddress(ArmEmitterContext context, Operand funcAddr, Operand guestAddress, bool isJump) - { - Operand nativeContextPtr = context.LoadArgument(OperandType.I64, 0); - context.Store(context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset())), guestAddress); - - EmitNativeCall(context, nativeContextPtr, funcAddr, isJump); - } - - private static void EmitBranchFallback(ArmEmitterContext context, Operand address, bool isJump) - { - Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address); - - EmitNativeCall(context, fallbackAddr, isJump); - } - - private static void EmitDynamicTableCall(ArmEmitterContext context, Operand tableAddress, Operand address, bool isJump) - { - // Loop over elements of the dynamic table. Unrolled loop. - - Operand endLabel = Label(); - Operand fallbackLabel = Label(); - - void EmitTableEntry(Operand entrySkipLabel) - { - // Try to take this entry in the table if its guest address equals 0. - Operand gotResult = context.CompareAndSwap(tableAddress, Const(0L), address); - - // Is the address ours? (either taken via CompareAndSwap (0), or what was already here) - context.BranchIfFalse(entrySkipLabel, - context.BitwiseOr( - context.ICompareEqual(gotResult, address), - context.ICompareEqual(gotResult, Const(0L))) - ); - - // It's ours, so what function is it pointing to? - Operand targetFunctionPtr = context.Add(tableAddress, Const(8L)); - Operand targetFunction = context.Load(OperandType.I64, targetFunctionPtr); - - // Call the function. - // We pass in the entry address as the guest address, as the entry may need to be updated by the - // indirect call stub. - EmitNativeCallWithGuestAddress(context, targetFunction, tableAddress, isJump); - - context.Branch(endLabel); - } - - // Currently this uses a size of 1, as higher values inflate code size for no real benefit. - for (int i = 0; i < JumpTable.DynamicTableElems; i++) - { - if (i == JumpTable.DynamicTableElems - 1) - { - // If this is the last entry, avoid emitting the additional label and add. - EmitTableEntry(fallbackLabel); - } - else - { - Operand nextLabel = Label(); - - EmitTableEntry(nextLabel); - - context.MarkLabel(nextLabel); - - // Move to the next table entry. - tableAddress = context.Add(tableAddress, Const((long)JumpTable.JumpTableStride)); - } - } - - context.MarkLabel(fallbackLabel); - - EmitBranchFallback(context, address, isJump); - - context.MarkLabel(endLabel); - } - - private static void EmitJumpTableBranch(ArmEmitterContext context, Operand address, bool isJump) - { - if (address.Type == OperandType.I32) - { - address = context.ZeroExtend32(OperandType.I64, address); - } - - context.StoreToContext(); - - // TODO: Constant folding. Indirect calls are slower in the best case and emit more code so we want to - // avoid them when possible. - bool isConst = address.Kind == OperandKind.Constant; - ulong constAddr = address.Value; - - if (!context.HighCq) - { - // Don't emit indirect calls or jumps if we're compiling in lowCq mode. This avoids wasting space on the - // jump and indirect tables. Just ask the translator for the function address. - EmitBranchFallback(context, address, isJump); - } - else if (!isConst) - { - // Virtual branch/call - store first used addresses on a small table for fast lookup. - int entry = context.JumpTable.ReserveDynamicEntry(context.EntryAddress, isJump); - - int jumpOffset = entry * JumpTable.JumpTableStride * JumpTable.DynamicTableElems; - - Operand dynTablePtr; - - if (Ptc.State == PtcState.Disabled) - { - dynTablePtr = Const(context.JumpTable.DynamicPointer.ToInt64() + jumpOffset); - } - else - { - dynTablePtr = Const(context.JumpTable.DynamicPointer.ToInt64(), true, Ptc.DynamicPointerIndex); - dynTablePtr = context.Add(dynTablePtr, Const((long)jumpOffset)); - } - - EmitDynamicTableCall(context, dynTablePtr, address, isJump); - } - else - { - int entry = context.JumpTable.ReserveTableEntry(context.EntryAddress, constAddr, isJump); - - int jumpOffset = entry * JumpTable.JumpTableStride + 8; // Offset directly to the host address. - - Operand tableEntryPtr; - - if (Ptc.State == PtcState.Disabled) - { - tableEntryPtr = Const(context.JumpTable.JumpPointer.ToInt64() + jumpOffset); - } - else - { - tableEntryPtr = Const(context.JumpTable.JumpPointer.ToInt64(), true, Ptc.JumpPointerIndex); - tableEntryPtr = context.Add(tableEntryPtr, Const((long)jumpOffset)); - } - - Operand funcAddr = context.Load(OperandType.I64, tableEntryPtr); - - // Call the function directly. If it's not present yet, this will call the direct call stub. - EmitNativeCallWithGuestAddress(context, funcAddr, address, isJump); - } - } } } diff --git a/ARMeilleure/Instructions/InstEmitMemoryHelper.cs b/ARMeilleure/Instructions/InstEmitMemoryHelper.cs index 2de123046..3bac68559 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryHelper.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryHelper.cs @@ -327,9 +327,9 @@ namespace ARMeilleure.Instructions Operand addrRotated = size != 0 ? context.RotateRight(address, Const(size)) : address; Operand addrShifted = context.ShiftRightUI(addrRotated, Const(PageBits - size)); - Operand pte = Ptc.State == PtcState.Disabled + Operand pte = !context.HasPtc ? Const(context.Memory.PageTablePointer.ToInt64()) - : Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex); + : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol); Operand pteOffset = context.BitwiseAnd(addrShifted, Const(addrShifted.Type, ptLevelMask)); @@ -411,9 +411,9 @@ namespace ARMeilleure.Instructions address = context.BitwiseAnd(address, mask); } - Operand baseAddr = Ptc.State == PtcState.Disabled + Operand baseAddr = !context.HasPtc ? Const(context.Memory.PageTablePointer.ToInt64()) - : Const(context.Memory.PageTablePointer.ToInt64(), true, Ptc.PageTablePointerIndex); + : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol); return context.Add(baseAddr, address); } diff --git a/ARMeilleure/Instructions/NativeInterface.cs b/ARMeilleure/Instructions/NativeInterface.cs index fa17d3349..02a22fa63 100644 --- a/ARMeilleure/Instructions/NativeInterface.cs +++ b/ARMeilleure/Instructions/NativeInterface.cs @@ -242,23 +242,6 @@ namespace ARMeilleure.Instructions return (ulong)function.FuncPtr.ToInt64(); } - public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress) - { - TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode); - - ulong ptr = (ulong)function.FuncPtr.ToInt64(); - - if (function.HighCq) - { - Debug.Assert(Context.Translator.JumpTable.CheckEntryFromAddressDynamicTable((IntPtr)entryAddress)); - - // Rewrite the host function address in the table to point to the highCq function. - Marshal.WriteInt64((IntPtr)entryAddress, 8, (long)ptr); - } - - return ptr; - } - public static bool CheckSynchronization() { Statistics.PauseTimer(); diff --git a/ARMeilleure/IntermediateRepresentation/Operand.cs b/ARMeilleure/IntermediateRepresentation/Operand.cs index ec0239392..64df416f5 100644 --- a/ARMeilleure/IntermediateRepresentation/Operand.cs +++ b/ARMeilleure/IntermediateRepresentation/Operand.cs @@ -1,3 +1,4 @@ +using ARMeilleure.Translation.PTC; using System; using System.Collections.Generic; using System.Diagnostics; @@ -12,12 +13,12 @@ namespace ARMeilleure.IntermediateRepresentation public ulong Value { get; private set; } - public bool Relocatable { get; private set; } - public int? PtcIndex { get; private set; } - public List Assignments { get; } public List Uses { get; } + public Symbol Symbol { get; private set; } + public bool Relocatable => Symbol.Type != SymbolType.None; + public Operand() { Assignments = new List(); @@ -34,16 +35,14 @@ namespace ARMeilleure.IntermediateRepresentation OperandKind kind, OperandType type = OperandType.None, ulong value = 0, - bool relocatable = false, - int? index = null) + Symbol symbol = default) { Kind = kind; Type = type; Value = value; - Relocatable = relocatable; - PtcIndex = index; + Symbol = symbol; Assignments.Clear(); Uses.Clear(); @@ -61,9 +60,14 @@ namespace ARMeilleure.IntermediateRepresentation return With(OperandKind.Constant, OperandType.I32, value); } - public Operand With(long value, bool relocatable = false, int? index = null) + public Operand With(long value) { - return With(OperandKind.Constant, OperandType.I64, (ulong)value, relocatable, index); + return With(OperandKind.Constant, OperandType.I64, (ulong)value); + } + + public Operand With(long value, Symbol symbol) + { + return With(OperandKind.Constant, OperandType.I64, (ulong)value, symbol); } public Operand With(ulong value) diff --git a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs index 1b748f6a2..420555a78 100644 --- a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs +++ b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs @@ -1,4 +1,5 @@ using ARMeilleure.Common; +using ARMeilleure.Translation.PTC; using System.Runtime.CompilerServices; namespace ARMeilleure.IntermediateRepresentation @@ -25,9 +26,14 @@ namespace ARMeilleure.IntermediateRepresentation return Operand().With(value); } - public static Operand Const(long value, bool relocatable = false, int? index = null) + public static Operand Const(long value) { - return Operand().With(value, relocatable, index); + return Operand().With(value); + } + + public static Operand Const(long value, Symbol symbol) + { + return Operand().With(value, symbol); } public static Operand Const(ulong value) @@ -35,9 +41,9 @@ namespace ARMeilleure.IntermediateRepresentation return Operand().With(value); } - public static unsafe Operand Const(ref T reference, int? index = null) + public static unsafe Operand Const(ref T reference, Symbol symbol = default) { - return Operand().With((long)Unsafe.AsPointer(ref reference), index != null, index); + return Operand().With((long)Unsafe.AsPointer(ref reference), symbol); } public static Operand ConstF(float value) diff --git a/ARMeilleure/Optimizations.cs b/ARMeilleure/Optimizations.cs index 0fd67c822..986b0c9ec 100644 --- a/ARMeilleure/Optimizations.cs +++ b/ARMeilleure/Optimizations.cs @@ -6,6 +6,9 @@ namespace ARMeilleure { public static bool FastFP { get; set; } = true; + public static bool AllowLcqInFunctionTable { get; set; } = true; + public static bool UseUnmanagedDispatchLoop { get; set; } = true; + public static bool UseSseIfAvailable { get; set; } = true; public static bool UseSse2IfAvailable { get; set; } = true; public static bool UseSse3IfAvailable { get; set; } = true; diff --git a/ARMeilleure/State/ExecutionContext.cs b/ARMeilleure/State/ExecutionContext.cs index a964f6baa..9a2215695 100644 --- a/ARMeilleure/State/ExecutionContext.cs +++ b/ARMeilleure/State/ExecutionContext.cs @@ -66,7 +66,11 @@ namespace ARMeilleure.State } } - public bool Running { get; private set; } + public bool Running + { + get => _nativeContext.GetRunning(); + private set => _nativeContext.SetRunning(value); + } public event EventHandler Interrupt; public event EventHandler Break; @@ -78,7 +82,6 @@ namespace ARMeilleure.State _hostTickFreq = 1.0 / Stopwatch.Frequency; _tickCounter = new Stopwatch(); - _tickCounter.Start(); } @@ -138,6 +141,7 @@ namespace ARMeilleure.State public void StopRunning() { Running = false; + _nativeContext.SetCounter(0); } diff --git a/ARMeilleure/State/NativeContext.cs b/ARMeilleure/State/NativeContext.cs index 09ec6cded..962783f5e 100644 --- a/ARMeilleure/State/NativeContext.cs +++ b/ARMeilleure/State/NativeContext.cs @@ -14,10 +14,11 @@ namespace ARMeilleure.State public fixed uint Flags[RegisterConsts.FlagsCount]; public fixed uint FpFlags[RegisterConsts.FpFlagsCount]; public int Counter; - public ulong CallAddress; + public ulong DispatchAddress; public ulong ExclusiveAddress; public ulong ExclusiveValueLow; public ulong ExclusiveValueHigh; + public int Running; } private static NativeCtxStorage _dummyStorage = new NativeCtxStorage(); @@ -117,6 +118,9 @@ namespace ARMeilleure.State public int GetCounter() => GetStorage().Counter; public void SetCounter(int value) => GetStorage().Counter = value; + public bool GetRunning() => GetStorage().Running != 0; + public void SetRunning(bool value) => GetStorage().Running = value ? 1 : 0; + public unsafe static int GetRegisterOffset(Register reg) { if (reg.Type == RegisterType.Integer) @@ -162,9 +166,9 @@ namespace ARMeilleure.State return StorageOffset(ref _dummyStorage, ref _dummyStorage.Counter); } - public static int GetCallAddressOffset() + public static int GetDispatchAddressOffset() { - return StorageOffset(ref _dummyStorage, ref _dummyStorage.CallAddress); + return StorageOffset(ref _dummyStorage, ref _dummyStorage.DispatchAddress); } public static int GetExclusiveAddressOffset() @@ -177,6 +181,11 @@ namespace ARMeilleure.State return StorageOffset(ref _dummyStorage, ref _dummyStorage.ExclusiveValueLow); } + public static int GetRunningOffset() + { + return StorageOffset(ref _dummyStorage, ref _dummyStorage.Running); + } + private static int StorageOffset(ref NativeCtxStorage storage, ref T target) { return (int)Unsafe.ByteOffset(ref Unsafe.As(ref storage), ref target); diff --git a/ARMeilleure/Translation/ArmEmitterContext.cs b/ARMeilleure/Translation/ArmEmitterContext.cs index ad44b0cfb..7a82b27b1 100644 --- a/ARMeilleure/Translation/ArmEmitterContext.cs +++ b/ARMeilleure/Translation/ArmEmitterContext.cs @@ -1,12 +1,14 @@ using ARMeilleure.Common; using ARMeilleure.Decoders; +using ARMeilleure.Diagnostics; using ARMeilleure.Instructions; using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Memory; using ARMeilleure.State; -using ARMeilleure.Translation.Cache; +using ARMeilleure.Translation.PTC; +using System; using System.Collections.Generic; - +using System.Reflection; using static ARMeilleure.IntermediateRepresentation.OperandHelper; namespace ARMeilleure.Translation @@ -41,8 +43,11 @@ namespace ARMeilleure.Translation public IMemoryManager Memory { get; } - public JumpTable JumpTable { get; } + public bool HasPtc { get; } + public EntryTable CountTable { get; } + public AddressTable FunctionTable { get; } + public TranslatorStubs Stubs { get; } public ulong EntryAddress { get; } public bool HighCq { get; } @@ -50,15 +55,18 @@ namespace ARMeilleure.Translation public ArmEmitterContext( IMemoryManager memory, - JumpTable jumpTable, EntryTable countTable, + AddressTable funcTable, + TranslatorStubs stubs, ulong entryAddress, bool highCq, Aarch32Mode mode) { + HasPtc = Ptc.State != PtcState.Disabled; Memory = memory; - JumpTable = jumpTable; CountTable = countTable; + FunctionTable = funcTable; + Stubs = stubs; EntryAddress = entryAddress; HighCq = highCq; Mode = mode; @@ -66,6 +74,27 @@ namespace ARMeilleure.Translation _labels = new Dictionary(); } + public override Operand Call(MethodInfo info, params Operand[] callArgs) + { + if (!HasPtc) + { + return base.Call(info, callArgs); + } + else + { + int index = Delegates.GetDelegateIndex(info); + IntPtr funcPtr = Delegates.GetDelegateFuncPtrByIndex(index); + + OperandType returnType = GetOperandType(info.ReturnType); + + Symbol symbol = new Symbol(SymbolType.DelegateTable, (ulong)index); + + Symbols.Add((ulong)funcPtr.ToInt64(), info.Name); + + return Call(Const(funcPtr.ToInt64(), symbol), returnType, callArgs); + } + } + public Operand GetLabel(ulong address) { if (!_labels.TryGetValue(address, out Operand label)) diff --git a/ARMeilleure/Translation/Cache/JitCache.cs b/ARMeilleure/Translation/Cache/JitCache.cs index db45c608b..c0d0a25de 100644 --- a/ARMeilleure/Translation/Cache/JitCache.cs +++ b/ARMeilleure/Translation/Cache/JitCache.cs @@ -25,6 +25,8 @@ namespace ARMeilleure.Translation.Cache private static readonly object _lock = new object(); private static bool _initialized; + public static IntPtr Base => _jitRegion.Pointer; + public static void Initialize(IJitMemoryAllocator allocator) { if (_initialized) return; diff --git a/ARMeilleure/Translation/Cache/JumpTable.cs b/ARMeilleure/Translation/Cache/JumpTable.cs deleted file mode 100644 index aa3b6caf3..000000000 --- a/ARMeilleure/Translation/Cache/JumpTable.cs +++ /dev/null @@ -1,279 +0,0 @@ -using ARMeilleure.Diagnostics; -using ARMeilleure.Memory; -using ARMeilleure.Translation.PTC; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.InteropServices; - -namespace ARMeilleure.Translation.Cache -{ - class JumpTable : IDisposable - { - // The jump table is a block of (guestAddress, hostAddress) function mappings. - // Each entry corresponds to one branch in a JIT compiled function. The entries are - // reserved specifically for each call. - // The Dependants dictionary can be used to update the hostAddress for any functions that change. - - public const int JumpTableStride = 16; // 8 byte guest address, 8 byte host address. - - private const int JumpTableSize = 1048576; - private const int JumpTableByteSize = JumpTableSize * JumpTableStride; - - // The dynamic table is also a block of (guestAddress, hostAddress) function mappings. - // The main difference is that indirect calls and jumps reserve _multiple_ entries on the table. - // These start out as all 0. When an indirect call is made, it tries to find the guest address on the table. - - // If we get to an empty address, the guestAddress is set to the call that we want. - - // If we get to a guestAddress that matches our own (or we just claimed it), the hostAddress is read. - // If it is non-zero, we immediately branch or call the host function. - // If it is 0, NativeInterface is called to find the rejited address of the call. - // If none is found, the hostAddress entry stays at 0. Otherwise, the new address is placed in the entry. - - // If the table size is exhausted and we didn't find our desired address, we fall back to requesting - // the function from the JIT. - - public const int DynamicTableElems = 1; - - public const int DynamicTableStride = DynamicTableElems * JumpTableStride; - - private const int DynamicTableSize = 1048576; - private const int DynamicTableByteSize = DynamicTableSize * DynamicTableStride; - - public const int DynamicEntryTag = 1 << 31; - - private readonly ReservedRegion _jumpRegion; - private readonly ReservedRegion _dynamicRegion; - - public IntPtr JumpPointer => _jumpRegion.Pointer; - public IntPtr DynamicPointer => _dynamicRegion.Pointer; - - public JumpTableEntryAllocator Table { get; } - public JumpTableEntryAllocator DynTable { get; } - - public ConcurrentDictionary Targets { get; } - public ConcurrentDictionary> Dependants { get; } // TODO: Attach to TranslatedFunction or a wrapper class. - public ConcurrentDictionary> Owners { get; } - - public JumpTable(IJitMemoryAllocator allocator) - { - _jumpRegion = new ReservedRegion(allocator, JumpTableByteSize); - _dynamicRegion = new ReservedRegion(allocator, DynamicTableByteSize); - - Table = new JumpTableEntryAllocator(); - DynTable = new JumpTableEntryAllocator(); - - Targets = new ConcurrentDictionary(); - Dependants = new ConcurrentDictionary>(); - Owners = new ConcurrentDictionary>(); - - Symbols.Add((ulong)_jumpRegion.Pointer.ToInt64(), JumpTableByteSize, JumpTableStride, "JMP_TABLE"); - Symbols.Add((ulong)_dynamicRegion.Pointer.ToInt64(), DynamicTableByteSize, DynamicTableStride, "DYN_TABLE"); - } - - public void Initialize(PtcJumpTable ptcJumpTable, ConcurrentDictionary funcs) - { - foreach (ulong guestAddress in ptcJumpTable.Targets) - { - if (funcs.TryGetValue(guestAddress, out TranslatedFunction func)) - { - Targets.TryAdd(guestAddress, func); - } - else - { - throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{guestAddress:X16})"); - } - } - - foreach (var kv in ptcJumpTable.Dependants) - { - Dependants.TryAdd(kv.Key, new List(kv.Value)); - } - - foreach (var kv in ptcJumpTable.Owners) - { - Owners.TryAdd(kv.Key, new List(kv.Value)); - } - } - - public void RegisterFunction(ulong address, TranslatedFunction func) - { - Targets.AddOrUpdate(address, func, (key, oldFunc) => func); - long funcPtr = func.FuncPtr.ToInt64(); - - // Update all jump table entries that target this address. - if (Dependants.TryGetValue(address, out List myDependants)) - { - lock (myDependants) - { - foreach (int entry in myDependants) - { - IntPtr addr = GetEntryAddressJumpTable(entry); - - Marshal.WriteInt64(addr, 8, funcPtr); - } - } - } - } - - public int ReserveTableEntry(ulong ownerGuestAddress, ulong address, bool isJump) - { - int entry = Table.AllocateEntry(); - - ExpandIfNeededJumpTable(entry); - - // Is the address we have already registered? If so, put the function address in the jump table. - // If not, it will point to the direct call stub. - long value = DirectCallStubs.DirectCallStub(isJump).ToInt64(); - if (Targets.TryGetValue(address, out TranslatedFunction func)) - { - value = func.FuncPtr.ToInt64(); - } - - // Make sure changes to the function at the target address update this jump table entry. - List targetDependants = Dependants.GetOrAdd(address, (addr) => new List()); - lock (targetDependants) - { - targetDependants.Add(entry); - } - - // Keep track of ownership for jump table entries. - List ownerEntries = Owners.GetOrAdd(ownerGuestAddress, (addr) => new List()); - lock (ownerEntries) - { - ownerEntries.Add(entry); - } - - IntPtr addr = GetEntryAddressJumpTable(entry); - - Marshal.WriteInt64(addr, 0, (long)address); - Marshal.WriteInt64(addr, 8, value); - - return entry; - } - - public int ReserveDynamicEntry(ulong ownerGuestAddress, bool isJump) - { - int entry = DynTable.AllocateEntry(); - - ExpandIfNeededDynamicTable(entry); - - // Keep track of ownership for jump table entries. - List ownerEntries = Owners.GetOrAdd(ownerGuestAddress, (addr) => new List()); - lock (ownerEntries) - { - ownerEntries.Add(entry | DynamicEntryTag); - } - - // Initialize all host function pointers to the indirect call stub. - IntPtr addr = GetEntryAddressDynamicTable(entry); - long stubPtr = DirectCallStubs.IndirectCallStub(isJump).ToInt64(); - - for (int i = 0; i < DynamicTableElems; i++) - { - Marshal.WriteInt64(addr, i * JumpTableStride + 8, stubPtr); - } - - return entry; - } - - // For future use. - public void RemoveFunctionEntries(ulong guestAddress) - { - Targets.TryRemove(guestAddress, out _); - Dependants.TryRemove(guestAddress, out _); - - if (Owners.TryRemove(guestAddress, out List entries)) - { - foreach (int entry in entries) - { - if ((entry & DynamicEntryTag) == 0) - { - IntPtr addr = GetEntryAddressJumpTable(entry); - - Marshal.WriteInt64(addr, 0, 0L); - Marshal.WriteInt64(addr, 8, 0L); - - Table.FreeEntry(entry); - } - else - { - IntPtr addr = GetEntryAddressDynamicTable(entry & ~DynamicEntryTag); - - for (int j = 0; j < DynamicTableElems; j++) - { - Marshal.WriteInt64(addr + j * JumpTableStride, 0, 0L); - Marshal.WriteInt64(addr + j * JumpTableStride, 8, 0L); - } - - DynTable.FreeEntry(entry & ~DynamicEntryTag); - } - } - } - } - - public void ExpandIfNeededJumpTable(int entry) - { - Debug.Assert(entry >= 0); - - if (entry < JumpTableSize) - { - _jumpRegion.ExpandIfNeeded((ulong)((entry + 1) * JumpTableStride)); - } - else - { - throw new OutOfMemoryException("JIT Direct Jump Table exhausted."); - } - } - - public void ExpandIfNeededDynamicTable(int entry) - { - Debug.Assert(entry >= 0); - - if (entry < DynamicTableSize) - { - _dynamicRegion.ExpandIfNeeded((ulong)((entry + 1) * DynamicTableStride)); - } - else - { - throw new OutOfMemoryException("JIT Dynamic Jump Table exhausted."); - } - } - - public IntPtr GetEntryAddressJumpTable(int entry) - { - Debug.Assert(Table.EntryIsValid(entry)); - - return _jumpRegion.Pointer + entry * JumpTableStride; - } - - public IntPtr GetEntryAddressDynamicTable(int entry) - { - Debug.Assert(DynTable.EntryIsValid(entry)); - - return _dynamicRegion.Pointer + entry * DynamicTableStride; - } - - public bool CheckEntryFromAddressJumpTable(IntPtr entryAddress) - { - int entry = Math.DivRem((int)((ulong)entryAddress - (ulong)_jumpRegion.Pointer), JumpTableStride, out int rem); - - return rem == 0 && Table.EntryIsValid(entry); - } - - public bool CheckEntryFromAddressDynamicTable(IntPtr entryAddress) - { - int entry = Math.DivRem((int)((ulong)entryAddress - (ulong)_dynamicRegion.Pointer), DynamicTableStride, out int rem); - - return rem == 0 && DynTable.EntryIsValid(entry); - } - - public void Dispose() - { - _jumpRegion.Dispose(); - _dynamicRegion.Dispose(); - } - } -} diff --git a/ARMeilleure/Translation/Cache/JumpTableEntryAllocator.cs b/ARMeilleure/Translation/Cache/JumpTableEntryAllocator.cs deleted file mode 100644 index ae2c075e1..000000000 --- a/ARMeilleure/Translation/Cache/JumpTableEntryAllocator.cs +++ /dev/null @@ -1,72 +0,0 @@ -using ARMeilleure.Common; -using System.Collections.Generic; -using System.Diagnostics; - -namespace ARMeilleure.Translation.Cache -{ - class JumpTableEntryAllocator - { - private readonly BitMap _bitmap; - private int _freeHint; - - public JumpTableEntryAllocator() - { - _bitmap = new BitMap(); - } - - public bool EntryIsValid(int entryIndex) - { - lock (_bitmap) - { - return _bitmap.IsSet(entryIndex); - } - } - - public void SetEntry(int entryIndex) - { - lock (_bitmap) - { - _bitmap.Set(entryIndex); - } - } - - public int AllocateEntry() - { - lock (_bitmap) - { - int entryIndex; - - if (!_bitmap.IsSet(_freeHint)) - { - entryIndex = _freeHint; - } - else - { - entryIndex = _bitmap.FindFirstUnset(); - } - - _freeHint = entryIndex + 1; - - bool wasSet = _bitmap.Set(entryIndex); - Debug.Assert(wasSet); - - return entryIndex; - } - } - - public void FreeEntry(int entryIndex) - { - lock (_bitmap) - { - _bitmap.Clear(entryIndex); - - _freeHint = entryIndex; - } - } - - public IEnumerable GetEntries() - { - return _bitmap; - } - } -} diff --git a/ARMeilleure/Translation/Delegates.cs b/ARMeilleure/Translation/Delegates.cs index a561d2653..f189eaf71 100644 --- a/ARMeilleure/Translation/Delegates.cs +++ b/ARMeilleure/Translation/Delegates.cs @@ -114,7 +114,6 @@ namespace ARMeilleure.Translation SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only. SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress))); - SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr))); SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only. SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidrEl0))); diff --git a/ARMeilleure/Translation/DirectCallStubs.cs b/ARMeilleure/Translation/DirectCallStubs.cs deleted file mode 100644 index 85af69018..000000000 --- a/ARMeilleure/Translation/DirectCallStubs.cs +++ /dev/null @@ -1,125 +0,0 @@ -using ARMeilleure.Instructions; -using ARMeilleure.IntermediateRepresentation; -using ARMeilleure.State; -using System; -using System.Diagnostics; -using System.Runtime.InteropServices; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; - -namespace ARMeilleure.Translation -{ - static class DirectCallStubs - { - private delegate long GuestFunction(IntPtr nativeContextPtr); - - private static IntPtr _directCallStubPtr; - private static IntPtr _directTailCallStubPtr; - private static IntPtr _indirectCallStubPtr; - private static IntPtr _indirectTailCallStubPtr; - - private static readonly object _lock = new object(); - private static bool _initialized; - - public static void InitializeStubs() - { - if (_initialized) return; - - lock (_lock) - { - if (_initialized) return; - - Translator.PreparePool(); - - _directCallStubPtr = Marshal.GetFunctionPointerForDelegate(GenerateDirectCallStub(false)); - _directTailCallStubPtr = Marshal.GetFunctionPointerForDelegate(GenerateDirectCallStub(true)); - _indirectCallStubPtr = Marshal.GetFunctionPointerForDelegate(GenerateIndirectCallStub(false)); - _indirectTailCallStubPtr = Marshal.GetFunctionPointerForDelegate(GenerateIndirectCallStub(true)); - - Translator.ResetPool(); - - Translator.DisposePools(); - - _initialized = true; - } - } - - public static IntPtr DirectCallStub(bool tailCall) - { - Debug.Assert(_initialized); - - return tailCall ? _directTailCallStubPtr : _directCallStubPtr; - } - - public static IntPtr IndirectCallStub(bool tailCall) - { - Debug.Assert(_initialized); - - return tailCall ? _indirectTailCallStubPtr : _indirectCallStubPtr; - } - - private static void EmitCall(EmitterContext context, Operand address, bool tailCall) - { - if (tailCall) - { - context.Tailcall(address, context.LoadArgument(OperandType.I64, 0)); - } - else - { - context.Return(context.Call(address, OperandType.I64, context.LoadArgument(OperandType.I64, 0))); - } - } - - /// - /// Generates a stub that is used to find function addresses. Used for direct calls when their jump table does not have the host address yet. - /// Takes a NativeContext like a translated guest function, and extracts the target address from the NativeContext. - /// When the target function is compiled in highCq, all table entries are updated to point to that function instead of this stub by the translator. - /// - private static GuestFunction GenerateDirectCallStub(bool tailCall) - { - EmitterContext context = new EmitterContext(); - - Operand nativeContextPtr = context.LoadArgument(OperandType.I64, 0); - - Operand address = context.Load(OperandType.I64, context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset()))); - - Operand functionAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address); - EmitCall(context, functionAddr, tailCall); - - ControlFlowGraph cfg = context.GetControlFlowGraph(); - - OperandType[] argTypes = new OperandType[] { OperandType.I64 }; - - return Compiler.Compile(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq); - } - - /// - /// Generates a stub that is used to find function addresses and add them to an indirect table. - /// Used for indirect calls entries (already claimed) when their jump table does not have the host address yet. - /// Takes a NativeContext like a translated guest function, and extracts the target indirect table entry from the NativeContext. - /// If the function we find is highCq, the entry in the table is updated to point to that function rather than this stub. - /// - private static GuestFunction GenerateIndirectCallStub(bool tailCall) - { - EmitterContext context = new EmitterContext(); - - Operand nativeContextPtr = context.LoadArgument(OperandType.I64, 0); - - Operand entryAddress = context.Load(OperandType.I64, context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset()))); - Operand address = context.Load(OperandType.I64, entryAddress); - - // We need to find the missing function. If the function is HighCq, then it replaces this stub in the indirect table. - // Either way, we call it afterwards. - Operand functionAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)), address, entryAddress); - - // Call and save the function. - EmitCall(context, functionAddr, tailCall); - - ControlFlowGraph cfg = context.GetControlFlowGraph(); - - OperandType[] argTypes = new OperandType[] { OperandType.I64 }; - - return Compiler.Compile(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq); - } - } -} diff --git a/ARMeilleure/Translation/DispatcherFunction.cs b/ARMeilleure/Translation/DispatcherFunction.cs new file mode 100644 index 000000000..e3ea21f67 --- /dev/null +++ b/ARMeilleure/Translation/DispatcherFunction.cs @@ -0,0 +1,6 @@ +using System; + +namespace ARMeilleure.Translation +{ + delegate void DispatcherFunction(IntPtr nativeContext, ulong startAddress); +} diff --git a/ARMeilleure/Translation/EmitterContext.cs b/ARMeilleure/Translation/EmitterContext.cs index cc2205cec..fbd9e6913 100644 --- a/ARMeilleure/Translation/EmitterContext.cs +++ b/ARMeilleure/Translation/EmitterContext.cs @@ -97,33 +97,18 @@ namespace ARMeilleure.Translation return Add(Instruction.ByteSwap, Local(op1.Type), op1); } - public Operand Call(MethodInfo info, params Operand[] callArgs) + public virtual Operand Call(MethodInfo info, params Operand[] callArgs) { - if (Ptc.State == PtcState.Disabled) - { - IntPtr funcPtr = Delegates.GetDelegateFuncPtr(info); + IntPtr funcPtr = Delegates.GetDelegateFuncPtr(info); - OperandType returnType = GetOperandType(info.ReturnType); + OperandType returnType = GetOperandType(info.ReturnType); - Symbols.Add((ulong)funcPtr.ToInt64(), info.Name); + Symbols.Add((ulong)funcPtr.ToInt64(), info.Name); - return Call(Const(funcPtr.ToInt64()), returnType, callArgs); - } - else - { - int index = Delegates.GetDelegateIndex(info); - - IntPtr funcPtr = Delegates.GetDelegateFuncPtrByIndex(index); - - OperandType returnType = GetOperandType(info.ReturnType); - - Symbols.Add((ulong)funcPtr.ToInt64(), info.Name); - - return Call(Const(funcPtr.ToInt64(), true, index), returnType, callArgs); - } + return Call(Const(funcPtr.ToInt64()), returnType, callArgs); } - private static OperandType GetOperandType(Type type) + protected static OperandType GetOperandType(Type type) { if (type == typeof(bool) || type == typeof(byte) || type == typeof(char) || type == typeof(short) || diff --git a/ARMeilleure/Translation/PTC/Ptc.cs b/ARMeilleure/Translation/PTC/Ptc.cs index ed4a003db..9f07ca011 100644 --- a/ARMeilleure/Translation/PTC/Ptc.cs +++ b/ARMeilleure/Translation/PTC/Ptc.cs @@ -28,7 +28,7 @@ namespace ARMeilleure.Translation.PTC private const string OuterHeaderMagicString = "PTCohd\0\0"; private const string InnerHeaderMagicString = "PTCihd\0\0"; - private const uint InternalVersion = 2289; //! To be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 2228; //! To be incremented manually for each change to the ARMeilleure project. private const string ActualDir = "0"; private const string BackupDir = "1"; @@ -36,10 +36,9 @@ namespace ARMeilleure.Translation.PTC private const string TitleIdTextDefault = "0000000000000000"; private const string DisplayVersionDefault = "0"; - internal const int PageTablePointerIndex = -1; // Must be a negative value. - internal const int JumpPointerIndex = -2; // Must be a negative value. - internal const int DynamicPointerIndex = -3; // Must be a negative value. - internal const int CountTableIndex = -4; // Must be a negative value. + internal static readonly Symbol PageTableSymbol = new(SymbolType.Special, 1); + internal static readonly Symbol CountTableSymbol = new(SymbolType.Special, 2); + internal static readonly Symbol DispatchStubSymbol = new(SymbolType.Special, 3); private const byte FillingByte = 0x00; private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest; @@ -59,8 +58,6 @@ namespace ARMeilleure.Translation.PTC private static bool _disposed; - internal static PtcJumpTable PtcJumpTable { get; private set; } - internal static string TitleIdText { get; private set; } internal static string DisplayVersion { get; private set; } @@ -89,8 +86,6 @@ namespace ARMeilleure.Translation.PTC _disposed = false; - PtcJumpTable = new PtcJumpTable(); - TitleIdText = TitleIdTextDefault; DisplayVersion = DisplayVersionDefault; @@ -348,20 +343,8 @@ namespace ARMeilleure.Translation.PTC return false; } - ReadOnlySpan ptcJumpTableBytes = new(stream.PositionPointer, innerHeader.PtcJumpTableLength); - stream.Seek(innerHeader.PtcJumpTableLength, SeekOrigin.Current); - Debug.Assert(stream.Position == stream.Length); - Hash128 ptcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes); - - if (innerHeader.PtcJumpTableHash != ptcJumpTableHash) - { - InvalidateCompressedStream(compressedStream); - - return false; - } - stream.Seek((long)Unsafe.SizeOf(), SeekOrigin.Begin); _infosStream.Write(infosBytes); @@ -375,8 +358,6 @@ namespace ARMeilleure.Translation.PTC _unwindInfosStream.Write(unwindInfosBytes); stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current); - PtcJumpTable = PtcJumpTable.Deserialize(stream); - Debug.Assert(stream.Position == stream.Length); } } @@ -422,7 +403,6 @@ namespace ARMeilleure.Translation.PTC finally { ResetCarriersIfNeeded(); - PtcJumpTable.ClearIfNeeded(); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; } @@ -442,7 +422,6 @@ namespace ARMeilleure.Translation.PTC innerHeader.CodesLength = _codesList.Length(); innerHeader.RelocsLength = (int)_relocsStream.Length; innerHeader.UnwindInfosLength = (int)_unwindInfosStream.Length; - innerHeader.PtcJumpTableLength = PtcJumpTable.GetSerializeSize(PtcJumpTable); OuterHeader outerHeader = new OuterHeader(); @@ -459,8 +438,7 @@ namespace ARMeilleure.Translation.PTC innerHeader.InfosLength + innerHeader.CodesLength + innerHeader.RelocsLength + - innerHeader.UnwindInfosLength + - innerHeader.PtcJumpTableLength; + innerHeader.UnwindInfosLength; outerHeader.SetHeaderHash(); @@ -486,16 +464,12 @@ namespace ARMeilleure.Translation.PTC ReadOnlySpan unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength); _unwindInfosStream.WriteTo(stream); - ReadOnlySpan ptcJumpTableBytes = new(stream.PositionPointer, innerHeader.PtcJumpTableLength); - PtcJumpTable.Serialize(stream, PtcJumpTable); - Debug.Assert(stream.Position == stream.Length); innerHeader.InfosHash = XXHash128.ComputeHash(infosBytes); innerHeader.CodesHash = XXHash128.ComputeHash(codesBytes); innerHeader.RelocsHash = XXHash128.ComputeHash(relocsBytes); innerHeader.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes); - innerHeader.PtcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes); innerHeader.SetHeaderHash(); @@ -505,7 +479,6 @@ namespace ARMeilleure.Translation.PTC translatedFuncsCount = GetEntriesCount(); ResetCarriersIfNeeded(); - PtcJumpTable.ClearIfNeeded(); using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate)) using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true)) @@ -545,11 +518,7 @@ namespace ARMeilleure.Translation.PTC } } - internal static void LoadTranslations( - ConcurrentDictionary funcs, - IMemoryManager memory, - JumpTable jumpTable, - EntryTable countTable) + internal static void LoadTranslations(Translator translator) { if (AreCarriersEmpty()) { @@ -580,7 +549,7 @@ namespace ARMeilleure.Translation.PTC continue; } - bool isEntryChanged = infoEntry.Hash != ComputeHash(memory, infoEntry.Address, infoEntry.GuestSize); + bool isEntryChanged = infoEntry.Hash != ComputeHash(translator.Memory, infoEntry.Address, infoEntry.GuestSize); if (isEntryChanged || (!infoEntry.HighCq && PtcProfiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) && value.HighCq)) { @@ -594,8 +563,6 @@ namespace ARMeilleure.Translation.PTC if (isEntryChanged) { - PtcJumpTable.Clean(infoEntry.Address); - Logger.Info?.Print(LogClass.Ptc, $"Invalidated translated function (address: 0x{infoEntry.Address:X16})"); } @@ -610,14 +577,16 @@ namespace ARMeilleure.Translation.PTC { RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount); - PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter); + PatchCode(translator, code, relocEntries, out callCounter); } UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader); TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq); - bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func); + translator.RegisterFunction(infoEntry.Address, func); + + bool isAddressUnique = translator.Functions.TryAdd(infoEntry.Address, func); Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique."); } @@ -630,12 +599,7 @@ namespace ARMeilleure.Translation.PTC throw new Exception("The length of a memory stream has changed, or its position has not reached or has exceeded its end."); } - jumpTable.Initialize(PtcJumpTable, funcs); - - PtcJumpTable.WriteJumpTable(jumpTable, funcs); - PtcJumpTable.WriteDynamicTable(jumpTable); - - Logger.Info?.Print(LogClass.Ptc, $"{funcs.Count} translated functions loaded"); + Logger.Info?.Print(LogClass.Ptc, $"{translator.Functions.Count} translated functions loaded"); } private static int GetEntriesCount() @@ -676,56 +640,63 @@ namespace ARMeilleure.Translation.PTC for (int i = 0; i < relocEntriesCount; i++) { int position = relocsReader.ReadInt32(); - int index = relocsReader.ReadInt32(); + SymbolType type = (SymbolType)relocsReader.ReadByte(); + ulong value = relocsReader.ReadUInt64(); - relocEntries[i] = new RelocEntry(position, index); + relocEntries[i] = new RelocEntry(position, new Symbol(type, value)); } return relocEntries; } - private static void PatchCode( - Span code, - RelocEntry[] relocEntries, - IntPtr pageTablePointer, - JumpTable jumpTable, - EntryTable countTable, - out Counter callCounter) + private static void PatchCode(Translator translator, Span code, RelocEntry[] relocEntries, out Counter callCounter) { callCounter = null; foreach (RelocEntry relocEntry in relocEntries) { - ulong imm; + IntPtr? imm = null; + Symbol symbol = relocEntry.Symbol; - if (relocEntry.Index == PageTablePointerIndex) + if (symbol.Type == SymbolType.FunctionTable) { - imm = (ulong)pageTablePointer.ToInt64(); - } - else if (relocEntry.Index == JumpPointerIndex) - { - imm = (ulong)jumpTable.JumpPointer.ToInt64(); - } - else if (relocEntry.Index == DynamicPointerIndex) - { - imm = (ulong)jumpTable.DynamicPointer.ToInt64(); - } - else if (relocEntry.Index == CountTableIndex) - { - callCounter = new Counter(countTable); + ulong guestAddress = symbol.Value; - unsafe { imm = (ulong)Unsafe.AsPointer(ref callCounter.Value); } + if (translator.FunctionTable.IsValid(guestAddress)) + { + unsafe { imm = (IntPtr)Unsafe.AsPointer(ref translator.FunctionTable.GetValue(guestAddress)); } + } } - else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr)) + else if (symbol.Type == SymbolType.DelegateTable) { - imm = (ulong)funcPtr.ToInt64(); + int index = (int)symbol.Value; + + if (Delegates.TryGetDelegateFuncPtrByIndex(index, out IntPtr funcPtr)) + { + imm = funcPtr; + } } - else + else if (symbol == PageTableSymbol) + { + imm = translator.Memory.PageTablePointer; + } + else if (symbol == CountTableSymbol) + { + callCounter = new Counter(translator.CountTable); + + unsafe { imm = (IntPtr)Unsafe.AsPointer(ref callCounter.Value); } + } + else if (symbol == DispatchStubSymbol) + { + imm = translator.Stubs.DispatchStub; + } + + if (imm == null) { throw new Exception($"Unexpected reloc entry {relocEntry}."); } - BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), imm); + BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), (ulong)imm.Value); } } @@ -798,13 +769,9 @@ namespace ARMeilleure.Translation.PTC } } - internal static void MakeAndSaveTranslations( - ConcurrentDictionary funcs, - IMemoryManager memory, - JumpTable jumpTable, - EntryTable countTable) + internal static void MakeAndSaveTranslations(Translator translator) { - var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs); + var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(translator.Functions); _translateCount = 0; _translateTotalCount = profiledFuncsToTranslate.Count; @@ -814,7 +781,6 @@ namespace ARMeilleure.Translation.PTC if (_translateTotalCount == 0 || degreeOfParallelism == 0) { ResetCarriersIfNeeded(); - PtcJumpTable.ClearIfNeeded(); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; @@ -844,19 +810,16 @@ namespace ARMeilleure.Translation.PTC Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); - TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.funcProfile.Mode, item.funcProfile.HighCq); + TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq); - bool isAddressUnique = funcs.TryAdd(address, func); + bool isAddressUnique = translator.Functions.TryAdd(address, func); Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique."); - if (func.HighCq) - { - jumpTable.RegisterFunction(address, func); - } - Interlocked.Increment(ref _translateCount); + translator.RegisterFunction(address, func); + if (State != PtcState.Enabled) { break; @@ -888,11 +851,6 @@ namespace ARMeilleure.Translation.PTC Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}"); - PtcJumpTable.Initialize(jumpTable); - - PtcJumpTable.ReadJumpTable(jumpTable); - PtcJumpTable.ReadDynamicTable(jumpTable); - Thread preSaveThread = new Thread(PreSave); preSaveThread.IsBackground = true; preSaveThread.Start(); @@ -1022,13 +980,11 @@ namespace ARMeilleure.Translation.PTC public long CodesLength; public int RelocsLength; public int UnwindInfosLength; - public int PtcJumpTableLength; public Hash128 InfosHash; public Hash128 CodesHash; public Hash128 RelocsHash; public Hash128 UnwindInfosHash; - public Hash128 PtcJumpTableHash; public Hash128 HeaderHash; diff --git a/ARMeilleure/Translation/PTC/PtcInfo.cs b/ARMeilleure/Translation/PTC/PtcInfo.cs index 920469c73..b28b2dc1f 100644 --- a/ARMeilleure/Translation/PTC/PtcInfo.cs +++ b/ARMeilleure/Translation/PTC/PtcInfo.cs @@ -30,7 +30,8 @@ namespace ARMeilleure.Translation.PTC public void WriteRelocEntry(RelocEntry relocEntry) { _relocWriter.Write((int)relocEntry.Position); - _relocWriter.Write((int)relocEntry.Index); + _relocWriter.Write((byte)relocEntry.Symbol.Type); + _relocWriter.Write((ulong)relocEntry.Symbol.Value); RelocEntriesCount++; } diff --git a/ARMeilleure/Translation/PTC/PtcJumpTable.cs b/ARMeilleure/Translation/PTC/PtcJumpTable.cs deleted file mode 100644 index 67719623e..000000000 --- a/ARMeilleure/Translation/PTC/PtcJumpTable.cs +++ /dev/null @@ -1,350 +0,0 @@ -using ARMeilleure.Translation.Cache; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Runtime.InteropServices; - -using static ARMeilleure.Translation.PTC.PtcFormatter; - -namespace ARMeilleure.Translation.PTC -{ - class PtcJumpTable - { - [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 16*/)] - public struct TableEntry - { - public int EntryIndex; - public long GuestAddress; - public TAddress HostAddress; - - public TableEntry(int entryIndex, long guestAddress, TAddress hostAddress) - { - EntryIndex = entryIndex; - GuestAddress = guestAddress; - HostAddress = hostAddress; - } - } - - public enum DirectHostAddress : int - { - CallStub = 0, - TailCallStub = 1, - Host = 2 - } - - public enum IndirectHostAddress : int - { - CallStub = 0, - TailCallStub = 1 - } - - private readonly List> _jumpTable; - private readonly List> _dynamicTable; - - public List Targets { get; } - public Dictionary> Dependants { get; } - public Dictionary> Owners { get; } - - public PtcJumpTable() - { - _jumpTable = new List>(); - _dynamicTable = new List>(); - - Targets = new List(); - Dependants = new Dictionary>(); - Owners = new Dictionary>(); - } - - public PtcJumpTable( - List> jumpTable, List> dynamicTable, - List targets, Dictionary> dependants, Dictionary> owners) - { - _jumpTable = jumpTable; - _dynamicTable = dynamicTable; - - Targets = targets; - Dependants = dependants; - Owners = owners; - } - - public static PtcJumpTable Deserialize(Stream stream) - { - var jumpTable = DeserializeList>(stream); - var dynamicTable = DeserializeList>(stream); - - var targets = DeserializeList(stream); - var dependants = DeserializeDictionary>(stream, (stream) => DeserializeList(stream)); - var owners = DeserializeDictionary>(stream, (stream) => DeserializeList(stream)); - - return new PtcJumpTable(jumpTable, dynamicTable, targets, dependants, owners); - } - - public static int GetSerializeSize(PtcJumpTable ptcJumpTable) - { - int size = 0; - - size += GetSerializeSizeList(ptcJumpTable._jumpTable); - size += GetSerializeSizeList(ptcJumpTable._dynamicTable); - - size += GetSerializeSizeList(ptcJumpTable.Targets); - size += GetSerializeSizeDictionary(ptcJumpTable.Dependants, (list) => GetSerializeSizeList(list)); - size += GetSerializeSizeDictionary(ptcJumpTable.Owners, (list) => GetSerializeSizeList(list)); - - return size; - } - - public static void Serialize(Stream stream, PtcJumpTable ptcJumpTable) - { - SerializeList(stream, ptcJumpTable._jumpTable); - SerializeList(stream, ptcJumpTable._dynamicTable); - - SerializeList(stream, ptcJumpTable.Targets); - SerializeDictionary(stream, ptcJumpTable.Dependants, (stream, list) => SerializeList(stream, list)); - SerializeDictionary(stream, ptcJumpTable.Owners, (stream, list) => SerializeList(stream, list)); - } - - public void Initialize(JumpTable jumpTable) - { - Targets.Clear(); - - foreach (ulong guestAddress in jumpTable.Targets.Keys) - { - Targets.Add(guestAddress); - } - - Dependants.Clear(); - - foreach (var kv in jumpTable.Dependants) - { - Dependants.Add(kv.Key, new List(kv.Value)); - } - - Owners.Clear(); - - foreach (var kv in jumpTable.Owners) - { - Owners.Add(kv.Key, new List(kv.Value)); - } - } - - public void Clean(ulong guestAddress) - { - if (Owners.TryGetValue(guestAddress, out List entries)) - { - foreach (int entry in entries) - { - if ((entry & JumpTable.DynamicEntryTag) == 0) - { - int removed = _jumpTable.RemoveAll(tableEntry => tableEntry.EntryIndex == entry); - - Debug.Assert(removed == 1); - } - else - { - if (JumpTable.DynamicTableElems > 1) - { - throw new NotSupportedException(); - } - - int removed = _dynamicTable.RemoveAll(tableEntry => tableEntry.EntryIndex == (entry & ~JumpTable.DynamicEntryTag)); - - Debug.Assert(removed == 1); - } - } - } - - Targets.Remove(guestAddress); - Dependants.Remove(guestAddress); - Owners.Remove(guestAddress); - } - - public void ClearIfNeeded() - { - if (_jumpTable.Count == 0 && _dynamicTable.Count == 0 && - Targets.Count == 0 && Dependants.Count == 0 && Owners.Count == 0) - { - return; - } - - _jumpTable.Clear(); - _jumpTable.TrimExcess(); - _dynamicTable.Clear(); - _dynamicTable.TrimExcess(); - - Targets.Clear(); - Targets.TrimExcess(); - Dependants.Clear(); - Dependants.TrimExcess(); - Owners.Clear(); - Owners.TrimExcess(); - } - - public void WriteJumpTable(JumpTable jumpTable, ConcurrentDictionary funcs) - { - // Writes internal state to jump table in-memory, after PtcJumpTable was deserialized. - - foreach (var tableEntry in _jumpTable) - { - long guestAddress = tableEntry.GuestAddress; - DirectHostAddress directHostAddress = tableEntry.HostAddress; - - long hostAddress; - - if (directHostAddress == DirectHostAddress.CallStub) - { - hostAddress = DirectCallStubs.DirectCallStub(false).ToInt64(); - } - else if (directHostAddress == DirectHostAddress.TailCallStub) - { - hostAddress = DirectCallStubs.DirectCallStub(true).ToInt64(); - } - else if (directHostAddress == DirectHostAddress.Host) - { - if (funcs.TryGetValue((ulong)guestAddress, out TranslatedFunction func)) - { - hostAddress = func.FuncPtr.ToInt64(); - } - else - { - if (!PtcProfiler.ProfiledFuncs.TryGetValue((ulong)guestAddress, out var value) || !value.HighCq) - { - throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{(ulong)guestAddress:X16})"); - } - - hostAddress = 0L; - } - } - else - { - throw new InvalidOperationException(nameof(directHostAddress)); - } - - int entry = tableEntry.EntryIndex; - - jumpTable.Table.SetEntry(entry); - jumpTable.ExpandIfNeededJumpTable(entry); - - IntPtr addr = jumpTable.GetEntryAddressJumpTable(entry); - - Marshal.WriteInt64(addr, 0, guestAddress); - Marshal.WriteInt64(addr, 8, hostAddress); - } - } - - public void WriteDynamicTable(JumpTable jumpTable) - { - // Writes internal state to jump table in-memory, after PtcJumpTable was deserialized. - - if (JumpTable.DynamicTableElems > 1) - { - throw new NotSupportedException(); - } - - foreach (var tableEntry in _dynamicTable) - { - long guestAddress = tableEntry.GuestAddress; - IndirectHostAddress indirectHostAddress = tableEntry.HostAddress; - - long hostAddress; - - if (indirectHostAddress == IndirectHostAddress.CallStub) - { - hostAddress = DirectCallStubs.IndirectCallStub(false).ToInt64(); - } - else if (indirectHostAddress == IndirectHostAddress.TailCallStub) - { - hostAddress = DirectCallStubs.IndirectCallStub(true).ToInt64(); - } - else - { - throw new InvalidOperationException(nameof(indirectHostAddress)); - } - - int entry = tableEntry.EntryIndex; - - jumpTable.DynTable.SetEntry(entry); - jumpTable.ExpandIfNeededDynamicTable(entry); - - IntPtr addr = jumpTable.GetEntryAddressDynamicTable(entry); - - Marshal.WriteInt64(addr, 0, guestAddress); - Marshal.WriteInt64(addr, 8, hostAddress); - } - } - - public void ReadJumpTable(JumpTable jumpTable) - { - // Reads in-memory jump table state and store internally for PtcJumpTable serialization. - - _jumpTable.Clear(); - - IEnumerable entries = jumpTable.Table.GetEntries(); - - foreach (int entry in entries) - { - IntPtr addr = jumpTable.GetEntryAddressJumpTable(entry); - - long guestAddress = Marshal.ReadInt64(addr, 0); - long hostAddress = Marshal.ReadInt64(addr, 8); - - DirectHostAddress directHostAddress; - - if (hostAddress == DirectCallStubs.DirectCallStub(false).ToInt64()) - { - directHostAddress = DirectHostAddress.CallStub; - } - else if (hostAddress == DirectCallStubs.DirectCallStub(true).ToInt64()) - { - directHostAddress = DirectHostAddress.TailCallStub; - } - else - { - directHostAddress = DirectHostAddress.Host; - } - - _jumpTable.Add(new TableEntry(entry, guestAddress, directHostAddress)); - } - } - - public void ReadDynamicTable(JumpTable jumpTable) - { - // Reads in-memory jump table state and store internally for PtcJumpTable serialization. - - if (JumpTable.DynamicTableElems > 1) - { - throw new NotSupportedException(); - } - - _dynamicTable.Clear(); - - IEnumerable entries = jumpTable.DynTable.GetEntries(); - - foreach (int entry in entries) - { - IntPtr addr = jumpTable.GetEntryAddressDynamicTable(entry); - - long guestAddress = Marshal.ReadInt64(addr, 0); - long hostAddress = Marshal.ReadInt64(addr, 8); - - IndirectHostAddress indirectHostAddress; - - if (hostAddress == DirectCallStubs.IndirectCallStub(false).ToInt64()) - { - indirectHostAddress = IndirectHostAddress.CallStub; - } - else if (hostAddress == DirectCallStubs.IndirectCallStub(true).ToInt64()) - { - indirectHostAddress = IndirectHostAddress.TailCallStub; - } - else - { - throw new InvalidOperationException($"({nameof(hostAddress)} = 0x{hostAddress:X16})"); - } - - _dynamicTable.Add(new TableEntry(entry, guestAddress, indirectHostAddress)); - } - } - } -} \ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/RelocEntry.cs b/ARMeilleure/Translation/PTC/RelocEntry.cs index 52d73db8b..545612d01 100644 --- a/ARMeilleure/Translation/PTC/RelocEntry.cs +++ b/ARMeilleure/Translation/PTC/RelocEntry.cs @@ -2,20 +2,20 @@ namespace ARMeilleure.Translation.PTC { struct RelocEntry { - public const int Stride = 8; // Bytes. + public const int Stride = 13; // Bytes. public int Position; - public int Index; + public Symbol Symbol; - public RelocEntry(int position, int index) + public RelocEntry(int position, Symbol symbol) { Position = position; - Index = index; + Symbol = symbol; } public override string ToString() { - return $"({nameof(Position)} = {Position}, {nameof(Index)} = {Index})"; + return $"({nameof(Position)} = {Position}, {nameof(Symbol)} = {Symbol})"; } } } \ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/Symbol.cs b/ARMeilleure/Translation/PTC/Symbol.cs new file mode 100644 index 000000000..f9d67742b --- /dev/null +++ b/ARMeilleure/Translation/PTC/Symbol.cs @@ -0,0 +1,100 @@ +using System; + +namespace ARMeilleure.Translation.PTC +{ + /// + /// Represents a symbol. + /// + struct Symbol + { + private readonly ulong _value; + + /// + /// Gets the of the . + /// + public SymbolType Type { get; } + + /// + /// Gets the value of the . + /// + /// is + public ulong Value + { + get + { + if (Type == SymbolType.None) + { + ThrowSymbolNone(); + } + + return _value; + } + } + + /// + /// Initializes a new instance of the structure with the specified and value. + /// + /// Type of symbol + /// Value of symbol + public Symbol(SymbolType type, ulong value) + { + (Type, _value) = (type, value); + } + + /// + /// Determines if the specified instances are equal. + /// + /// First instance + /// Second instance + /// if equal; otherwise + public static bool operator ==(Symbol a, Symbol b) + { + return a.Equals(b); + } + + /// + /// Determines if the specified instances are not equal. + /// + /// First instance + /// Second instance + /// if not equal; otherwise + /// + public static bool operator !=(Symbol a, Symbol b) + { + return !(a == b); + } + + /// + /// Determines if the specified is equal to this instance. + /// + /// Other instance + /// if equal; otherwise + public bool Equals(Symbol other) + { + return other.Type == Type && other._value == _value; + } + + /// + public override bool Equals(object obj) + { + return obj is Symbol sym && Equals(sym); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(Type, _value); + } + + /// + public override string ToString() + { + return $"{Type}:{_value}"; + } + + private static void ThrowSymbolNone() + { + throw new InvalidOperationException("Symbol refers to nothing."); + } + } +} diff --git a/ARMeilleure/Translation/PTC/SymbolType.cs b/ARMeilleure/Translation/PTC/SymbolType.cs new file mode 100644 index 000000000..cd7b6c1c6 --- /dev/null +++ b/ARMeilleure/Translation/PTC/SymbolType.cs @@ -0,0 +1,28 @@ +namespace ARMeilleure.Translation.PTC +{ + /// + /// Types of . + /// + enum SymbolType : byte + { + /// + /// Refers to nothing, i.e no symbol. + /// + None, + + /// + /// Refers to an entry in . + /// + DelegateTable, + + /// + /// Refers to an entry in . + /// + FunctionTable, + + /// + /// Refers to a special symbol which is handled by . + /// + Special + } +} diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs index eeeb517f6..2110a4e35 100644 --- a/ARMeilleure/Translation/Translator.cs +++ b/ARMeilleure/Translation/Translator.cs @@ -24,12 +24,27 @@ namespace ARMeilleure.Translation { public class Translator { - private const int CountTableCapacity = 4 * 1024 * 1024; + private static readonly AddressTable.Level[] Levels64Bit = + new AddressTable.Level[] + { + new(31, 17), + new(23, 8), + new(15, 8), + new( 7, 8), + new( 2, 5) + }; + + private static readonly AddressTable.Level[] Levels32Bit = + new AddressTable.Level[] + { + new(31, 17), + new(23, 8), + new(15, 8), + new( 7, 8), + new( 1, 6) + }; private readonly IJitMemoryAllocator _allocator; - private readonly IMemoryManager _memory; - - private readonly ConcurrentDictionary _funcs; private readonly ConcurrentQueue> _oldFuncs; private readonly ConcurrentDictionary _backgroundSet; @@ -37,21 +52,22 @@ namespace ARMeilleure.Translation private readonly AutoResetEvent _backgroundTranslatorEvent; private readonly ReaderWriterLock _backgroundTranslatorLock; - private JumpTable _jumpTable; - internal JumpTable JumpTable => _jumpTable; + internal ConcurrentDictionary Functions { get; } + internal AddressTable FunctionTable { get; } internal EntryTable CountTable { get; } + internal TranslatorStubs Stubs { get; } + internal IMemoryManager Memory { get; } private volatile int _threadCount; // FIXME: Remove this once the init logic of the emulator will be redone. public static readonly ManualResetEvent IsReadyForTranslation = new(false); - public Translator(IJitMemoryAllocator allocator, IMemoryManager memory) + public Translator(IJitMemoryAllocator allocator, IMemoryManager memory, bool for64Bits) { _allocator = allocator; - _memory = memory; + Memory = memory; - _funcs = new ConcurrentDictionary(); _oldFuncs = new ConcurrentQueue>(); _backgroundSet = new ConcurrentDictionary(); @@ -59,11 +75,14 @@ namespace ARMeilleure.Translation _backgroundTranslatorEvent = new AutoResetEvent(false); _backgroundTranslatorLock = new ReaderWriterLock(); - CountTable = new EntryTable(); - JitCache.Initialize(allocator); - DirectCallStubs.InitializeStubs(); + CountTable = new EntryTable(); + Functions = new ConcurrentDictionary(); + FunctionTable = new AddressTable(for64Bits ? Levels64Bit : Levels32Bit); + Stubs = new TranslatorStubs(this); + + FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub; if (memory.Type.IsHostMapped()) { @@ -80,27 +99,21 @@ namespace ARMeilleure.Translation if (_backgroundStack.TryPop(out RejitRequest request) && _backgroundSet.TryRemove(request.Address, out _)) { - TranslatedFunction func = Translate( - _memory, - _jumpTable, - CountTable, - request.Address, - request.Mode, - highCq: true); + TranslatedFunction func = Translate(request.Address, request.Mode, highCq: true); - _funcs.AddOrUpdate(request.Address, func, (key, oldFunc) => + Functions.AddOrUpdate(request.Address, func, (key, oldFunc) => { EnqueueForDeletion(key, oldFunc); return func; }); - _jumpTable.RegisterFunction(request.Address, func); - if (PtcProfiler.Enabled) { PtcProfiler.UpdateEntry(request.Address, request.Mode, highCq: true); } + RegisterFunction(request.Address, func); + _backgroundTranslatorLock.ReleaseReaderLock(); } else @@ -120,14 +133,11 @@ namespace ARMeilleure.Translation { IsReadyForTranslation.WaitOne(); - Debug.Assert(_jumpTable == null); - _jumpTable = new JumpTable(_allocator); - if (Ptc.State == PtcState.Enabled) { - Debug.Assert(_funcs.Count == 0); - Ptc.LoadTranslations(_funcs, _memory, _jumpTable, CountTable); - Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable); + Debug.Assert(Functions.Count == 0); + Ptc.LoadTranslations(this); + Ptc.MakeAndSaveTranslations(this); } PtcProfiler.Start(); @@ -160,13 +170,20 @@ namespace ARMeilleure.Translation Statistics.InitializeTimer(); - NativeInterface.RegisterThread(context, _memory, this); + NativeInterface.RegisterThread(context, Memory, this); - do + if (Optimizations.UseUnmanagedDispatchLoop) { - address = ExecuteSingle(context, address); + Stubs.DispatchLoop(context.NativeContextPtr, address); + } + else + { + do + { + address = ExecuteSingle(context, address); + } + while (context.Running && address != 0); } - while (context.Running && address != 0); NativeInterface.UnregisterThread(); @@ -178,9 +195,8 @@ namespace ARMeilleure.Translation DisposePools(); - _jumpTable.Dispose(); - _jumpTable = null; - + Stubs.Dispose(); + FunctionTable.Dispose(); CountTable.Dispose(); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; @@ -202,40 +218,51 @@ namespace ARMeilleure.Translation internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode) { - if (!_funcs.TryGetValue(address, out TranslatedFunction func)) + if (!Functions.TryGetValue(address, out TranslatedFunction func)) { - func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false); + func = Translate(address, mode, highCq: false); - TranslatedFunction getFunc = _funcs.GetOrAdd(address, func); + TranslatedFunction oldFunc = Functions.GetOrAdd(address, func); - if (getFunc != func) + if (oldFunc != func) { JitCache.Unmap(func.FuncPtr); - func = getFunc; + func = oldFunc; } if (PtcProfiler.Enabled) { PtcProfiler.AddEntry(address, mode, highCq: false); } + + RegisterFunction(address, func); } return func; } - internal static TranslatedFunction Translate( - IMemoryManager memory, - JumpTable jumpTable, - EntryTable countTable, - ulong address, - ExecutionMode mode, - bool highCq) + internal void RegisterFunction(ulong guestAddress, TranslatedFunction func) { - var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User); + if (FunctionTable.IsValid(guestAddress) && (Optimizations.AllowLcqInFunctionTable || func.HighCq)) + { + Volatile.Write(ref FunctionTable.GetValue(guestAddress), (ulong)func.FuncPtr); + } + } + + internal TranslatedFunction Translate(ulong address, ExecutionMode mode, bool highCq) + { + var context = new ArmEmitterContext( + Memory, + CountTable, + FunctionTable, + Stubs, + address, + highCq, + mode: Aarch32Mode.User); Logger.StartPass(PassName.Decoding); - Block[] blocks = Decoder.Decode(memory, address, mode, highCq, singleBlock: false); + Block[] blocks = Decoder.Decode(Memory, address, mode, highCq, singleBlock: false); Logger.EndPass(PassName.Decoding); @@ -268,7 +295,7 @@ namespace ARMeilleure.Translation GuestFunction func; - if (Ptc.State == PtcState.Disabled) + if (!context.HasPtc) { func = Compiler.Compile(cfg, argTypes, OperandType.I64, options); @@ -282,7 +309,7 @@ namespace ARMeilleure.Translation ResetPool(highCq ? 1 : 0); - Hash128 hash = Ptc.ComputeHash(memory, address, funcSize); + Hash128 hash = Ptc.ComputeHash(Memory, address, funcSize); Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, hash, highCq, ptcInfo); } @@ -360,7 +387,11 @@ namespace ARMeilleure.Translation if (block.Exit) { - InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address)); + // Left option here as it may be useful if we need to return to managed rather than tail call in + // future. (eg. for debug) + bool useReturns = false; + + InstEmitFlowHelper.EmitVirtualJump(context, Const(block.Address), isReturn: useReturns); } else { @@ -416,7 +447,10 @@ namespace ARMeilleure.Translation Operand lblEnd = Label(); - Operand address = Const(ref counter.Value, Ptc.CountTableIndex); + Operand address = !context.HasPtc ? + Const(ref counter.Value) : + Const(ref counter.Value, Ptc.CountTableSymbol); + Operand curCount = context.Load(OperandType.I32, address); Operand count = context.Add(curCount, Const(1)); context.Store(address, count); @@ -477,14 +511,14 @@ namespace ARMeilleure.Translation // Ensure no attempt will be made to compile new functions due to rejit. ClearRejitQueue(allowRequeue: false); - foreach (var func in _funcs.Values) + foreach (var func in Functions.Values) { JitCache.Unmap(func.FuncPtr); func.CallCounter?.Dispose(); } - _funcs.Clear(); + Functions.Clear(); while (_oldFuncs.TryDequeue(out var kv)) { @@ -502,7 +536,7 @@ namespace ARMeilleure.Translation { while (_backgroundStack.TryPop(out var request)) { - if (_funcs.TryGetValue(request.Address, out var func) && func.CallCounter != null) + if (Functions.TryGetValue(request.Address, out var func) && func.CallCounter != null) { Volatile.Write(ref func.CallCounter.Value, 0); } diff --git a/ARMeilleure/Translation/TranslatorStubs.cs b/ARMeilleure/Translation/TranslatorStubs.cs new file mode 100644 index 000000000..aff2ac7e4 --- /dev/null +++ b/ARMeilleure/Translation/TranslatorStubs.cs @@ -0,0 +1,248 @@ +using ARMeilleure.Instructions; +using ARMeilleure.IntermediateRepresentation; +using ARMeilleure.State; +using ARMeilleure.Translation.Cache; +using System; +using System.Reflection; +using System.Runtime.InteropServices; +using static ARMeilleure.IntermediateRepresentation.OperandHelper; + +namespace ARMeilleure.Translation +{ + /// + /// Represents a stub manager. + /// + class TranslatorStubs : IDisposable + { + private static readonly Lazy _slowDispatchStub = new(GenerateSlowDispatchStub, isThreadSafe: true); + + private bool _disposed; + + private readonly Translator _translator; + private readonly Lazy _dispatchStub; + private readonly Lazy _dispatchLoop; + + /// + /// Gets the dispatch stub. + /// + /// instance was disposed + public IntPtr DispatchStub + { + get + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + return _dispatchStub.Value; + } + } + + /// + /// Gets the slow dispatch stub. + /// + /// instance was disposed + public IntPtr SlowDispatchStub + { + get + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + return _slowDispatchStub.Value; + } + } + + /// + /// Gets the dispatch loop function. + /// + /// instance was disposed + public DispatcherFunction DispatchLoop + { + get + { + if (_disposed) + { + throw new ObjectDisposedException(null); + } + + return _dispatchLoop.Value; + } + } + + /// + /// Initializes a new instance of the class with the specified + /// instance. + /// + /// instance to use + /// is null + public TranslatorStubs(Translator translator) + { + _translator = translator ?? throw new ArgumentNullException(nameof(translator)); + _dispatchStub = new(GenerateDispatchStub, isThreadSafe: true); + _dispatchLoop = new(GenerateDispatchLoop, isThreadSafe: true); + } + + /// + /// Releases all resources used by the instance. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases all unmanaged and optionally managed resources used by the instance. + /// + /// to dispose managed resources also; otherwise just unmanaged resouces + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (_dispatchStub.IsValueCreated) + { + JitCache.Unmap(_dispatchStub.Value); + } + + if (_dispatchLoop.IsValueCreated) + { + JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value)); + } + + _disposed = true; + } + } + + /// + /// Frees resources used by the instance. + /// + ~TranslatorStubs() + { + Dispose(false); + } + + /// + /// Generates a . + /// + /// Generated + private IntPtr GenerateDispatchStub() + { + var context = new EmitterContext(); + + Operand lblFallback = Label(); + Operand lblEnd = Label(); + + // Load the target guest address from the native context. + Operand nativeContext = context.LoadArgument(OperandType.I64, 0); + Operand guestAddress = context.Load(OperandType.I64, + context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset()))); + + // Check if guest address is within range of the AddressTable. + Operand masked = context.BitwiseAnd(guestAddress, Const(~_translator.FunctionTable.Mask)); + context.BranchIfTrue(lblFallback, masked); + + Operand index = null; + Operand page = Const((long)_translator.FunctionTable.Base); + + for (int i = 0; i < _translator.FunctionTable.Levels.Length; i++) + { + ref var level = ref _translator.FunctionTable.Levels[i]; + + // level.Mask is not used directly because it is more often bigger than 32-bits, so it will not + // be encoded as an immediate on x86's bitwise and operation. + Operand mask = Const(level.Mask >> level.Index); + + index = context.BitwiseAnd(context.ShiftRightUI(guestAddress, Const(level.Index)), mask); + + if (i < _translator.FunctionTable.Levels.Length - 1) + { + page = context.Load(OperandType.I64, context.Add(page, context.ShiftLeft(index, Const(3)))); + context.BranchIfFalse(lblFallback, page); + } + } + + Operand hostAddress; + Operand hostAddressAddr = context.Add(page, context.ShiftLeft(index, Const(3))); + hostAddress = context.Load(OperandType.I64, hostAddressAddr); + context.Tailcall(hostAddress, nativeContext); + + context.MarkLabel(lblFallback); + hostAddress = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), guestAddress); + context.Tailcall(hostAddress, nativeContext); + + var cfg = context.GetControlFlowGraph(); + var retType = OperandType.I64; + var argTypes = new[] { OperandType.I64 }; + + var func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq); + + return Marshal.GetFunctionPointerForDelegate(func); + } + + /// + /// Generates a . + /// + /// Generated + private static IntPtr GenerateSlowDispatchStub() + { + var context = new EmitterContext(); + + // Load the target guest address from the native context. + Operand nativeContext = context.LoadArgument(OperandType.I64, 0); + Operand guestAddress = context.Load(OperandType.I64, + context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset()))); + + MethodInfo getFuncAddress = typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)); + Operand hostAddress = context.Call(getFuncAddress, guestAddress); + context.Tailcall(hostAddress, nativeContext); + + var cfg = context.GetControlFlowGraph(); + var retType = OperandType.I64; + var argTypes = new[] { OperandType.I64 }; + + var func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq); + + return Marshal.GetFunctionPointerForDelegate(func); + } + + /// + /// Generates a function. + /// + /// function + private DispatcherFunction GenerateDispatchLoop() + { + var context = new EmitterContext(); + + Operand beginLbl = Label(); + Operand endLbl = Label(); + + Operand nativeContext = context.LoadArgument(OperandType.I64, 0); + Operand guestAddress = context.Copy( + context.AllocateLocal(OperandType.I64), + context.LoadArgument(OperandType.I64, 1)); + + Operand runningAddress = context.Add(nativeContext, Const((ulong)NativeContext.GetRunningOffset())); + Operand dispatchAddress = context.Add(nativeContext, Const((ulong)NativeContext.GetDispatchAddressOffset())); + + context.MarkLabel(beginLbl); + context.Store(dispatchAddress, guestAddress); + context.Copy(guestAddress, context.Call(Const((ulong)DispatchStub), OperandType.I64, nativeContext)); + context.BranchIfFalse(endLbl, guestAddress); + context.BranchIfFalse(endLbl, context.Load(OperandType.I32, runningAddress)); + context.Branch(beginLbl); + + context.MarkLabel(endLbl); + context.Return(); + + var cfg = context.GetControlFlowGraph(); + var retType = OperandType.None; + var argTypes = new[] { OperandType.I64, OperandType.I64 }; + + return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq); + } + } +} diff --git a/Ryujinx.Cpu/CpuContext.cs b/Ryujinx.Cpu/CpuContext.cs index 407353fdf..3e422546a 100644 --- a/Ryujinx.Cpu/CpuContext.cs +++ b/Ryujinx.Cpu/CpuContext.cs @@ -8,9 +8,9 @@ namespace Ryujinx.Cpu { private readonly Translator _translator; - public CpuContext(IMemoryManager memory) + public CpuContext(IMemoryManager memory, bool for64Bit) { - _translator = new Translator(new JitMemoryAllocator(), memory); + _translator = new Translator(new JitMemoryAllocator(), memory, for64Bit); memory.UnmapEvent += UnmapHandler; } diff --git a/Ryujinx.HLE/HOS/ArmProcessContext.cs b/Ryujinx.HLE/HOS/ArmProcessContext.cs index ae5fe601f..457d12181 100644 --- a/Ryujinx.HLE/HOS/ArmProcessContext.cs +++ b/Ryujinx.HLE/HOS/ArmProcessContext.cs @@ -3,7 +3,6 @@ using ARMeilleure.State; using Ryujinx.Cpu; using Ryujinx.HLE.HOS.Kernel.Process; using Ryujinx.Memory; -using System; namespace Ryujinx.HLE.HOS { @@ -14,7 +13,7 @@ namespace Ryujinx.HLE.HOS public IVirtualMemoryManager AddressSpace => _memoryManager; - public ArmProcessContext(T memoryManager) + public ArmProcessContext(T memoryManager, bool for64Bit) { if (memoryManager is IRefCounted rc) { @@ -22,7 +21,7 @@ namespace Ryujinx.HLE.HOS } _memoryManager = memoryManager; - _cpuContext = new CpuContext(memoryManager); + _cpuContext = new CpuContext(memoryManager, for64Bit); } public void Execute(ExecutionContext context, ulong codeAddress) diff --git a/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs index 14617cf22..04d06e1f9 100644 --- a/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs +++ b/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs @@ -9,19 +9,19 @@ namespace Ryujinx.HLE.HOS { class ArmProcessContextFactory : IProcessContextFactory { - public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler) + public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit) { MemoryManagerMode mode = context.Device.Configuration.MemoryManagerMode; switch (mode) { case MemoryManagerMode.SoftwarePageTable: - return new ArmProcessContext(new MemoryManager(addressSpaceSize, invalidAccessHandler)); + return new ArmProcessContext(new MemoryManager(addressSpaceSize, invalidAccessHandler), for64Bit); case MemoryManagerMode.HostMapped: case MemoryManagerMode.HostMappedUnsafe: bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe; - return new ArmProcessContext(new MemoryManagerHostMapped(addressSpaceSize, unsafeMode, invalidAccessHandler)); + return new ArmProcessContext(new MemoryManagerHostMapped(addressSpaceSize, unsafeMode, invalidAccessHandler), for64Bit); default: throw new ArgumentOutOfRangeException(); diff --git a/Ryujinx.HLE/HOS/Kernel/Process/IProcessContextFactory.cs b/Ryujinx.HLE/HOS/Kernel/Process/IProcessContextFactory.cs index e9fbf618a..fbd6c139d 100644 --- a/Ryujinx.HLE/HOS/Kernel/Process/IProcessContextFactory.cs +++ b/Ryujinx.HLE/HOS/Kernel/Process/IProcessContextFactory.cs @@ -1,10 +1,9 @@ -using Ryujinx.Cpu; -using Ryujinx.Memory; +using Ryujinx.Memory; namespace Ryujinx.HLE.HOS.Kernel.Process { interface IProcessContextFactory { - IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler); + IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit); } } diff --git a/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs b/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs index f2ba675f9..90cd01f01 100644 --- a/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs +++ b/Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs @@ -1049,7 +1049,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Process _ => 39 }; - Context = _contextFactory.Create(KernelContext, 1UL << addrSpaceBits, InvalidAccessHandler); + bool for64Bit = flags.HasFlag(ProcessCreationFlags.Is64Bit); + + Context = _contextFactory.Create(KernelContext, 1UL << addrSpaceBits, InvalidAccessHandler, for64Bit); // TODO: This should eventually be removed. // The GPU shouldn't depend on the CPU memory manager at all. diff --git a/Ryujinx.HLE/HOS/Kernel/Process/ProcessContextFactory.cs b/Ryujinx.HLE/HOS/Kernel/Process/ProcessContextFactory.cs index 29860b3bb..5920fe44d 100644 --- a/Ryujinx.HLE/HOS/Kernel/Process/ProcessContextFactory.cs +++ b/Ryujinx.HLE/HOS/Kernel/Process/ProcessContextFactory.cs @@ -1,11 +1,10 @@ -using Ryujinx.Cpu; -using Ryujinx.Memory; +using Ryujinx.Memory; namespace Ryujinx.HLE.HOS.Kernel.Process { class ProcessContextFactory : IProcessContextFactory { - public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler) + public IProcessContext Create(KernelContext context, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit) { return new ProcessContext(new AddressSpaceManager(addressSpaceSize)); } diff --git a/Ryujinx.Tests/Cpu/CpuTest.cs b/Ryujinx.Tests/Cpu/CpuTest.cs index e823ea17b..26f0d286c 100644 --- a/Ryujinx.Tests/Cpu/CpuTest.cs +++ b/Ryujinx.Tests/Cpu/CpuTest.cs @@ -1,3 +1,4 @@ +using ARMeilleure; using ARMeilleure.State; using ARMeilleure.Translation; using NUnit.Framework; @@ -60,7 +61,12 @@ namespace Ryujinx.Tests.Cpu _context = CpuContext.CreateExecutionContext(); Translator.IsReadyForTranslation.Set(); - _cpuContext = new CpuContext(_memory); + _cpuContext = new CpuContext(_memory, for64Bit: true); + + // Prevent registering LCQ functions in the FunctionTable to avoid initializing and populating the table, + // which improves test durations. + Optimizations.AllowLcqInFunctionTable = false; + Optimizations.UseUnmanagedDispatchLoop = false; if (_unicornAvailable) { diff --git a/Ryujinx.Tests/Cpu/CpuTest32.cs b/Ryujinx.Tests/Cpu/CpuTest32.cs index 380c86e85..5d24af39c 100644 --- a/Ryujinx.Tests/Cpu/CpuTest32.cs +++ b/Ryujinx.Tests/Cpu/CpuTest32.cs @@ -1,4 +1,5 @@ -using ARMeilleure.State; +using ARMeilleure; +using ARMeilleure.State; using ARMeilleure.Translation; using NUnit.Framework; using Ryujinx.Cpu; @@ -56,7 +57,12 @@ namespace Ryujinx.Tests.Cpu _context.IsAarch32 = true; Translator.IsReadyForTranslation.Set(); - _cpuContext = new CpuContext(_memory); + _cpuContext = new CpuContext(_memory, for64Bit: false); + + // Prevent registering LCQ functions in the FunctionTable to avoid initializing and populating the table, + // which improves test durations. + Optimizations.AllowLcqInFunctionTable = false; + Optimizations.UseUnmanagedDispatchLoop = false; if (_unicornAvailable) {