Implement virtual buffer dependencies (#6190)
* Implement virtual buffer copies * Introduce TranslateAndCreateMultiBuffersPhysicalOnly, use it for copy and clear * Rename VirtualBufferCache to VirtualRangeCache * Fix potential issue where virtual range could exist in the cache, without a physical buffer * Fix bug that could cause copy with negative size on CopyToDependantVirtualBuffer * Remove virtual copy back for SyncAction * GetData XML docs * Make field readonly * Fix virtual buffer modification tracking * Remove CopyFromDependantVirtualBuffers from ExternalFlush * Move things around a little to avoid perf impact - Inline null check for CopyFromDependantVirtualBuffers - Remove extra method call for SynchronizeMemoryWithVirtualCopyBack, prefer calling CopyFromDependantVirtualBuffers separately * Fix up XML doc --------- Co-authored-by: riperiperi <rhy3756547@hotmail.com>
This commit is contained in:
gdkchan
GitHub
parent
ba91f5d401
commit
167f50bbcd
5 changed files with 573 additions and 49 deletions
|
|
@ -5,6 +5,8 @@ using Ryujinx.Memory.Tracking;
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
{
|
||||
|
|
@ -65,6 +67,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
private readonly Action<ulong, ulong> _loadDelegate;
|
||||
private readonly Action<ulong, ulong> _modifiedDelegate;
|
||||
|
||||
private HashSet<MultiRangeBuffer> _virtualDependencies;
|
||||
private readonly ReaderWriterLockSlim _virtualDependenciesLock;
|
||||
|
||||
private int _sequenceNumber;
|
||||
|
||||
private readonly bool _useGranular;
|
||||
|
|
@ -152,6 +157,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
_externalFlushDelegate = new RegionSignal(ExternalFlush);
|
||||
_loadDelegate = new Action<ulong, ulong>(LoadRegion);
|
||||
_modifiedDelegate = new Action<ulong, ulong>(RegionModified);
|
||||
|
||||
_virtualDependenciesLock = new ReaderWriterLockSlim();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -220,6 +227,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </remarks>
|
||||
/// <param name="address">Start address of the range to synchronize</param>
|
||||
/// <param name="size">Size in bytes of the range to synchronize</param>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SynchronizeMemory(ulong address, ulong size)
|
||||
{
|
||||
if (_useGranular)
|
||||
|
|
@ -239,6 +247,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
else
|
||||
{
|
||||
_context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size));
|
||||
CopyToDependantVirtualBuffers();
|
||||
}
|
||||
|
||||
_sequenceNumber = _context.SequenceNumber;
|
||||
|
|
@ -460,6 +469,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
int offset = (int)(mAddress - Address);
|
||||
|
||||
_context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize));
|
||||
|
||||
CopyToDependantVirtualBuffers(mAddress, mSize);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -520,6 +531,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="dstOffset">The offset of the destination buffer to copy into</param>
|
||||
public void CopyTo(Buffer destination, int dstOffset)
|
||||
{
|
||||
CopyFromDependantVirtualBuffers();
|
||||
_context.Renderer.Pipeline.CopyBuffer(Handle, destination.Handle, 0, dstOffset, (int)Size);
|
||||
}
|
||||
|
||||
|
|
@ -536,7 +548,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
|
||||
|
||||
// TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
|
||||
_physicalMemory.WriteUntracked(address, data.Get());
|
||||
_physicalMemory.WriteUntracked(address, CopyFromDependantVirtualBuffers(data.Get(), address, size));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -617,6 +629,207 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
UnmappedSequence++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a virtual buffer dependency, indicating that a virtual buffer depends on data from this buffer.
|
||||
/// </summary>
|
||||
/// <param name="virtualBuffer">Dependant virtual buffer</param>
|
||||
public void AddVirtualDependency(MultiRangeBuffer virtualBuffer)
|
||||
{
|
||||
_virtualDependenciesLock.EnterWriteLock();
|
||||
|
||||
try
|
||||
{
|
||||
(_virtualDependencies ??= new()).Add(virtualBuffer);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_virtualDependenciesLock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a virtual buffer dependency, indicating that a virtual buffer no longer depends on data from this buffer.
|
||||
/// </summary>
|
||||
/// <param name="virtualBuffer">Dependant virtual buffer</param>
|
||||
public void RemoveVirtualDependency(MultiRangeBuffer virtualBuffer)
|
||||
{
|
||||
_virtualDependenciesLock.EnterWriteLock();
|
||||
|
||||
try
|
||||
{
|
||||
if (_virtualDependencies != null)
|
||||
{
|
||||
_virtualDependencies.Remove(virtualBuffer);
|
||||
|
||||
if (_virtualDependencies.Count == 0)
|
||||
{
|
||||
_virtualDependencies = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_virtualDependenciesLock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies the buffer data to all virtual buffers that depends on it.
|
||||
/// </summary>
|
||||
public void CopyToDependantVirtualBuffers()
|
||||
{
|
||||
CopyToDependantVirtualBuffers(Address, Size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies the buffer data inside the specifide range to all virtual buffers that depends on it.
|
||||
/// </summary>
|
||||
/// <param name="address">Address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
public void CopyToDependantVirtualBuffers(ulong address, ulong size)
|
||||
{
|
||||
if (_virtualDependencies != null)
|
||||
{
|
||||
foreach (var virtualBuffer in _virtualDependencies)
|
||||
{
|
||||
CopyToDependantVirtualBuffer(virtualBuffer, address, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies all modified ranges from all virtual buffers back into this buffer.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void CopyFromDependantVirtualBuffers()
|
||||
{
|
||||
if (_virtualDependencies != null)
|
||||
{
|
||||
CopyFromDependantVirtualBuffersImpl();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies all modified ranges from all virtual buffers back into this buffer.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
private void CopyFromDependantVirtualBuffersImpl()
|
||||
{
|
||||
foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber))
|
||||
{
|
||||
virtualBuffer.ConsumeModifiedRegion(this, (mAddress, mSize) =>
|
||||
{
|
||||
// Get offset inside both this and the virtual buffer.
|
||||
// Note that sometimes there is no right answer for the virtual offset,
|
||||
// as the same physical range might be mapped multiple times inside a virtual buffer.
|
||||
// We just assume it does not happen in practice as it can only be implemented correctly
|
||||
// when the host has support for proper sparse mapping.
|
||||
|
||||
ulong mEndAddress = mAddress + mSize;
|
||||
mAddress = Math.Max(mAddress, Address);
|
||||
mSize = Math.Min(mEndAddress, EndAddress) - mAddress;
|
||||
|
||||
int physicalOffset = (int)(mAddress - Address);
|
||||
int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize));
|
||||
|
||||
_context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)mSize);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies all overlapping modified ranges from all virtual buffers back into this buffer, and returns an updated span with the data.
|
||||
/// </summary>
|
||||
/// <param name="dataSpan">Span where the unmodified data will be taken from for the output</param>
|
||||
/// <param name="address">Address of the region to copy</param>
|
||||
/// <param name="size">Size of the region to copy in bytes</param>
|
||||
/// <returns>A span with <paramref name="dataSpan"/>, and the data for all modified ranges if any</returns>
|
||||
private ReadOnlySpan<byte> CopyFromDependantVirtualBuffers(ReadOnlySpan<byte> dataSpan, ulong address, ulong size)
|
||||
{
|
||||
_virtualDependenciesLock.EnterReadLock();
|
||||
|
||||
try
|
||||
{
|
||||
if (_virtualDependencies != null)
|
||||
{
|
||||
byte[] storage = dataSpan.ToArray();
|
||||
|
||||
foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber))
|
||||
{
|
||||
virtualBuffer.ConsumeModifiedRegion(address, size, (mAddress, mSize) =>
|
||||
{
|
||||
// Get offset inside both this and the virtual buffer.
|
||||
// Note that sometimes there is no right answer for the virtual offset,
|
||||
// as the same physical range might be mapped multiple times inside a virtual buffer.
|
||||
// We just assume it does not happen in practice as it can only be implemented correctly
|
||||
// when the host has support for proper sparse mapping.
|
||||
|
||||
ulong mEndAddress = mAddress + mSize;
|
||||
mAddress = Math.Max(mAddress, address);
|
||||
mSize = Math.Min(mEndAddress, address + size) - mAddress;
|
||||
|
||||
int physicalOffset = (int)(mAddress - Address);
|
||||
int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize));
|
||||
|
||||
_context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)size);
|
||||
virtualBuffer.GetData(storage.AsSpan().Slice((int)(mAddress - address), (int)mSize), virtualOffset, (int)mSize);
|
||||
});
|
||||
}
|
||||
|
||||
dataSpan = storage;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_virtualDependenciesLock.ExitReadLock();
|
||||
}
|
||||
|
||||
return dataSpan;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies the buffer data to the specified virtual buffer.
|
||||
/// </summary>
|
||||
/// <param name="virtualBuffer">Virtual buffer to copy the data into</param>
|
||||
public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer)
|
||||
{
|
||||
CopyToDependantVirtualBuffer(virtualBuffer, Address, Size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies the buffer data inside the given range to the specified virtual buffer.
|
||||
/// </summary>
|
||||
/// <param name="virtualBuffer">Virtual buffer to copy the data into</param>
|
||||
/// <param name="address">Address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer, ulong address, ulong size)
|
||||
{
|
||||
// Broadcast data to all ranges of the virtual buffer that are contained inside this buffer.
|
||||
|
||||
ulong lastOffset = 0;
|
||||
|
||||
while (virtualBuffer.TryGetPhysicalOffset(this, lastOffset, out ulong srcOffset, out ulong dstOffset, out ulong copySize))
|
||||
{
|
||||
ulong innerOffset = address - Address;
|
||||
ulong innerEndOffset = (address + size) - Address;
|
||||
|
||||
lastOffset = dstOffset + copySize;
|
||||
|
||||
// Clamp range to the specified range.
|
||||
ulong copySrcOffset = Math.Max(srcOffset, innerOffset);
|
||||
ulong copySrcEndOffset = Math.Min(innerEndOffset, srcOffset + copySize);
|
||||
|
||||
if (copySrcEndOffset > copySrcOffset)
|
||||
{
|
||||
copySize = copySrcEndOffset - copySrcOffset;
|
||||
dstOffset += copySrcOffset - srcOffset;
|
||||
srcOffset = copySrcOffset;
|
||||
|
||||
_context.Renderer.Pipeline.CopyBuffer(Handle, virtualBuffer.Handle, (int)srcOffset, (int)dstOffset, (int)copySize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Increments the buffer reference count.
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ using Ryujinx.Memory.Range;
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
{
|
||||
|
|
@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache;
|
||||
private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache;
|
||||
private bool _pruneCaches;
|
||||
private int _virtualModifiedSequenceNumber;
|
||||
|
||||
public event Action NotifyBuffersModified;
|
||||
|
||||
|
|
@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
|
||||
/// <summary>
|
||||
/// Performs address translation of the GPU virtual address, and creates
|
||||
/// new buffers, if needed, for the specified range.
|
||||
/// new physical and virtual buffers, if needed, for the specified range.
|
||||
/// </summary>
|
||||
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
|
||||
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
||||
|
|
@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
return new MultiRange(MemoryManager.PteUnmapped, size);
|
||||
}
|
||||
|
||||
bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
|
||||
|
||||
// Fast path not taken for non-contiguous ranges,
|
||||
// since multi-range buffers are not coalesced, so a buffer that covers
|
||||
// the entire cached range might not actually exist.
|
||||
if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
|
||||
if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
|
||||
range.Count == 1)
|
||||
{
|
||||
return range;
|
||||
|
|
@ -154,6 +154,50 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
return range;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs address translation of the GPU virtual address, and creates
|
||||
/// new physical buffers, if needed, for the specified range.
|
||||
/// </summary>
|
||||
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
|
||||
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
||||
/// <param name="size">Size in bytes of the buffer</param>
|
||||
/// <returns>Physical ranges of the buffer, after address translation</returns>
|
||||
public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size)
|
||||
{
|
||||
if (gpuVa == 0)
|
||||
{
|
||||
return new MultiRange(MemoryManager.PteUnmapped, size);
|
||||
}
|
||||
|
||||
// Fast path not taken for non-contiguous ranges,
|
||||
// since multi-range buffers are not coalesced, so a buffer that covers
|
||||
// the entire cached range might not actually exist.
|
||||
if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
|
||||
range.Count == 1)
|
||||
{
|
||||
return range;
|
||||
}
|
||||
|
||||
for (int i = 0; i < range.Count; i++)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(i);
|
||||
|
||||
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||
{
|
||||
if (range.Count > 1)
|
||||
{
|
||||
CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
CreateBuffer(subRange.Address, subRange.Size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return range;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new buffer for the specified range, if it does not yet exist.
|
||||
/// This can be used to ensure the existance of a buffer.
|
||||
|
|
@ -263,41 +307,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
}
|
||||
}
|
||||
|
||||
BufferRange[] storages = new BufferRange[range.Count];
|
||||
MultiRangeBuffer multiRangeBuffer;
|
||||
|
||||
MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
|
||||
|
||||
ulong alignmentMask = SparseBufferAlignmentSize - 1;
|
||||
|
||||
for (int i = 0; i < range.Count; i++)
|
||||
if (_context.Capabilities.SupportsSparseBuffer)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(i);
|
||||
BufferRange[] storages = new BufferRange[range.Count];
|
||||
|
||||
for (int i = 0; i < range.Count; i++)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(i);
|
||||
|
||||
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||
{
|
||||
ulong endAddress = subRange.Address + subRange.Size;
|
||||
|
||||
ulong alignedAddress = subRange.Address & ~alignmentMask;
|
||||
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
|
||||
ulong alignedSize = alignedEndAddress - alignedAddress;
|
||||
|
||||
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
|
||||
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
|
||||
|
||||
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
|
||||
storages[i] = bufferRange;
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
|
||||
|
||||
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
|
||||
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
|
||||
}
|
||||
}
|
||||
|
||||
multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int i = 0; i < range.Count; i++)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(i);
|
||||
|
||||
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||
{
|
||||
ulong endAddress = subRange.Address + subRange.Size;
|
||||
|
||||
ulong alignedAddress = subRange.Address & ~alignmentMask;
|
||||
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
|
||||
ulong alignedSize = alignedEndAddress - alignedAddress;
|
||||
|
||||
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
|
||||
|
||||
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
|
||||
}
|
||||
}
|
||||
|
||||
multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges));
|
||||
|
||||
UpdateVirtualBufferDependencies(multiRangeBuffer);
|
||||
}
|
||||
|
||||
_multiRangeBuffers.Add(multiRangeBuffer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds two-way dependencies to all physical buffers contained within a given virtual buffer.
|
||||
/// </summary>
|
||||
/// <param name="virtualBuffer">Virtual buffer to have dependencies added</param>
|
||||
private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer)
|
||||
{
|
||||
virtualBuffer.ClearPhysicalDependencies();
|
||||
|
||||
ulong dstOffset = 0;
|
||||
|
||||
HashSet<Buffer> physicalBuffers = new();
|
||||
|
||||
for (int i = 0; i < virtualBuffer.Range.Count; i++)
|
||||
{
|
||||
MemoryRange subRange = virtualBuffer.Range.GetSubRange(i);
|
||||
|
||||
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||
{
|
||||
ulong endAddress = subRange.Address + subRange.Size;
|
||||
Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
|
||||
|
||||
ulong alignedAddress = subRange.Address & ~alignmentMask;
|
||||
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
|
||||
ulong alignedSize = alignedEndAddress - alignedAddress;
|
||||
|
||||
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
|
||||
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
|
||||
|
||||
storages[i] = bufferRange;
|
||||
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
|
||||
virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size);
|
||||
physicalBuffers.Add(buffer);
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
|
||||
|
||||
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
|
||||
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
|
||||
}
|
||||
dstOffset += subRange.Size;
|
||||
}
|
||||
|
||||
MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
|
||||
|
||||
_multiRangeBuffers.Add(multiRangeBuffer);
|
||||
foreach (var buffer in physicalBuffers)
|
||||
{
|
||||
buffer.CopyToDependantVirtualBuffer(virtualBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="size">Size in bytes of the copy</param>
|
||||
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
|
||||
{
|
||||
MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
|
||||
MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
|
||||
MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size);
|
||||
MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size);
|
||||
|
||||
if (srcRange.Count == 1 && dstRange.Count == 1)
|
||||
{
|
||||
|
|
@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
dstBuffer.ClearModified(dstAddress, size);
|
||||
memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer);
|
||||
}
|
||||
|
||||
dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// <param name="value">Value to be written into the buffer</param>
|
||||
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
|
||||
{
|
||||
MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
|
||||
MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size);
|
||||
|
||||
for (int index = 0; index < range.Count; index++)
|
||||
{
|
||||
|
|
@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
|
||||
|
||||
memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
|
||||
|
||||
buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
}
|
||||
}
|
||||
|
||||
if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer)
|
||||
{
|
||||
buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber);
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
|
@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
{
|
||||
buffer = _buffers.FindFirstOverlap(address, size);
|
||||
|
||||
buffer.CopyFromDependantVirtualBuffers();
|
||||
buffer.SynchronizeMemory(address, size);
|
||||
|
||||
if (write)
|
||||
|
|
@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
if (range.Count == 1)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(0);
|
||||
SynchronizeBufferRange(subRange.Address, subRange.Size);
|
||||
SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int index = 0; index < range.Count; index++)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(index);
|
||||
SynchronizeBufferRange(subRange.Address, subRange.Size);
|
||||
SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
/// <param name="address">Start address of the memory range</param>
|
||||
/// <param name="size">Size in bytes of the memory range</param>
|
||||
private void SynchronizeBufferRange(ulong address, ulong size)
|
||||
/// <param name="copyBackVirtual">Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified</param>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual)
|
||||
{
|
||||
if (size != 0)
|
||||
{
|
||||
Buffer buffer = _buffers.FindFirstOverlap(address, size);
|
||||
|
||||
if (copyBackVirtual)
|
||||
{
|
||||
buffer.CopyFromDependantVirtualBuffers();
|
||||
}
|
||||
|
||||
buffer.SynchronizeMemory(address, size);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,9 +40,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
internal PhysicalMemory Physical { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Virtual buffer cache.
|
||||
/// Virtual range cache.
|
||||
/// </summary>
|
||||
internal VirtualBufferCache VirtualBufferCache { get; }
|
||||
internal VirtualRangeCache VirtualRangeCache { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Cache of GPU counters.
|
||||
|
|
@ -56,12 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
internal MemoryManager(PhysicalMemory physicalMemory)
|
||||
{
|
||||
Physical = physicalMemory;
|
||||
VirtualBufferCache = new VirtualBufferCache(this);
|
||||
VirtualRangeCache = new VirtualRangeCache(this);
|
||||
CounterCache = new CounterCache();
|
||||
_pageTable = new ulong[PtLvl0Size][];
|
||||
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
|
||||
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
|
||||
MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler;
|
||||
MemoryUnmapped += VirtualRangeCache.MemoryUnmappedHandler;
|
||||
MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
using Ryujinx.Graphics.GAL;
|
||||
using Ryujinx.Memory.Range;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
{
|
||||
|
|
@ -21,12 +22,73 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
public MultiRange Range { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Ever increasing counter value indicating when the buffer was modified relative to other buffers.
|
||||
/// </summary>
|
||||
public int ModificationSequenceNumber { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Physical buffer dependency entry.
|
||||
/// </summary>
|
||||
private readonly struct PhysicalDependency
|
||||
{
|
||||
/// <summary>
|
||||
/// Physical buffer.
|
||||
/// </summary>
|
||||
public readonly Buffer PhysicalBuffer;
|
||||
|
||||
/// <summary>
|
||||
/// Offset of the range on the physical buffer.
|
||||
/// </summary>
|
||||
public readonly ulong PhysicalOffset;
|
||||
|
||||
/// <summary>
|
||||
/// Offset of the range on the virtual buffer.
|
||||
/// </summary>
|
||||
public readonly ulong VirtualOffset;
|
||||
|
||||
/// <summary>
|
||||
/// Size of the range.
|
||||
/// </summary>
|
||||
public readonly ulong Size;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new physical dependency.
|
||||
/// </summary>
|
||||
/// <param name="physicalBuffer">Physical buffer</param>
|
||||
/// <param name="physicalOffset">Offset of the range on the physical buffer</param>
|
||||
/// <param name="virtualOffset">Offset of the range on the virtual buffer</param>
|
||||
/// <param name="size">Size of the range</param>
|
||||
public PhysicalDependency(Buffer physicalBuffer, ulong physicalOffset, ulong virtualOffset, ulong size)
|
||||
{
|
||||
PhysicalBuffer = physicalBuffer;
|
||||
PhysicalOffset = physicalOffset;
|
||||
VirtualOffset = virtualOffset;
|
||||
Size = size;
|
||||
}
|
||||
}
|
||||
|
||||
private List<PhysicalDependency> _dependencies;
|
||||
private BufferModifiedRangeList _modifiedRanges = null;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the buffer.
|
||||
/// </summary>
|
||||
/// <param name="context">GPU context that the buffer belongs to</param>
|
||||
/// <param name="range">Range of memory where the data is mapped</param>
|
||||
/// <param name="storages">Backing memory for the buffers</param>
|
||||
public MultiRangeBuffer(GpuContext context, MultiRange range)
|
||||
{
|
||||
_context = context;
|
||||
Range = range;
|
||||
Handle = context.Renderer.CreateBuffer((int)range.GetSize());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the buffer.
|
||||
/// </summary>
|
||||
/// <param name="context">GPU context that the buffer belongs to</param>
|
||||
/// <param name="range">Range of memory where the data is mapped</param>
|
||||
/// <param name="storages">Backing memory for the buffer</param>
|
||||
public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
|
||||
{
|
||||
_context = context;
|
||||
|
|
@ -49,11 +111,134 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
return new BufferRange(Handle, offset, (int)range.GetSize());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes all physical buffer dependencies.
|
||||
/// </summary>
|
||||
public void ClearPhysicalDependencies()
|
||||
{
|
||||
_dependencies?.Clear();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a physical buffer dependency.
|
||||
/// </summary>
|
||||
/// <param name="buffer">Physical buffer to be added</param>
|
||||
/// <param name="rangeAddress">Address inside the physical buffer where the virtual buffer range is located</param>
|
||||
/// <param name="dstOffset">Offset inside the virtual buffer where the physical range is located</param>
|
||||
/// <param name="rangeSize">Size of the range in bytes</param>
|
||||
public void AddPhysicalDependency(Buffer buffer, ulong rangeAddress, ulong dstOffset, ulong rangeSize)
|
||||
{
|
||||
(_dependencies ??= new()).Add(new(buffer, rangeAddress - buffer.Address, dstOffset, rangeSize));
|
||||
buffer.AddVirtualDependency(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to get the physical range corresponding to the given physical buffer.
|
||||
/// </summary>
|
||||
/// <param name="buffer">Physical buffer</param>
|
||||
/// <param name="minimumVirtOffset">Minimum virtual offset that a range match can have</param>
|
||||
/// <param name="physicalOffset">Physical offset of the match</param>
|
||||
/// <param name="virtualOffset">Virtual offset of the match, always greater than or equal <paramref name="minimumVirtOffset"/></param>
|
||||
/// <param name="size">Size of the range match</param>
|
||||
/// <returns>True if a match was found for the given parameters, false otherwise</returns>
|
||||
public bool TryGetPhysicalOffset(Buffer buffer, ulong minimumVirtOffset, out ulong physicalOffset, out ulong virtualOffset, out ulong size)
|
||||
{
|
||||
physicalOffset = 0;
|
||||
virtualOffset = 0;
|
||||
size = 0;
|
||||
|
||||
if (_dependencies != null)
|
||||
{
|
||||
foreach (var dependency in _dependencies)
|
||||
{
|
||||
if (dependency.PhysicalBuffer == buffer && dependency.VirtualOffset >= minimumVirtOffset)
|
||||
{
|
||||
physicalOffset = dependency.PhysicalOffset;
|
||||
virtualOffset = dependency.VirtualOffset;
|
||||
size = dependency.Size;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a modified virtual memory range.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is only required when the host does not support sparse buffers, otherwise only physical buffers need to track modification.
|
||||
/// </remarks>
|
||||
/// <param name="range">Modified range</param>
|
||||
/// <param name="modifiedSequenceNumber">ModificationSequenceNumber</param>
|
||||
public void AddModifiedRegion(MultiRange range, int modifiedSequenceNumber)
|
||||
{
|
||||
_modifiedRanges ??= new(_context, null, null);
|
||||
|
||||
for (int i = 0; i < range.Count; i++)
|
||||
{
|
||||
MemoryRange subRange = range.GetSubRange(i);
|
||||
|
||||
_modifiedRanges.SignalModified(subRange.Address, subRange.Size);
|
||||
}
|
||||
|
||||
ModificationSequenceNumber = modifiedSequenceNumber;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="buffer"/>.
|
||||
/// </summary>
|
||||
/// <param name="buffer">Buffer to have its range checked</param>
|
||||
/// <param name="rangeAction">Action to perform for modified ranges</param>
|
||||
public void ConsumeModifiedRegion(Buffer buffer, Action<ulong, ulong> rangeAction)
|
||||
{
|
||||
ConsumeModifiedRegion(buffer.Address, buffer.Size, rangeAction);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="address"/> and <paramref name="size"/>.
|
||||
/// </summary>
|
||||
/// <param name="address">Address of the region to consume</param>
|
||||
/// <param name="size">Size of the region to consume</param>
|
||||
/// <param name="rangeAction">Action to perform for modified ranges</param>
|
||||
public void ConsumeModifiedRegion(ulong address, ulong size, Action<ulong, ulong> rangeAction)
|
||||
{
|
||||
if (_modifiedRanges != null)
|
||||
{
|
||||
_modifiedRanges.GetRanges(address, size, rangeAction);
|
||||
_modifiedRanges.Clear(address, size);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets data from the specified region of the buffer, and places it on <paramref name="output"/>.
|
||||
/// </summary>
|
||||
/// <param name="output">Span to put the data into</param>
|
||||
/// <param name="offset">Offset of the buffer to get the data from</param>
|
||||
/// <param name="size">Size of the data in bytes</param>
|
||||
public void GetData(Span<byte> output, int offset, int size)
|
||||
{
|
||||
using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, size);
|
||||
data.Get().CopyTo(output);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the host buffer.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_dependencies != null)
|
||||
{
|
||||
foreach (var dependency in _dependencies)
|
||||
{
|
||||
dependency.PhysicalBuffer.RemoveVirtualDependency(this);
|
||||
}
|
||||
|
||||
_dependencies = null;
|
||||
}
|
||||
|
||||
_context.Renderer.DeleteBuffer(Handle);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ using System.Threading;
|
|||
namespace Ryujinx.Graphics.Gpu.Memory
|
||||
{
|
||||
/// <summary>
|
||||
/// Virtual buffer cache.
|
||||
/// Virtual range cache.
|
||||
/// </summary>
|
||||
class VirtualBufferCache
|
||||
class VirtualRangeCache
|
||||
{
|
||||
private readonly MemoryManager _memoryManager;
|
||||
|
||||
|
|
@ -68,10 +68,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
private int _hasDeferredUnmaps;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the virtual buffer cache.
|
||||
/// Creates a new instance of the virtual range cache.
|
||||
/// </summary>
|
||||
/// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param>
|
||||
public VirtualBufferCache(MemoryManager memoryManager)
|
||||
/// <param name="memoryManager">Memory manager that the virtual range cache belongs to</param>
|
||||
public VirtualRangeCache(MemoryManager memoryManager)
|
||||
{
|
||||
_memoryManager = memoryManager;
|
||||
_virtualRanges = new RangeList<VirtualRange>();
|
||||
|
|
@ -102,10 +102,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
/// </summary>
|
||||
/// <param name="gpuVa">GPU virtual address to get the physical range from</param>
|
||||
/// <param name="size">Size in bytes of the region</param>
|
||||
/// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
|
||||
/// <param name="range">Physical range for the specified GPU virtual region</param>
|
||||
/// <returns>True if the range already existed, false if a new one was created and added</returns>
|
||||
public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range)
|
||||
public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range)
|
||||
{
|
||||
VirtualRange[] overlaps = _virtualRangeOverlaps;
|
||||
int overlapsCount;
|
||||
|
|
@ -158,7 +157,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
}
|
||||
else
|
||||
{
|
||||
found = true;
|
||||
found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range);
|
||||
range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
|
||||
}
|
||||
}
|
||||
|
|
@ -175,11 +174,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
|||
ShrinkOverlapsBufferIfNeeded();
|
||||
|
||||
// If the the range is not properly aligned for sparse mapping,
|
||||
// or if the host does not support sparse mapping, let's just
|
||||
// force it to a single range.
|
||||
// let's just force it to a single range.
|
||||
// This might cause issues in some applications that uses sparse
|
||||
// mappings.
|
||||
if (!IsSparseAligned(range) || !supportsSparse)
|
||||
if (!IsSparseAligned(range))
|
||||
{
|
||||
range = new MultiRange(range.GetSubRange(0).Address, size);
|
||||
}
|
||||
Loading…
Reference in a new issue