diff --git a/docs/bink2-bridge.md b/docs/bink2-bridge.md new file mode 100644 index 0000000..6ea35a5 --- /dev/null +++ b/docs/bink2-bridge.md @@ -0,0 +1,46 @@ + + +# Bink 2 bridge + +Demon's Souls plays Bink 2 (.bk2) files through a Bink implementation linked +directly into eboot.bin. It does not use libSceVideodec, therefore an HLE video +decoder cannot observe or replace those frames. + +SharpEmu observes successful guest .bk2 opens and, when a Bink bridge is +available, presents its decoded BGRA frames at the normal guest-flip boundary. +This preserves the game's own timing and lets the host Vulkan presenter display +the movie without trying to execute the PS5-specific Bink GPU decode path. + +Without an adapter, Bink movies are skipped by default: their open call returns +not-found so games that mark cinematics as optional progress to their next +state instead of waiting on an empty Bink GPU texture. + +Set SHARPEMU_BINK_MODE=dummy to retain the open and show a built-in, +non-decoded placeholder frame. This requires no SDK, but is a visual diagnostic +only; it does not decode the movie or alter its game logic. Set +SHARPEMU_BINK_MODE=native to force native bridge mode. + +## Supplying the adapter + +Bink 2 is proprietary. Obtain a compatible Mac Bink 2 SDK from RAD Game Tools, +then compile sharpemu_bink2_bridge.c against the SDK's bink.h and Mac library. +The adapter deliberately contains only a three-function C ABI so the managed +emulator never depends on RAD's private binary ABI. + +Place the resulting libsharpemu_bink2_bridge.dylib next to the SharpEmu +executable, or point to it explicitly: + + SHARPEMU_BINK2_BRIDGE=/absolute/path/libsharpemu_bink2_bridge.dylib \ + ./SharpEmu /path/to/eboot.bin + +The expected exports are sharpemu_bink2_open_utf8, +sharpemu_bink2_decode_next_bgra, and sharpemu_bink2_close. The supplied +adapter opens one movie, exposes BGRA pixels, and advances after each decoded +frame. The managed side validates dimensions and retains ownership of the +destination buffer. + +If the bridge is absent in native mode, SharpEmu logs one informational line +and retains the existing guest rendering path. diff --git a/native/bink2-bridge/sharpemu_bink2_bridge.c b/native/bink2-bridge/sharpemu_bink2_bridge.c new file mode 100644 index 0000000..a2db193 --- /dev/null +++ b/native/bink2-bridge/sharpemu_bink2_bridge.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2026 SharpEmu Emulator Project + * SPDX-License-Identifier: GPL-2.0-or-later + * + * Build this small adapter with a licensed RAD Bink 2 SDK. The SDK and its + * headers are not distributed by SharpEmu. See docs/bink2-bridge.md. + */ + +#include +#include "bink.h" + +typedef struct sharpemu_bink2_info { + uint32_t width; + uint32_t height; + uint32_t frames_per_second_numerator; + uint32_t frames_per_second_denominator; +} sharpemu_bink2_info; + +int sharpemu_bink2_open_utf8(const char *path, HBINK *movie, sharpemu_bink2_info *info) { + HBINK bink; + if (!path || !movie || !info) return 0; + + bink = BinkOpen(path, 0); + if (!bink) return 0; + + *movie = bink; + info->width = bink->Width; + info->height = bink->Height; + info->frames_per_second_numerator = bink->FrameRate; + info->frames_per_second_denominator = bink->FrameRateDiv; + return 1; +} + +int sharpemu_bink2_decode_next_bgra(HBINK movie, uint8_t *destination, + uint32_t stride, uint32_t destination_bytes) { + uint64_t needed; + if (!movie || !destination || stride < movie->Width * 4) return 0; + needed = (uint64_t)stride * movie->Height; + if (needed > destination_bytes) return 0; + + /* Async Bink I/O has not filled the next frame yet; retry on the next host present. */ + if (BinkWait(movie)) return 0; + BinkDoFrame(movie); + BinkCopyToBuffer(movie, destination, stride, movie->Height, 0, 0, BINKSURFACE32RA); + BinkNextFrame(movie); + return 1; +} + +void sharpemu_bink2_close(HBINK movie) { + if (movie) BinkClose(movie); +} diff --git a/src/SharpEmu.CLI/Program.cs b/src/SharpEmu.CLI/Program.cs index 26e56b3..c0dd2bb 100644 --- a/src/SharpEmu.CLI/Program.cs +++ b/src/SharpEmu.CLI/Program.cs @@ -45,6 +45,11 @@ internal static partial class Program [STAThread] private static int Main(string[] args) { + // Avoid blocking full collections while guest and render threads are + // running, and establish the GC mode before the runtime reserves the + // fixed guest address-space window. + System.Runtime.GCSettings.LatencyMode = System.Runtime.GCLatencyMode.SustainedLowLatency; + try { return Run(args); @@ -84,6 +89,7 @@ internal static partial class Program { if (OperatingSystem.IsMacOS()) { + ConfigureMoltenVkDefaults(); PreloadMacVulkanLoader(); } @@ -151,6 +157,26 @@ internal static partial class Program return false; } + /// + /// Applies MoltenVK performance defaults before the Vulkan loader is + /// loaded. Existing user-provided values always take precedence. + /// + private static void ConfigureMoltenVkDefaults() + { + try + { + _ = MacSetEnv("MVK_CONFIG_SYNCHRONOUS_QUEUE_SUBMITS", "0", 0); + _ = MacSetEnv("MVK_CONFIG_SHOULD_MAXIMIZE_CONCURRENT_COMPILATION", "1", 0); + _ = MacSetEnv("MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS", "1", 0); + _ = MacSetEnv("MVK_CONFIG_RESUME_LOST_DEVICE", "1", 0); + } + catch (Exception exception) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Failed to set MoltenVK defaults: {exception.Message}"); + } + } + /// /// Makes a Vulkan loader visible to GLFW's dlopen("libvulkan.1.dylib"). /// Homebrew's Vulkan libraries are arm64-only and cannot load into this @@ -1319,4 +1345,7 @@ internal static partial class Program uint creationDisposition, uint flagsAndAttributes, nint templateFile); + + [DllImport("libSystem", EntryPoint = "setenv")] + private static extern int MacSetEnv(string name, string value, int overwrite); } diff --git a/src/SharpEmu.CLI/SharpEmu.CLI.csproj b/src/SharpEmu.CLI/SharpEmu.CLI.csproj index 219d12f..5b9ddc8 100644 --- a/src/SharpEmu.CLI/SharpEmu.CLI.csproj +++ b/src/SharpEmu.CLI/SharpEmu.CLI.csproj @@ -25,7 +25,11 @@ SPDX-License-Identifier: GPL-2.0-or-later true enable true - true + + false true true diff --git a/src/SharpEmu.Core/Cpu/CpuDispatcher.cs b/src/SharpEmu.Core/Cpu/CpuDispatcher.cs index 28e1cb1..02fad8c 100644 --- a/src/SharpEmu.Core/Cpu/CpuDispatcher.cs +++ b/src/SharpEmu.Core/Cpu/CpuDispatcher.cs @@ -7,15 +7,11 @@ using SharpEmu.Core.Cpu.Native; using SharpEmu.Core.Loader; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.HLE.Host; -using SharpEmu.Logging; namespace SharpEmu.Core.Cpu; public sealed class CpuDispatcher : ICpuDispatcher, IDisposable { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Dispatcher"); - private enum EntryFrameKind { ProcessEntry, @@ -31,9 +27,9 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable private static readonly ulong TlsBaseAddress = OperatingSystem.IsWindows() ? 0x7FFE_0000_0000UL : 0x6FFE_0000_0000UL; private const ulong TlsSize = 0x0001_0000UL; // The static TLS blocks live at negative offsets from the TCB (FreeBSD - // amd64 variant II); libc.prx alone reaches beyond -0x1700, so give the - // prefix a full 64KB on POSIX. Windows keeps its historical 4KB prefix. - private static readonly ulong TlsPrefixSize = OperatingSystem.IsWindows() ? 0x0000_1000UL : 0x0001_0000UL; + // amd64 variant II). Keep every host in sync with GuestTlsTemplate's + // startup reservation; PS5 modules routinely reach beyond one host page. + private const ulong TlsPrefixSize = GuestTlsTemplate.StartupStaticTlsReservation; private static readonly ulong BootstrapStubBaseAddress = OperatingSystem.IsWindows() ? 0x7FFD_F000_0000UL : 0x6FFD_F000_0000UL; private static readonly ulong BootstrapPayloadBaseAddress = OperatingSystem.IsWindows() ? 0x7FFD_E000_0000UL : 0x6FFD_E000_0000UL; private static readonly ulong DynlibFallbackStubBaseAddress = OperatingSystem.IsWindows() ? 0x7FFD_D000_0000UL : 0x6FFD_D000_0000UL; @@ -49,19 +45,16 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable ]; private readonly IVirtualMemory _virtualMemory; private readonly IModuleManager _moduleManager; - private readonly IHostPlatform? _hostPlatform; private INativeCpuBackend? _nativeCpuBackend; public CpuDispatcher( IVirtualMemory virtualMemory, IModuleManager moduleManager, - INativeCpuBackend? nativeCpuBackend = null, - IHostPlatform? hostPlatform = null) + INativeCpuBackend? nativeCpuBackend = null) { _virtualMemory = virtualMemory ?? throw new ArgumentNullException(nameof(virtualMemory)); _moduleManager = moduleManager ?? throw new ArgumentNullException(nameof(moduleManager)); _nativeCpuBackend = nativeCpuBackend; - _hostPlatform = hostPlatform; } public ulong? LastEntryPoint { get; private set; } @@ -94,8 +87,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable string processImageName = "eboot.bin", CpuExecutionOptions executionOptions = default) { - Log.Debug("=== DispatchEntry START ==="); - Log.Debug($"entryPoint=0x{entryPoint:X16}, generation={generation}"); + Console.Error.WriteLine("[DISPATCHER] === DispatchEntry START ==="); + Console.Error.WriteLine($"[DISPATCHER] entryPoint=0x{entryPoint:X16}, generation={generation}"); try { @@ -103,7 +96,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable } catch (Exception ex) { - Log.Critical($"FATAL EXCEPTION in DispatchEntry: {ex.GetType().Name}: {ex.Message}", ex); + Console.Error.WriteLine($"[DISPATCHER] FATAL EXCEPTION in DispatchEntry: {ex.GetType().Name}: {ex.Message}"); + Console.Error.WriteLine($"[DISPATCHER] Stack trace: {ex.StackTrace}"); throw; } } @@ -116,8 +110,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable string moduleName = "module", CpuExecutionOptions executionOptions = default) { - Log.Debug("=== DispatchModuleInitializer START ==="); - Log.Debug($"moduleInit=0x{entryPoint:X16}, generation={generation}, module={moduleName}"); + Console.Error.WriteLine("[DISPATCHER] === DispatchModuleInitializer START ==="); + Console.Error.WriteLine($"[DISPATCHER] moduleInit=0x{entryPoint:X16}, generation={generation}, module={moduleName}"); try { @@ -132,7 +126,8 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable } catch (Exception ex) { - Log.Critical($"FATAL EXCEPTION in DispatchModuleInitializer: {ex.GetType().Name}: {ex.Message}", ex); + Console.Error.WriteLine($"[DISPATCHER] FATAL EXCEPTION in DispatchModuleInitializer: {ex.GetType().Name}: {ex.Message}"); + Console.Error.WriteLine($"[DISPATCHER] Stack trace: {ex.StackTrace}"); throw; } } @@ -146,7 +141,7 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable CpuExecutionOptions executionOptions = default, EntryFrameKind frameKind = EntryFrameKind.ProcessEntry) { - Log.Debug("DispatchEntryCore STARTING..."); + Console.Error.WriteLine("[DISPATCHER] DispatchEntryCore STARTING..."); LastEntryPoint = entryPoint; LastTrapInfo = null; @@ -277,7 +272,7 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable entryFrameDiagnostic, Environment.NewLine, "CpuEngine: native-only"); - _nativeCpuBackend ??= new DirectExecutionBackend(_moduleManager, _hostPlatform); + _nativeCpuBackend ??= new DirectExecutionBackend(_moduleManager); if (_nativeCpuBackend.TryExecute( context, entryPoint, @@ -318,7 +313,7 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable LastMilestoneLog, Environment.NewLine, $"CpuEngine native-only failed: {backendError}"); - Log.Error($"Native backend FAILED: {backendError}"); + Console.Error.WriteLine($"[DISPATCHER] Native backend FAILED: {backendError}"); return FailEarly( OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_IMPLEMENTED, CpuExitReason.NativeBackendUnavailable); @@ -377,11 +372,19 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable private static bool InitializeTls(CpuContext context, ulong tlsBase) { - return context.TryWriteUInt64(tlsBase - 0xF0, 0) && - context.TryWriteUInt64(tlsBase + 0x00, tlsBase) && - context.TryWriteUInt64(tlsBase + 0x10, tlsBase) && - context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBABEUL) && - context.TryWriteUInt64(tlsBase + 0x60, tlsBase); + if (!context.TryWriteUInt64(tlsBase - 0xF0, 0) || + !context.TryWriteUInt64(tlsBase + 0x00, tlsBase) || + !context.TryWriteUInt64(tlsBase + 0x10, tlsBase) || + !context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBA00UL) || + !context.TryWriteUInt64(tlsBase + 0x60, tlsBase)) + { + return false; + } + + // Seed the static TLS block below the thread pointer with the main + // module's initialized thread-locals (variant II layout). + SharpEmu.HLE.GuestTlsTemplate.SeedThreadBlock(context, tlsBase); + return true; } private static bool InitializeGuestFrameChainSentinel(CpuContext context) @@ -407,35 +410,54 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable ulong programExitHandlerAddress) { var imageName = string.IsNullOrWhiteSpace(processImageName) ? "eboot.bin" : processImageName; - var encodedNameLength = Encoding.UTF8.GetByteCount(imageName); - Span argv0Buffer = encodedNameLength + 1 <= 512 - ? stackalloc byte[encodedNameLength + 1] - : new byte[encodedNameLength + 1]; - if (Encoding.UTF8.GetBytes(imageName.AsSpan(), argv0Buffer) != encodedNameLength) + var arguments = new List(3) { imageName }; + var configuredArguments = Environment.GetEnvironmentVariable("SHARPEMU_GUEST_ARGS"); + if (!string.IsNullOrWhiteSpace(configuredArguments)) { - return false; + // The PS5 entry-parameter ABI exposes three inline argv pointers. + // Two compatibility arguments are therefore safe without changing + // the fixed 0x20-byte structure expected by existing titles. + var compatibilityArguments = configuredArguments.Split( + (char[]?)null, + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + arguments.AddRange(compatibilityArguments.Take(2)); } - argv0Buffer[encodedNameLength] = 0; var cursor = context[CpuRegister.Rsp]; - - var argv0Address = AlignDown(cursor - (ulong)argv0Buffer.Length, 16); - if (!context.Memory.TryWrite(argv0Address, argv0Buffer)) + var argumentAddresses = new ulong[arguments.Count]; + for (var index = arguments.Count - 1; index >= 0; index--) { - return false; + var encoded = Encoding.UTF8.GetBytes(arguments[index] + '\0'); + cursor = AlignDown(cursor - (ulong)encoded.Length, 16); + if (!context.Memory.TryWrite(cursor, encoded)) + { + return false; + } + + argumentAddresses[index] = cursor; } const ulong entryParamsSize = 0x20; - var entryParamsAddress = AlignDown(argv0Address - entryParamsSize, 16); - if (!context.TryWriteUInt32(entryParamsAddress + 0x00, 1) || - !context.TryWriteUInt32(entryParamsAddress + 0x04, 0) || - !context.TryWriteUInt64(entryParamsAddress + 0x08, argv0Address) || - !context.TryWriteUInt64(entryParamsAddress + 0x10, 0) || - !context.TryWriteUInt64(entryParamsAddress + 0x18, 0)) + var entryParamsAddress = AlignDown(cursor - entryParamsSize, 16); + if (!TryWriteUInt32(context, entryParamsAddress + 0x00, (uint)arguments.Count) || + !TryWriteUInt32(context, entryParamsAddress + 0x04, 0) || + !context.TryWriteUInt64(entryParamsAddress + 0x08, argumentAddresses[0]) || + !context.TryWriteUInt64( + entryParamsAddress + 0x10, + argumentAddresses.Length > 1 ? argumentAddresses[1] : 0) || + !context.TryWriteUInt64( + entryParamsAddress + 0x18, + argumentAddresses.Length > 2 ? argumentAddresses[2] : 0)) { return false; } + if (arguments.Count > 1) + { + Console.Error.WriteLine( + $"[DISPATCHER] Guest arguments: {string.Join(' ', arguments.Skip(1))}"); + } + var entryStackPointer = entryParamsAddress - sizeof(ulong); if (!context.TryWriteUInt64(entryStackPointer, 0)) { @@ -468,6 +490,13 @@ public sealed class CpuDispatcher : ICpuDispatcher, IDisposable return value & ~(alignment - 1); } + private static bool TryWriteUInt32(CpuContext context, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return context.Memory.TryWrite(address, buffer); + } + private static string BuildEntryFrameDiagnostic( ulong entryPoint, CpuContext context, diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs index 2bc8b5e..c741472 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Diagnostics.cs @@ -7,8 +7,8 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using SharpEmu.Core.Cpu.Disasm; using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Logging; namespace SharpEmu.Core.Cpu.Native; @@ -17,6 +17,55 @@ public sealed partial class DirectExecutionBackend { private static readonly ConcurrentDictionary _knownExecutablePages = new(); + private static readonly bool _perfHleHistogram = + string.Equals(System.Environment.GetEnvironmentVariable("SHARPEMU_PERF_HLE"), "1", System.StringComparison.Ordinal); + private static readonly System.Collections.Concurrent.ConcurrentDictionary _perfHleCounts = new(); + private static long _perfHleTotal; + private static long _perfHleDispatchTicks; + + private static void RecordPerfHleDispatchTime(long ticks) + { + var total = System.Threading.Interlocked.Add(ref _perfHleDispatchTicks, ticks); + var calls = System.Threading.Interlocked.Read(ref _perfHleTotal); + if (calls > 0 && calls % 500000 == 0) + { + var avgUs = (double)total / System.Diagnostics.Stopwatch.Frequency * 1_000_000.0 / calls; + System.Console.Error.WriteLine($"[PERF][HLE] managed_dispatch_avg={avgUs:F3}us total_managed_s={(double)total / System.Diagnostics.Stopwatch.Frequency:F2}"); + } + } + + private static readonly bool _perfHleNoDict = + string.Equals(System.Environment.GetEnvironmentVariable("SHARPEMU_PERF_HLE_NODICT"), "1", System.StringComparison.Ordinal); + + private static void RecordPerfHleCall(string name) + { + var total = System.Threading.Interlocked.Increment(ref _perfHleTotal); + if (!_perfHleNoDict) + { + _perfHleCounts.AddOrUpdate(name, 1, static (_, v) => v + 1); + } + + if (total % 500000 == 0 && !_perfHleNoDict) + { + // Snapshot via foreach (a safe moving enumerator) before sorting. + // LINQ over a ConcurrentDictionary uses ICollection.CopyTo, which + // throws ArgumentException if another thread adds a key between the + // Count read and the copy — that exception was being swallowed into + // a CPU_TRAP return and crashing the guest. + var snapshot = new System.Collections.Generic.List>(_perfHleCounts.Count + 16); + foreach (var kvp in _perfHleCounts) + { + snapshot.Add(kvp); + } + + var top = snapshot + .OrderByDescending(kvp => kvp.Value) + .Take(20) + .Select(kvp => $"{kvp.Key}={kvp.Value}"); + System.Console.Error.WriteLine($"[PERF][HLE] total={total} top: {string.Join(", ", top)}"); + } + } + private void RecordRecentImportTrace( long dispatchIndex, string nid, @@ -25,101 +74,40 @@ public sealed partial class DirectExecutionBackend ulong arg1, ulong arg2) { - _recentImportTrace[_recentImportTraceWriteIndex] = new RecentImportTraceEntry( + var trace = _recentImportTrace; + trace[_recentImportTraceWriteIndex] = new RecentImportTraceEntry( dispatchIndex, nid, returnRip, arg0, arg1, - arg2); - _recentImportTraceWriteIndex = (_recentImportTraceWriteIndex + 1) % _recentImportTrace.Length; - if (_recentImportTraceCount < _recentImportTrace.Length) + arg2, + GuestThreadExecution.CurrentGuestThreadHandle, + Environment.CurrentManagedThreadId); + _recentImportTraceWriteIndex = (_recentImportTraceWriteIndex + 1) % trace.Length; + if (_recentImportTraceCount < trace.Length) { _recentImportTraceCount++; } } - private void RecordDeferredBootstrapTrace( - long dispatchIndex, - ulong op, - ulong symbolPointer, - ulong outputPointer, - ulong returnRip) - { - lock (_deferredBootstrapTraceGate) - { - _deferredBootstrapTrace[_deferredBootstrapTraceWriteIndex] = new DeferredBootstrapTraceEntry( - dispatchIndex, - op, - symbolPointer, - outputPointer, - returnRip); - _deferredBootstrapTraceWriteIndex = - (_deferredBootstrapTraceWriteIndex + 1) % _deferredBootstrapTrace.Length; - if (_deferredBootstrapTraceCount < _deferredBootstrapTrace.Length) - { - _deferredBootstrapTraceCount++; - } - } - } - - private void DrainDeferredBootstrapTraces() - { - if (!_logBootstrap) - { - return; - } - - DeferredBootstrapTraceEntry[] pending; - lock (_deferredBootstrapTraceGate) - { - if (_deferredBootstrapTraceCount == 0) - { - return; - } - - pending = new DeferredBootstrapTraceEntry[_deferredBootstrapTraceCount]; - var readIndex = (_deferredBootstrapTraceWriteIndex - _deferredBootstrapTraceCount + - _deferredBootstrapTrace.Length) % _deferredBootstrapTrace.Length; - for (var i = 0; i < _deferredBootstrapTraceCount; i++) - { - pending[i] = _deferredBootstrapTrace[(readIndex + i) % _deferredBootstrapTrace.Length]; - } - - _deferredBootstrapTraceCount = 0; - } - - foreach (var entry in pending) - { - var symbolText = ""; - if (TryReadAsciiZ(entry.SymbolPointer, 256, out var sym)) - { - symbolText = sym; - } - - Console.Error.WriteLine( - $"[LOADER][TRACE] bootstrap_call#{entry.DispatchIndex}: op=0x{entry.Op:X16} " + - $"sym_ptr=0x{entry.SymbolPointer:X16} sym='{symbolText}' " + - $"out_ptr=0x{entry.OutputPointer:X16} ret=0x{entry.ReturnRip:X16}"); - } - } - private void DumpRecentImportTrace() { - if (_recentImportTraceCount == 0) + var trace = _recentImportTrace; + if (trace is null || _recentImportTraceCount == 0) { return; } - Log.Info($" Recent import calls ({_recentImportTraceCount}):"); - int num = (_recentImportTraceWriteIndex - _recentImportTraceCount + _recentImportTrace.Length) % _recentImportTrace.Length; + Log.Info($" Recent import calls for managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} ({_recentImportTraceCount}):"); + int num = (_recentImportTraceWriteIndex - _recentImportTraceCount + trace.Length) % trace.Length; for (int i = 0; i < _recentImportTraceCount; i++) { - int num2 = (num + i) % _recentImportTrace.Length; - var entry = _recentImportTrace[num2]; + int num2 = (num + i) % trace.Length; + var entry = trace[num2]; if (!string.IsNullOrEmpty(entry.Nid)) { Log.Info( - $" #{entry.DispatchIndex} nid={entry.Nid} ret=0x{entry.ReturnRip:X16} " + + $" #{entry.DispatchIndex} managed={entry.ManagedThreadId} guest=0x{entry.GuestThreadHandle:X16} nid={entry.Nid} ret=0x{entry.ReturnRip:X16} " + $"rdi=0x{entry.Arg0:X16} rsi=0x{entry.Arg1:X16} rdx=0x{entry.Arg2:X16}"); } } @@ -135,9 +123,8 @@ public sealed partial class DirectExecutionBackend int num2 = 0; List list = new List(16); ulong num3 = scanStart; - var hostMemory = ResolveDiagnosticsHostMemory(); - HostRegionInfo lpBuffer; - while (num3 < scanEnd && hostMemory.Query(num3, out lpBuffer)) + MEMORY_BASIC_INFORMATION64 lpBuffer; + while (num3 < scanEnd && VirtualQuery((void*)num3, out lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) != 0) { ulong baseAddress = lpBuffer.BaseAddress; ulong num4 = baseAddress + lpBuffer.RegionSize; @@ -147,7 +134,7 @@ public sealed partial class DirectExecutionBackend } ulong value = Math.Max(num3, baseAddress); ulong num5 = Math.Min(num4, scanEnd); - if (lpBuffer.State == HostRegionState.Committed && IsReadableProtection(lpBuffer.RawProtection) && !IsExecutableProtection(lpBuffer.RawProtection)) + if (lpBuffer.State == 4096 && IsReadableProtection(lpBuffer.Protect) && !IsExecutableProtection(lpBuffer.Protect)) { ulong num6 = AlignUp(value, 8uL); for (ulong num7 = num6; num7 + 8 <= num5; num7 += 8) @@ -186,6 +173,50 @@ public sealed partial class DirectExecutionBackend { return; } + const int preludeSize = 192; + Span prelude = stackalloc byte[preludeSize]; + if (returnRip >= preludeSize && cpuContext.Memory.TryRead(returnRip - preludeSize, prelude)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] Import#{dispatchIndex} pre-return bytes @0x{returnRip - preludeSize:X16}: " + + BitConverter.ToString(prelude.ToArray()).Replace("-", " ")); + + List? bestCallChain = null; + var preludeAddress = returnRip - preludeSize; + for (var startOffset = 0; startOffset < preludeSize; startOffset++) + { + var cursor = preludeAddress + (ulong)startOffset; + var candidate = new List(); + while (cursor < returnRip && candidate.Count < 96 && + IcedDecoder.TryReadGuestBytes(cpuContext.Memory, cursor, 15, out var instructionBytes) && + IcedDecoder.TryDecode(cursor, instructionBytes, out var instruction) && + instruction.Length > 0 && + cursor + (ulong)instruction.Length <= returnRip) + { + candidate.Add(instruction); + cursor += (ulong)instruction.Length; + } + + if (cursor == returnRip && + candidate.Count > 0 && + string.Equals(candidate[^1].Mnemonic, "Call", StringComparison.OrdinalIgnoreCase) && + (bestCallChain is null || candidate.Count > bestCallChain.Count)) + { + bestCallChain = candidate; + } + } + + if (bestCallChain is not null) + { + Console.Error.WriteLine($"[LOADER][TRACE] Import#{dispatchIndex} pre-return disassembly:"); + foreach (var instruction in bestCallChain.TakeLast(32)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] 0x{instruction.Rip:X16}: {instruction.Text} " + + $"bytes={IcedDecoder.FormatBytes(instruction.Bytes)}"); + } + } + } Span destination = stackalloc byte[128]; if (!cpuContext.Memory.TryRead(returnRip, destination)) { @@ -237,15 +268,14 @@ public sealed partial class DirectExecutionBackend ulong callRip = returnRip + (ulong)i; ulong target = unchecked((ulong)((long)(callRip + 5) + rel32)); Log.Debug($"Import#{dispatchIndex} near-call @{callRip:X16}: target=0x{target:X16}"); - var importEntries = _importEntries; - for (int importIndex = 0; importIndex < importEntries.Length; importIndex++) + for (int importIndex = 0; importIndex < _importEntries.Length; importIndex++) { - if (importEntries[importIndex].Address != target) + if (_importEntries[importIndex].Address != target) { continue; } - string nid = importEntries[importIndex].Nid; + string nid = _importEntries[importIndex].Nid; if (_moduleManager.TryGetExport(nid, out var export)) { Log.Debug( @@ -272,14 +302,14 @@ public sealed partial class DirectExecutionBackend { Log.Debug( $"Import#{dispatchIndex} near-call PLT slot: [0x{slot:X16}] = 0x{slotTarget:X16}"); - for (int importIndex = 0; importIndex < importEntries.Length; importIndex++) + for (int importIndex = 0; importIndex < _importEntries.Length; importIndex++) { - if (importEntries[importIndex].Address != slotTarget) + if (_importEntries[importIndex].Address != slotTarget) { continue; } - string nid = importEntries[importIndex].Nid; + string nid = _importEntries[importIndex].Nid; if (_moduleManager.TryGetExport(nid, out var export)) { Log.Debug( @@ -303,6 +333,28 @@ public sealed partial class DirectExecutionBackend return value == 65534 || value == 4294967294u || value == 18446744073709551614uL; } + private static ulong ParseOptionalHexAddress(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return 0; + } + + var text = value.Trim(); + if (text.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + text = text[2..]; + } + + return ulong.TryParse( + text, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) + ? address + : 0; + } + private static bool IsPlausibleReturnAddress(ulong address) { return address >= 12884901888L && address < 17592186044416L && !IsUnresolvedSentinel(address); @@ -352,7 +404,7 @@ public sealed partial class DirectExecutionBackend { return false; } - if (!ResolveDiagnosticsHostMemory().Query(address, out var lpBuffer)) + if (VirtualQuery((void*)address, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } @@ -361,7 +413,7 @@ public sealed partial class DirectExecutionBackend { return false; } - if (lpBuffer.State != HostRegionState.Committed || !IsReadableProtection(lpBuffer.RawProtection)) + if (lpBuffer.State != 4096 || !IsReadableProtection(lpBuffer.Protect)) { return false; } @@ -393,12 +445,12 @@ public sealed partial class DirectExecutionBackend return true; } - if (!ResolveDiagnosticsHostMemory().Query(address, out var lpBuffer)) + if (VirtualQuery((void*)address, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } - var executable = lpBuffer.State == HostRegionState.Committed && IsExecutableProtection(lpBuffer.RawProtection); + var executable = lpBuffer.State == 4096 && IsExecutableProtection(lpBuffer.Protect); if (executable) { _knownExecutablePages.TryAdd(pageAddress, 0); @@ -417,14 +469,6 @@ public sealed partial class DirectExecutionBackend return (value + num) & ~num; } - // Diagnostics helpers are static (reachable from static handler paths), so - // they use the platform injected into the backend active on this thread and - // fall back to the process-wide singleton only when no run is bound. - private static IHostMemory ResolveDiagnosticsHostMemory() - { - return _activeExecutionBackend?._hostMemory ?? HostPlatform.Current.Memory; - } - private static bool IsReadableProtection(uint protect) { if ((protect & 0x100) != 0 || (protect & 1) != 0) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs index 7c96cab..9ea5199 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Exceptions.cs @@ -9,9 +9,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Threading; using SharpEmu.Core.Cpu.Disasm; -using SharpEmu.Core.Cpu.Native.Windows; using SharpEmu.HLE; -using SharpEmu.HLE.Host; namespace SharpEmu.Core.Cpu.Native; @@ -19,6 +17,8 @@ public sealed partial class DirectExecutionBackend { private const ulong LazyCommitWindowBytes = 0x0200_0000UL; private static int _lazyCommitTraceCount; + private static int _guestAllocatorHoleRecoveries; + private static int _auxiliaryThreadExecuteFaultRecoveries; private unsafe void SetupExceptionHandler() { @@ -30,12 +30,12 @@ public sealed partial class DirectExecutionBackend if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_RAW_HANDLER"), "1", StringComparison.Ordinal)) { - _rawExceptionHandlerStub = _faultHandling.CreateHandlerThunk(RawVectoredHandlerPtrManaged, _hostRspSlotTlsIndex, _tlsGetValueAddress); + _rawExceptionHandlerStub = CreateExceptionHandlerTrampoline(RawVectoredHandlerPtrManaged); if (_rawExceptionHandlerStub == 0) { throw new InvalidOperationException("Failed to create raw exception handler trampoline"); } - _rawExceptionHandler = _faultHandling.AddFirstChanceHandler(_rawExceptionHandlerStub); + _rawExceptionHandler = (nint)AddVectoredExceptionHandler(1u, _rawExceptionHandlerStub); Console.Error.WriteLine($"[LOADER][INFO] Raw exception handler installed: 0x{_rawExceptionHandler:X16}"); } else @@ -45,22 +45,22 @@ public sealed partial class DirectExecutionBackend _handlerDelegate = VectoredHandler; _handlerHandle = GCHandle.Alloc(_handlerDelegate); - _exceptionHandlerStub = _faultHandling.CreateHandlerThunk(Marshal.GetFunctionPointerForDelegate(_handlerDelegate), _hostRspSlotTlsIndex, _tlsGetValueAddress); + _exceptionHandlerStub = CreateExceptionHandlerTrampoline(Marshal.GetFunctionPointerForDelegate(_handlerDelegate)); if (_exceptionHandlerStub == 0) { throw new InvalidOperationException("Failed to create exception handler trampoline"); } - _exceptionHandler = _faultHandling.AddFirstChanceHandler(_exceptionHandlerStub); + _exceptionHandler = (nint)AddVectoredExceptionHandler(1u, _exceptionHandlerStub); Console.Error.WriteLine($"[LOADER][INFO] Exception handler installed: 0x{_exceptionHandler:X16}"); _unhandledFilterDelegate = UnhandledExceptionFilter; _unhandledFilterHandle = GCHandle.Alloc(_unhandledFilterDelegate); - _unhandledFilterStub = _faultHandling.CreateHandlerThunk(Marshal.GetFunctionPointerForDelegate(_unhandledFilterDelegate), _hostRspSlotTlsIndex, _tlsGetValueAddress); + _unhandledFilterStub = CreateExceptionHandlerTrampoline(Marshal.GetFunctionPointerForDelegate(_unhandledFilterDelegate)); if (_unhandledFilterStub == 0) { throw new InvalidOperationException("Failed to create unhandled exception filter trampoline"); } - _faultHandling.SetUnhandledFilter(_unhandledFilterStub); + SetUnhandledExceptionFilter(_unhandledFilterStub); } private unsafe int UnhandledExceptionFilter(void* exceptionInfo) @@ -68,8 +68,8 @@ public sealed partial class DirectExecutionBackend try { EXCEPTION_RECORD* exceptionRecord = ((EXCEPTION_POINTERS*)exceptionInfo)->ExceptionRecord; - ulong rip = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, CTX_RIP); - ulong rsp = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, CTX_RSP); + ulong rip = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, 248); + ulong rsp = ReadCtxU64(((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord, 152); Console.Error.WriteLine("[LOADER][FATAL] Unhandled exception filter fired."); Console.Error.WriteLine($"[LOADER][FATAL] Code: 0x{exceptionRecord->ExceptionCode:X8}"); Console.Error.WriteLine($"[LOADER][FATAL] Exception Address: 0x{(ulong)(nint)exceptionRecord->ExceptionAddress:X16}"); @@ -108,19 +108,23 @@ public sealed partial class DirectExecutionBackend return 0; } - ulong rip = ReadCtxU64(contextRecord, CTX_RIP); - ulong rsp = ReadCtxU64(contextRecord, CTX_RSP); - - // Thread-mode probe: a hardware exception raised while this thread is inside - // the managed import gateway means the VEH->managed reentry happened from - // cooperative GC mode — a ReversePInvokeBadTransition candidate. - if (LogThreadMode && _threadModeGatewayDepth > 0) + ulong rip = ReadCtxU64(contextRecord, 248); + ulong rsp = ReadCtxU64(contextRecord, 152); + if (TryRecoverGuestInt41(exceptionCode, contextRecord, rip)) { - TraceThreadMode( - $"veh_in_gateway code=0x{exceptionCode:X8} rip=0x{rip:X16} gateway_depth={_threadModeGatewayDepth}"); + return -1; + } + if (TryRecoverAuxiliaryThreadExecuteFault(exceptionRecord, contextRecord, rip)) + { + return -1; } - if (exceptionCode == WindowsFaultCodes.AccessViolation && TryHandleLazyCommittedPage(exceptionRecord, rip, rsp)) + if (exceptionCode == 3221225477u && TryHandleLazyCommittedPage(exceptionRecord, rip, rsp)) + { + return -1; + } + if (exceptionCode == 3221225477u && + TryRecoverGuestAllocatorHole(exceptionRecord, contextRecord, rip)) { return -1; } @@ -135,10 +139,10 @@ public sealed partial class DirectExecutionBackend switch (exceptionCode) { - case WindowsFaultCodes.AccessViolation: + case 3221225477u: LogAccessViolationTrace(exceptionAddress, exceptionRecord); break; - case WindowsFaultCodes.FastFail: + case 3221226505u: { ulong p0 = exceptionRecord->NumberParameters >= 1 ? (*exceptionRecord->ExceptionInformation) : 0; ulong p1 = exceptionRecord->NumberParameters >= 2 ? exceptionRecord->ExceptionInformation[1] : 0; @@ -148,27 +152,56 @@ public sealed partial class DirectExecutionBackend } } - ulong rax = ReadCtxU64(contextRecord, CTX_RAX); - ulong rbx = ReadCtxU64(contextRecord, CTX_RBX); - ulong rcx = ReadCtxU64(contextRecord, CTX_RCX); - ulong rdx = ReadCtxU64(contextRecord, CTX_RDX); - ulong rsi = ReadCtxU64(contextRecord, CTX_RSI); - ulong rdi = ReadCtxU64(contextRecord, CTX_RDI); - ulong rbp = ReadCtxU64(contextRecord, CTX_RBP); - ulong r8 = ReadCtxU64(contextRecord, CTX_R8); - ulong r9 = ReadCtxU64(contextRecord, CTX_R9); - ulong r10 = ReadCtxU64(contextRecord, CTX_R10); - ulong r11 = ReadCtxU64(contextRecord, CTX_R11); - ulong r12 = ReadCtxU64(contextRecord, CTX_R12); - ulong r13 = ReadCtxU64(contextRecord, CTX_R13); - ulong r14 = ReadCtxU64(contextRecord, CTX_R14); - ulong r15 = ReadCtxU64(contextRecord, CTX_R15); + ulong rax = ReadCtxU64(contextRecord, 120); + ulong rbx = ReadCtxU64(contextRecord, 144); + ulong rcx = ReadCtxU64(contextRecord, 128); + ulong rdx = ReadCtxU64(contextRecord, 136); + ulong rsi = ReadCtxU64(contextRecord, 168); + ulong rdi = ReadCtxU64(contextRecord, 176); + ulong rbp = ReadCtxU64(contextRecord, 160); + ulong r8 = ReadCtxU64(contextRecord, 184); + ulong r9 = ReadCtxU64(contextRecord, 192); + ulong r10 = ReadCtxU64(contextRecord, 200); + ulong r11 = ReadCtxU64(contextRecord, 208); + ulong r12 = ReadCtxU64(contextRecord, 216); + ulong r13 = ReadCtxU64(contextRecord, 224); + ulong r14 = ReadCtxU64(contextRecord, 232); + ulong r15 = ReadCtxU64(contextRecord, 240); Console.Error.WriteLine("[LOADER][INFO] ========================================="); Console.Error.WriteLine("[LOADER][INFO] NATIVE EXCEPTION CAUGHT!"); Console.Error.WriteLine($"[LOADER][INFO] Code: 0x{exceptionCode:X8}"); Console.Error.WriteLine($"[LOADER][INFO] Exception Address: 0x{exceptionAddress:X16}"); Console.Error.WriteLine($"[LOADER][INFO] RIP: 0x{rip:X16}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Host thread: managed={Environment.CurrentManagedThreadId} " + + $"name='{Thread.CurrentThread.Name ?? ""}'"); + if (_activeGuestThreadState is { } activeGuestThread) + { + Console.Error.WriteLine( + $"[LOADER][INFO] Guest thread: handle=0x{activeGuestThread.ThreadHandle:X16} " + + $"name='{activeGuestThread.Name}' state={activeGuestThread.State} " + + $"last_import={activeGuestThread.LastImportNid ?? ""} " + + $"last_ret=0x{activeGuestThread.LastReturnRip:X16}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Last import registers: " + + $"rax=0x{Volatile.Read(ref activeGuestThread.LastImportRax):X16} " + + $"result_valid={Volatile.Read(ref activeGuestThread.LastImportResultValid) != 0} " + + $"rdi=0x{activeGuestThread.LastImportRdi:X16} " + + $"rsi=0x{activeGuestThread.LastImportRsi:X16} " + + $"rdx=0x{activeGuestThread.LastImportRdx:X16} " + + $"rcx=0x{activeGuestThread.LastImportRcx:X16} " + + $"r8=0x{activeGuestThread.LastImportR8:X16} " + + $"r9=0x{activeGuestThread.LastImportR9:X16}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Last import stack args: " + + $"0=0x{activeGuestThread.LastImportStack0:X16} " + + $"1=0x{activeGuestThread.LastImportStack1:X16} " + + $"2=0x{activeGuestThread.LastImportStack2:X16} " + + $"3=0x{activeGuestThread.LastImportStack3:X16} " + + $"4=0x{activeGuestThread.LastImportStack4:X16} " + + $"5=0x{activeGuestThread.LastImportStack5:X16}"); + } if (TryFormatNearestRuntimeSymbol(rip, out string symbol)) { Console.Error.WriteLine("[LOADER][INFO] RIP symbol: " + symbol); @@ -193,7 +226,7 @@ public sealed partial class DirectExecutionBackend ulong accessType = 0; ulong target = 0; - if (exceptionCode == WindowsFaultCodes.AccessViolation && exceptionRecord->NumberParameters >= 2) + if (exceptionCode == 3221225477u && exceptionRecord->NumberParameters >= 2) { accessType = *exceptionRecord->ExceptionInformation; target = exceptionRecord->ExceptionInformation[1]; @@ -206,9 +239,9 @@ public sealed partial class DirectExecutionBackend }; Console.Error.WriteLine("[LOADER][INFO] AV access: " + accessText); Console.Error.WriteLine($"[LOADER][INFO] AV target: 0x{target:X16}"); - if (_hostMemory.Query(target, out var mbi)) + if (VirtualQuery((void*)target, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) != 0) { - Console.Error.WriteLine($"[LOADER][INFO] AV target region: base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} state=0x{mbi.RawState:X08} protect=0x{mbi.RawProtection:X08}"); + Console.Error.WriteLine($"[LOADER][INFO] AV target region: base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} state=0x{mbi.State:X08} protect=0x{mbi.Protect:X08}"); } } @@ -225,13 +258,46 @@ public sealed partial class DirectExecutionBackend Console.Error.WriteLine($"[LOADER][INFO] [rsp+0x{i * 8:X2}] @0x{stackAddr:X16} = 0x{value:X16}"); } + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DUMP_FAULT_STACK_WINDOW"), + "1", + StringComparison.Ordinal)) + { + Console.Error.WriteLine("[LOADER][INFO] Full fault stack window (RSP-0x300..RSP+0x100):"); + var windowStart = rsp >= 0x300 ? rsp - 0x300 : 0; + for (var stackAddr = windowStart; stackAddr < rsp + 0x100; stackAddr += 8) + { + if (!TryReadHostQword(stackAddr, out var value)) + { + continue; + } + + var relative = unchecked((long)(stackAddr - rsp)); + var relativeText = relative >= 0 + ? $"+0x{relative:X}" + : $"-0x{-relative:X}"; + var symbolText = TryFormatNearestRuntimeSymbol(value, out var stackSymbol) + ? $" [{stackSymbol}]" + : string.Empty; + Console.Error.WriteLine( + $"[LOADER][INFO] [rsp{relativeText}] " + + $"@0x{stackAddr:X16} = 0x{value:X16}{symbolText}"); + } + } + + DumpPointerWindow("fault-register-rbx", rbx, 0x60); + DumpPointerWindow("fault-register-rsi", rsi, 0x60); + DumpPointerWindow("fault-register-rdi", rdi, 0x60); + DumpPointerWindow("fault-register-r13", r13, 0x60); + DumpPointerWindow("fault-register-r14", r14, 0x60); + try { Console.Error.WriteLine("[LOADER][INFO] Frame chain (RBP walk):"); ulong frame = rbp; for (int i = 0; i < 12; i++) { - if (frame < 140733193388032L || frame > 140737488355327L) + if (frame < 0x10000) { break; } @@ -256,7 +322,7 @@ public sealed partial class DirectExecutionBackend switch (exceptionCode) { - case WindowsFaultCodes.AccessViolation: + case 3221225477u: Console.Error.WriteLine("[LOADER][ERROR] Type: Access Violation"); Console.Error.WriteLine("[LOADER][ERROR] This usually means:"); Console.Error.WriteLine("[LOADER][ERROR] - Guest code called an unmapped import"); @@ -290,22 +356,57 @@ public sealed partial class DirectExecutionBackend { Console.Error.WriteLine("[LOADER][INFO] Code window [RIP-0x20..]: " + BitConverter.ToString(window).Replace("-", " ")); } + for (var stackIndex = 0; stackIndex < 16; stackIndex++) + { + byte[] stackSlot = new byte[8]; + if (!TryReadHostBytes(rsp + (ulong)(stackIndex * 8), stackSlot)) + { + continue; + } + var candidate = BitConverter.ToUInt64(stackSlot); + if (candidate < _entryPoint || candidate >= _entryPoint + 0x10000000 || candidate < 24) + { + continue; + } + byte[] callSiteWindow = new byte[48]; + if (TryReadHostBytes(candidate - 24, callSiteWindow)) + { + Console.Error.WriteLine( + $"[LOADER][INFO] Stack guest-code candidate [rsp+0x{stackIndex * 8:X2}]=0x{candidate:X16}, bytes [-0x18..]: " + + BitConverter.ToString(callSiteWindow).Replace("-", " ")); + } + } } else { Console.Error.WriteLine("[LOADER][ERROR] Could not read code at RIP"); } DumpRecentImportTrace(); - DumpGuestDisasmDiagnostics(rip, rbp); + DumpGuestDisasmDiagnostics(rip, rbp, rsp); + DumpGuestRegisterWindowDiagnostics( + rax, rbx, rcx, rdx, rsi, rdi, rbp, rsp, + r8, r9, r10, r11, r12, r13, r14, r15); DumpGuestReferenceDiagnostics(); DumpGuestPointerWindowDiagnostics(); break; - case WindowsFaultCodes.Breakpoint: + case 2147483651u: Console.Error.WriteLine("[LOADER][WARNING] Type: Breakpoint (int3)"); Console.Error.WriteLine("[LOADER][WARNING] Unexpected breakpoint in direct-bridge mode"); break; - case WindowsFaultCodes.IllegalInstruction: + case 1073741845u: + Console.Error.WriteLine("[LOADER][ERROR] Type: Abort (SIGABRT)"); + DumpRecentImportTrace(); + DumpGuestDisasmDiagnostics(rip, rbp, rsp); + break; + case 3221225501u: Console.Error.WriteLine("[LOADER][INFO] Type: Illegal Instruction"); + byte[] illegalCode = new byte[16]; + if (TryReadHostBytes(rip, illegalCode)) + { + Console.Error.WriteLine("[LOADER][INFO] Code at RIP: " + BitConverter.ToString(illegalCode).Replace("-", " ")); + } + DumpRecentImportTrace(); + DumpGuestDisasmDiagnostics(rip, rbp, rsp); break; } @@ -319,6 +420,108 @@ public sealed partial class DirectExecutionBackend } } + private unsafe bool TryRecoverAuxiliaryThreadExecuteFault( + EXCEPTION_RECORD* exceptionRecord, + void* contextRecord, + ulong rip) + { + if (exceptionRecord->ExceptionCode != 3221225477u || + rip >= 0x0000000800000000UL || + _activeGuestThreadState is not { Name: "tbb_thead" } activeThread) + { + return false; + } + + var hostExit = ActiveEntryReturnSentinelRip; + if (hostExit < 0x10000) + { + hostExit = unchecked((ulong)_guestReturnStub); + } + if (hostExit < 0x10000) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Could not recover auxiliary TBB execute fault: target=0x{rip:X16} " + + $"active_exit=0x{ActiveEntryReturnSentinelRip:X16} guest_return_stub=0x{unchecked((ulong)_guestReturnStub):X16}"); + return false; + } + + _ = TryPatchActiveGuestReturnSlot(hostExit); + WriteCtxU64(contextRecord, 120, 0); + WriteCtxU64(contextRecord, 248, hostExit); + var recovery = Interlocked.Increment(ref _auxiliaryThreadExecuteFaultRecoveries); + Console.Error.WriteLine( + $"[LOADER][WARN] Recovered auxiliary TBB execute fault #{recovery}: " + + $"thread=0x{activeThread.ThreadHandle:X16} target=0x{rip:X16} -> host_exit=0x{hostExit:X16}"); + return true; + } + + private unsafe bool TryRecoverGuestInt41(uint exceptionCode, void* contextRecord, ulong rip) + { + if (!_ignoreGuestInt41 || exceptionCode != 3221225477u || rip < 0x10000) + { + return false; + } + + byte[] opcode = new byte[2]; + if (!TryReadHostBytes(rip, opcode) || opcode[0] != 0xCD || opcode[1] != 0x41) + { + return false; + } + + var count = Interlocked.Increment(ref _ignoredGuestInt41Count); + WriteCtxU64(contextRecord, 248, rip + 2); + if (count <= 16 || count % 65536 == 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Ignored guest int 0x41 trap #{count} at 0x{rip:X16} (SHARPEMU_IGNORE_INT41=1)"); + Console.Error.Flush(); + } + return true; + } + + private unsafe static bool TryRecoverGuestAllocatorHole( + EXCEPTION_RECORD* exceptionRecord, + void* contextRecord, + ulong rip) + { + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_GUEST_ALLOCATOR_HOLE_RECOVERY"), + "1", + StringComparison.Ordinal) || + exceptionRecord->NumberParameters < 2 || + exceptionRecord->ExceptionInformation[0] != 0 || + exceptionRecord->ExceptionInformation[1] != 8 || + ReadCtxU64(contextRecord, CTX_RDI) != 0 || + rip < 0x10000) + { + return false; + } + + // Demon's Souls occasionally leaves an empty payload in a locked pool + // tree node. The allocator dereferences payload+8 before reaching its + // existing empty-pool fallback. Match the instruction stream instead of + // a title-specific absolute address, then resume at that fallback so the + // lock is released and the allocator can try its next backing pool. + const ulong allocatorHoleSignature = 0x634CFF568D08778BUL; + if (*(ulong*)rip != allocatorHoleSignature || *((byte*)rip + 8) != 0xF2) + { + return false; + } + + const ulong emptyPoolFallbackDelta = 0x8E; + WriteCtxU64(contextRecord, CTX_RIP, rip + emptyPoolFallbackDelta); + var recovery = Interlocked.Increment(ref _guestAllocatorHoleRecoveries); + if (recovery <= 16 || (recovery & (recovery - 1)) == 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Guest allocator empty-node adapter recovery #{recovery}: " + + $"rip=0x{rip:X16} -> 0x{rip + emptyPoolFallbackDelta:X16}"); + Console.Error.Flush(); + } + + return true; + } + private static bool IsBenignHostDebugException(uint exceptionCode) { return exceptionCode is DBG_PRINTEXCEPTION_C or DBG_PRINTEXCEPTION_WIDE_C or MS_VC_THREADNAME_EXCEPTION; @@ -337,8 +540,8 @@ public sealed partial class DirectExecutionBackend EXCEPTION_POINTERS* pointers = (EXCEPTION_POINTERS*)exceptionInfo; EXCEPTION_RECORD* record = pointers->ExceptionRecord; void* contextRecord = pointers->ContextRecord; - ulong rip = contextRecord != null ? ReadCtxU64(contextRecord, CTX_RIP) : 0; - ulong rsp = contextRecord != null ? ReadCtxU64(contextRecord, CTX_RSP) : 0; + ulong rip = contextRecord != null ? ReadCtxU64(contextRecord, 248) : 0; + ulong rsp = contextRecord != null ? ReadCtxU64(contextRecord, 152) : 0; ulong accessType = record->NumberParameters >= 1 ? *record->ExceptionInformation : 0; ulong target = record->NumberParameters >= 2 ? record->ExceptionInformation[1] : 0; Console.Error.WriteLine( @@ -398,7 +601,7 @@ public sealed partial class DirectExecutionBackend } } - private void DumpGuestDisasmDiagnostics(ulong rip, ulong rbp) + private void DumpGuestDisasmDiagnostics(ulong rip, ulong rbp, ulong rsp) { if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_DISASM"), "1", StringComparison.Ordinal)) { @@ -410,12 +613,20 @@ public sealed partial class DirectExecutionBackend DumpGuestInstructionStream("fault-prelude", rip - 0x20, 24); } + // Optimized guest code frequently omits frame pointers. The return + // address at RSP is then more useful than an RBP walk and identifies the + // exact call site that supplied the faulting arguments. + if (TryReadHostQword(rsp, out var stackReturn) && stackReturn >= 0x60) + { + DumpGuestInstructionStream("stack-return-prelude", stackReturn - 0x60, 40); + } + try { ulong frame = rbp; for (int i = 0; i < 3; i++) { - if (frame < 140733193388032L || frame > 140737488355327L) + if (frame < 0x10000) { break; } @@ -484,7 +695,7 @@ public sealed partial class DirectExecutionBackend ulong address = scanBase; while (address < scanEnd) { - if (!_hostMemory.Query(address, out var mbi)) + if (VirtualQuery((void*)address, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { break; } @@ -496,9 +707,9 @@ public sealed partial class DirectExecutionBackend break; } - if (mbi.State == HostRegionState.Committed && - IsReadableProtection(mbi.RawProtection) && - IsExecutableProtection(mbi.RawProtection)) + if (mbi.State == MEM_COMMIT && + IsReadableProtection(mbi.Protect) && + IsExecutableProtection(mbi.Protect)) { ScanExecutableRegionForTargetReferences(regionBase, regionEnd, targetList, hitCounts, maxHitsPerTarget); } @@ -559,6 +770,55 @@ public sealed partial class DirectExecutionBackend } } + private void DumpGuestRegisterWindowDiagnostics( + ulong rax, + ulong rbx, + ulong rcx, + ulong rdx, + ulong rsi, + ulong rdi, + ulong rbp, + ulong rsp, + ulong r8, + ulong r9, + ulong r10, + ulong r11, + ulong r12, + ulong r13, + ulong r14, + ulong r15) + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_REGISTER_WINDOWS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + // A register can be the only surviving reference to the object or + // argument array that caused a native guest fault. Capture a compact + // window while the process is alive so the post-mortem log can + // distinguish an absent object from a partially initialized one. + var registers = new (string Name, ulong Value)[] + { + ("rax", rax), ("rbx", rbx), ("rcx", rcx), ("rdx", rdx), + ("rsi", rsi), ("rdi", rdi), ("rbp", rbp), ("rsp", rsp), + ("r8", r8), ("r9", r9), ("r10", r10), ("r11", r11), + ("r12", r12), ("r13", r13), ("r14", r14), ("r15", r15), + }; + var seen = new HashSet(); + foreach (var (name, value) in registers) + { + if (value < 0x10000 || !seen.Add(value)) + { + continue; + } + + DumpPointerWindow($"register-{name}", value, 0x80); + } + } + private void ScanExecutableRegionForTargetReferences( ulong regionBase, ulong regionEnd, @@ -803,13 +1063,13 @@ public sealed partial class DirectExecutionBackend return false; } - if (!_hostMemory.Query(address, out var mbi)) + if (VirtualQuery((void*)address, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } ulong regionEnd = mbi.BaseAddress + mbi.RegionSize; - if (mbi.State != HostRegionState.Committed || !IsReadableProtection(mbi.RawProtection) || regionEnd <= address || address > regionEnd - 8) + if (mbi.State != MEM_COMMIT || !IsReadableProtection(mbi.Protect) || regionEnd <= address || address > regionEnd - 8) { return false; } @@ -848,7 +1108,7 @@ public sealed partial class DirectExecutionBackend } } - private unsafe bool TryReadHostBytes(ulong address, byte[] buffer) + private unsafe static bool TryReadHostBytes(ulong address, byte[] buffer) { if (address < 65536) { @@ -861,9 +1121,9 @@ public sealed partial class DirectExecutionBackend ulong end = address + (ulong)buffer.Length; for (ulong page = address & 0xFFFFFFFFFFFFF000uL; page < end; page += 4096) { - if (!_hostMemory.Query(page, out var mbi) || - mbi.State != HostRegionState.Committed || - !IsReadableProtection(mbi.RawProtection)) + if (VirtualQuery((void*)page, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0 || + mbi.State != MEM_COMMIT || + !IsReadableProtection(mbi.Protect)) { return false; } @@ -976,25 +1236,25 @@ public sealed partial class DirectExecutionBackend { return false; } - if (!_hostMemory.Query(faultAddress, out var mbi)) + if (VirtualQuery((void*)faultAddress, out var mbi, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0) { return false; } ulong pageBase = faultAddress & 0xFFFFFFFFFFFFF000uL; - uint commitProtect = ResolveLazyCommitProtection(accessType, mbi.RawAllocationProtection); + uint commitProtect = ResolveLazyCommitProtection(accessType, mbi.AllocationProtect); int traceIndex = Interlocked.Increment(ref _lazyCommitTraceCount); bool traceLazyCommit = ShouldTraceLazyCommit(traceIndex); if (traceLazyCommit) { - Console.Error.WriteLine($"[LOADER][TRACE] lazy-query#{traceIndex}: fault=0x{faultAddress:X16} owner={owner} rip=0x{rip:X16} rsp=0x{rsp:X16} state=0x{mbi.RawState:X08} base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} alloc=0x{mbi.RawAllocationProtection:X08} prot=0x{mbi.RawProtection:X08}"); + Console.Error.WriteLine($"[LOADER][TRACE] lazy-query#{traceIndex}: fault=0x{faultAddress:X16} owner={owner} rip=0x{rip:X16} rsp=0x{rsp:X16} state=0x{mbi.State:X08} base=0x{mbi.BaseAddress:X16} size=0x{mbi.RegionSize:X16} alloc=0x{mbi.AllocationProtect:X08} prot=0x{mbi.Protect:X08}"); } - if (mbi.State == HostRegionState.Committed && IsAccessCompatible(accessType, mbi.RawProtection)) + if (mbi.State == 4096 && IsAccessCompatible(accessType, mbi.Protect)) { if (traceLazyCommit) { - Console.Error.WriteLine($"[LOADER][TRACE] lazy-commit-race#{traceIndex}: fault=0x{faultAddress:X16} protect=0x{mbi.RawProtection:X08}"); + Console.Error.WriteLine($"[LOADER][TRACE] lazy-commit-race#{traceIndex}: fault=0x{faultAddress:X16} protect=0x{mbi.Protect:X08}"); } return true; } @@ -1003,10 +1263,10 @@ public sealed partial class DirectExecutionBackend ulong committedBase = 0; ulong committedSize = 0; - if (mbi.State == HostRegionState.Free) + if (mbi.State == 65536) { if (TryGetLazyCommitWindow(faultAddress, mbi.BaseAddress, mbi.RegionSize, out var windowBase, out var windowSize) && - TryReserveThenCommit(_hostMemory, windowBase, windowSize, windowBase, windowSize, commitProtect)) + TryReserveThenCommit(windowBase, windowSize, windowBase, windowSize, commitProtect)) { committed = true; committedBase = windowBase; @@ -1015,7 +1275,7 @@ public sealed partial class DirectExecutionBackend else { ulong largeBase = faultAddress & 0xFFFFFFFFFFE00000uL; - if (TryReserveThenCommit(_hostMemory, largeBase, 2097152uL, largeBase, 2097152uL, commitProtect)) + if (TryReserveThenCommit(largeBase, 2097152uL, largeBase, 2097152uL, commitProtect)) { committed = true; committedBase = largeBase; @@ -1026,13 +1286,13 @@ public sealed partial class DirectExecutionBackend if (!committed) { ulong region64kBase = faultAddress & 0xFFFFFFFFFFFF0000uL; - if (TryReserveThenCommit(_hostMemory, region64kBase, 65536uL, region64kBase, 65536uL, commitProtect)) + if (TryReserveThenCommit(region64kBase, 65536uL, region64kBase, 65536uL, commitProtect)) { committed = true; committedBase = region64kBase; committedSize = 65536uL; } - else if (TryReserveThenCommit(_hostMemory, pageBase, 4096uL, pageBase, 4096uL, commitProtect)) + else if (TryReserveThenCommit(pageBase, 4096uL, pageBase, 4096uL, commitProtect)) { committed = true; committedBase = pageBase; @@ -1045,7 +1305,7 @@ public sealed partial class DirectExecutionBackend return false; } - TryCommitRange(_hostMemory, pageBase + 4096, 4096uL, commitProtect); + TryCommitRange(pageBase + 4096, 4096uL, commitProtect); if (traceLazyCommit) { Console.Error.WriteLine($"[LOADER][TRACE] lazy-reserve-commit#{traceIndex}: addr=0x{committedBase:X16} size=0x{committedSize:X16} access={accessType} protect=0x{commitProtect:X8}"); @@ -1053,13 +1313,13 @@ public sealed partial class DirectExecutionBackend return true; } - if (mbi.State != HostRegionState.Reserved) + if (mbi.State != 8192) { return false; } if (TryGetLazyCommitWindow(faultAddress, mbi.BaseAddress, mbi.RegionSize, out var commitWindowBase, out var commitWindowSize) && - TryCommitRange(_hostMemory, commitWindowBase, commitWindowSize, commitProtect)) + TryCommitRange(commitWindowBase, commitWindowSize, commitProtect)) { committed = true; committedBase = commitWindowBase; @@ -1068,7 +1328,7 @@ public sealed partial class DirectExecutionBackend else { ulong largeCommitBase = faultAddress & 0xFFFFFFFFFFE00000uL; - if (TryCommitRange(_hostMemory, largeCommitBase, 2097152uL, commitProtect)) + if (TryCommitRange(largeCommitBase, 2097152uL, commitProtect)) { committed = true; committedBase = largeCommitBase; @@ -1079,19 +1339,19 @@ public sealed partial class DirectExecutionBackend if (!committed) { ulong region64kBase = faultAddress & 0xFFFFFFFFFFFF0000uL; - if (TryCommitRange(_hostMemory, region64kBase, 65536uL, commitProtect)) + if (TryCommitRange(region64kBase, 65536uL, commitProtect)) { committed = true; committedBase = region64kBase; committedSize = 65536uL; } - else if (TryCommitRange(_hostMemory, pageBase, 8192uL, commitProtect)) + else if (TryCommitRange(pageBase, 8192uL, commitProtect)) { committed = true; committedBase = pageBase; committedSize = 8192uL; } - else if (TryCommitRange(_hostMemory, pageBase, 4096uL, commitProtect)) + else if (TryCommitRange(pageBase, 4096uL, commitProtect)) { committed = true; committedBase = pageBase; @@ -1104,7 +1364,7 @@ public sealed partial class DirectExecutionBackend return false; } - TryCommitRange(_hostMemory, pageBase + 4096, 4096uL, commitProtect); + TryCommitRange(pageBase + 4096, 4096uL, commitProtect); if (traceLazyCommit) { Console.Error.WriteLine($"[LOADER][TRACE] lazy-commit#{traceIndex}: addr=0x{committedBase:X16} size=0x{committedSize:X16} access={accessType} protect=0x{commitProtect:X8}"); @@ -1145,33 +1405,31 @@ public sealed partial class DirectExecutionBackend return true; } - // The commit protection is one of the two raw values ResolveLazyCommitProtection - // produces (0x40 RWX / 0x04 RW); the enum mapping reproduces those exactly. - static bool TryCommitRange(IHostMemory hostMemory, ulong baseAddress, ulong length, uint protection) + static unsafe bool TryCommitRange(ulong baseAddress, ulong length, uint protection) { if (length == 0) { return false; } - return hostMemory.Commit(baseAddress, length, protection == 64u ? HostPageProtection.ReadWriteExecute : HostPageProtection.ReadWrite); + return VirtualAlloc((void*)baseAddress, (nuint)length, 4096u, protection) != null; } - static bool TryReserveRange(IHostMemory hostMemory, ulong baseAddress, ulong length) + static unsafe bool TryReserveRange(ulong baseAddress, ulong length) { if (length == 0) { return false; } - return hostMemory.Reserve(baseAddress, length, HostPageProtection.ReadWrite) != 0; + return VirtualAlloc((void*)baseAddress, (nuint)length, 8192u, 4u) != null; } - static bool TryReserveThenCommit(IHostMemory hostMemory, ulong reserveAddress, ulong reserveSize, ulong commitAddress, ulong commitSize, uint protection) + static bool TryReserveThenCommit(ulong reserveAddress, ulong reserveSize, ulong commitAddress, ulong commitSize, uint protection) { - if (!TryReserveRange(hostMemory, reserveAddress, reserveSize)) + if (!TryReserveRange(reserveAddress, reserveSize)) { return false; } - return TryCommitRange(hostMemory, commitAddress, commitSize, protection); + return TryCommitRange(commitAddress, commitSize, protection); } static bool IsAccessCompatible(ulong accessType, uint protection) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs index 74b5d56..e691704 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.Imports.cs @@ -2,36 +2,43 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; using System.Threading; using SharpEmu.Core.Cpu; -using SharpEmu.Core.Cpu.Native.Windows; -using SharpEmu.Core.Loader; using SharpEmu.HLE; -using SharpEmu.HLE.Host; +using SharpEmu.Libs.Kernel; namespace SharpEmu.Core.Cpu.Native; public sealed partial class DirectExecutionBackend { + // The native import trampoline keeps the original guest GPR stack layout at + // argPackPtr and stores volatile SysV-only state immediately below it. This + // lets the managed gateway observe AL (the variadic vector-argument count) + // and all eight vector argument registers without changing the return-slot + // offsets used by the guest scheduler. + private const int ImportSavedRaxOffset = -176; + private const int ImportSavedR10Offset = -168; + private const int ImportSavedR11Offset = -160; + private const int ImportSavedMxcsrOffset = -152; + private const int ImportSavedFpuControlOffset = -148; + private const int ImportSavedXmmOffset = -128; + private const int ImportVectorRegisterCount = 8; + private const ulong StackCheckGuardValue = 0xC0DEC0DECAFEBA00UL; + private static long _canaryReturnRecoveries; + private readonly object _importResultLogSampleGate = new(); private readonly Dictionary _importResultLogSamples = new(StringComparer.Ordinal); + private int _il2CppExceptionDiagnosticCount; private static ulong ImportDispatchGatewayManaged(nint backendHandle, int importIndex, nint argPackPtr) { - if (LogThreadMode) - { - _threadModeGatewayCalls++; - _threadModeGatewayDepth++; - if (!_threadModeGatewayFirstLogged) - { - _threadModeGatewayFirstLogged = true; - TraceThreadMode($"gateway_first import={importIndex} total={_threadModeGatewayCalls}"); - } - } try { if (!(GCHandle.FromIntPtr(backendHandle).Target is DirectExecutionBackend directExecutionBackend)) @@ -41,6 +48,14 @@ public sealed partial class DirectExecutionBackend return 18446744071562199042uL; } + if (_perfHleHistogram) + { + var startTicks = System.Diagnostics.Stopwatch.GetTimestamp(); + var r = directExecutionBackend.DispatchImport(importIndex, argPackPtr); + RecordPerfHleDispatchTime(System.Diagnostics.Stopwatch.GetTimestamp() - startTicks); + return r; + } + return directExecutionBackend.DispatchImport(importIndex, argPackPtr); } catch (Exception ex) @@ -49,13 +64,6 @@ public sealed partial class DirectExecutionBackend $"[LOADER][ERROR] ImportDispatchGatewayManaged exception: {ex.GetType().Name}: {ex.Message}"); return 18446744071562199298uL; } - finally - { - if (LogThreadMode) - { - _threadModeGatewayDepth--; - } - } } private unsafe static int RawVectoredHandlerManaged(void* exceptionInfo) @@ -71,35 +79,68 @@ public sealed partial class DirectExecutionBackend private unsafe static int TryRecoverUnresolvedSentinel(void* exceptionInfo) { EXCEPTION_RECORD* exceptionRecord = ((EXCEPTION_POINTERS*)exceptionInfo)->ExceptionRecord; - if (exceptionRecord->ExceptionCode != WindowsFaultCodes.AccessViolation) + if (exceptionRecord->ExceptionCode != 3221225477u) { return 0; } void* contextRecord = ((EXCEPTION_POINTERS*)exceptionInfo)->ContextRecord; - ulong value = ReadCtxU64(contextRecord, CTX_RIP); + ulong value = ReadCtxU64(contextRecord, 248); ulong value2 = (ulong)exceptionRecord->ExceptionAddress; + if (value == StackCheckGuardValue && TryRecoverCanaryReturn(contextRecord)) + { + return -1; + } if (!IsUnresolvedSentinel(value) && !IsUnresolvedSentinel(value2)) { return 0; } - ulong rsp = ReadCtxU64(contextRecord, CTX_RSP); - WriteCtxU64(contextRecord, CTX_RAX, 0uL); + ulong rsp = ReadCtxU64(contextRecord, 152); + WriteCtxU64(contextRecord, 120, 0uL); if (TryGetPlausibleReturnFromStack(rsp, out var returnRip, out var nextRsp)) { - WriteCtxU64(contextRecord, CTX_RSP, nextRsp); - WriteCtxU64(contextRecord, CTX_RIP, returnRip); + WriteCtxU64(contextRecord, 152, nextRsp); + WriteCtxU64(contextRecord, 248, returnRip); Interlocked.Increment(ref _rawSentinelRecoveries); - if (LogThreadMode) - { - TraceThreadMode( - $"sentinel_recover rip=0x{value:X16} -> 0x{returnRip:X16} rsp=0x{rsp:X16} -> 0x{nextRsp:X16} " + - $"gateway_depth={_threadModeGatewayDepth}"); - } return -1; } return 0; } + private unsafe static bool TryRecoverCanaryReturn(void* contextRecord) + { + var rsp = ReadCtxU64(contextRecord, CTX_RSP); + var interruptedReturn = ReadCtxU64(contextRecord, CTX_RBP); + var interruptedFrame = rsp + 0x18; + if (!IsLikelyReturnAddress(interruptedReturn) || + rsp < sizeof(ulong) || + !TryReadStackU64(interruptedFrame, out var callerRbp) || + !TryReadStackU64(rsp + 0x20, out var callerReturn) || + callerRbp <= rsp || callerRbp - rsp > 0x10000 || + !IsLikelyReturnAddress(callerReturn)) + { + return false; + } + + // The guest unwind reached this callback return one stack slot late: the + // final pop loaded the interrupted return into rbp and ret consumed the + // stack guard. Resume at that return with the outer frame pointer rebuilt + // from the still-intact caller frame on the stack. + WriteCtxU64(contextRecord, CTX_RBP, interruptedFrame); + WriteCtxU64(contextRecord, CTX_RSP, rsp - sizeof(ulong)); + WriteCtxU64(contextRecord, CTX_RIP, interruptedReturn); + var recoveryCount = Interlocked.Increment(ref _canaryReturnRecoveries); + if (recoveryCount <= 4 || recoveryCount % 1000 == 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Recovered malformed canary return #{recoveryCount}: " + + $"resume=0x{interruptedReturn:X16} rsp=0x{rsp - sizeof(ulong):X16} " + + $"rbp=0x{interruptedFrame:X16} caller_rbp=0x{callerRbp:X16} " + + $"caller=0x{callerReturn:X16}"); + Console.Error.Flush(); + } + return true; + } + private unsafe ulong DispatchImport(int importIndex, nint argPackPtr) { long num = NextImportDispatchIndex(); @@ -113,32 +154,30 @@ public sealed partial class DirectExecutionBackend LastError = "Import dispatch called without active CPU context"; return 18446744071562199298uL; } - var importEntries = _importEntries; - if ((uint)importIndex >= (uint)importEntries.Length) + if ((uint)importIndex >= (uint)_importEntries.Length) { LastError = $"Import dispatch index out of range: {importIndex}"; return 18446744071562199042uL; } - ImportStubEntry importStubEntry = importEntries[importIndex]; + ImportStubEntry importStubEntry = _importEntries[importIndex]; + if (_perfHleHistogram) + { + RecordPerfHleCall(importStubEntry.Export?.Name ?? importStubEntry.Nid); + } int num2 = Volatile.Read(in _rawSentinelRecoveries); if (num2 != _lastReportedRawSentinelRecoveries) { Console.Error.WriteLine($"[LOADER][TRACE] Raw sentinel recoveries: {num2} (last import index={importIndex})"); _lastReportedRawSentinelRecoveries = num2; } - // Leaf imports take a scalar-only fast path that reads its own operands and - // intentionally bypasses the SysV variadic XMM marshalling below. This is safe - // only while the leaf set contains no float-variadic or float-returning function. - // The constraint and the audited list live on IsLeafImport — do not add a - // function that consumes xmm0-7 args or returns in xmm0 to the leaf set; - // such functions must stay on the full gateway path below. - if (IsLeafImport(importStubEntry.Nid) && + if (importStubEntry.IsLeaf && TryDispatchLeafImport(cpuContext, importStubEntry, argPackPtr, num, out var leafResult)) { return leafResult; } cpuContext.Rip = importStubEntry.Address; + LoadImportVolatileArguments(cpuContext, argPackPtr); cpuContext[CpuRegister.Rdi] = *(ulong*)argPackPtr; cpuContext[CpuRegister.Rsi] = *(ulong*)(argPackPtr + 8); cpuContext[CpuRegister.Rdx] = *(ulong*)(argPackPtr + 16); @@ -151,26 +190,7 @@ public sealed partial class DirectExecutionBackend cpuContext[CpuRegister.R13] = *(ulong*)(argPackPtr + 72); cpuContext[CpuRegister.R14] = *(ulong*)(argPackPtr + 80); cpuContext[CpuRegister.R15] = *(ulong*)(argPackPtr + 88); - // The trampoline spills the SysV variadic XMM save area (xmm0..xmm7) into the - // 0x80 bytes immediately below the GP argpack, so variadic float args (printf - // %f, and powf/logf inputs) reach the handler. xmm{i} is at argPackPtr-0x80+i*16. - for (var xmmIndex = 0; xmmIndex < 8; xmmIndex++) - { - var xmmSlot = argPackPtr - 0x80 + (xmmIndex * 16); - cpuContext.SetXmmRegister( - xmmIndex, - *(ulong*)xmmSlot, - *(ulong*)(xmmSlot + 8)); - } cpuContext[CpuRegister.Rsp] = (ulong)argPackPtr + 96uL; - if (importStubEntry.Kind == ImportStubKind.BootstrapBridge) - { - NormalizeKernelDynlibDlsymArguments(cpuContext, out _, out _); - *(ulong*)argPackPtr = cpuContext[CpuRegister.Rdi]; - *(ulong*)(argPackPtr + 8) = cpuContext[CpuRegister.Rsi]; - *(ulong*)(argPackPtr + 16) = cpuContext[CpuRegister.Rdx]; - } - ulong value = cpuContext[CpuRegister.Rdi]; ulong value2 = cpuContext[CpuRegister.Rsi]; ulong num3 = cpuContext[CpuRegister.Rdx]; @@ -184,6 +204,25 @@ public sealed partial class DirectExecutionBackend ulong value7 = cpuContext[CpuRegister.R14]; ulong value8 = cpuContext[CpuRegister.R15]; ulong num7 = *(ulong*)(argPackPtr + 96); + var importStackPointer = (ulong)argPackPtr + 96; + var probeTarget = (_probeImportReturnAddress != 0 && num7 == _probeImportReturnAddress) || + (string.Equals(importStubEntry.Nid, "2Z+PpY6CaJg", StringComparison.Ordinal) && + importStackPointer >= 0x00006FFFAC1FF000UL && + importStackPointer < 0x00006FFFAC200000UL); + if (probeTarget && + Interlocked.Increment(ref _probeImportReturnAddressCount) <= 2048) + { + var frameValue = TryReadStackU64(value4, out var savedRbp) ? savedRbp : 0; + var frameReturn = TryReadStackU64(value4 + sizeof(ulong), out var savedReturn) + ? savedReturn + : 0; + Console.Error.WriteLine( + $"[LOADER][TRACE] import-return-address-probe " + + $"thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} " + + $"nid={importStubEntry.Nid} ret=0x{num7:X16} " + + $"rsp=0x{(ulong)argPackPtr + 96:X16} rbp=0x{value4:X16} " + + $"saved_rbp=0x{frameValue:X16} saved_ret=0x{frameReturn:X16}"); + } var isGuestWorker = GuestThreadExecution.IsGuestThread; if (!IsLikelyReturnAddress(num7)) { @@ -199,11 +238,54 @@ public sealed partial class DirectExecutionBackend } } } + // Diagnostic compatibility escape hatch for a guest stack-protector + // failure whose noreturn call is immediately followed by UD2. Returning + // normally from the HLE export would execute that UD2; redirect this one + // well-known compiler epilogue back through its register/stack unwind. + // Keep the byte-pattern check strict so the opt-in cannot guess at an + // unrelated function layout. + if (string.Equals(importStubEntry.Nid, "Ou3iL1abvng", StringComparison.Ordinal) && + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_IGNORE_STACK_CHK"), + "1", + StringComparison.Ordinal) && + num7 >= 0x20) + { + var returnCode = (byte*)num7; + if (returnCode[0] == 0x0F && returnCode[1] == 0x0B && + returnCode[-22] == 0x75 && returnCode[-21] == 0x0F && + returnCode[-20] == 0x48 && returnCode[-19] == 0x83 && + returnCode[-18] == 0xC4) + { + var recoveredReturn = num7 - 20; + *(ulong*)(argPackPtr + 96) = recoveredReturn; + cpuContext[CpuRegister.Rax] = 0; + Console.Error.WriteLine( + $"[LOADER][WARN] Recovered guest stack-check epilogue " + + $"ret=0x{num7:X16} -> 0x{recoveredReturn:X16}"); + return 0; + } + } if (_activeGuestThreadState is { } activeGuestThreadState) { Interlocked.Increment(ref activeGuestThreadState.ImportCount); - Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); + activeGuestThreadState.LastImportRdi = value; + activeGuestThreadState.LastImportRsi = value2; + activeGuestThreadState.LastImportRdx = num3; + activeGuestThreadState.LastImportRcx = num4; + activeGuestThreadState.LastImportR8 = num5; + activeGuestThreadState.LastImportR9 = num6; + activeGuestThreadState.LastImportStack0 = ReadImportStackArgument(argPackPtr, 0); + activeGuestThreadState.LastImportStack1 = ReadImportStackArgument(argPackPtr, 1); + activeGuestThreadState.LastImportStack2 = ReadImportStackArgument(argPackPtr, 2); + activeGuestThreadState.LastImportStack3 = ReadImportStackArgument(argPackPtr, 3); + activeGuestThreadState.LastImportStack4 = ReadImportStackArgument(argPackPtr, 4); + activeGuestThreadState.LastImportStack5 = ReadImportStackArgument(argPackPtr, 5); + Volatile.Write(ref activeGuestThreadState.LastImportResultValid, 0); Volatile.Write(ref activeGuestThreadState.LastReturnRip, num7); + // Publish the NID last so readers cannot pair a new import name with + // the preceding import's argument snapshot. + Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); } if (_logStrlenBursts) { @@ -212,7 +294,8 @@ public sealed partial class DirectExecutionBackend } if (!string.IsNullOrWhiteSpace(_probeImportReturn) && (string.Equals(_probeImportReturn, "*", StringComparison.Ordinal) || - string.Equals(_probeImportReturn, importStubEntry.Nid, StringComparison.Ordinal))) + string.Equals(_probeImportReturn, importStubEntry.Nid, StringComparison.Ordinal)) && + Interlocked.Increment(ref _probeImportReturnAddressCount) <= 8) { ProbeReturnRip(num7, num); } @@ -221,55 +304,37 @@ public sealed partial class DirectExecutionBackend TraceGuestContext( $"import dispatch={num} nid={importStubEntry.Nid} ret=0x{num7:X16} managed={Environment.CurrentManagedThreadId} guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16} active={HasActiveExecutionThread}"); } + if (_logBootstrap && string.Equals(importStubEntry.Nid, RuntimeStubNids.BootstrapBridge, StringComparison.Ordinal)) + { + string symbolText = ""; + if (TryReadAsciiZ(value2, 256, out var sym)) + { + symbolText = sym; + } + Console.Error.WriteLine( + $"[LOADER][TRACE] bootstrap_call#{num}: op=0x{value:X16} sym_ptr=0x{value2:X16} sym='{symbolText}' out_ptr=0x{num3:X16} ret=0x{num7:X16}"); + } if (!isGuestWorker && !ActiveForcedGuestExit && - ShouldForceGuestExitOnImportLoop(importStubEntry.Nid, num7, num, value, value2) && + ShouldForceGuestExitOnImportLoop(in importStubEntry, num7, num, value, value2) && TryForceGuestExitToHostStub(argPackPtr, num, num7, importStubEntry.Nid)) { cpuContext[CpuRegister.Rax] = 1uL; return 1uL; } - // Backend teardown in progress: unwind this worker to the host instead of - // dispatching. RunGuestThread parks it as Blocked with no wake handler, its - // host thread exits, and RequestGuestThreadTeardown can finish before any - // executable memory is freed. - if (isGuestWorker && - _guestTeardownRequested && - TryYieldGuestThreadToHostStub(argPackPtr, num, num7, importStubEntry.Nid, "backend teardown")) - { - cpuContext[CpuRegister.Rax] = 0uL; - return 0uL; - } - if (_hostShutdownRequested) - { - if (isGuestWorker && - TryYieldGuestThreadToHostStub(argPackPtr, num, num7, importStubEntry.Nid, "host shutdown")) - { - cpuContext[CpuRegister.Rax] = 0uL; - return 0uL; - } - - if (!isGuestWorker && - TryAbortGuestForHostShutdown(argPackPtr, num, num7)) - { - cpuContext[CpuRegister.Rax] = 1uL; - return 1uL; - } - } - bool flag0 = ShouldSuppressStrlenTrace(importStubEntry.Nid); + bool flag0 = importStubEntry.SuppressStrlenTrace; bool flag = num7 >= 2156221920u && num7 <= 2156225024u; bool flag2 = num7 >= 2156351360u && num7 <= 2156352080u; bool flag3 = num >= 1020 && num <= 1040; bool flag4 = !string.IsNullOrWhiteSpace(_importFilter); bool flag5 = false; ExportedFunction? matchedExport = importStubEntry.Export; - var traceFlags = importStubEntry.TraceFlags; bool periodicTrace = num <= 128 || (num >= 240 && num <= 400) || (num >= 900 && num <= 1300) || num % 100000 == 0L || - ((traceFlags & ImportStubTraceFlags.PeriodicEvery1000) != 0 && (num <= 256 || num % 1000 == 0L)) || - ((traceFlags & ImportStubTraceFlags.PeriodicEvery128) != 0 && (num <= 256 || num % 128 == 0)) || + (importStubEntry.Nid == "tsvEmnenz48" && (num <= 256 || num % 1000 == 0L)) || + (importStubEntry.Nid == "rTXw65xmLIA" && (num <= 256 || num % 128 == 0)) || flag || flag2 || flag3; @@ -330,21 +395,33 @@ public sealed partial class DirectExecutionBackend cpuContext[CpuRegister.Rsi], cpuContext[CpuRegister.Rdx]); } - if ((traceFlags & ImportStubTraceFlags.Memset) != 0 && num <= 256) + if (importStubEntry.Nid == "8zTFvBIAIN8" && num <= 256) { Console.Error.WriteLine($"[LOADER][TRACE] memset#{num}: dst=0x{cpuContext[CpuRegister.Rdi]:X16} val=0x{cpuContext[CpuRegister.Rsi] & 0xFF:X2} len=0x{cpuContext[CpuRegister.Rdx]:X16} ret=0x{num7:X16}"); } - if ((traceFlags & ImportStubTraceFlags.CxaAtexit) != 0 && num <= 64) + if (importStubEntry.Nid == "tsvEmnenz48" && num <= 64) { Console.Error.WriteLine($"[LOADER][TRACE] __cxa_atexit#{num}: func=0x{cpuContext[CpuRegister.Rdi]:X16} arg=0x{cpuContext[CpuRegister.Rsi]:X16} dso=0x{cpuContext[CpuRegister.Rdx]:X16} ret=0x{num7:X16}"); } - if ((traceFlags & ImportStubTraceFlags.RawArgs) != 0) + if (importStubEntry.Nid == "bzQExy189ZI" || importStubEntry.Nid == "8G2LB+A3rzg") { Console.Error.WriteLine($"[LOADER][TRACE] {importStubEntry.Nid}#{num}: rdi=0x{cpuContext[CpuRegister.Rdi]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} rdx=0x{cpuContext[CpuRegister.Rdx]:X16} ret=0x{num7:X16}"); } if (flag6 || flag || flag2 || flag3) { Console.Error.WriteLine($"[LOADER][TRACE] ImportCtx#{num}: nid={importStubEntry.Nid} ret=0x{num7:X16} rdi=0x{cpuContext[CpuRegister.Rdi]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} rdx=0x{cpuContext[CpuRegister.Rdx]:X16} rcx=0x{cpuContext[CpuRegister.Rcx]:X16}"); + if (flag6) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] ImportArgs#{num}: " + + $"r8=0x{cpuContext[CpuRegister.R8]:X16} r9=0x{cpuContext[CpuRegister.R9]:X16} " + + $"stack0=0x{ReadImportStackArgument(argPackPtr, 0):X16} " + + $"stack1=0x{ReadImportStackArgument(argPackPtr, 1):X16} " + + $"stack2=0x{ReadImportStackArgument(argPackPtr, 2):X16} " + + $"stack3=0x{ReadImportStackArgument(argPackPtr, 3):X16} " + + $"stack4=0x{ReadImportStackArgument(argPackPtr, 4):X16} " + + $"stack5=0x{ReadImportStackArgument(argPackPtr, 5):X16}"); + } Console.Error.WriteLine($"[LOADER][TRACE] ImportNV#{num}: rbx=0x{value3:X16} rbp=0x{value4:X16} r12=0x{value5:X16} r13=0x{value6:X16} r14=0x{value7:X16} r15=0x{value8:X16}"); if (flag3) { @@ -368,17 +445,30 @@ public sealed partial class DirectExecutionBackend Console.Error.Flush(); } } - if ((traceFlags & ImportStubTraceFlags.StackChkFail) != 0) + if (importStubEntry.Nid == "Ou3iL1abvng") { if (_logStackCheck) { - var savedGuardAddress = value4 >= 0x10 ? value4 - 0x10 : 0; - var guardKnown = TryReadUInt64Compat(value3, out var guardValue); - var savedKnown = TryReadUInt64Compat(savedGuardAddress, out var savedGuardValue); + var rbxGuardKnown = TryReadUInt64Compat(value3, out var rbxGuardValue); + var r12GuardKnown = TryReadUInt64Compat(value5, out var r12GuardValue); Console.Error.WriteLine( - $"[LOADER][TRACE] stack_chk_diag#{num}: ret=0x{num7:X16} guard_ptr=0x{value3:X16} " + - $"guard={(guardKnown ? $"0x{guardValue:X16}" : "?")} saved@0x{savedGuardAddress:X16}={(savedKnown ? $"0x{savedGuardValue:X16}" : "?")} " + + $"[LOADER][TRACE] stack_chk_diag#{num}: ret=0x{num7:X16} " + + $"rbx=0x{value3:X16}:{(rbxGuardKnown ? $"0x{rbxGuardValue:X16}" : "?")} " + + $"r12=0x{value5:X16}:{(r12GuardKnown ? $"0x{r12GuardValue:X16}" : "?")} " + $"rbp=0x{value4:X16} rsp=0x{((ulong)argPackPtr + 96uL):X16}"); + + // Stack-protector layouts vary by compiler and function. Emit the + // bounded caller-frame tail instead of assuming a fixed canary + // offset; this also exposes an adjacent ABI output buffer overwrite. + for (var frameOffset = 0x10; frameOffset <= 0x80; frameOffset += sizeof(ulong)) + { + var slotAddress = value4 >= (ulong)frameOffset ? value4 - (ulong)frameOffset : 0; + if (slotAddress != 0 && TryReadUInt64Compat(slotAddress, out var slotValue)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] stack_chk_frame#{num}: [rbp-0x{frameOffset:X}]=0x{slotValue:X16}"); + } + } } try { @@ -400,20 +490,16 @@ public sealed partial class DirectExecutionBackend ActiveGuestReturnSlotAddress); try { - if (importStubEntry.Kind == ImportStubKind.BootstrapBridge) + if (string.Equals(importStubEntry.Nid, RuntimeStubNids.BootstrapBridge, StringComparison.Ordinal)) { - if (_logBootstrap) - { - RecordDeferredBootstrapTrace(num, value, value2, num3, num7); - } - orbisGen2Result = DispatchBootstrapBridge(); } - else if (importStubEntry.Kind == ImportStubKind.KernelDynlibDlsym) + else if (string.Equals(importStubEntry.Nid, RuntimeStubNids.KernelDynlibDlsym, StringComparison.Ordinal) || + string.Equals(importStubEntry.Nid, "LwG8g3niqwA", StringComparison.Ordinal)) { orbisGen2Result = DispatchKernelDynlibDlsym(); } - else if (importStubEntry.Kind == ImportStubKind.Il2CppApiLookupSymbol) + else if (string.Equals(importStubEntry.Nid, "r8mvOaWdi28", StringComparison.Ordinal)) { orbisGen2Result = DispatchIl2CppApiLookupSymbol(); } @@ -428,26 +514,21 @@ public sealed partial class DirectExecutionBackend } orbisGen2Result = (OrbisGen2Result)returnValue; } - else if (importStubEntry.Export is { } mismatchedExport) - { - dispatchResolved = true; - orbisGen2Result = OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_IMPLEMENTED; - cpuContext[CpuRegister.Rax] = unchecked((ulong)(int)orbisGen2Result); - Console.Error.WriteLine( - $"[LOADER][WARN] Import#{num} not implemented for generation {cpuContext.TargetGeneration}: " + - $"nid={importStubEntry.Nid} targets={mismatchedExport.Target} ret=0x{num7:X16}"); - } - else - { - dispatchResolved = false; - orbisGen2Result = OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - cpuContext[CpuRegister.Rax] = unchecked((ulong)(int)orbisGen2Result); - } + else + { + dispatchResolved = false; + orbisGen2Result = OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + cpuContext[CpuRegister.Rax] = unchecked((ulong)(int)orbisGen2Result); + } } finally { GuestThreadExecution.RestoreImportCallFrame(previousImportCallFrame); } + DeliverPendingGuestExceptionAtSafePoint( + cpuContext, + CaptureImportBoundaryContinuation(cpuContext, argPackPtr, num7)); + StoreImportVectorReturn(cpuContext, argPackPtr); if (dispatchResolved && orbisGen2Result == OrbisGen2Result.ORBIS_GEN2_OK && string.Equals(importStubEntry.Nid, "BohYr-F7-is", StringComparison.Ordinal)) @@ -457,6 +538,15 @@ public sealed partial class DirectExecutionBackend if (!dispatchResolved) { LastError = "Missing HLE export for NID: " + importStubEntry.Nid; + if (string.Equals(importStubEntry.Nid, "cfwBSQyr5Ys", StringComparison.Ordinal) && + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_IL2CPP_EXCEPTION"), + "1", + StringComparison.Ordinal) && + Interlocked.Increment(ref _il2CppExceptionDiagnosticCount) <= 4) + { + DumpIl2CppExceptionDiagnostic(cpuContext, value, num7); + } Console.Error.WriteLine( $"[LOADER][WARN] Import#{num} unresolved: nid={importStubEntry.Nid} ret=0x{num7:X16} " + $"rdi=0x{value:X16} rsi=0x{value2:X16} rdx=0x{num3:X16} rcx=0x{num4:X16} r8=0x{num5:X16} r9=0x{num6:X16}"); @@ -516,6 +606,11 @@ public sealed partial class DirectExecutionBackend $"fiber=0x{GuestThreadExecution.CurrentFiberAddress:X16}"); } + if (_activeGuestThreadState is { } transferGuestThreadState) + { + Volatile.Write(ref transferGuestThreadState.LastImportRax, transferTarget.Rax); + Volatile.Write(ref transferGuestThreadState.LastImportResultValid, 1); + } return unchecked((ulong)transferFrame); } if (GuestThreadExecution.TryConsumeCurrentEntryExit(out var exitValue, out var exitReason)) @@ -559,15 +654,31 @@ public sealed partial class DirectExecutionBackend Console.Error.Flush(); } } - // Publish the handler's XMM0 back into the argpack's xmm0 save slot; the - // trampoline epilogue reloads it into the guest's XMM0, delivering float/double - // return values (powf/logf/wcstod). Harmless for int/pointer returns (XMM is - // volatile across a SysV call, so the guest never relies on a preserved XMM0). - cpuContext.GetXmmRegister(0, out var returnXmm0Low, out var returnXmm0High); - *(ulong*)(argPackPtr - 0x80) = returnXmm0Low; - *(ulong*)(argPackPtr - 0x80 + 8) = returnXmm0High; - DrainDeferredBootstrapTraces(); - return cpuContext[CpuRegister.Rax]; + var guestReturnValue = cpuContext[CpuRegister.Rax]; + if (probeTarget) + { + ulong finalReturnSlot; + try + { + finalReturnSlot = *(ulong*)(argPackPtr + 96); + } + catch + { + finalReturnSlot = 0; + } + Console.Error.WriteLine( + $"[LOADER][TRACE] import-return-address-probe-exit " + + $"thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} " + + $"nid={importStubEntry.Nid} original=0x{num7:X16} " + + $"final=0x{finalReturnSlot:X16} rsp=0x{(ulong)argPackPtr + 96:X16} " + + $"yield={ActiveGuestThreadYieldRequested}"); + } + if (_activeGuestThreadState is { } completedGuestThreadState) + { + Volatile.Write(ref completedGuestThreadState.LastImportRax, guestReturnValue); + Volatile.Write(ref completedGuestThreadState.LastImportResultValid, 1); + } + return guestReturnValue; } catch (Exception ex) { @@ -575,11 +686,539 @@ public sealed partial class DirectExecutionBackend Console.Error.WriteLine($"[LOADER][ERROR] {LastError}"); Console.Error.WriteLine($"[LOADER][ERROR] {ex.StackTrace}"); cpuContext[CpuRegister.Rax] = 18446744071562199298uL; - DrainDeferredBootstrapTraces(); - return 18446744071562199298uL; + if (_activeGuestThreadState is { } failedGuestThreadState) + { + Volatile.Write(ref failedGuestThreadState.LastImportRax, cpuContext[CpuRegister.Rax]); + Volatile.Write(ref failedGuestThreadState.LastImportResultValid, 1); + } + return cpuContext[CpuRegister.Rax]; } } + private static void DumpIl2CppExceptionDiagnostic( + CpuContext cpuContext, + ulong wrapperAddress, + ulong returnAddress) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.wrapper=0x{wrapperAddress:X16} " + + $"ret=0x{returnAddress:X16}"); + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.registers " + + $"rbx=0x{cpuContext[CpuRegister.Rbx]:X16} " + + $"r12=0x{cpuContext[CpuRegister.R12]:X16} " + + $"r13=0x{cpuContext[CpuRegister.R13]:X16} " + + $"r14=0x{cpuContext[CpuRegister.R14]:X16} " + + $"r15=0x{cpuContext[CpuRegister.R15]:X16}"); + if (GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame)) + { + DumpGuestCodePointers(cpuContext, "stack", frame.ResumeRsp, 0x1000); + } + DumpGuestFramePointerChain(cpuContext, cpuContext[CpuRegister.Rbp]); + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_PS5_USER_SLOTS"), + "1", + StringComparison.Ordinal)) + { + for (var slot = 0; slot < 4; slot++) + { + DumpGuestQwords( + cpuContext, + $"ps5_user_slot[{slot}]", + 0x0000000801A73110 + (ulong)(slot * 0x51C8), + 0x60); + } + } + + if (!TryReadGuestU64(cpuContext, wrapperAddress, out var exceptionAddress)) + { + Console.Error.WriteLine("[LOADER][TRACE] il2cpp_exception.wrapper unreadable"); + return; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.object=0x{exceptionAddress:X16}"); + DumpGuestQwords(cpuContext, "exception", exceptionAddress, 0x90); + + // Il2CppException begins with Il2CppObject (klass, monitor), followed by + // trace_ips, inner_ex and message. Decode both the documented message + // slot and nearby object pointers because Unity revisions have appended + // fields without changing the wrapper itself. + for (var offset = 0; offset <= 0x80; offset += 8) + { + if (!TryReadGuestU64(cpuContext, exceptionAddress + (ulong)offset, out var candidate) || + candidate < 0x10000) + { + continue; + } + + if (TryReadIl2CppString(cpuContext, candidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.string+0x{offset:X2}=" + + $"'{text}'"); + } + } + + if (TryReadGuestU64(cpuContext, exceptionAddress + 0x38, out var traceIpsAddress)) + { + DumpIl2CppPointerArray(cpuContext, "trace_ips", traceIpsAddress); + } + + if (TryReadGuestU64(cpuContext, exceptionAddress, out var klassAddress)) + { + DumpGuestQwords(cpuContext, "exception_class", klassAddress, 0x100); + for (var offset = 0; offset <= 0xF8; offset += 8) + { + if (!TryReadGuestU64(cpuContext, klassAddress + (ulong)offset, out var candidate) || + candidate < 0x10000) + { + continue; + } + + if (TryReadGuestCString(cpuContext, candidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.class_string+0x{offset:X2}=" + + $"'{text}'"); + } + } + } + } + + private static void DumpGuestCodePointers( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + return; + } + + var logged = 0; + for (var offset = 0; offset <= buffer.Length - 8 && logged < 128; offset += 8) + { + var candidate = BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset, 8)); + if (candidate < 0x0000000800000000 || candidate >= 0x0000001000000000) + { + continue; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}+0x{offset:X3}=0x{candidate:X16}"); + logged++; + } + } + + private static void DumpGuestFramePointerChain(CpuContext cpuContext, ulong framePointer) + { + for (var depth = 0; depth < 64 && framePointer >= 0x10000; depth++) + { + if (!TryReadGuestU64(cpuContext, framePointer, out var nextFrame) || + !TryReadGuestU64(cpuContext, framePointer + 8, out var returnRip)) + { + break; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.frame[{depth}] " + + $"rbp=0x{framePointer:X16} ret=0x{returnRip:X16}"); + if (framePointer >= 0x40) + { + DumpGuestQwords( + cpuContext, + $"frame[{depth}]_locals", + framePointer - 0x40, + 0x60); + if (depth <= 10 && + TryReadGuestU64(cpuContext, framePointer - 0x20, out var savedRbx) && + savedRbx >= 0x0000000100000000 && + savedRbx < 0x0000000800000000) + { + DumpGuestQwords( + cpuContext, + $"frame[{depth}]_saved_rbx_object", + savedRbx, + 0x50); + if (depth is >= 4 and <= 12) + { + DumpIl2CppObjectGraph( + cpuContext, + $"frame[{depth}]_saved_rbx", + savedRbx); + } + } + DumpIl2CppStringsInRange( + cpuContext, + $"frame[{depth}]", + framePointer >= 0x100 ? framePointer - 0x100 : 0, + 0x140); + DumpIl2CppObjectsInRange( + cpuContext, + $"frame[{depth}]", + framePointer >= 0x100 ? framePointer - 0x100 : 0, + 0x140); + } + if (nextFrame <= framePointer || nextFrame - framePointer > 0x100000) + { + break; + } + + framePointer = nextFrame; + } + } + + private static void DumpIl2CppObjectsInRange( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (address == 0 || !cpuContext.Memory.TryRead(address, buffer)) + { + return; + } + + for (var offset = 0; offset <= buffer.Length - 8; offset += 8) + { + var candidate = BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset, 8)); + if (candidate < 0x0000000100000000 || + candidate >= 0x0000000800000000 || + !TryReadGuestU64(cpuContext, candidate, out var klass) || + klass < 0x0000000100000000 || + klass >= 0x0000000800000000 || + !TryReadGuestU64(cpuContext, klass + 0x10, out var nameAddress) || + !TryReadGuestCString(cpuContext, nameAddress, out var name)) + { + continue; + } + + var nameSpace = string.Empty; + if (TryReadGuestU64(cpuContext, klass + 0x18, out var namespaceAddress)) + { + _ = TryReadGuestCString(cpuContext, namespaceAddress, out nameSpace); + } + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_object@{offset:X3}=" + + $"0x{candidate:X16} {nameSpace}.{name}"); + if (string.Equals(name, "ControllerMap_Editor", StringComparison.Ordinal)) + { + DumpGuestQwords(cpuContext, $"{label}_controller_map", candidate, 0x40); + for (var fieldOffset = 0x20; fieldOffset <= 0x28; fieldOffset += 8) + { + if (TryReadGuestU64(cpuContext, candidate + (ulong)fieldOffset, out var stringAddress) && + TryReadIl2CppString(cpuContext, stringAddress, out var fieldText)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_controller_map+0x{fieldOffset:X2}=" + + $"'{fieldText}'"); + } + } + } + } + } + + private static void DumpIl2CppStringsInRange( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (address == 0 || !cpuContext.Memory.TryRead(address, buffer)) + { + return; + } + + for (var offset = 0; offset <= buffer.Length - 8; offset += 8) + { + var candidate = BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset, 8)); + if (candidate < 0x0000000100000000 || + candidate >= 0x0000000800000000 || + !TryReadIl2CppString(cpuContext, candidate, out var text) || + text.Length == 0) + { + continue; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_string@{offset:X3}=" + + $"0x{candidate:X16} '{text}'"); + } + } + + private static void DumpIl2CppObjectGraph( + CpuContext cpuContext, + string label, + ulong objectAddress) + { + if (!TryReadGuestU64(cpuContext, objectAddress, out var klassAddress)) + { + return; + } + + DumpGuestQwords(cpuContext, $"{label}_class", klassAddress, 0x400); + for (var offset = 0; offset <= 0xF8; offset += 8) + { + if (TryReadGuestU64(cpuContext, klassAddress + (ulong)offset, out var candidate) && + candidate >= 0x10000 && + TryReadGuestCString(cpuContext, candidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_class_string+0x{offset:X2}='{text}'"); + } + } + for (var offset = 0x10; offset <= 0x48; offset += 8) + { + if (!TryReadGuestU64(cpuContext, objectAddress + (ulong)offset, out var candidate) || + candidate < 0x0000000100000000 || + candidate >= 0x0000000800000000) + { + continue; + } + + DumpGuestQwords( + cpuContext, + $"{label}_field+0x{offset:X2}", + candidate, + 0x80); + if (TryReadGuestU64(cpuContext, candidate, out var candidateClass)) + { + DumpGuestQwords( + cpuContext, + $"{label}_field+0x{offset:X2}_class", + candidateClass, + 0x400); + for (var classOffset = 0; classOffset <= 0xF8; classOffset += 8) + { + if (TryReadGuestU64( + cpuContext, + candidateClass + (ulong)classOffset, + out var classCandidate) && + classCandidate >= 0x10000 && + TryReadGuestCString(cpuContext, classCandidate, out var text)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}_field+0x{offset:X2}" + + $"_class_string+0x{classOffset:X2}='{text}'"); + } + } + } + } + } + + private static void DumpIl2CppPointerArray( + CpuContext cpuContext, + string label, + ulong address) + { + if (address < 0x10000 || + !TryReadGuestU64(cpuContext, address + 0x18, out var rawLength)) + { + return; + } + + var length = (int)Math.Min(rawLength, 128); + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}=0x{address:X16} " + + $"length={rawLength}"); + for (var index = 0; index < length; index++) + { + if (!TryReadGuestU64(cpuContext, address + 0x20 + (ulong)(index * 8), out var value)) + { + break; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}[{index}]=0x{value:X16}"); + } + } + + private static void DumpGuestQwords( + CpuContext cpuContext, + string label, + ulong address, + int byteCount) + { + var buffer = new byte[byteCount]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}=unreadable@0x{address:X16}"); + return; + } + + for (var offset = 0; offset < buffer.Length; offset += 32) + { + var values = new string[Math.Min(4, (buffer.Length - offset) / 8)]; + for (var i = 0; i < values.Length; i++) + { + values[i] = $"{BinaryPrimitives.ReadUInt64LittleEndian(buffer.AsSpan(offset + i * 8, 8)):X16}"; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] il2cpp_exception.{label}+0x{offset:X2}: " + + string.Join(" ", values)); + } + } + + private static bool TryReadGuestU64(CpuContext cpuContext, ulong address, out ulong value) + { + Span buffer = stackalloc byte[8]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + + private static bool TryReadIl2CppString( + CpuContext cpuContext, + ulong address, + out string text) + { + text = string.Empty; + Span header = stackalloc byte[20]; + if (!cpuContext.Memory.TryRead(address, header)) + { + return false; + } + + var length = BinaryPrimitives.ReadInt32LittleEndian(header[16..]); + if (length <= 0 || length > 2048) + { + return false; + } + + var bytes = new byte[length * 2]; + if (!cpuContext.Memory.TryRead(address + 20, bytes)) + { + return false; + } + + text = SanitizeDiagnosticText(Encoding.Unicode.GetString(bytes)); + return text.Length != 0; + } + + private static bool TryReadGuestCString( + CpuContext cpuContext, + ulong address, + out string text) + { + text = string.Empty; + var buffer = new byte[256]; + if (!cpuContext.Memory.TryRead(address, buffer)) + { + return false; + } + + var length = Array.IndexOf(buffer, (byte)0); + if (length <= 0) + { + return false; + } + + for (var i = 0; i < length; i++) + { + if (buffer[i] is < 0x20 or > 0x7E) + { + return false; + } + } + + text = Encoding.UTF8.GetString(buffer, 0, length); + return true; + } + + private static string SanitizeDiagnosticText(string text) + { + var builder = new StringBuilder(Math.Min(text.Length, 512)); + foreach (var character in text) + { + if (builder.Length >= 512) + { + break; + } + + builder.Append(char.IsControl(character) ? ' ' : character); + } + + return builder.ToString().Trim(); + } + + private unsafe static void LoadImportVolatileArguments(CpuContext cpuContext, nint argPackPtr) + { + cpuContext[CpuRegister.Rax] = *(ulong*)(argPackPtr + ImportSavedRaxOffset); + cpuContext[CpuRegister.R10] = *(ulong*)(argPackPtr + ImportSavedR10Offset); + cpuContext[CpuRegister.R11] = *(ulong*)(argPackPtr + ImportSavedR11Offset); + cpuContext.Mxcsr = *(uint*)(argPackPtr + ImportSavedMxcsrOffset); + cpuContext.FpuControlWord = *(ushort*)(argPackPtr + ImportSavedFpuControlOffset); + for (var registerIndex = 0; registerIndex < ImportVectorRegisterCount; registerIndex++) + { + var registerAddress = argPackPtr + ImportSavedXmmOffset + (registerIndex * 16); + cpuContext.SetXmmRegister( + registerIndex, + *(ulong*)registerAddress, + *(ulong*)(registerAddress + 8)); + } + } + + private unsafe static void StoreImportVectorReturn(CpuContext cpuContext, nint argPackPtr) + { + // AMD64 returns scalar/vector floating-point values in XMM0 and may use + // XMM1 for the second eightbyte of a classified aggregate. + for (var registerIndex = 0; registerIndex < 2; registerIndex++) + { + cpuContext.GetXmmRegister(registerIndex, out var low, out var high); + var registerAddress = argPackPtr + ImportSavedXmmOffset + (registerIndex * 16); + *(ulong*)registerAddress = low; + *(ulong*)(registerAddress + 8) = high; + } + } + + private static ulong ReadImportStackArgument(nint argPackPtr, int index) + { + var address = checked((ulong)argPackPtr + 104UL + (ulong)index * sizeof(ulong)); + return TryReadHostQword(address, out var value) ? value : 0; + } + + private static GuestCpuContinuation CaptureImportBoundaryContinuation( + CpuContext context, + nint argPackPtr, + ulong returnRip) => + new( + Rip: returnRip, + Rsp: (ulong)argPackPtr + 104UL, + ReturnSlotAddress: (ulong)argPackPtr + 96UL, + Rflags: context.Rflags, + FsBase: context.FsBase, + GsBase: context.GsBase, + Rax: context[CpuRegister.Rax], + Rcx: context[CpuRegister.Rcx], + Rdx: context[CpuRegister.Rdx], + Rbx: context[CpuRegister.Rbx], + Rbp: context[CpuRegister.Rbp], + Rsi: context[CpuRegister.Rsi], + Rdi: context[CpuRegister.Rdi], + R8: context[CpuRegister.R8], + R9: context[CpuRegister.R9], + R10: context[CpuRegister.R10], + R11: context[CpuRegister.R11], + R12: context[CpuRegister.R12], + R13: context[CpuRegister.R13], + R14: context[CpuRegister.R14], + R15: context[CpuRegister.R15], + FpuControlWord: context.FpuControlWord, + Mxcsr: context.Mxcsr, + RestoreFullFpuState: false); + private unsafe bool TryDispatchLeafImport( CpuContext cpuContext, ImportStubEntry importStubEntry, @@ -596,7 +1235,20 @@ public sealed partial class DirectExecutionBackend var arg0 = *(ulong*)argPackPtr; var returnRip = *(ulong*)(argPackPtr + 96); + var leafStackPointer = (ulong)argPackPtr + 96UL; + var probeLeafReturn = _logAllImports && + string.Equals(importStubEntry.Nid, "2Z+PpY6CaJg", StringComparison.Ordinal) && + leafStackPointer >= 0x00006FFFAC1FF000UL && + leafStackPointer < 0x00006FFFAC200000UL; + if (probeLeafReturn) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] leaf-return-probe-enter nid={importStubEntry.Nid} " + + $"ret=0x{returnRip:X16} rsp=0x{leafStackPointer:X16} " + + $"active_slot=0x{ActiveGuestReturnSlotAddress:X16}"); + } cpuContext.Rip = importStubEntry.Address; + LoadImportVolatileArguments(cpuContext, argPackPtr); cpuContext[CpuRegister.Rdi] = arg0; cpuContext[CpuRegister.Rsi] = *(ulong*)(argPackPtr + 8); cpuContext[CpuRegister.Rdx] = *(ulong*)(argPackPtr + 16); @@ -614,8 +1266,21 @@ public sealed partial class DirectExecutionBackend if (_activeGuestThreadState is { } activeGuestThreadState) { Interlocked.Increment(ref activeGuestThreadState.ImportCount); - Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); + activeGuestThreadState.LastImportRdi = arg0; + activeGuestThreadState.LastImportRsi = *(ulong*)(argPackPtr + 8); + activeGuestThreadState.LastImportRdx = *(ulong*)(argPackPtr + 16); + activeGuestThreadState.LastImportRcx = *(ulong*)(argPackPtr + 24); + activeGuestThreadState.LastImportR8 = *(ulong*)(argPackPtr + 32); + activeGuestThreadState.LastImportR9 = *(ulong*)(argPackPtr + 40); + activeGuestThreadState.LastImportStack0 = ReadImportStackArgument(argPackPtr, 0); + activeGuestThreadState.LastImportStack1 = ReadImportStackArgument(argPackPtr, 1); + activeGuestThreadState.LastImportStack2 = ReadImportStackArgument(argPackPtr, 2); + activeGuestThreadState.LastImportStack3 = ReadImportStackArgument(argPackPtr, 3); + activeGuestThreadState.LastImportStack4 = ReadImportStackArgument(argPackPtr, 4); + activeGuestThreadState.LastImportStack5 = ReadImportStackArgument(argPackPtr, 5); + Volatile.Write(ref activeGuestThreadState.LastImportResultValid, 0); Volatile.Write(ref activeGuestThreadState.LastReturnRip, returnRip); + Volatile.Write(ref activeGuestThreadState.LastImportNid, importStubEntry.Nid); } if (dispatchIndex % 100000 == 0) { @@ -656,6 +1321,10 @@ public sealed partial class DirectExecutionBackend GuestThreadExecution.RestoreImportCallFrame(previousImportCallFrame); } } + DeliverPendingGuestExceptionAtSafePoint( + cpuContext, + CaptureImportBoundaryContinuation(cpuContext, argPackPtr, returnRip)); + StoreImportVectorReturn(cpuContext, argPackPtr); if (returnValue != (int)OrbisGen2Result.ORBIS_GEN2_OK) { @@ -671,13 +1340,15 @@ public sealed partial class DirectExecutionBackend } } - if (GuestThreadExecution.TryConsumeCurrentThreadBlock( + var consumedThreadBlock = GuestThreadExecution.TryConsumeCurrentThreadBlock( out var blockReason, out var blockContinuation, out var hasBlockContinuation, out var blockWakeKey, - out var blockWaiter, - out var blockDeadlineTimestamp) && + out var blockResumeHandler, + out var blockWakeHandler, + out var blockDeadlineTimestamp); + if (consumedThreadBlock && TryYieldGuestThreadToHostStub(argPackPtr, dispatchIndex, returnRip, importStubEntry.Nid, blockReason)) { if (hasBlockContinuation) @@ -686,24 +1357,31 @@ public sealed partial class DirectExecutionBackend GuestThreadExecution.CurrentGuestThreadHandle, blockContinuation, blockWakeKey, - blockWaiter, + blockResumeHandler, + blockWakeHandler, blockDeadlineTimestamp); } cpuContext[CpuRegister.Rax] = 0uL; } + if (probeLeafReturn) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] leaf-return-probe-exit nid={importStubEntry.Nid} " + + $"original=0x{returnRip:X16} final=0x{*(ulong*)(argPackPtr + 96):X16} " + + $"rsp=0x{leafStackPointer:X16} active_slot=0x{ActiveGuestReturnSlotAddress:X16} " + + $"block={consumedThreadBlock} yield={ActiveGuestThreadYieldRequested}"); + } result = cpuContext[CpuRegister.Rax]; + if (_activeGuestThreadState is { } completedGuestThreadState) + { + Volatile.Write(ref completedGuestThreadState.LastImportRax, result); + Volatile.Write(ref completedGuestThreadState.LastImportResultValid, 1); + } return true; } - // Subset of the leaf set that additionally skips the import-call-frame - // bookkeeping. The same scalar-only (no XMM args, no XMM return) constraint - // as IsLeafImport applies — see the note there before adding entries. - // NOTE: this filter is only consulted after IsLeafImport accepts the NID; - // entries listed here but not in IsLeafImport (scePadRead, scePadOpen, - // sceUserServiceGetEvent, sceUserServiceGetPlatformPrivacySetting, - // pthread_mutex_trylock) currently take the full gateway path. private static bool IsNoBlockLeafImport(string nid) => nid is "8aI7R7WaOlc" or // sceAmprCommandBufferConstructor @@ -719,19 +1397,21 @@ public sealed partial class DirectExecutionBackend "C+IEj+BsAFM" or // sceAmprMeasureCommandSizeWriteAddressOnCompletion "tZDDEo2tE5k" or // sceAmprCommandBufferGetSize "GnxKOHEawhk" or // sceAmprCommandBufferGetCurrentOffset + "gzndltBEzWc" or // sceAmprCommandBufferGetNumCommands "H896Pt-yB4I" or // sceAmprCommandBufferWriteKernelEventQueue_04_00 "sJXyWHjP-F8" or // sceAmprCommandBufferWriteAddressOnCompletion + "mPpPxv5CZt4" or // sceSystemServiceGetHdrToneMapLuminance + "1FZBKy8HeNU" or // sceVideoOutGetVblankStatus "ASoW5WE-UPo" or // sceKernelAprSubmitCommandBufferAndGetResult "rqwFKI4PAiM" or // sceKernelAprWaitCommandBuffer "eE4Szl8sil8" or // sceKernelAprSubmitCommandBuffer "qvMUCyyaCSI" or // sceKernelAprSubmitCommandBufferAndGetId "Q2V+iqvjgC0" or // vsnprintf - "AV6ipCNa4Rw" or // strcasecmp "q1cHNfGycLI" or // scePadRead "xk0AcarP3V4" or // scePadOpen "yH17Q6NWtVg" or // sceUserServiceGetEvent "D-CzAxQL0XI" or // sceUserServiceGetPlatformPrivacySetting - "K-jXhbt2gn4"; // pthread_mutex_trylock (scePthreadMutexTrylock is upoVrzMHFeE) + "K-jXhbt2gn4"; // scePthreadMutexTrylock private bool ShouldLogImportResult(string nid, OrbisGen2Result result) { @@ -750,12 +1430,12 @@ public sealed partial class DirectExecutionBackend var expectedEqueueTimeout = string.Equals(nid, "fzyMKs9kim0", StringComparison.Ordinal) && result == OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; - var expectedEventFlagTimeout = - string.Equals(nid, "JTvBflhYazQ", StringComparison.Ordinal) && - result == OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; var expectedMutexTrylockBusy = string.Equals(nid, "K-jXhbt2gn4", StringComparison.Ordinal) && result == OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + var expectedNetAcceptWouldBlock = + string.Equals(nid, "PIWqhn9oSxc", StringComparison.Ordinal) && + resultValue == unchecked((int)0x80410123); var expectedUserServiceNoEvent = string.Equals(nid, "yH17Q6NWtVg", StringComparison.Ordinal) && resultValue == unchecked((int)0x80960007); @@ -765,8 +1445,8 @@ public sealed partial class DirectExecutionBackend if (!expectedFileProbeMiss && !expectedTimedWaitTimeout && !expectedEqueueTimeout && - !expectedEventFlagTimeout && !expectedMutexTrylockBusy && + !expectedNetAcceptWouldBlock && !expectedUserServiceNoEvent && !expectedPrivacyInvalidParameter) { @@ -802,45 +1482,25 @@ public sealed partial class DirectExecutionBackend "1G3lF1Gg1k8" or // sceKernelOpen "gEpBkcwxUjw"; // sceKernelAprResolveFilepathsToIdsAndFileSizes - // LEAF-IMPORT REGISTRATION — scalar-only constraint. - // - // TryDispatchLeafImport is a fast path that never copies the trampoline's XMM - // save area into CpuContext and never publishes an XMM0 return back to the - // guest (see DispatchImport). Every NID listed here must therefore satisfy BOTH: - // 1. no float/double parameters (nothing passed in xmm0..xmm7), and - // 2. no float/double return value (nothing returned in xmm0). - // Integer/pointer arguments and returns only. va_list-based functions - // (vsnprintf) are safe: SysV va_list floats are read from the caller-built - // reg_save_area in guest memory, not from the callee's incoming XMM registers. - // - // If a function that consumes XMM arguments or returns in xmm0 (e.g. powf, - // logf, wcstod, sceVideoOutColorSettingsSetGamma_) is ever added here, its - // float arguments will silently arrive stale and its return value will be - // dropped. Such functions must stay on the full gateway path — do not list them. - // - // Audited 2026-07-11 (PR #59): every NID below maps to a registered - // SysAbiExport whose handler neither reads nor writes CpuContext XMM state - // and whose known guest signature is integer/pointer only. private bool IsLeafImport(string nid) { - if (nid == "1jfXLRVzisc") // sceKernelUsleep + if (nid == "1jfXLRVzisc") { return !_logUsleep; } - // Mutex lock uses this block-capable leaf path. Keep it out of the no-block subset. + // These leaf operations cannot park the current guest thread. Unlocks may + // wake another cooperative thread, but the scheduler drain is independent + // of the caller's import-boundary bookkeeping. return nid is - "9UK1vLZQft4" or // scePthreadMutexLock - "7H0iTOciTLo" or // pthread_mutex_lock "tn3VlD0hG60" or // scePthreadMutexUnlock "2Z+PpY6CaJg" or // pthread_mutex_unlock - "EgmLo6EWgso" or // pthread_rwlock_unlock - "+L98PIbGttk" or // scePthreadRwlockUnlock - "q1cHNfGycLI" or // scePadRead + "EgmLo6EWgso" or // scePthreadRwlockUnlock + "+L98PIbGttk" or // pthread_rwlock_unlock "8aI7R7WaOlc" or // sceAmprCommandBufferConstructor "zgXifHT9ErY" or // sceVideoOutIsFlipPending "V++UgBtQhn0" or // sceAgcGetDataPacketPayloadAddress - "qj7QZpgr9Uw" or // sceAgcUnknownQj7QZpgr9Uw (unknown NID; observed scalar: command-buffer pointer in, packet address out) + "qj7QZpgr9Uw" or // Gen5 graphics type-2 packet "LtTouSCZjHM" or // sceAgcCbNop "k3GhuSNmBLU" or // sceAgcCbDispatch "UZbQjYAwwXM" or // sceAgcCbSetShRegistersDirect @@ -862,25 +1522,34 @@ public sealed partial class DirectExecutionBackend "H7uZqCoNuWk" or // sceAgcDcbPopMarker "IxYiarKlXxM" or // sceAgcDmaDataPatchSetDstAddressOrOffset "3KDcnM3lrcU" or // sceAgcWaitRegMemPatchAddress + "n485EBnIWmk" or // sceAgcWaitRegMemPatchCompareFunction + "7nOoijNPvEU" or // sceAgcWaitRegMemPatchReference + "hXAnLgDHCoI" or // sceAgcWaitRegMemPatchMask "0fWWK5uG9rQ" or // sceAgcQueueEndOfPipeActionPatchAddress - "a8uLzYY--tM" or // sceAmprAprCommandBufferConstructor - "Qs1xtplKo0U" or // sceAmprAprCommandBufferDestructor - "GuchCTefuZw" or // sceAmprCommandBufferDestructor - "N-FSPA4S3nI" or // sceAmprCommandBufferSetBuffer - "baQO9ez2gL4" or // sceAmprCommandBufferReset - "ULvXMDz56po" or // sceAmprCommandBufferClearBuffer - "mQ16-QdKv7k" or // sceAmprAprCommandBufferReadFile - "vWU-odnS+fU" or // sceAmprMeasureCommandSizeReadFile - "sSAUCCU1dv4" or // sceAmprMeasureCommandSizeWriteKernelEventQueue_04_00 - "C+IEj+BsAFM" or // sceAmprMeasureCommandSizeWriteAddressOnCompletion - "tZDDEo2tE5k" or // sceAmprCommandBufferGetSize - "GnxKOHEawhk" or // sceAmprCommandBufferGetCurrentOffset - "H896Pt-yB4I" or // sceAmprCommandBufferWriteKernelEventQueue_04_00 - "sJXyWHjP-F8" or // sceAmprCommandBufferWriteAddressOnCompletion - "ASoW5WE-UPo" or // sceKernelAprSubmitCommandBufferAndGetResult - "rqwFKI4PAiM" or // sceKernelAprWaitCommandBuffer - "eE4Szl8sil8" or // sceKernelAprSubmitCommandBuffer - "qvMUCyyaCSI" or // sceKernelAprSubmitCommandBufferAndGetId + "J8YCgfKAMQs" or // sceAgcQueueEndOfPipeActionPatchGcrCntl + "MlEw1feXcjg" or // sceAgcQueueEndOfPipeActionPatchData + "T9fjQIINoeE" or // sceAgcQueueEndOfPipeActionPatchType + "a8uLzYY--tM" or + "Qs1xtplKo0U" or + "GuchCTefuZw" or + "N-FSPA4S3nI" or + "baQO9ez2gL4" or + "ULvXMDz56po" or + "mQ16-QdKv7k" or + "vWU-odnS+fU" or + "sSAUCCU1dv4" or + "C+IEj+BsAFM" or + "tZDDEo2tE5k" or + "GnxKOHEawhk" or + "gzndltBEzWc" or + "H896Pt-yB4I" or + "sJXyWHjP-F8" or + "mPpPxv5CZt4" or + "1FZBKy8HeNU" or + "ASoW5WE-UPo" or + "rqwFKI4PAiM" or + "eE4Szl8sil8" or + "qvMUCyyaCSI" or "Vo5V8KAwCmk" or // sceSystemServiceHideSplashScreen "TywrFKCoLGY" or // sceSaveDataInitialize3 "dyIhnXq-0SM" or // sceSaveDataDirNameSearch @@ -895,11 +1564,10 @@ public sealed partial class DirectExecutionBackend "Q2V+iqvjgC0" or // vsnprintf "j4ViWNHEgww" or // strlen "5jNubw4vlAA" or // strnlen - "LHMrG7e8G78" or // wcsmisc + "LHMrG7e8G78" or // wcslen "WkkeywLJcgU" or // wcslen "Ovb2dSJOAuE" or // strcmp "aesyjrHVWy4" or // strncmp - "AV6ipCNa4Rw" or // strcasecmp "pNtJdE3x49E" or // wcscmp "fV2xHER+bKE" or // wcscoll "E8wCoUEbfzk" or // wcsncmp @@ -920,12 +1588,7 @@ public sealed partial class DirectExecutionBackend "vz+pg2zdopI" or // sceKernelGetEventUserData "mJ7aghmgvfc" or // sceKernelGetEventId "23CPPI1tyBY" or // sceKernelGetEventFilter - "kwGyyjohI50" or // sceKernelGetEventData - // _Getpctype is a per-character ctype table lookup; the embedded mcpp preprocessor - // re-fetches the table pointer on every isdigit()/isalpha() check, so classifying a - // large input legitimately calls it hundreds of thousands of times back-to-back - - // real, finite work that would otherwise trip the loop guard. - "sUP1hBaouOw"; // _Getpctype + "kwGyyjohI50"; // sceKernelGetEventData } private long NextImportDispatchIndex() @@ -989,31 +1652,6 @@ public sealed partial class DirectExecutionBackend return true; } - private unsafe bool TryAbortGuestForHostShutdown(nint argPackPtr, long dispatchIndex, ulong returnRip) - { - ulong hostExit = ActiveEntryReturnSentinelRip; - if (hostExit < 65536 || !TryPatchActiveGuestReturnSlot(hostExit)) - { - return false; - } - try - { - *(ulong*)(argPackPtr + 96) = hostExit; - } - catch - { - return false; - } - ActiveForcedGuestExit = true; - if (string.IsNullOrWhiteSpace(LastError)) - { - LastError = "Host shutdown requested."; - } - Console.Error.WriteLine( - $"[LOADER][INFO] Guest unwind for host shutdown at import#{dispatchIndex} ret=0x{returnRip:X16} -> host_exit=0x{hostExit:X16}"); - return true; - } - private unsafe bool TryCompleteGuestEntryToHostStub(nint argPackPtr, long dispatchIndex, ulong returnRip, string nid, string reason, ulong value) { ulong hostExit = ActiveEntryReturnSentinelRip; @@ -1068,7 +1706,7 @@ public sealed partial class DirectExecutionBackend ActiveCpuContext.TryWriteUInt64(returnSlotAddress, hostExit); } - private bool ShouldForceGuestExitOnImportLoop(string nid, ulong returnRip, long dispatchIndex, ulong arg0, ulong arg1) + private bool ShouldForceGuestExitOnImportLoop(in ImportStubEntry entry, ulong returnRip, long dispatchIndex, ulong arg0, ulong arg1) { if (dispatchIndex < 1200) { @@ -1078,18 +1716,18 @@ public sealed partial class DirectExecutionBackend { return false; } - if (IsImportLoopGuardBoundary(nid)) + if (entry.IsLoopGuardBoundary) { ResetImportLoopPattern(); return false; } - if (!_importNidHashCache.TryGetValue(nid, out var value)) - { - value = StableHash64(nid); - _importNidHashCache[nid] = value; - } + var value = entry.NidHash; RecordImportLoopSignature(value, returnRip, BuildImportLoopSignature(value, returnRip, arg0, arg1)); - if ((dispatchIndex & 0x3F) != 0) + // The O(period x repeats) pattern scan is a boot/hang watchdog, not a + // steady-state feature; sampling every 256th dispatch keeps its cost + // off the hot path while still tripping within a couple of thousand + // dispatches of a genuine import loop. + if ((dispatchIndex & 0xFF) != 0) { return false; } @@ -1120,15 +1758,12 @@ public sealed partial class DirectExecutionBackend } private static bool IsImportLoopGuardBoundary(string nid) => - nid switch - { - "1jfXLRVzisc" => true, // sceKernelUsleep - "QcteRwbsnV0" => true, // usleep - "n88vx3C5nW8" => true, // gettimeofday - "Zxa0VhQVTsk" => true, // sceKernelWaitSema - "T72hz6ffq08" => true, // scePthreadYield - _ => false - }; + nid is + "1jfXLRVzisc" or // sceKernelUsleep + "WKAXJ4XBPQ4" or // scePthreadCondWait + "BmMjYxmew1w" or // scePthreadCondTimedwait + "Op8TBGY5KHg" or // pthread_cond_wait + "27bAgiJmOh0"; // pthread_cond_timedwait private void ResetImportLoopPattern() { @@ -1368,483 +2003,66 @@ public sealed partial class DirectExecutionBackend { return OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - - NormalizeKernelDynlibDlsymArguments(cpuContext, out var symbolNameAddress, out var outputAddress); - try + ulong symbolNameAddress = cpuContext[CpuRegister.Rsi]; + ulong outputAddress = cpuContext[CpuRegister.Rdx]; + if (!TryReadAsciiZ(symbolNameAddress, 512, out var symbolName)) { - if (!TryReadAsciiZ(symbolNameAddress, 512, out var symbolName) || - !TryResolveDlsymGuestAddress(symbolName, out var resolvedAddress) || - outputAddress == 0 || - !TryWriteUInt64Compat(outputAddress, resolvedAddress)) - { - return CompleteKernelDynlibDlsymFailure(cpuContext, outputAddress); - } + cpuContext[CpuRegister.Rax] = 18446744073709551615uL; + return OrbisGen2Result.ORBIS_GEN2_OK; } - catch + var moduleHandle = unchecked((int)cpuContext[CpuRegister.Rdi]); + if (!TryResolveModuleSymbolAddress(moduleHandle, symbolName, out var resolvedAddress) && + !TryResolveRuntimeSymbolAddress(symbolName, out resolvedAddress) && + !TryResolveRuntimeSymbolAddress(ComputePsNid(symbolName), out resolvedAddress) && + !TryResolveRuntimeSymbolAlias(symbolName, out resolvedAddress)) { - return CompleteKernelDynlibDlsymFailure(cpuContext, outputAddress); + Console.Error.WriteLine( + $"[LOADER][WARN] sceKernelDlsym failed: handle=0x{cpuContext[CpuRegister.Rdi]:X} symbol='{symbolName}'"); + cpuContext[CpuRegister.Rax] = 18446744073709551615uL; + return OrbisGen2Result.ORBIS_GEN2_OK; + } + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_DLSYM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] sceKernelDlsym: handle=0x{moduleHandle:X} symbol='{symbolName}' -> 0x{resolvedAddress:X16}"); + } + if (outputAddress == 0L || !TryWriteUInt64Compat(outputAddress, resolvedAddress)) + { + cpuContext[CpuRegister.Rax] = 18446744073709551615uL; + return OrbisGen2Result.ORBIS_GEN2_OK; } - cpuContext[CpuRegister.Rax] = 0uL; return OrbisGen2Result.ORBIS_GEN2_OK; } - private static void NormalizeKernelDynlibDlsymArguments( - CpuContext cpuContext, - out ulong symbolNameAddress, - out ulong outputAddress) + private static bool TryResolveModuleSymbolAddress(int moduleHandle, string symbolName, out ulong address) { - var handle = cpuContext[CpuRegister.Rdi]; - symbolNameAddress = cpuContext[CpuRegister.Rsi]; - outputAddress = cpuContext[CpuRegister.Rdx]; - - // Standalone bootstrap loaders sometimes call through the bridge with - // (symbol_ptr, handle, out) while sceKernelDlsym is (handle, symbol_ptr, out). - // Heuristic only: valid when RSI looks like a small handle and RDI is a guest pointer. - if (symbolNameAddress < 0x10000 && - IsPlausibleDynlibSymbolPointer(handle)) - { - symbolNameAddress = handle; - handle = cpuContext[CpuRegister.Rsi]; - cpuContext[CpuRegister.Rdi] = handle; - cpuContext[CpuRegister.Rsi] = symbolNameAddress; - } - } - - private static bool IsPlausibleDynlibSymbolPointer(ulong address) - { - return address >= 0x10000 && address < 0x0000_8000_0000_0000UL; - } - - private OrbisGen2Result CompleteKernelDynlibDlsymFailure(CpuContext cpuContext, ulong outputAddress) - { - if (outputAddress != 0) - { - _ = TryWriteUInt64Compat(outputAddress, 0); - } - - cpuContext[CpuRegister.Rax] = ulong.MaxValue; - return OrbisGen2Result.ORBIS_GEN2_OK; - } - - private void ResetLazyDlsymStubState() - { - lock (_lazyDlsymStubGate) - { - _lazyDlsymStubCache.Clear(); - _lazyImportStubPoolMapped = false; - _lazyImportStubPoolBase = 0; - _lazyImportStubNextSlot = 0; - _lazyImportStubPoolLimit = 0; - } - } - - private bool TryResolveDlsymGuestAddress(string symbolName, out ulong guestAddress) - { - guestAddress = 0; - if (string.IsNullOrWhiteSpace(symbolName)) - { - return false; - } - - // Tier 0: ELF runtime symbols and aliases. - if (TryResolveRuntimeSymbolAddress(symbolName, out guestAddress) || - TryResolveRuntimeSymbolAlias(symbolName, out guestAddress)) + if (KernelModuleRegistry.TryResolveModuleSymbol(moduleHandle, symbolName, out address)) { return true; } - // Tier 1: existing import stub entries (duplicate prevention). - if (TryFindImportStubGuestAddress(symbolName, out guestAddress)) - { - return true; - } - - var hasAerolibSymbol = Aerolib.Instance.TryGetByExportName(symbolName, out var hleSymbol); - if (hasAerolibSymbol) - { - if (HleDataSymbols.TryGetAddress(hleSymbol.Nid, out _)) - { - return false; - } - - if (TryResolveRuntimeSymbolAddress(hleSymbol.Nid, out guestAddress) || - TryResolveRuntimeSymbolAddress(hleSymbol.ExportName, out guestAddress) || - TryFindImportStubGuestAddress(hleSymbol.Nid, out guestAddress) || - TryFindImportStubGuestAddress(hleSymbol.ExportName, out guestAddress)) - { - return true; - } - } - else if (Aerolib.Instance.TryGetByNid(symbolName, out hleSymbol)) - { - if (HleDataSymbols.TryGetAddress(hleSymbol.Nid, out _)) - { - return false; - } - - if (TryFindImportStubGuestAddress(hleSymbol.Nid, out guestAddress) || - TryFindImportStubGuestAddress(hleSymbol.ExportName, out guestAddress)) - { - return true; - } - - hasAerolibSymbol = true; - } - - // Tier 3: lazy materialization for callable HLE symbols missing from ELF imports. - if (!TryResolveLazyDispatchTarget(symbolName, hasAerolibSymbol, in hleSymbol, out var dispatchNid, out var export)) - { - return false; - } - - return TryGetOrCreateLazyImportStub( - dispatchNid, - symbolName, - hasAerolibSymbol ? hleSymbol : null, - export, - out guestAddress); + var nid = ComputePsNid(symbolName); + return KernelModuleRegistry.TryResolveModuleSymbol(moduleHandle, nid, out address); } - private bool TryFindImportStubGuestAddress(string identifier, out ulong guestAddress) + private static string ComputePsNid(string symbolName) { - guestAddress = 0; - if (string.IsNullOrWhiteSpace(identifier)) - { - return false; - } - - var importEntries = _importEntries; - for (var i = 0; i < importEntries.Length; i++) - { - var entry = importEntries[i]; - if (!ImportStubEntryMatchesIdentifier(entry, identifier)) - { - continue; - } - - if (entry.Address >= 0x10000) - { - guestAddress = entry.Address; - return true; - } - } - - return false; - } - - private static bool ImportStubEntryMatchesIdentifier(in ImportStubEntry entry, string identifier) - { - if (string.Equals(entry.Nid, identifier, StringComparison.Ordinal)) - { - return true; - } - - if (entry.Export is not { } export) - { - return false; - } - - return string.Equals(export.Name, identifier, StringComparison.Ordinal) || - string.Equals(export.Nid, identifier, StringComparison.Ordinal); - } - - private static bool IsKernelDynlibDlsymIdentifier(string identifier) - { - return string.Equals(identifier, "sceKernelDlsym", StringComparison.Ordinal) || - string.Equals(identifier, KernelDynlibDlsymAerolibNid, StringComparison.Ordinal) || - string.Equals(identifier, RuntimeStubNids.KernelDynlibDlsym, StringComparison.Ordinal); - } - - private bool TryResolveLazyDispatchTarget( - string symbolName, - bool hasAerolibSymbol, - in SysAbiSymbol hleSymbol, - out string dispatchNid, - out ExportedFunction? export) - { - export = null; - dispatchNid = string.Empty; - - if (IsKernelDynlibDlsymIdentifier(symbolName)) - { - dispatchNid = KernelDynlibDlsymAerolibNid; - _ = _moduleManager.TryGetExport(dispatchNid, out export); - return true; - } - - if (!hasAerolibSymbol) - { - return false; - } - - if (HleDataSymbols.TryGetAddress(hleSymbol.Nid, out _)) - { - return false; - } - - if (_moduleManager.TryGetExport(hleSymbol.Nid, out export)) - { - dispatchNid = hleSymbol.Nid; - return true; - } - - return false; - } - - private bool TryGetOrCreateLazyImportStub( - string dispatchNid, - string symbolName, - SysAbiSymbol? hleSymbol, - ExportedFunction? export, - out ulong guestAddress) - { - guestAddress = 0; - if (string.IsNullOrWhiteSpace(dispatchNid)) - { - return false; - } - - lock (_lazyDlsymStubGate) - { - if (TryGetCachedLazyDlsymAddress(dispatchNid, symbolName, out guestAddress)) - { - return true; - } - - if (!EnsureLazyImportStubPoolMapped()) - { - LogLazyImportStubMaterializationFailure( - "import stub region unresolved", - dispatchNid, - symbolName); - return false; - } - - if (!TryAllocateLazyImportStubSlot(out guestAddress)) - { - LogLazyImportStubMaterializationFailure( - "lazy import stub pool exhausted", - dispatchNid, - symbolName); - return false; - } - - var previousEntries = _importEntries; - var importIndex = previousEntries.Length; - var newEntries = new ImportStubEntry[importIndex + 1]; - if (importIndex > 0) - { - Array.Copy(previousEntries, newEntries, importIndex); - } - - newEntries[importIndex] = new ImportStubEntry(guestAddress, dispatchNid, export); - - var hostTrampoline = CreateImportHandlerTrampoline(importIndex); - if (hostTrampoline == 0 || - !PatchImportStub((nint)(long)guestAddress, hostTrampoline)) - { - guestAddress = 0; - LogLazyImportStubMaterializationFailure( - "failed to patch lazy import stub trampoline", - dispatchNid, - symbolName); - return false; - } - - _importEntries = newEntries; - - RegisterDlsymRuntimeSymbolKeys(symbolName, dispatchNid, hleSymbol, guestAddress); - return true; - } - } - - private bool TryGetCachedLazyDlsymAddress(string dispatchNid, string symbolName, out ulong guestAddress) - { - if (_lazyDlsymStubCache.TryGetValue(dispatchNid, out guestAddress) || - _lazyDlsymStubCache.TryGetValue(symbolName, out guestAddress)) - { - return guestAddress >= 0x10000; - } - - return false; - } - - private void RegisterDlsymRuntimeSymbolKeys( - string symbolName, - string dispatchNid, - SysAbiSymbol? hleSymbol, - ulong guestAddress) - { - RegisterDlsymRuntimeSymbolKey(symbolName, guestAddress); - RegisterDlsymRuntimeSymbolKey(dispatchNid, guestAddress); - - if (IsKernelDynlibDlsymIdentifier(symbolName) || IsKernelDynlibDlsymIdentifier(dispatchNid)) - { - RegisterDlsymRuntimeSymbolKey(RuntimeStubNids.KernelDynlibDlsym, guestAddress); - RegisterDlsymRuntimeSymbolKey(KernelDynlibDlsymAerolibNid, guestAddress); - RegisterDlsymRuntimeSymbolKey("sceKernelDlsym", guestAddress); - } - - if (hleSymbol.HasValue) - { - RegisterDlsymRuntimeSymbolKey(hleSymbol.Value.Nid, guestAddress); - RegisterDlsymRuntimeSymbolKey(hleSymbol.Value.ExportName, guestAddress); - } - } - - private void RegisterDlsymRuntimeSymbolKey(string key, ulong guestAddress) - { - if (string.IsNullOrWhiteSpace(key) || !IsRuntimeSymbolAddressUsable(guestAddress)) - { - return; - } - - _runtimeSymbolsByName[key] = guestAddress; - _lazyDlsymStubCache[key] = guestAddress; - } - - private void LogLazyImportStubMaterializationFailure(string reason, string dispatchNid, string symbolName) - { - Console.Error.WriteLine( - $"[LOADER][WARN] Lazy import stub materialization failed ({reason}): " + - $"nid={dispatchNid} symbol='{symbolName}'"); - } - - private unsafe bool TryResolveImportStubRegionBounds( - ImportStubEntry[] importEntries, - out ulong regionBase, - out ulong regionLimit) - { - regionBase = 0; - regionLimit = 0; - - for (var candidateIndex = 0; candidateIndex < 64; candidateIndex++) - { - var candidateBase = ImportStubRegionCanonicalBase - - (ulong)candidateIndex * ImportStubRegionAddressStride; - if (!_hostMemory.Query(candidateBase, out var memoryInfo) || - memoryInfo.RegionSize == 0 || - memoryInfo.State != HostRegionState.Committed) - { - continue; - } - - var candidateLimit = candidateBase + memoryInfo.RegionSize; - var hasStub = false; - for (var i = 0; i < importEntries.Length; i++) - { - var entryAddress = importEntries[i].Address; - if (entryAddress < candidateBase || entryAddress >= candidateLimit) - { - continue; - } - - if ((entryAddress - candidateBase) % LazyImportStubSlotSize != 0) - { - continue; - } - - hasStub = true; - break; - } - - if (!hasStub) - { - continue; - } - - regionBase = candidateBase; - regionLimit = candidateLimit; - return true; - } - - ulong maxStubEnd = 0; - for (var i = 0; i < importEntries.Length; i++) - { - var entryAddress = importEntries[i].Address; - if (entryAddress < ImportStubRegionCanonicalBase) - { - continue; - } - - if ((entryAddress - ImportStubRegionCanonicalBase) % LazyImportStubSlotSize != 0) - { - continue; - } - - var entryEnd = entryAddress + LazyImportStubSlotSize; - if (entryEnd > maxStubEnd) - { - maxStubEnd = entryEnd; - regionBase = ImportStubRegionCanonicalBase; - } - } - - if (regionBase == 0 || maxStubEnd <= regionBase) - { - return false; - } - - var spanBytes = maxStubEnd - regionBase; - regionLimit = regionBase + AlignUp(spanBytes, ImportStubRegionPageSize); - return regionLimit > regionBase; - } - - private bool EnsureLazyImportStubPoolMapped() - { - if (_lazyImportStubPoolMapped && _lazyImportStubPoolBase != 0) - { - return true; - } - - var importEntries = _importEntries; - if (!TryResolveImportStubRegionBounds(importEntries, out var importStubRegionBase, out var importStubRegionLimit)) - { - return false; - } - - ulong nextSlot = importStubRegionBase; - for (var i = 0; i < importEntries.Length; i++) - { - var entryAddress = importEntries[i].Address; - if (entryAddress < importStubRegionBase || entryAddress >= importStubRegionLimit) - { - continue; - } - - var entryEnd = entryAddress + LazyImportStubSlotSize; - if (entryEnd > nextSlot) - { - nextSlot = entryEnd; - } - } - - if (nextSlot >= importStubRegionLimit) - { - return false; - } - - _lazyImportStubPoolBase = importStubRegionBase; - _lazyImportStubNextSlot = nextSlot; - _lazyImportStubPoolLimit = importStubRegionLimit; - _lazyImportStubPoolMapped = true; - return true; - } - - private bool TryAllocateLazyImportStubSlot(out ulong guestAddress) - { - guestAddress = 0; - if (!_lazyImportStubPoolMapped || - _lazyImportStubNextSlot < _lazyImportStubPoolBase || - _lazyImportStubNextSlot + LazyImportStubSlotSize > _lazyImportStubPoolLimit) - { - return false; - } - - guestAddress = _lazyImportStubNextSlot; - _lazyImportStubNextSlot += LazyImportStubSlotSize; - return true; + ReadOnlySpan salt = + [ + 0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, + 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30, + ]; + var nameBytes = Encoding.UTF8.GetBytes(symbolName); + var input = new byte[nameBytes.Length + salt.Length]; + nameBytes.CopyTo(input, 0); + salt.CopyTo(input.AsSpan(nameBytes.Length)); + Span digest = stackalloc byte[20]; + SHA1.HashData(input, digest); + var value = BinaryPrimitives.ReadUInt64LittleEndian(digest); + Span bigEndianValue = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64BigEndian(bigEndianValue, value); + return Convert.ToBase64String(bigEndianValue).TrimEnd('=').Replace('/', '-'); } private bool TryResolveRuntimeSymbolAlias(string symbolName, out ulong address) @@ -1911,20 +2129,29 @@ public sealed partial class DirectExecutionBackend return OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + ulong bridgeHandle = cpuContext[CpuRegister.Rdi]; + ulong symbolNameAddress = cpuContext[CpuRegister.Rsi]; ulong outputAddress = cpuContext[CpuRegister.Rdx]; - try + _ = TryReadAsciiZ(symbolNameAddress, 512, out var symbolName); + + OrbisGen2Result result = DispatchKernelDynlibDlsym(); + if (result != OrbisGen2Result.ORBIS_GEN2_OK) { - OrbisGen2Result result = DispatchKernelDynlibDlsym(); - if (result != OrbisGen2Result.ORBIS_GEN2_OK) - { - return result; - } + return result; } - catch + if (_logBootstrap) { - return CompleteKernelDynlibDlsymFailure(cpuContext, outputAddress); + Console.Error.WriteLine( + $"[LOADER][TRACE] bootstrap_dispatch: handle=0x{bridgeHandle:X16} symbol='{symbolName}' out=0x{outputAddress:X16} rax=0x{cpuContext[CpuRegister.Rax]:X16}"); } + if (cpuContext[CpuRegister.Rax] == 0uL) + { + return OrbisGen2Result.ORBIS_GEN2_OK; + } + + Console.Error.WriteLine( + $"[LOADER][WARN] bootstrap_bridge unresolved: handle=0x{bridgeHandle:X} symbol='{symbolName}' out=0x{outputAddress:X16}"); return OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1966,15 +2193,14 @@ public sealed partial class DirectExecutionBackend return false; } - // Reads stay byte-by-byte through TryReadByteCompat (its Marshal.ReadByte - // fallback must probe exactly up to the terminator), but the bytes land in a - // stack buffer instead of a List + ToArray per symbol resolution. const int StackBufferLength = 512; - byte[]? rented = maxLength > StackBufferLength ? System.Buffers.ArrayPool.Shared.Rent(maxLength) : null; + byte[]? rented = maxLength > StackBufferLength + ? System.Buffers.ArrayPool.Shared.Rent(maxLength) + : null; Span buffer = rented is null ? stackalloc byte[StackBufferLength] : rented; try { - for (int i = 0; i < maxLength; i++) + for (var i = 0; i < maxLength; i++) { if (!TryReadByteCompat(address + (ulong)i, buffer.Slice(i, 1))) { @@ -2077,7 +2303,7 @@ public sealed partial class DirectExecutionBackend uint flNewProtect = default(uint); try { - if (Marshal.ReadByte(num2) != 232 || !_hostMemory.Protect((ulong)(void*)num, 5u, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (Marshal.ReadByte(num2) != 232 || !VirtualProtect((void*)num, 5u, 64u, &flNewProtect)) { return; } @@ -2085,7 +2311,7 @@ public sealed partial class DirectExecutionBackend { Marshal.WriteByte(num2 + i, 144); } - _hostMemory.FlushInstructionCache((ulong)(void*)num, 5u); + FlushInstructionCache(GetCurrentProcess(), (void*)num, 5u); _patchedEa020eLookupCall = true; Console.Error.WriteLine($"[LOADER][WARNING] Import#{dispatchIndex}: patched hash-lookup call at 0x{num:X16} -> NOP*5"); } @@ -2096,7 +2322,7 @@ public sealed partial class DirectExecutionBackend { if (flNewProtect != 0) { - _hostMemory.ProtectRaw((ulong)(void*)num, 5u, flNewProtect, out flNewProtect); + VirtualProtect((void*)num, 5u, flNewProtect, &flNewProtect); } } } diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs index 43e8414..38a468d 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.NativeWorker.cs @@ -6,8 +6,6 @@ using System.Collections.Generic; using System.Runtime.InteropServices; using System.Threading; using SharpEmu.HLE; -using SharpEmu.HLE.Host; -using SharpEmu.HLE.Host.Posix; namespace SharpEmu.Core.Cpu.Native; @@ -37,6 +35,20 @@ public sealed partial class DirectExecutionBackend private bool _nativeWorkersDisposed; private int _nativeWorkerCreationFailedLogged; + private const uint StackSizeParamIsAReservation = 0x00010000u; + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern nint CreateThread( + nint lpThreadAttributes, + nuint dwStackSize, + nint lpStartAddress, + nint lpParameter, + uint dwCreationFlags, + out uint lpThreadId); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern uint WaitForSingleObject(nint hHandle, uint dwMilliseconds); + // Runs an emitted guest entry stub. Preferred path is a pooled native worker // thread; falls back to the historical inline calli (guest frames above this // thread's managed frames) when workers are disabled or unavailable. @@ -49,7 +61,7 @@ public sealed partial class DirectExecutionBackend var worker = RentNativeGuestExecutor(); if (worker is null) { - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, (nint)hostRspSlot); + TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot); return CallNativeEntry(entryStub); } try @@ -80,7 +92,10 @@ public sealed partial class DirectExecutionBackend private NativeGuestExecutor? RentNativeGuestExecutor() { - if (NativeGuestWorkersDisabled) + // NativeGuestExecutor emits a Win32 wait loop and creates it with + // kernel32!CreateThread. POSIX hosts use the established inline entry + // path until the worker loop has a pthread/eventfd implementation. + if (!OperatingSystem.IsWindows() || NativeGuestWorkersDisabled) { return null; } @@ -234,7 +249,7 @@ public sealed partial class DirectExecutionBackend public static NativeGuestExecutor? TryCreate(DirectExecutionBackend backend) { - if (!EnsureHostRuntimeExports(backend._hostSymbols)) + if (!EnsureKernel32Exports()) { return null; } @@ -247,27 +262,32 @@ public sealed partial class DirectExecutionBackend return executor; } - private static bool EnsureHostRuntimeExports(IHostSymbolResolver symbols) + private static bool EnsureKernel32Exports() { if (_exitThreadAddress != 0) { return _waitForSingleObjectAddress != 0 && _setEventAddress != 0; } - _waitForSingleObjectAddress = symbols.GetAddress(HostRuntimeFunction.WaitForSingleObject); - _setEventAddress = symbols.GetAddress(HostRuntimeFunction.SetEvent); - _exitThreadAddress = symbols.GetAddress(HostRuntimeFunction.ExitThread); + nint kernel32 = GetModuleHandle("kernel32.dll"); + if (kernel32 == 0) + { + return false; + } + _waitForSingleObjectAddress = GetProcAddress(kernel32, "WaitForSingleObject"); + _setEventAddress = GetProcAddress(kernel32, "SetEvent"); + _exitThreadAddress = GetProcAddress(kernel32, "ExitThread"); return _waitForSingleObjectAddress != 0 && _setEventAddress != 0 && _exitThreadAddress != 0; } private bool Initialize() { _selfHandle = GCHandle.Alloc(this); - _controlBlock = (void*)_backend._hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWrite); + _controlBlock = VirtualAlloc(null, 4096u, 12288u, 4u); if (_controlBlock == null) { return false; } - _loopStub = (void*)_backend._hostMemory.Allocate(0, LoopStubSize, HostPageProtection.ReadWriteExecute); + _loopStub = VirtualAlloc(null, LoopStubSize, 12288u, 64u); if (_loopStub == null) { return false; @@ -375,15 +395,17 @@ public sealed partial class DirectExecutionBackend *(int*)(code + skipJump) = skipEntryOffset - (skipJump + sizeof(int)); uint oldProtect = 0; - if (!_backend._hostMemory.Protect((ulong)_loopStub, LoopStubSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(_loopStub, LoopStubSize, 32u, &oldProtect)) { return false; } - _backend._hostMemory.FlushInstructionCache((ulong)_loopStub, LoopStubSize); - _threadHandle = _backend._hostThreading.CreateNativeThread( - (nint)_loopStub, + FlushInstructionCache(GetCurrentProcess(), _loopStub, LoopStubSize); + _threadHandle = CreateThread( 0, WorkerStackReservation, + (nint)_loopStub, + 0, + StackSizeParamIsAReservation, out _nativeThreadId); if (_threadHandle == 0) { @@ -511,7 +533,7 @@ public sealed partial class DirectExecutionBackend _prevYieldRequested = _activeGuestThreadYieldRequested; _prevYieldReason = _activeGuestThreadYieldReason; _prevState = _activeGuestThreadState; - _prevHostRspSlot = backend._hostThreading.GetTlsValue(backend._hostRspSlotTlsIndex); + _prevHostRspSlot = TlsGetValue(backend._hostRspSlotTlsIndex); _prevGuestThreadHandle = GuestThreadExecution.EnterGuestThread(_runGuestThreadHandle); _entered = true; _activeExecutionBackend = backend; @@ -523,11 +545,11 @@ public sealed partial class DirectExecutionBackend _activeGuestThreadYieldReason = null; _activeGuestThreadState = _runState; backend.BindTlsBase(_runContext!); - backend._hostThreading.SetTlsValue(backend._hostRspSlotTlsIndex, _runHostRspSlot); + TlsSetValue(backend._hostRspSlotTlsIndex, _runHostRspSlot); if (_runState is { } state) { _prevHostThreadId = Volatile.Read(ref state.HostThreadId); - Volatile.Write(ref state.HostThreadId, unchecked((int)backend._hostThreading.CurrentThreadId)); + Volatile.Write(ref state.HostThreadId, unchecked((int)GetCurrentThreadId())); } if (_runAffinityMask != 0) { @@ -557,7 +579,7 @@ public sealed partial class DirectExecutionBackend { Volatile.Write(ref state.HostThreadId, _prevHostThreadId); } - _backend._hostThreading.SetTlsValue(_backend._hostRspSlotTlsIndex, _prevHostRspSlot); + TlsSetValue(_backend._hostRspSlotTlsIndex, _prevHostRspSlot); GuestThreadExecution.RestoreGuestThread(_prevGuestThreadHandle); _activeExecutionBackend = _prevBackend; _activeCpuContext = _prevContext; @@ -594,8 +616,8 @@ public sealed partial class DirectExecutionBackend var exited = _threadHandle == 0; if (_threadHandle != 0) { - exited = _backend._hostThreading.WaitForThreadExit(_threadHandle, 1000u); - _backend._hostThreading.CloseThreadHandle(_threadHandle); + exited = WaitForSingleObject(_threadHandle, 1000u) == 0u; + CloseHandle(_threadHandle); _threadHandle = 0; } if (!exited) @@ -609,12 +631,12 @@ public sealed partial class DirectExecutionBackend } if (_loopStub != null) { - _backend._hostMemory.Free((ulong)_loopStub); + VirtualFree(_loopStub, 0u, 32768u); _loopStub = null; } if (_controlBlock != null) { - _backend._hostMemory.Free((ulong)_controlBlock); + VirtualFree(_controlBlock, 0u, 32768u); _controlBlock = null; } if (_selfHandle.IsAllocated) diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs index d9dbf82..a91b3f1 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.PosixSignals.cs @@ -4,7 +4,7 @@ using System; using System.Runtime.InteropServices; using System.Threading; -using SharpEmu.Core.Cpu.Native.Windows; +using SharpEmu.HLE; namespace SharpEmu.Core.Cpu.Native; @@ -21,6 +21,8 @@ public sealed unsafe partial class DirectExecutionBackend // runtime keeps turning its own faults into managed exceptions. private const int PosixSigIll = 4; + private const int PosixSigTrap = 5; + private const int PosixSigAbort = 6; private const int PosixSigSegv = 11; private static readonly int PosixSigBus = OperatingSystem.IsMacOS() ? 10 : 7; @@ -62,6 +64,9 @@ public sealed unsafe partial class DirectExecutionBackend private static bool _posixSignalWarmup; private static readonly nint[] _posixPreviousActions = new nint[32]; private static int _posixSignalTraceCount; + private static long _perfSignalCount; + private static readonly bool _perfSignalCounter = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_PERF_MEM"), "1", StringComparison.Ordinal); [ThreadStatic] private static int _posixSignalHandlerDepth; @@ -88,10 +93,13 @@ public sealed unsafe partial class DirectExecutionBackend } WarmUpPosixSignalPath(); + SharpEmu.HLE.GuestImageWriteTracker.WarmUp(); if (!InstallPosixSignalHandler(PosixSigSegv) || !InstallPosixSignalHandler(PosixSigBus) || - !InstallPosixSignalHandler(PosixSigIll)) + !InstallPosixSignalHandler(PosixSigIll) || + !InstallPosixSignalHandler(PosixSigTrap) || + !InstallPosixSignalHandler(PosixSigAbort)) { throw new InvalidOperationException("Failed to install POSIX fault signal handlers"); } @@ -132,8 +140,8 @@ public sealed unsafe partial class DirectExecutionBackend // action and sigaction(0, ...) fails with EINVAL). EXCEPTION_RECORD record = default; record.ExceptionCode = DBG_PRINTEXCEPTION_C; - byte* contextRecord = stackalloc byte[Win64ContextOffsets.Size]; - new Span(contextRecord, Win64ContextOffsets.Size).Clear(); + byte* contextRecord = stackalloc byte[Win64ContextSize]; + new Span(contextRecord, Win64ContextSize).Clear(); EXCEPTION_POINTERS pointers; pointers.ExceptionRecord = &record; pointers.ContextRecord = contextRecord; @@ -190,8 +198,27 @@ public sealed unsafe partial class DirectExecutionBackend } _posixSignalHandlerDepth++; + if (_perfSignalCounter) + { + var n = Interlocked.Increment(ref _perfSignalCount); + if (n % 100000 == 0) + { + Console.Error.WriteLine($"[PERF][MEM] posix_faults={n}"); + } + } try { + // Guest-image write tracking runs first: it only needs the fault + // address (safe for host and guest threads alike) and must resume + // the faulting write immediately after restoring write access. + if (signal != PosixSigIll && + siginfo != 0 && + SharpEmu.HLE.GuestImageWriteTracker.TryHandleWriteFault( + *(ulong*)((byte*)siginfo + PosixSigInfoAddressOffset))) + { + return; + } + if (TryHandlePosixFault(signal, siginfo, ucontext)) { return; @@ -217,8 +244,8 @@ public sealed unsafe partial class DirectExecutionBackend return false; } - byte* contextRecord = stackalloc byte[Win64ContextOffsets.Size]; - new Span(contextRecord, Win64ContextOffsets.Size).Clear(); + byte* contextRecord = stackalloc byte[Win64ContextSize]; + new Span(contextRecord, Win64ContextSize).Clear(); int[] offsets = PosixRegisterOffsets; for (int i = 0; i < offsets.Length; i++) { @@ -231,6 +258,14 @@ public sealed unsafe partial class DirectExecutionBackend { record.ExceptionCode = 3221225501u; } + else if (signal == PosixSigTrap) + { + record.ExceptionCode = 2147483651u; + } + else if (signal == PosixSigAbort) + { + record.ExceptionCode = 1073741845u; + } else { ulong faultAddress = GetPosixFaultAddress(siginfo, registers); diff --git a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs index d8d27a1..df70098 100644 --- a/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs +++ b/src/SharpEmu.Core/Cpu/Native/DirectExecutionBackend.cs @@ -2,52 +2,39 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System; -using System.Collections.Concurrent; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; using System.Threading; using SharpEmu.Core.Cpu; -using SharpEmu.Core.Cpu.Native.Windows; using SharpEmu.Core.Loader; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.HLE.Host; -using SharpEmu.HLE.Host.Posix; -using SharpEmu.Logging; namespace SharpEmu.Core.Cpu.Native; public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, IGuestThreadScheduler, IDisposable { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Native"); + private static readonly SharpEmu.Logging.SharpEmuLogger Log = SharpEmu.Logging.SharpEmuLog.For("Native"); + + private static readonly bool LogThreadMode = + string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_THREAD_MODE"), "1", StringComparison.Ordinal); + + private static void TraceThreadMode(string message) + { + Console.Error.WriteLine( + $"[THREADMODE] {message} tid={GetCurrentThreadId()} managed={Environment.CurrentManagedThreadId}"); + Console.Error.Flush(); + } + private const int ImportLoopHistoryLength = 2048; private const int ImportLoopWideDiversityWindow = 768; private const int DefaultImportLoopGuardSeconds = 5; - private enum ImportStubKind : byte - { - Normal = 0, - BootstrapBridge = 1, - KernelDynlibDlsym = 2, - Il2CppApiLookupSymbol = 3, - } - - [Flags] - private enum ImportStubTraceFlags : byte - { - None = 0, - Memset = 1 << 0, - CxaAtexit = 1 << 1, - RawArgs = 1 << 2, - StackChkFail = 1 << 3, - PeriodicEvery1000 = 1 << 4, - PeriodicEvery128 = 1 << 5, - } - private readonly struct ImportStubEntry { public ulong Address { get; } @@ -56,36 +43,34 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public ExportedFunction? Export { get; } - public ImportStubKind Kind { get; } + // Precomputed per-import classification: DispatchImport runs for + // every guest HLE call, so per-call string pattern matches and NID + // hashing are hoisted to stub-setup time. + public bool IsLeaf { get; } - public ImportStubTraceFlags TraceFlags { get; } + public bool SuppressStrlenTrace { get; } - public ImportStubEntry(ulong address, string nid, ExportedFunction? export) + public bool IsLoopGuardBoundary { get; } + + public ulong NidHash { get; } + + public ImportStubEntry( + ulong address, + string nid, + ExportedFunction? export, + bool isLeaf, + bool suppressStrlenTrace, + bool isLoopGuardBoundary, + ulong nidHash) { Address = address; Nid = nid; Export = export; - Kind = ClassifyKind(nid); - TraceFlags = ClassifyTraceFlags(nid); + IsLeaf = isLeaf; + SuppressStrlenTrace = suppressStrlenTrace; + IsLoopGuardBoundary = isLoopGuardBoundary; + NidHash = nidHash; } - - private static ImportStubKind ClassifyKind(string nid) => nid switch - { - RuntimeStubNids.BootstrapBridge => ImportStubKind.BootstrapBridge, - RuntimeStubNids.KernelDynlibDlsym or "LwG8g3niqwA" => ImportStubKind.KernelDynlibDlsym, - "r8mvOaWdi28" => ImportStubKind.Il2CppApiLookupSymbol, - _ => ImportStubKind.Normal, - }; - - private static ImportStubTraceFlags ClassifyTraceFlags(string nid) => nid switch - { - "8zTFvBIAIN8" => ImportStubTraceFlags.Memset, - "tsvEmnenz48" => ImportStubTraceFlags.CxaAtexit | ImportStubTraceFlags.PeriodicEvery1000, - "bzQExy189ZI" or "8G2LB+A3rzg" => ImportStubTraceFlags.RawArgs, - "Ou3iL1abvng" => ImportStubTraceFlags.StackChkFail, - "rTXw65xmLIA" => ImportStubTraceFlags.PeriodicEvery128, - _ => ImportStubTraceFlags.None, - }; } private readonly record struct RecentImportTraceEntry( @@ -94,14 +79,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ulong ReturnRip, ulong Arg0, ulong Arg1, - ulong Arg2); - - private readonly record struct DeferredBootstrapTraceEntry( - long DispatchIndex, - ulong Op, - ulong SymbolPointer, - ulong OutputPointer, - ulong ReturnRip); + ulong Arg2, + ulong GuestThreadHandle, + int ManagedThreadId); #pragma warning disable CS0649 private struct EXCEPTION_POINTERS @@ -129,6 +109,29 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private delegate int ExceptionHandlerDelegate(void* exceptionInfo); +#pragma warning disable CS0649 + private struct MEMORY_BASIC_INFORMATION64 + { + public ulong BaseAddress; + + public ulong AllocationBase; + + public uint AllocationProtect; + + public uint __alignment1; + + public ulong RegionSize; + + public uint State; + + public uint Protect; + + public uint Type; + + public uint __alignment2; + } +#pragma warning restore CS0649 + private const ulong SYSTEM_RESERVED = 34359738368uL; private const ulong CODE_BASE_OFFSET = 4294967296uL; @@ -139,29 +142,32 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private const ulong GuestImageScanEnd = 36507222016uL; - // The 0x7FFx window is Windows-specific; dyld and Rosetta reserve that - // range on macOS, so POSIX guest threads use the lower 0x6FFx window. - // The POSIX stack base sits a further 1GB down: the import-stub region - // descends from 0x7000_0000_0000 on the same 16MB grid and reaches - // 0x6FFF_C000_0000 at its 64-module limit, which would otherwise consume - // the top stack slots (on Windows the two bands are ~15TB apart). - private static readonly ulong GuestThreadStackBaseAddress = - OperatingSystem.IsWindows() ? 0x7FFF_E000_0000UL : 0x6FFF_A000_0000UL; + // See CpuDispatcher: the 0x7FFx window is Windows-only; POSIX hosts + // (dyld shared cache, Rosetta 2 runtime) use 0x6FFx instead. + private static readonly ulong GuestThreadStackBaseAddress = OperatingSystem.IsWindows() ? 0x7FFF_E000_0000UL : 0x6FFF_E000_0000UL; - private static readonly ulong GuestThreadTlsBaseAddress = - OperatingSystem.IsWindows() ? 0x7FFE_0000_0000UL : 0x6FFE_0000_0000UL; + private static readonly ulong GuestThreadTlsBaseAddress = OperatingSystem.IsWindows() ? 0x7FFE_0000_0000UL : 0x6FFE_0000_0000UL; private const ulong GuestThreadStackSize = 0x0020_0000UL; private const ulong GuestThreadTlsSize = 0x0001_0000UL; // Matches CpuDispatcher.TlsPrefixSize: static TLS blocks sit below the - // TCB, and libc.prx already reaches beyond -0x1700 on POSIX. - private static readonly ulong GuestThreadTlsPrefixSize = OperatingSystem.IsWindows() ? 0x0000_1000UL : 0x0001_0000UL; + // thread pointer and PS5 modules can reach beyond one host page. + private const ulong GuestThreadTlsPrefixSize = GuestTlsTemplate.StartupStaticTlsReservation; private const ulong GuestThreadRegionStride = 0x0100_0000UL; - private const int GuestThreadRegionSlotCount = 256; + // Unity titles routinely create more than 64 workers once native plugins, + // lighting, streaming, and audio are active at the same time. Keep a broad + // deterministic address window for their stack and TLS regions. + private const int GuestThreadRegionSlots = 1024; + + [ThreadStatic] + private static List<(IVirtualMemory Memory, ulong Base)>? _nestedGuestCallbackStacks; + + [ThreadStatic] + private static int _nestedGuestCallbackDepth; private const uint PAGE_EXECUTE_READWRITE = 64u; @@ -262,7 +268,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private readonly List _importHandlerTrampolines = new List(); - private const int GuestContextTransferFrameQwords = 15; + private const int GuestContextTransferFrameQwords = 20; private readonly object _guestContextTransferStubGate = new(); @@ -278,31 +284,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private KeyValuePair[] _runtimeSymbolsByAddress = Array.Empty>(); - private readonly ConcurrentDictionary _runtimeSymbolsByName = - new(StringComparer.Ordinal); - - // Keep in sync with SelfLoader import-stub mapping constants. - private const ulong ImportStubRegionCanonicalBase = 0x0000_7000_0000_0000UL; - - private const ulong ImportStubRegionAddressStride = 0x0000_0000_0100_0000UL; - - private const ulong LazyImportStubSlotSize = 0x10; - - private const ulong ImportStubRegionPageSize = 0x1000UL; - - private const string KernelDynlibDlsymAerolibNid = "LwG8g3niqwA"; - - private readonly object _lazyDlsymStubGate = new(); - - private readonly Dictionary _lazyDlsymStubCache = new(StringComparer.Ordinal); - - private ulong _lazyImportStubPoolBase; - - private ulong _lazyImportStubNextSlot; - - private ulong _lazyImportStubPoolLimit; - - private bool _lazyImportStubPoolMapped; + private readonly Dictionary _runtimeSymbolsByName = new Dictionary(StringComparer.Ordinal); private readonly RecentImportTraceEntry[] _recentImportTrace = new RecentImportTraceEntry[64]; @@ -310,14 +292,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private int _recentImportTraceWriteIndex; - private readonly DeferredBootstrapTraceEntry[] _deferredBootstrapTrace = new DeferredBootstrapTraceEntry[32]; - - private int _deferredBootstrapTraceCount; - - private int _deferredBootstrapTraceWriteIndex; - - private readonly object _deferredBootstrapTraceGate = new(); - private readonly string[] _distinctImportNidHistory = new string[128]; private int _distinctImportNidHistoryCount; @@ -336,6 +310,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private bool _logGuestContext; + private bool _ignoreGuestInt41; + + private int _ignoredGuestInt41Count; + private bool _logGuestThreads; private bool _logUsleep; @@ -352,6 +330,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private string? _probeImportReturn; + private ulong _probeImportReturnAddress; + + private long _probeImportReturnAddressCount; + private string? _importFilter; private bool _disableImportLoopGuard; @@ -398,7 +380,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private long _importLoopPatternStartTimestamp; - private readonly Dictionary _importNidHashCache = new Dictionary(StringComparer.Ordinal); private enum GuestThreadRunState { @@ -427,11 +408,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public string Name { get; init; } = string.Empty; - public int Priority { get; init; } + public int Priority { get; set; } - public ulong AffinityMask { get; init; } + public ulong AffinityMask { get; set; } - public CpuContext Context { get; init; } = null!; + public CpuContext Context { get; set; } = null!; + + public bool IsExternalExecutor { get; init; } + + public ulong StackBase { get; init; } + + public ulong StackSize { get; init; } + + public ulong ExceptionStackBase { get; set; } public GuestThreadRunState State { get; set; } @@ -448,6 +437,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I // Stays set through the wake transition; Resume() consumes it when the thread pumps. public IGuestThreadBlockWaiter? BlockWaiter { get; set; } + public Func? BlockResumeHandler { get; set; } + + public Func? BlockWakeHandler { get; set; } + public long BlockDeadlineTimestamp { get; set; } public long ImportCount; @@ -456,13 +449,55 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public ulong LastReturnRip; + // Busy guest workers overwrite the global recent-import ring. Preserve + // the most recent complete SysV call frame per guest thread so native + // fault diagnostics can identify the exact preceding HLE invocation. + public ulong LastImportRdi; + public ulong LastImportRsi; + public ulong LastImportRdx; + public ulong LastImportRcx; + public ulong LastImportR8; + public ulong LastImportR9; + public ulong LastImportStack0; + public ulong LastImportStack1; + public ulong LastImportStack2; + public ulong LastImportStack3; + public ulong LastImportStack4; + public ulong LastImportStack5; + public ulong LastImportRax; + public int LastImportResultValid; + public Thread? HostThread { get; set; } public int HostThreadId; + // State may become Ready as soon as another guest thread satisfies this + // thread's wait, while the host executor that yielded it is still + // unwinding. Keep executor ownership separate from State so a Ready + // continuation cannot be claimed concurrently with that unwind. + public bool ExecutorActive { get; set; } + + public bool ExceptionDeliveryActive { get; set; } + + public long ExecutorClaimDeferrals { get; set; } + public GuestContinuationRunner? ContinuationRunner { get; set; } + + public GuestExecutionRunner? ExecutionRunner { get; set; } } + private sealed class ExternalGuestThreadState + { + public CpuContext Context { get; set; } = null!; + + public ulong ExceptionStackBase { get; set; } + } + + private readonly record struct PendingGuestException( + ulong Handler, + int ExceptionType, + ulong ExceptionStackBase); + private sealed class GuestContinuationRunner : IDisposable { private readonly ulong _guestThreadHandle; @@ -501,10 +536,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private void ThreadMain() { var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(_guestThreadHandle); - if (LogThreadMode) - { - TraceThreadMode($"runner_start guest=0x{_guestThreadHandle:X16}"); - } try { while (true) @@ -515,11 +546,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return; } - if (LogThreadMode) - { - _threadModeCycleId = Interlocked.Increment(ref _threadModeCycleCounter); - TraceThreadMode($"runner_run guest=0x{_guestThreadHandle:X16}"); - } try { _work?.Invoke(); @@ -532,10 +558,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - if (LogThreadMode) - { - TraceThreadMode($"runner_stop guest=0x{_guestThreadHandle:X16}"); - } GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); } } @@ -553,6 +575,86 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } } + // A guest pthread may block and resume hundreds of times per second. Creating + // a new host Thread for every resume is especially costly for audio workers + // and eventually floods macOS with short-lived FMOD threads. Keep one dormant + // host executor per guest pthread and signal it for each runnable slice. + private sealed class GuestExecutionRunner : IDisposable + { + private readonly object _gate = new(); + private readonly AutoResetEvent _workAvailable = new(false); + private readonly Thread _thread; + private Action? _work; + private volatile bool _stopping; + + public GuestExecutionRunner(ulong guestThreadHandle, string name, ThreadPriority priority) + { + _thread = new Thread(() => ThreadMain(guestThreadHandle)) + { + IsBackground = true, + Name = $"SharpEmu-{name}", + Priority = priority, + }; + _thread.Start(); + } + + public void Schedule(Action work) + { + lock (_gate) + { + if (_stopping) + { + return; + } + if (_work is not null) + { + throw new InvalidOperationException("Guest execution runner already has pending work."); + } + _work = work; + _workAvailable.Set(); + } + } + + private void ThreadMain(ulong guestThreadHandle) + { + var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(guestThreadHandle); + try + { + while (true) + { + _workAvailable.WaitOne(); + if (_stopping) + { + return; + } + + Action? work; + lock (_gate) + { + work = _work; + _work = null; + } + work?.Invoke(); + } + } + finally + { + GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); + } + } + + public void Dispose() + { + _stopping = true; + _workAvailable.Set(); + if (!ReferenceEquals(Thread.CurrentThread, _thread)) + { + _thread.Join(500); + } + _workAvailable.Dispose(); + } + } + private readonly record struct LazyCommitRange(ulong BaseAddress, ulong Size); private readonly object _guestThreadGate = new object(); @@ -591,22 +693,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private readonly Queue _readyGuestThreads = new Queue(); - // Once set, guest worker threads are unwound to the host at their next import - // dispatch and Pump refuses to start new ones. This must happen before the - // runtime frees trampolines or guest memory: workers that keep running native - // guest code during teardown execute freed pages (observed as an execute-AV in - // a MEM_FREE module region plus a CLR "UnmanagedCallersOnly" fatal from a Pump - // thread entering a freed stub). - private volatile bool _guestTeardownRequested; - - private volatile bool _hostShutdownRequested; - - private static volatile DirectExecutionBackend? _activeSessionBackend; - private int _readyGuestThreadCount; private readonly Dictionary _guestThreads = new Dictionary(); + private readonly Dictionary _externalGuestThreads = new Dictionary(); + + [ThreadStatic] + private static ulong _currentExternalGuestThreadHandle; + + private readonly Dictionary _pendingGuestExceptions = new Dictionary(); + + private readonly HashSet _activeGuestExceptionDeliveries = new HashSet(); + private int _guestThreadPumpDepth; private bool _guestThreadYieldRequested; @@ -631,20 +730,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private Thread? _stallWatchdogThread; + private volatile bool _readyDispatchStop; + + private Thread? _readyDispatchThread; + private GCHandle _selfHandle; private nint _selfHandlePtr; - private readonly IHostPlatform _hostPlatform; - - private readonly IHostThreading _hostThreading; - - private readonly IHostSymbolResolver _hostSymbols; - - private readonly IHostMemory _hostMemory; - - private readonly IHostFaultHandling _faultHandling; - private const int MinTlsPatchInstructionBytes = 9; private delegate ulong ImportGatewayDelegate(nint backendHandle, int importIndex, nint argPackPtr); @@ -653,17 +746,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static readonly RawExceptionHandlerDelegate RawVectoredHandlerDelegateInstance = RawVectoredHandlerManaged; private static readonly RawExceptionHandlerDelegate RawUnhandledFilterDelegateInstance = RawUnhandledFilterManaged; - private static readonly nint ImportGatewayPtr = ResolveImportGatewayPtr(); + private static readonly nint ImportGatewayPtr = ResolveWin64CallbackPtr( + Marshal.GetFunctionPointerForDelegate(ImportGatewayDelegateInstance)); - // Emitted import trampolines use the Win64 ABI. Managed callbacks use the - // host ABI, so POSIX needs a small Win64-to-SysV register-shuffling thunk. - private static nint ResolveImportGatewayPtr() - { - var managedPtr = Marshal.GetFunctionPointerForDelegate(ImportGatewayDelegateInstance); - return OperatingSystem.IsWindows() - ? managedPtr - : PosixHostStubs.CreateWin64ToSysVThunk(managedPtr); - } + // Emitted trampolines call managed callbacks with the Win64 ABI. On + // Windows the runtime already compiles them that way; on POSIX .NET they + // are SysV, so route through a Win64->SysV thunk. + private static nint ResolveWin64CallbackPtr(nint sysvPtr) => + OperatingSystem.IsWindows() ? sysvPtr : PosixHostStubs.CreateWin64ToSysVThunk(sysvPtr); private static readonly nint RawVectoredHandlerPtrManaged = Marshal.GetFunctionPointerForDelegate(RawVectoredHandlerDelegateInstance); @@ -671,41 +761,41 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static readonly nint RawUnhandledFilterPtrManaged = Marshal.GetFunctionPointerForDelegate(RawUnhandledFilterDelegateInstance); - private const int CTX_MXCSR = Win64ContextOffsets.Mxcsr; + private const int CTX_MXCSR = 52; - private const int CTX_RAX = Win64ContextOffsets.Rax; + private const int CTX_RAX = 120; - private const int CTX_RCX = Win64ContextOffsets.Rcx; + private const int CTX_RCX = 128; - private const int CTX_RDX = Win64ContextOffsets.Rdx; + private const int CTX_RDX = 136; - private const int CTX_RBX = Win64ContextOffsets.Rbx; + private const int CTX_RBX = 144; - private const int CTX_RSP = Win64ContextOffsets.Rsp; + private const int CTX_RSP = 152; - private const int CTX_RBP = Win64ContextOffsets.Rbp; + private const int CTX_RBP = 160; - private const int CTX_RSI = Win64ContextOffsets.Rsi; + private const int CTX_RSI = 168; - private const int CTX_RDI = Win64ContextOffsets.Rdi; + private const int CTX_RDI = 176; - private const int CTX_R8 = Win64ContextOffsets.R8; + private const int CTX_R8 = 184; - private const int CTX_R9 = Win64ContextOffsets.R9; + private const int CTX_R9 = 192; - private const int CTX_R10 = Win64ContextOffsets.R10; + private const int CTX_R10 = 200; - private const int CTX_R11 = Win64ContextOffsets.R11; + private const int CTX_R11 = 208; - private const int CTX_R12 = Win64ContextOffsets.R12; + private const int CTX_R12 = 216; - private const int CTX_R13 = Win64ContextOffsets.R13; + private const int CTX_R13 = 224; - private const int CTX_R14 = Win64ContextOffsets.R14; + private const int CTX_R14 = 232; - private const int CTX_R15 = Win64ContextOffsets.R15; + private const int CTX_R15 = 240; - private const int CTX_RIP = Win64ContextOffsets.Rip; + private const int CTX_RIP = 248; private ExceptionHandlerDelegate? _handlerDelegate; @@ -720,37 +810,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static int _nestedVehTraceCount; - // SHARPEMU_LOG_THREAD_MODE=1 — GC thread-mode corruption investigation. Traces - // every managed<->guest transition per guest-thread run cycle so the last event - // before a ReversePInvokeBadTransition FailFast identifies the corrupting path. - private static readonly bool LogThreadMode = - string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_THREAD_MODE"), "1", StringComparison.Ordinal); + private const uint MEM_COMMIT = 4096u; - private static long _threadModeCycleCounter; + private const uint MEM_RESERVE = 8192u; - [ThreadStatic] - private static long _threadModeCycleId; + private const uint MEM_FREE = 65536u; - [ThreadStatic] - private static int _threadModeGatewayDepth; - - [ThreadStatic] - private static long _threadModeGatewayCalls; - - [ThreadStatic] - private static bool _threadModeGatewayFirstLogged; - - private static void TraceThreadMode(string message) - { - // Prefer the platform injected into the backend bound to this thread; - // the ambient is set on every guest/import/VEH transition this tracer - // observes. The singleton fallback only covers pre-run traces, where a - // constructed backend implies the platform resolved successfully. - var threading = _activeExecutionBackend?._hostThreading ?? HostPlatform.Current.Threading; - Console.Error.WriteLine( - $"[THREADMODE] {message} cycle={_threadModeCycleId} tid={threading.CurrentThreadId} managed={Environment.CurrentManagedThreadId}"); - Console.Error.Flush(); - } + private const uint MEM_RELEASE = 32768u; private const uint PAGE_EXECUTE = 16u; @@ -770,6 +836,16 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private const uint HostXmmSaveAreaSize = 0xA0u; + private const uint ContextAmd64ControlInteger = 0x00100003u; + + private const uint ThreadGetContext = 0x0008u; + + private const uint ThreadSuspendResume = 0x0002u; + + private const int Win64ContextSize = 0x4D0; + + private const int Win64ContextFlagsOffset = 0x30; + private readonly record struct HostThreadContextSnapshot( bool IsValid, ulong Rip, @@ -792,17 +868,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe static int CallNativeEntry(void* entry) { var nativeEntry = (delegate* unmanaged[Cdecl])entry; - if (!LogThreadMode) - { - return nativeEntry(); - } - - _threadModeGatewayFirstLogged = false; - var dispatchesBefore = _threadModeGatewayCalls; - TraceThreadMode($"native_enter entry=0x{(ulong)entry:X16}"); - var result = nativeEntry(); - TraceThreadMode($"native_exit result=0x{result:X8} dispatches={_threadModeGatewayCalls - dispatchesBefore}"); - return result; + return nativeEntry(); } private unsafe static void WriteCtxU64(void* contextRecord, int offset, ulong value) @@ -845,8 +911,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private bool ActiveForcedGuestExit { - get => _hostShutdownRequested || - (HasActiveExecutionThread ? _activeForcedGuestExit : _forcedGuestExit); + get => HasActiveExecutionThread ? _activeForcedGuestExit : _forcedGuestExit; set { if (HasActiveExecutionThread) @@ -910,41 +975,47 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _activeGuestThreadYieldReason = previousYieldReason; } - public unsafe DirectExecutionBackend(IModuleManager moduleManager, IHostPlatform? hostPlatform = null, IHostFaultHandling? faultHandling = null) + public unsafe DirectExecutionBackend(IModuleManager moduleManager) { _moduleManager = moduleManager ?? throw new ArgumentNullException("moduleManager"); - _hostPlatform = hostPlatform ?? HostPlatform.Current; - _hostThreading = _hostPlatform.Threading; - _hostSymbols = _hostPlatform.Symbols; - _hostMemory = _hostPlatform.Memory; - _faultHandling = faultHandling ?? (OperatingSystem.IsWindows() - ? new WindowsFaultHandling(_hostMemory) - : NullHostFaultHandling.Instance); _selfHandle = GCHandle.Alloc(this); _selfHandlePtr = GCHandle.ToIntPtr(_selfHandle); - _guestTlsBaseTlsIndex = _hostThreading.AllocateTlsSlot(); - _hostRspSlotTlsIndex = _hostThreading.AllocateTlsSlot(); + _guestTlsBaseTlsIndex = TlsAlloc(); + _hostRspSlotTlsIndex = TlsAlloc(); if (_guestTlsBaseTlsIndex == uint.MaxValue || _hostRspSlotTlsIndex == uint.MaxValue) { throw new OutOfMemoryException("Failed to allocate native TLS slots"); } - _tlsGetValueAddress = _hostSymbols.GetAddress(HostRuntimeFunction.TlsGetValue); - if (_tlsGetValueAddress == 0) + if (OperatingSystem.IsWindows()) { - throw new InvalidOperationException("Failed to resolve kernel32!TlsGetValue"); + nint kernel32 = GetModuleHandle("kernel32.dll"); + _tlsGetValueAddress = kernel32 != 0 ? GetProcAddress(kernel32, "TlsGetValue") : 0; + if (_tlsGetValueAddress == 0) + { + throw new InvalidOperationException("Failed to resolve kernel32!TlsGetValue"); + } + _queryPerformanceCounterAddress = kernel32 != 0 ? GetProcAddress(kernel32, "QueryPerformanceCounter") : 0; + if (_queryPerformanceCounterAddress == 0) + { + throw new InvalidOperationException("Failed to resolve kernel32!QueryPerformanceCounter"); + } + _switchToThreadAddress = kernel32 != 0 ? GetProcAddress(kernel32, "SwitchToThread") : 0; + _sleepAddress = kernel32 != 0 ? GetProcAddress(kernel32, "Sleep") : 0; + if (_switchToThreadAddress == 0 || _sleepAddress == 0) + { + throw new InvalidOperationException("Failed to resolve kernel32 thread timing functions"); + } } - _queryPerformanceCounterAddress = _hostSymbols.GetAddress(HostRuntimeFunction.QueryPerformanceCounter); - if (_queryPerformanceCounterAddress == 0) + else { - throw new InvalidOperationException("Failed to resolve kernel32!QueryPerformanceCounter"); + // Win64-ABI-compatible helper stubs so the emitted call sites do + // not need to change per platform. + _tlsGetValueAddress = PosixHostStubs.TlsGetValueStubAddress; + _queryPerformanceCounterAddress = PosixHostStubs.QueryPerformanceCounterStubAddress; + _switchToThreadAddress = PosixHostStubs.SwitchToThreadStubAddress; + _sleepAddress = PosixHostStubs.SleepStubAddress; } - _switchToThreadAddress = _hostSymbols.GetAddress(HostRuntimeFunction.SwitchToThread); - _sleepAddress = _hostSymbols.GetAddress(HostRuntimeFunction.Sleep); - if (_switchToThreadAddress == 0 || _sleepAddress == 0) - { - throw new InvalidOperationException("Failed to resolve kernel32 thread timing functions"); - } - _tlsBaseAddress = (nint)_hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWrite); + _tlsBaseAddress = (nint)VirtualAlloc(null, 4096u, 12288u, 4u); if (_tlsBaseAddress == 0) { throw new OutOfMemoryException("Failed to allocate TLS base"); @@ -952,7 +1023,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _ownedTlsBaseAddress = _tlsBaseAddress; _ownsTlsBaseAddress = true; SeedTlsLayout(_tlsBaseAddress); - _hostRspSlotStorage = (nint)_hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWrite); + _hostRspSlotStorage = (nint)VirtualAlloc(null, 4096u, 12288u, 4u); if (_hostRspSlotStorage == 0) { throw new OutOfMemoryException("Failed to allocate host stack slot storage"); @@ -981,14 +1052,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I result = OrbisGen2Result.ORBIS_GEN2_OK; LastError = null; InitializeRuntimeSymbolIndex(runtimeSymbols); - ResetLazyDlsymStubState(); _recentImportTraceCount = 0; _recentImportTraceWriteIndex = 0; - lock (_deferredBootstrapTraceGate) - { - _deferredBootstrapTraceCount = 0; - _deferredBootstrapTraceWriteIndex = 0; - } _distinctImportNidHistoryCount = 0; _distinctImportNidHistoryWriteIndex = 0; _lastDistinctImportNid = string.Empty; @@ -998,6 +1063,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _logStrlenBursts = _logStrlenImports || string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_STRLEN_BURSTS"), "1", StringComparison.Ordinal); _logGuestContext = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_CONTEXT"), "1", StringComparison.Ordinal); + _ignoreGuestInt41 = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_IGNORE_INT41"), "1", StringComparison.Ordinal); + _ignoredGuestInt41Count = 0; _logGuestThreads = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_THREADS"), "1", StringComparison.Ordinal); _logUsleep = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_USLEEP"), "1", StringComparison.Ordinal); _logBootstrap = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_BOOTSTRAP"), "1", StringComparison.Ordinal); @@ -1006,6 +1073,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _logImportRecent = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_IMPORT_RECENT"), "1", StringComparison.Ordinal); _logStackCheck = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_STACK_CHK"), "1", StringComparison.Ordinal); _probeImportReturn = Environment.GetEnvironmentVariable("SHARPEMU_PROBE_IMPORT_RET"); + _probeImportReturnAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_PROBE_IMPORT_RET_ADDRESS")); + _probeImportReturnAddressCount = 0; _importFilter = Environment.GetEnvironmentVariable("SHARPEMU_LOG_IMPORT_FILTER"); _disableImportLoopGuard = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_IMPORT_LOOP_GUARD"), @@ -1014,15 +1084,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _importLoopGuardSeconds = GetImportLoopGuardSeconds(); _entryReturnSentinelRip = 0uL; _forcedGuestExit = false; - _hostShutdownRequested = false; - _guestTeardownRequested = false; - _activeSessionBackend = this; HostSessionControl.SetShutdownHandler(RequestHostShutdown); _importLoopSignatureCount = 0; _importLoopSignatureWriteIndex = 0; _importLoopPatternHits = 0; _importLoopPatternStartTimestamp = 0; - _importNidHashCache.Clear(); lock (_importResultLogSampleGate) { _importResultLogSamples.Clear(); @@ -1035,6 +1101,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _contextualUnresolvedReturnSites.Clear(); _stallWatchdogTriggered = 0; _stallWatchdogStop = false; + _readyDispatchStop = false; _patchedEa020eLookupCall = false; MarkExecutionProgress(); BindTlsBase(context); @@ -1066,11 +1133,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I finally { HostSessionControl.SetShutdownHandler(null); - if (ReferenceEquals(_activeSessionBackend, this)) - { - _activeSessionBackend = null; - } - DrainDeferredBootstrapTraces(); GuestThreadExecution.Scheduler = previousGuestThreadScheduler; Console.Error.WriteLine("[LOADER][INFO] === Execute END (LastError: " + (LastError ?? "null") + ") ==="); } @@ -1078,9 +1140,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I internal void RequestHostShutdown(string reason) { - _hostShutdownRequested = true; _forcedGuestExit = true; - _guestTeardownRequested = true; LastError = string.IsNullOrWhiteSpace(reason) ? "Host shutdown requested." : $"Host shutdown requested: {reason}"; @@ -1099,7 +1159,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I foreach (var (num4, text2) in importStubs) { _ = _moduleManager.TryGetExport(text2, out var resolvedExport); - _importEntries[num] = new ImportStubEntry(num4, text2, resolvedExport); + _importEntries[num] = new ImportStubEntry( + num4, + text2, + resolvedExport, + IsLeafImport(text2), + ShouldSuppressStrlenTrace(text2), + IsImportLoopGuardBoundary(text2), + StableHash64(text2)); if ((num4 >= 34393242112L && num4 <= 34393242624L) || (num4 >= 34393258496L && num4 <= 34393259008L)) { if (resolvedExport is not null) @@ -1113,7 +1180,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } if (TryResolveDirectImportTarget(text2, out var targetAddress, out var resolvedSymbol) && !hashSet.Contains(targetAddress)) { - Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Direct bridge for {text2} -> 0x{targetAddress:X16}"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Direct bridge for {text2} -> 0x{targetAddress:X16}"); + } if (!PatchImportStub((nint)(long)num4, (nint)(long)targetAddress)) { LastError = $"Failed to patch direct import stub at 0x{num4:X16}"; @@ -1146,7 +1216,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I LastError = "Failed to create import trampoline for NID " + text2; return false; } - Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Trampoline for {text2} -> 0x{num5:X16}"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] SetupImportStubs: Trampoline for {text2} -> 0x{num5:X16}"); + } if (!PatchImportStub((nint)num4, num5)) { LastError = $"Failed to patch import stub at 0x{num4:X16}"; @@ -1333,72 +1406,21 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I 0x75, 0xF5, 0xC3, ], - // memcpy: guarded native copy. The first "rep movsb" intrinsic had no bounds/null - // checking and crashed with a read at -1 right after a null-dst memset recovery in - // the NGS2 audio streaming code path, so it was temporarily pulled in favor of the - // safe C# HLE memcpy. That detour costs a full import dispatch per call - far too - // slow for a function this hot - so this stub keeps the native leaf path and adds - // the same guards as memset below: it silently returns dst without copying when dst - // or src is null/low-page (< 0x10000) or outside canonical user space, or when len - // is 0 or absurd (> 512MB). "Q3VBxCXhUHs" => [ - 0x48, 0x89, 0xF8, // mov rax, rdi (return dst) - 0x48, 0x81, 0xFF, 0x00, 0x00, 0x01, 0x00, // cmp rdi, 0x10000 - 0x72, 0x31, // jb done - 0x49, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, // mov r8, 0x800000000000 - 0x4C, 0x39, 0xC7, // cmp rdi, r8 - 0x73, 0x22, // jae done - 0x48, 0x81, 0xFE, 0x00, 0x00, 0x01, 0x00, // cmp rsi, 0x10000 - 0x72, 0x19, // jb done - 0x4C, 0x39, 0xC6, // cmp rsi, r8 - 0x73, 0x14, // jae done - 0x48, 0x81, 0xFA, 0x00, 0x00, 0x00, 0x20, // cmp rdx, 0x20000000 - 0x77, 0x0B, // ja done - 0x48, 0x85, 0xD2, // test rdx, rdx - 0x74, 0x06, // jz done - 0x48, 0x89, 0xD1, // mov rcx, rdx - 0xFC, // cld - 0xF3, 0xA4, // rep movsb - 0xC3, // done: ret - ], - // memset: guarded native fill. An earlier unguarded version crashed with a write AV - // at address 0 (NGS2 audio streaming init memsets a never-populated buffer field), - // so this one mirrors the HLE guards and silently returns dst without writing when - // dst is null/low-page (< 0x10000), dst is outside canonical user space, or len is - // absurd (> 512MB, e.g. the 0x27060035 / sign-extended values NGS2 passes). Routing - // memset through the HLE trampoline instead is not viable: parse/streaming loops - // issue hundreds of thousands of small memsets back-to-back, which crawls at - // dispatch speed and looks like a repeating-import hang to the loop guard. - // _sigprocmask: the HLE handler (KernelRuntimeCompatExports.Sigprocmask) is a pure - // no-op returning 0 that never writes oldset, so this is behavior-identical. The - // game's bundled libc queries the mask (set=NULL) once per iteration in its font/ - // parse loops - hundreds of thousands of back-to-back calls that both crawl at - // dispatch speed and read as a repeating-import hang to the loop guard. - "6xVpy0Fdq+I" => - [ - 0x31, 0xC0, // xor eax, eax - 0xC3, // ret + 0x48, 0x89, 0xF8, + 0x48, 0x89, 0xD1, + 0xF3, 0xA4, + 0xC3, ], "8zTFvBIAIN8" => [ - 0x48, 0x89, 0xF8, // mov rax, rdi (return dst) - 0x48, 0x81, 0xFF, 0x00, 0x00, 0x01, 0x00, // cmp rdi, 0x10000 - 0x72, 0x2B, // jb done - 0x49, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, // mov r8, 0x800000000000 - 0x4C, 0x39, 0xC7, // cmp rdi, r8 - 0x73, 0x1C, // jae done - 0x48, 0x81, 0xFA, 0x00, 0x00, 0x00, 0x20, // cmp rdx, 0x20000000 - 0x77, 0x13, // ja done - 0x48, 0x85, 0xD2, // test rdx, rdx - 0x74, 0x0E, // jz done - 0x48, 0x89, 0xD1, // mov rcx, rdx - 0x49, 0x89, 0xF9, // mov r9, rdi - 0x89, 0xF0, // mov eax, esi - 0xFC, // cld - 0xF3, 0xAA, // rep stosb - 0x4C, 0x89, 0xC8, // mov rax, r9 - 0xC3, // done: ret + 0x49, 0x89, 0xF8, + 0x48, 0x89, 0xF0, + 0x48, 0x89, 0xD1, + 0xF3, 0xAA, + 0x4C, 0x89, 0xC0, + 0xC3, ], _ => default, }; @@ -1409,7 +1431,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } const uint intrinsicAllocationSize = 128u; - void* memory = (void*)_hostMemory.Allocate(0, intrinsicAllocationSize, HostPageProtection.ReadWriteExecute); + void* memory = VirtualAlloc(null, intrinsicAllocationSize, 12288u, 64u); if (memory == null) { address = 0; @@ -1427,14 +1449,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I *(nint*)((byte*)memory + 64) = _sleepAddress; } uint oldProtect = 0; - if (!_hostMemory.Protect((ulong)memory, intrinsicAllocationSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(memory, intrinsicAllocationSize, 32u, &oldProtect)) { - _hostMemory.Free((ulong)memory); + VirtualFree(memory, 0u, 32768u); address = 0; return false; } - _hostMemory.FlushInstructionCache((ulong)memory, (nuint)code.Length); + FlushInstructionCache(GetCurrentProcess(), memory, (nuint)code.Length); address = (nint)memory; _importHandlerTrampolines.Add(address); return true; @@ -1457,7 +1479,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { if (IsKernelLibrary(export.LibraryName)) { - Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} ({export.LibraryName}:{export.Name}) -> HLE (kernel library)"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} ({export.LibraryName}:{export.Name}) -> HLE (kernel library)"); + } return false; } if (!IsLibcLibrary(export.LibraryName) || !PreferLleForLibcExport(export.Name)) @@ -1482,13 +1507,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } - Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} not in HLE table, checking runtime symbols..."); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} not in HLE table, checking runtime symbols..."); + } if (TryResolveRuntimeSymbolAddress(nid, out var directValue) && IsDirectImportTargetUsable(directValue)) { targetAddress = directValue; resolvedSymbol = nid; - Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} -> runtime symbol 0x{targetAddress:X16}"); + if (_logAllImports) + { + Console.Error.WriteLine($"[LOADER][DEBUG] TryResolveDirectImportTarget: {nid} -> runtime symbol 0x{targetAddress:X16}"); + } return true; } @@ -1617,25 +1648,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { return exportName switch { + "memcpy" or "memmove" or - // memset/memcpy excluded: the raw LLE routines have no null/bounds guard and crash - // with an access violation on bad pointers (observed hit during Quake's CL_Init, - // where a still-unidentified upstream bug calls memcpy/memset with a null - // destination). Both are instead served by the guarded native intrinsics in - // TryCreateNativeImportIntrinsic, which fail safely without leaving the leaf path. - "memcmp" or - // _Getpctype must come from the game's own Dinkumware libc when one is bundled: - // it returns a pointer to that CRT's ctype bitmask table, whose bit layout - // (_DI=0x20, _SP=0x04, _BB=0x80, ...) differs from the MSVC-style table the HLE - // fallback used to serve. Serving the wrong layout made the bundled printf engine - // render every Sys_Error message as an empty string (isdigit misfired during - // %-directive parsing) and made mcpp drop 'a'-'f' from identifiers ("texture" -> - // "txtur", the 0x80 bit reads as _BB/control there). _Getptolower/_Getptoupper - // already resolve to the bundled module because no HLE export shadows them; this - // keeps _Getpctype consistent with them. It is a pure accessor returning a pointer - // to a const table, so it is also the cheapest possible LLE call - important - // because parsers hit it once per input character. - "_Getpctype" => true, + "memset" or + "memcmp" => true, _ => false, }; } @@ -1695,7 +1711,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context.FsBase = (ulong)num; context.GsBase = (ulong)num; SeedTlsLayout(num); - _hostThreading.SetTlsValue(_guestTlsBaseTlsIndex, num); + TlsSetValue(_guestTlsBaseTlsIndex, num); } } @@ -1719,7 +1735,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } uint oldProtect = default; - if (!_hostMemory.Protect((ulong)(void*)_tlsHandlerAddress, 16u, HostPageProtection.ReadWriteExecute, out oldProtect)) + if (!VirtualProtect((void*)_tlsHandlerAddress, 16u, 64u, &oldProtect)) { return; } @@ -1730,8 +1746,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)_tlsHandlerAddress, 16u, oldProtect, out oldProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)_tlsHandlerAddress, 16u); + VirtualProtect((void*)_tlsHandlerAddress, 16u, oldProtect, &oldProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)_tlsHandlerAddress, 16u); } } @@ -1749,6 +1765,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I error = $"invalid guest context transfer target rip=0x{target.Rip:X16} rsp=0x{target.Rsp:X16}"; return false; } + if (target.Rsp < sizeof(ulong) || + ActiveCpuContext is not { } activeContext || + !activeContext.TryWriteUInt64(target.Rsp - sizeof(ulong), target.Rip)) + { + error = $"guest context transfer slot is not writable at 0x{target.Rsp - sizeof(ulong):X16}"; + return false; + } transferStub = GetOrCreateGuestContextTransferStub(); if (transferStub == 0) @@ -1776,10 +1799,15 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I frame[8] = target.Rdi; frame[9] = target.R8; frame[10] = target.R9; - frame[11] = target.R12; - frame[12] = target.R13; - frame[13] = target.R14; - frame[14] = target.R15; + frame[11] = target.R10; + frame[12] = target.R11; + frame[13] = target.R12; + frame[14] = target.R13; + frame[15] = target.R14; + frame[16] = target.R15; + frame[17] = target.Mxcsr == 0 ? 0x1F80u : target.Mxcsr; + frame[18] = target.FpuControlWord == 0 ? 0x037Fu : target.FpuControlWord; + frame[19] = target.RestoreFullFpuState ? 1u : 0u; return true; } @@ -1797,8 +1825,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return _guestContextTransferStub; } - const uint stubSize = 128; - var code = (byte*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + const uint stubSize = 256; + var code = (byte*)VirtualAlloc(null, stubSize, 12288u, 64u); if (code == null) { return 0; @@ -1813,10 +1841,47 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Emit((byte)(0x40 | ((register & 7) << 3) | 0x03)); Emit(displacement); } + void EmitLoadFromR11Disp32(int register, int displacement) + { + Emit((byte)(0x49 | (register >= 8 ? 0x04 : 0x00))); + Emit(0x8B); + Emit((byte)(0x80 | ((register & 7) << 3) | 0x03)); + *(int*)(code + offset) = displacement; + offset += sizeof(int); + } Emit(0x49); Emit(0x89); Emit(0xC3); // mov r11, rax - EmitLoadFromR11(10, 0); // target RIP - EmitLoadFromR11(4, 8); // rsp + // A new >=3.50 fiber receives the SDK-defined MXCSR verbatim. A + // resumed fiber follows _sceFiberLongJmp: preserve status bits 0-5 + // while restoring the saved control bits. + Emit(0x49); Emit(0x83); Emit(0xBB); // cmp qword [r11+152],0 + *(int*)(code + offset) = 152; offset += sizeof(int); Emit(0x00); + Emit(0x0F); Emit(0x84); // je merge_status + var mergeBranch = offset; offset += sizeof(int); + Emit(0x41); Emit(0x0F); Emit(0xAE); Emit(0x93); // ldmxcsr [r11+136] + *(int*)(code + offset) = 136; offset += sizeof(int); + Emit(0xE9); // jmp mxcsr_done + var doneBranch = offset; offset += sizeof(int); + var mergeLabel = offset; + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // sub rsp,8 + Emit(0x0F); Emit(0xAE); Emit(0x1C); Emit(0x24); // stmxcsr [rsp] + Emit(0x41); Emit(0x8B); Emit(0x83); // mov eax,[r11+136] + *(int*)(code + offset) = 136; offset += sizeof(int); + Emit(0x25); *(uint*)(code + offset) = 0xFFFFFFC0u; offset += sizeof(uint); // and eax,~0x3f + Emit(0x8B); Emit(0x0C); Emit(0x24); // mov ecx,[rsp] + Emit(0x83); Emit(0xE1); Emit(0x3F); // and ecx,0x3f + Emit(0x09); Emit(0xC8); // or eax,ecx + Emit(0x89); Emit(0x04); Emit(0x24); // mov [rsp],eax + Emit(0x0F); Emit(0xAE); Emit(0x14); Emit(0x24); // ldmxcsr [rsp] + Emit(0x48); Emit(0x83); Emit(0xC4); Emit(0x08); // add rsp,8 + var mxcsrDoneLabel = offset; + *(int*)(code + mergeBranch) = mergeLabel - (mergeBranch + sizeof(int)); + *(int*)(code + doneBranch) = mxcsrDoneLabel - (doneBranch + sizeof(int)); + Emit(0x41); Emit(0xD9); Emit(0xAB); // fldcw [r11+144] + *(int*)(code + offset) = 144; offset += sizeof(int); + + EmitLoadFromR11(4, 8); // resume rsp + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // point at transfer return slot EmitLoadFromR11(1, 24); // rcx EmitLoadFromR11(2, 32); // rdx EmitLoadFromR11(3, 40); // rbx @@ -1825,21 +1890,24 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I EmitLoadFromR11(7, 64); // rdi EmitLoadFromR11(8, 72); // r8 EmitLoadFromR11(9, 80); // r9 - EmitLoadFromR11(12, 88); // r12 - EmitLoadFromR11(13, 96); // r13 - EmitLoadFromR11(14, 104); // r14 - EmitLoadFromR11(15, 112); // r15 + EmitLoadFromR11(10, 88); // r10 + EmitLoadFromR11(12, 104); // r12 + EmitLoadFromR11(13, 112); // r13 + EmitLoadFromR11(14, 120); // r14 + EmitLoadFromR11Disp32(15, 128); // r15 EmitLoadFromR11(0, 16); // rax - Emit(0x41); Emit(0xFF); Emit(0xE2); // jmp r10 + EmitLoadFromR11(11, 96); // r11 (last: frame pointer) + Emit(0xC3); // ret through [resume_rsp-8] + Debug.Assert(offset <= stubSize, "Guest context transfer stub exceeded its allocation."); uint oldProtect = 0; - if (!_hostMemory.Protect((ulong)code, stubSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(code, stubSize, 32u, &oldProtect)) { - _hostMemory.Free((ulong)code); + VirtualFree(code, 0u, 32768u); return 0; } - _hostMemory.FlushInstructionCache((ulong)code, stubSize); + FlushInstructionCache(GetCurrentProcess(), code, stubSize); Volatile.Write(ref _guestContextTransferStub, (nint)code); return (nint)code; } @@ -1847,7 +1915,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe nint CreateImportHandlerTrampoline(int importIndex) { - void* ptr = (void*)_hostMemory.Allocate(0, 256u, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, 512u, 12288u, 64u); if (ptr == null) { return 0; @@ -1875,21 +1943,41 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 82; ptr2[num++] = 86; ptr2[num++] = 87; - // sub rsp, 0x80 — reserve 8*16 bytes for the SysV variadic XMM save area - ptr2[num++] = 0x48; ptr2[num++] = 0x81; ptr2[num++] = 0xEC; - ptr2[num++] = 0x80; ptr2[num++] = 0x00; ptr2[num++] = 0x00; ptr2[num++] = 0x00; - // movdqu [rsp + i*0x10], xmm{i} for i = 0..7 (F3 0F 7F /r, SIB=0x24 base=rsp, disp8) - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x44; ptr2[num++] = 0x24; ptr2[num++] = 0x00; // xmm0 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x4C; ptr2[num++] = 0x24; ptr2[num++] = 0x10; // xmm1 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x54; ptr2[num++] = 0x24; ptr2[num++] = 0x20; // xmm2 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x5C; ptr2[num++] = 0x24; ptr2[num++] = 0x30; // xmm3 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x64; ptr2[num++] = 0x24; ptr2[num++] = 0x40; // xmm4 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x6C; ptr2[num++] = 0x24; ptr2[num++] = 0x50; // xmm5 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x74; ptr2[num++] = 0x24; ptr2[num++] = 0x60; // xmm6 - ptr2[num++] = 0xF3; ptr2[num++] = 0x0F; ptr2[num++] = 0x7F; ptr2[num++] = 0x7C; ptr2[num++] = 0x24; ptr2[num++] = 0x70; // xmm7 - // lea r12, [rsp + 0x80] — r12 = argpack base (the 12 pushed GP regs), past the XMM area - ptr2[num++] = 0x4C; ptr2[num++] = 0x8D; ptr2[num++] = 0xA4; ptr2[num++] = 0x24; - ptr2[num++] = 0x80; ptr2[num++] = 0x00; ptr2[num++] = 0x00; ptr2[num++] = 0x00; + // Preserve incoming guest RAX/AL and XMM0-XMM7 below the existing + // GPR argument pack. The original pack still begins with RDI and its + // return address remains at +0x60. + ptr2[num++] = 0x48; + ptr2[num++] = 0x81; + ptr2[num++] = 0xEC; + *(uint*)(ptr2 + num) = 0xB0; + num += 4; + ptr2[num++] = 0x48; + ptr2[num++] = 0x89; + ptr2[num++] = 0x04; + ptr2[num++] = 0x24; + // Preserve the remaining volatile guest machine context before any + // host call is made. libSceFiber's setjmp/longjmp contract includes + // R10/R11 and the x87/MXCSR control state. + ptr2[num++] = 0x4C; ptr2[num++] = 0x89; ptr2[num++] = 0x54; ptr2[num++] = 0x24; ptr2[num++] = 0x08; // mov [rsp+8],r10 + ptr2[num++] = 0x4C; ptr2[num++] = 0x89; ptr2[num++] = 0x5C; ptr2[num++] = 0x24; ptr2[num++] = 0x10; // mov [rsp+16],r11 + ptr2[num++] = 0x0F; ptr2[num++] = 0xAE; ptr2[num++] = 0x5C; ptr2[num++] = 0x24; ptr2[num++] = 0x18; // stmxcsr [rsp+24] + ptr2[num++] = 0xD9; ptr2[num++] = 0x7C; ptr2[num++] = 0x24; ptr2[num++] = 0x1C; // fnstcw [rsp+28] + for (var xmm = 0; xmm < 8; xmm++) + { + ptr2[num++] = 0xF3; + ptr2[num++] = 0x0F; + ptr2[num++] = 0x7F; + ptr2[num++] = (byte)(0x84 | (xmm << 3)); + ptr2[num++] = 0x24; + *(uint*)(ptr2 + num) = (uint)(0x30 + (xmm * 0x10)); + num += 4; + } + ptr2[num++] = 0x4C; + ptr2[num++] = 0x8D; + ptr2[num++] = 0xA4; + ptr2[num++] = 0x24; + *(uint*)(ptr2 + num) = 0xB0; + num += 4; ptr2[num++] = 72; ptr2[num++] = 131; ptr2[num++] = 236; @@ -1907,6 +1995,12 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 131; ptr2[num++] = 196; ptr2[num++] = 40; + ptr2[num++] = 0x4C; + ptr2[num++] = 0x8D; + ptr2[num++] = 0xA4; + ptr2[num++] = 0x24; + *(uint*)(ptr2 + num) = 0xB0; + num += 4; ptr2[num++] = 73; ptr2[num++] = 137; ptr2[num++] = 195; @@ -1916,7 +2010,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 72; ptr2[num++] = 131; ptr2[num++] = 236; - ptr2[num++] = 40; + ptr2[num++] = 56; + ptr2[num++] = 0x4C; ptr2[num++] = 0x89; ptr2[num++] = 0x64; ptr2[num++] = 0x24; ptr2[num++] = 0x28; // mov [rsp+0x28],r12 ptr2[num++] = 72; ptr2[num++] = 185; *(long*)(ptr2 + num) = _selfHandlePtr; @@ -1933,15 +2028,24 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I num += 8; ptr2[num++] = byte.MaxValue; ptr2[num++] = 208; + ptr2[num++] = 0x4C; ptr2[num++] = 0x8B; ptr2[num++] = 0x64; ptr2[num++] = 0x24; ptr2[num++] = 0x28; // mov r12,[rsp+0x28] ptr2[num++] = 72; ptr2[num++] = 131; ptr2[num++] = 196; - ptr2[num++] = 40; - // movdqu xmm0, [r12 - 0x80] — reload the return XMM0 the gateway wrote into the - // argpack's xmm0 save slot (float/double returns: powf/logf/wcstod). SysV/Win64 - // XMM regs are volatile across calls, so an unconditional reload is ABI-safe. - ptr2[num++] = 0xF3; ptr2[num++] = 0x41; ptr2[num++] = 0x0F; ptr2[num++] = 0x6F; - ptr2[num++] = 0x84; ptr2[num++] = 0x24; ptr2[num++] = 0x80; ptr2[num++] = 0xFF; ptr2[num++] = 0xFF; ptr2[num++] = 0xFF; + ptr2[num++] = 56; + // Materialize SysV vector return registers written by the managed HLE + // gateway before restoring the guest stack. + for (var xmm = 0; xmm < 2; xmm++) + { + ptr2[num++] = 0xF3; + ptr2[num++] = 0x41; + ptr2[num++] = 0x0F; + ptr2[num++] = 0x6F; + ptr2[num++] = (byte)(0x84 | (xmm << 3)); + ptr2[num++] = 0x24; + *(int*)(ptr2 + num) = -0x80 + (xmm * 0x10); + num += 4; + } ptr2[num++] = 76; ptr2[num++] = 137; ptr2[num++] = 228; @@ -1964,14 +2068,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[num++] = 65; ptr2[num++] = 95; ptr2[num++] = 195; - uint num2 = default(uint); - if (!_hostMemory.Protect((ulong)ptr, 256u, HostPageProtection.ReadExecute, out num2)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for import dispatch stub at 0x{(nint)ptr:X16}"); - return 0; - } - _hostMemory.FlushInstructionCache((ulong)ptr, 256u); - return (nint)ptr; + Debug.Assert(num <= 512, "Import handler trampoline exceeded its allocation."); + uint num2 = default(uint); + VirtualProtect(ptr, 512u, 32u, &num2); + FlushInstructionCache(GetCurrentProcess(), ptr, 512u); + return (nint)ptr; } catch { @@ -1982,7 +2083,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe bool PatchImportStub(nint address, nint trampoline) { uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, 16u, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, 16u, 64u, &flNewProtect)) { Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for import stub at 0x{address:X16}"); return false; @@ -2002,8 +2103,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, 16u, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, 16u); + VirtualProtect((void*)address, 16u, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, 16u); } } @@ -2013,7 +2114,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { if (importHandlerTrampoline != 0) { - _hostMemory.Free((ulong)importHandlerTrampoline); + VirtualFree((void*)importHandlerTrampoline, 0u, 32768u); } } _importHandlerTrampolines.Clear(); @@ -2024,7 +2125,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _tlsHandlerAddress = (nint)TryAllocateNearEntry(TlsHandlerRegionSize); if (_tlsHandlerAddress == 0) { - _tlsHandlerAddress = (nint)_hostMemory.Allocate(0, TlsHandlerRegionSize, HostPageProtection.ReadWriteExecute); + _tlsHandlerAddress = (nint)VirtualAlloc(null, TlsHandlerRegionSize, 12288u, 64u); } if (_tlsHandlerAddress == 0) { @@ -2080,18 +2181,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I tlsHandlerAddress[num++] = 0xC3; // ret _tlsPatchStubOffset = (num + 15) & ~15; uint num2 = default(uint); - if (!_hostMemory.Protect((ulong)(void*)_tlsHandlerAddress, TlsHandlerRegionSize, HostPageProtection.ReadExecute, out num2)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for TLS handler at 0x{_tlsHandlerAddress:X16}"); - return; - } - _hostMemory.FlushInstructionCache((ulong)(void*)_tlsHandlerAddress, TlsHandlerRegionSize); + VirtualProtect((void*)_tlsHandlerAddress, TlsHandlerRegionSize, 32u, &num2); + FlushInstructionCache(GetCurrentProcess(), (void*)_tlsHandlerAddress, TlsHandlerRegionSize); Console.Error.WriteLine($"[LOADER][INFO] TLS handler at 0x{_tlsHandlerAddress:X16}"); } - private unsafe nint CreateUnresolvedReturnStub() + private unsafe static nint CreateUnresolvedReturnStub() { - void* ptr = (void*)_hostMemory.Allocate(0, 4096u, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, 4096u, 12288u, 64u); if (ptr == null) { return 0; @@ -2105,19 +2202,15 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr2[i] = 144; } uint num = default(uint); - if (!_hostMemory.Protect((ulong)ptr, 4096u, HostPageProtection.ReadExecute, out num)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for unresolved return stub at 0x{(nint)ptr:X16}"); - return 0; - } - _hostMemory.FlushInstructionCache((ulong)ptr, 16u); + VirtualProtect(ptr, 4096u, 32u, &num); + FlushInstructionCache(GetCurrentProcess(), ptr, 16u); return (nint)ptr; } private unsafe nint CreateGuestReturnStub() { const uint stubSize = 256u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) { return 0; @@ -2125,16 +2218,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I byte* code = (byte*)ptr; int offset = 0; - // TlsGetValue returns its TLS pointer in RAX. Preserve the guest return value - // above the 32-byte Windows shadow space while keeping the call site aligned. - EmitByte(code, ref offset, 0x48); // sub rsp, 0x30 + EmitByte(code, ref offset, 0x48); // sub rsp, 0x20 EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); - EmitByte(code, ref offset, 0x30); - EmitByte(code, ref offset, 0x48); // mov [rsp+0x20], rax - EmitByte(code, ref offset, 0x89); - EmitByte(code, ref offset, 0x44); - EmitByte(code, ref offset, 0x24); EmitByte(code, ref offset, 0x20); EmitByte(code, ref offset, 0xB9); // mov ecx, tlsIndex EmitUInt32(code, ref offset, _hostRspSlotTlsIndex); @@ -2144,21 +2230,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I offset += sizeof(ulong); EmitByte(code, ref offset, 0xFF); // call rax EmitByte(code, ref offset, 0xD0); - EmitByte(code, ref offset, 0x49); // mov r11, rax - EmitByte(code, ref offset, 0x89); - EmitByte(code, ref offset, 0xC3); - EmitByte(code, ref offset, 0x48); // mov rax, [rsp+0x20] - EmitByte(code, ref offset, 0x8B); - EmitByte(code, ref offset, 0x44); - EmitByte(code, ref offset, 0x24); - EmitByte(code, ref offset, 0x20); - EmitByte(code, ref offset, 0x48); // add rsp, 0x30 + EmitByte(code, ref offset, 0x48); // add rsp, 0x20 EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); - EmitByte(code, ref offset, 0x30); - EmitByte(code, ref offset, 0x49); // mov rsp, [r11] + EmitByte(code, ref offset, 0x20); + EmitByte(code, ref offset, 0x48); // mov rsp, [rax] EmitByte(code, ref offset, 0x8B); - EmitByte(code, ref offset, 0x23); + EmitByte(code, ref offset, 0x20); EmitHostNonvolatileXmmRestore(code, ref offset); EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5F); EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5E); @@ -2171,12 +2249,104 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I EmitByte(code, ref offset, 0xC3); uint oldProtect = default; - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadExecute, out oldProtect)) + if (!VirtualProtect(ptr, stubSize, 32u, &oldProtect)) { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for guest return stub at 0x{(nint)ptr:X16}"); + VirtualFree(ptr, 0u, 32768u); return 0; } - _hostMemory.FlushInstructionCache((ulong)ptr, (nuint)offset); + FlushInstructionCache(GetCurrentProcess(), ptr, (nuint)offset); + return (nint)ptr; + } + + private unsafe nint CreateExceptionHandlerTrampoline(nint managedHandler) + { + const uint stubSize = 256u; + void* ptr = VirtualAlloc(null, stubSize, 12288u, 64u); + if (ptr == null) + { + return 0; + } + + byte* code = (byte*)ptr; + int offset = 0; + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x54); // push r12 + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x55); // push r13 + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE4); // mov r12, rsp + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xCD); // mov r13, rcx + EmitByte(code, ref offset, 0x65); EmitByte(code, ref offset, 0x48); // mov rax, gs:[8] + EmitByte(code, ref offset, 0x8B); EmitByte(code, ref offset, 0x04); EmitByte(code, ref offset, 0x25); + EmitUInt32(code, ref offset, 8u); + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x39); EmitByte(code, ref offset, 0xC4); // cmp r12, rax + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x83); // jae guestStack + int aboveStackJump = offset; + EmitUInt32(code, ref offset, 0u); + EmitByte(code, ref offset, 0x65); EmitByte(code, ref offset, 0x48); // mov rax, gs:[0x10] + EmitByte(code, ref offset, 0x8B); EmitByte(code, ref offset, 0x04); EmitByte(code, ref offset, 0x25); + EmitUInt32(code, ref offset, 0x10u); + EmitByte(code, ref offset, 0x49); EmitByte(code, ref offset, 0x39); EmitByte(code, ref offset, 0xC4); // cmp r12, rax + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x82); // jb guestStack + int belowStackJump = offset; + EmitUInt32(code, ref offset, 0u); + + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE9); // mov rcx, r13 + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0xB8); + *(nint*)(code + offset) = managedHandler; + offset += sizeof(nint); + EmitByte(code, ref offset, 0xFF); EmitByte(code, ref offset, 0xD0); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0xE9); + int hostRestoreJump = offset; + EmitUInt32(code, ref offset, 0u); + + int guestStackOffset = offset; + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0xB9); + EmitUInt32(code, ref offset, _hostRspSlotTlsIndex); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0xB8); + *(nint*)(code + offset) = _tlsGetValueAddress; + offset += sizeof(nint); + EmitByte(code, ref offset, 0xFF); EmitByte(code, ref offset, 0xD0); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x85); EmitByte(code, ref offset, 0xC0); // test rax, rax + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x84); + int missingTlsJump = offset; + EmitUInt32(code, ref offset, 0u); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x8B); EmitByte(code, ref offset, 0x18); // mov r11, [rax] + EmitByte(code, ref offset, 0x4D); EmitByte(code, ref offset, 0x85); EmitByte(code, ref offset, 0xDB); // test r11, r11 + EmitByte(code, ref offset, 0x0F); EmitByte(code, ref offset, 0x84); + int missingHostStackJump = offset; + EmitUInt32(code, ref offset, 0u); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xDC); // mov rsp, r11 + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xEC); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE9); // mov rcx, r13 + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0xB8); + *(nint*)(code + offset) = managedHandler; + offset += sizeof(nint); + EmitByte(code, ref offset, 0xFF); EmitByte(code, ref offset, 0xD0); + EmitByte(code, ref offset, 0x48); EmitByte(code, ref offset, 0x83); EmitByte(code, ref offset, 0xC4); EmitByte(code, ref offset, 0x28); + EmitByte(code, ref offset, 0xE9); + int guestRestoreJump = offset; + EmitUInt32(code, ref offset, 0u); + + int passThroughOffset = offset; + EmitByte(code, ref offset, 0x31); EmitByte(code, ref offset, 0xC0); // xor eax, eax + int restoreOffset = offset; + EmitByte(code, ref offset, 0x4C); EmitByte(code, ref offset, 0x89); EmitByte(code, ref offset, 0xE4); // mov rsp, r12 + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5D); + EmitByte(code, ref offset, 0x41); EmitByte(code, ref offset, 0x5C); + EmitByte(code, ref offset, 0xC3); + + *(int*)(code + aboveStackJump) = guestStackOffset - (aboveStackJump + sizeof(int)); + *(int*)(code + belowStackJump) = guestStackOffset - (belowStackJump + sizeof(int)); + *(int*)(code + hostRestoreJump) = restoreOffset - (hostRestoreJump + sizeof(int)); + *(int*)(code + missingTlsJump) = passThroughOffset - (missingTlsJump + sizeof(int)); + *(int*)(code + missingHostStackJump) = passThroughOffset - (missingHostStackJump + sizeof(int)); + *(int*)(code + guestRestoreJump) = restoreOffset - (guestRestoreJump + sizeof(int)); + + uint oldProtect = default; + VirtualProtect(ptr, stubSize, 32u, &oldProtect); + FlushInstructionCache(GetCurrentProcess(), ptr, (nuint)offset); return (nint)ptr; } @@ -2198,7 +2368,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return null; } - private unsafe bool TryAllocAt(ulong baseAddress, long signedDelta, nuint size, out void* memory) + private unsafe static bool TryAllocAt(ulong baseAddress, long signedDelta, nuint size, out void* memory) { memory = null; ulong num; @@ -2219,7 +2389,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } num = baseAddress - num2; } - void* ptr = (void*)_hostMemory.Allocate(num, size, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc((void*)num, size, 12288u, 64u); if (ptr == null) { return false; @@ -2230,15 +2400,18 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe void PatchTlsPatterns() { - const ulong MaxScanBytes = 33554432uL; + // Large Gen5 executables can keep valid code well past the first 32 MiB. + // Astro Bot, for example, has an FS:[0] TLS load near +0x70A0000. + const ulong MaxScanBytes = 134217728uL; ulong num = _entryPoint; ulong num2 = num + MaxScanBytes; int num3 = 0; int num4 = 0; int num9 = 0; + int sse4aPatchCount = 0; while (num < num2) { - if (!_hostMemory.Query(num, out var lpBuffer) || lpBuffer.RegionSize == 0) + if (VirtualQuery((void*)num, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) == 0 || lpBuffer.RegionSize == 0) { num += 4096uL; continue; @@ -2249,8 +2422,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { num6 = num2; } - uint num7 = lpBuffer.RawProtection & 0xFF; - bool flag = lpBuffer.State == HostRegionState.Committed && (lpBuffer.RawProtection & PAGE_GUARD) == 0 && num7 != PAGE_NOACCESS; + uint num7 = lpBuffer.Protect & 0xFF; + bool flag = lpBuffer.State == 4096 && (lpBuffer.Protect & PAGE_GUARD) == 0 && num7 != PAGE_NOACCESS; bool flag2 = num7 == PAGE_EXECUTE || num7 == 32 || num7 == 64 || num7 == PAGE_EXECUTE_WRITECOPY; if (flag && flag2 && num6 > num5 + MinTlsPatchInstructionBytes) { @@ -2268,6 +2441,10 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { num9++; } + else if (remainingBytes >= 12 && TryPatchSse4aExtrqBlend(address, ptr + i)) + { + sse4aPatchCount++; + } else if (TryPatchStackCanaryInstruction(address, ptr + i)) { num4++; @@ -2276,7 +2453,51 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } num = num6 > num ? num6 : num + 4096uL; } - Console.Error.WriteLine($"[LOADER][INFO] Patched {num3} TLS loads, {num9} TLS stores, {num4} stack-canary accesses"); + Console.Error.WriteLine($"[LOADER][INFO] Patched {num3} TLS loads, {num9} TLS stores, {num4} stack-canary accesses, {sse4aPatchCount} SSE4a EXTRQ blends"); + } + + private unsafe bool TryPatchSse4aExtrqBlend(nint address, byte* source) + { + // Rosetta does not implement AMD SSE4a EXTRQ. This exact sequence masks + // xmm2 to its low 40 bits, then copies the resulting second dword into + // xmm0. PEXTRB/PINSRD provides the same observable result in 12 bytes: + // extract source byte 4 and insert the zero-extended value into lane 1. + ReadOnlySpan pattern = + [ + 0x66, 0x0F, 0x78, 0xC2, 0x28, 0x00, + 0xC4, 0xE3, 0x79, 0x02, 0xC2, 0x02, + ]; + for (var i = 0; i < pattern.Length; i++) + { + if (source[i] != pattern[i]) + { + return false; + } + } + + ReadOnlySpan replacement = + [ + 0x66, 0x0F, 0x3A, 0x14, 0xD0, 0x04, + 0x66, 0x0F, 0x3A, 0x22, 0xC0, 0x01, + ]; + uint oldProtect = 0; + if (!VirtualProtect((void*)address, (nuint)replacement.Length, 64u, &oldProtect)) + { + return false; + } + try + { + for (var i = 0; i < replacement.Length; i++) + { + ((byte*)address)[i] = replacement[i]; + } + } + finally + { + VirtualProtect((void*)address, (nuint)replacement.Length, oldProtect, &oldProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)replacement.Length); + } + return true; } private unsafe bool IsPatternMatch(byte* ptr, byte[] pattern) @@ -2335,7 +2556,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } byte b5 = (byte)(0xC0 | ((num4 & 7) << 3) | (num4 & 7)); uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, (nuint)num2, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, (nuint)num2, 64u, &flNewProtect)) { return false; } @@ -2351,8 +2572,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, (nuint)num2, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, (nuint)num2); + VirtualProtect((void*)address, (nuint)num2, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)num2); } return true; } @@ -2419,7 +2640,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private unsafe bool PatchTlsLoadInstruction(nint address, int instructionLength, int destinationRegister) { uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, (nuint)instructionLength, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, (nuint)instructionLength, 64u, &flNewProtect)) { return false; } @@ -2453,8 +2674,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, (nuint)instructionLength, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, (nuint)instructionLength); + VirtualProtect((void*)address, (nuint)instructionLength, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)instructionLength); } } @@ -2506,12 +2727,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ptr[num2++] = 144; } uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)num, 32u, HostPageProtection.ReadExecute, out flNewProtect)) - { - Console.Error.WriteLine($"[LOADER][ERROR] VirtualProtect failed for TLS store helper at 0x{num:X16}"); - return 0; - } - _hostMemory.FlushInstructionCache((ulong)(void*)num, 32u); + VirtualProtect((void*)num, 32u, 32u, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)num, 32u); return num; } @@ -2530,7 +2747,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I nint result = _tlsHandlerAddress + _tlsPatchStubOffset; _tlsPatchStubOffset += num; uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)result, (nuint)num, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)result, (nuint)num, 64u, &flNewProtect)) { return 0; } @@ -2544,7 +2761,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } uint flNewProtect = default(uint); - if (!_hostMemory.Protect((ulong)(void*)address, (nuint)instructionLength, HostPageProtection.ReadWriteExecute, out flNewProtect)) + if (!VirtualProtect((void*)address, (nuint)instructionLength, 64u, &flNewProtect)) { return false; } @@ -2565,17 +2782,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostMemory.ProtectRaw((ulong)(void*)address, (nuint)instructionLength, flNewProtect, out flNewProtect); - _hostMemory.FlushInstructionCache((ulong)(void*)address, (nuint)instructionLength); + VirtualProtect((void*)address, (nuint)instructionLength, flNewProtect, &flNewProtect); + FlushInstructionCache(GetCurrentProcess(), (void*)address, (nuint)instructionLength); } return true; } private unsafe void TryPreReservePrtAperture(ulong baseAddress, ulong size) { - if (_hostMemory.Query(baseAddress, out var lpBuffer) && lpBuffer.State != HostRegionState.Free) + if (VirtualQuery((void*)baseAddress, out var lpBuffer, (nuint)sizeof(MEMORY_BASIC_INFORMATION64)) != 0 && lpBuffer.State != 65536) { - Console.Error.WriteLine($"[LOADER][INFO] PRT aperture at 0x{baseAddress:X16} already in use (state=0x{lpBuffer.RawState:X}), will use lazy-commit"); + Console.Error.WriteLine($"[LOADER][INFO] PRT aperture at 0x{baseAddress:X16} already in use (state=0x{lpBuffer.State:X}), will use lazy-commit"); return; } ulong num = baseAddress; @@ -2587,7 +2804,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { ulong val = num2 - num; num5 = (nuint)Math.Min(2097152uL, val); - if (_hostMemory.Reserve(num, num5, HostPageProtection.ReadWrite) != 0) + void* ptr = VirtualAlloc((void*)num, num5, 8192u, 4u); + if (ptr != null) { num3++; } @@ -2609,7 +2827,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I int num8 = 0; for (; num6 < num7; num6 += 2097152) { - if (_hostMemory.Commit(num6, 2097152u, HostPageProtection.ReadWrite)) + void* ptr2 = VirtualAlloc((void*)num6, 2097152u, 4096u, 4u); + if (ptr2 != null) { num8++; } @@ -2706,11 +2925,44 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I $"entry=0x{thread.EntryPoint:X16} arg=0x{thread.Argument:X16} priority={thread.Priority} " + $"host_priority={MapGuestThreadPriority(thread.Priority)} affinity=0x{thread.AffinityMask:X}"); Pump(creatorContext, "pthread_create"); + // Pump is suppressed while another cooperative dispatch is active. The + // background dispatcher would eventually observe this thread, but an + // immediate authoritative drain avoids making thread creation depend on + // the approximate ready-count polling hint. + DispatchReadyGuestThreads(); return true; } public bool SupportsGuestContextTransfer => true; + public void RegisterGuestThreadContext(ulong threadHandle, CpuContext context) + { + if (threadHandle == 0) + { + return; + } + + lock (_guestThreadGate) + { + _currentExternalGuestThreadHandle = threadHandle; + if (_guestThreads.ContainsKey(threadHandle)) + { + return; + } + + if (_externalGuestThreads.TryGetValue(threadHandle, out var existing)) + { + existing.Context = context; + return; + } + + _externalGuestThreads[threadHandle] = new ExternalGuestThreadState + { + Context = context, + }; + } + } + public bool TryJoinThread( CpuContext callerContext, ulong threadHandle, @@ -2767,21 +3019,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I if (hostThread is not null && !ReferenceEquals(hostThread, Thread.CurrentThread)) { - // The handle is published before the host thread starts. - if ((hostThread.ThreadState & System.Threading.ThreadState.Unstarted) != 0) - { - Thread.Sleep(1); - continue; - } - - try - { - hostThread.Join(joinPollMilliseconds); - } - catch (ThreadStateException) - { - Thread.Sleep(joinPollMilliseconds); - } + hostThread.Join(1); } else { @@ -2801,10 +3039,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I public void Pump(CpuContext callerContext, string reason) { _ = callerContext; - if (_guestTeardownRequested) - { - return; - } var runSynchronously = string.Equals(reason, "entry_return", StringComparison.Ordinal); WakeExpiredBlockedGuestThreads(); if (Volatile.Read(ref _readyGuestThreadCount) == 0) @@ -2822,17 +3056,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I GuestThreadState? thread = null; using (LockGate("Pump.dequeue")) { - while (_readyGuestThreads.Count > 0) - { - var candidate = _readyGuestThreads.Dequeue(); - Interlocked.Decrement(ref _readyGuestThreadCount); - if (candidate.State == GuestThreadRunState.Ready) - { - thread = candidate; - thread.State = GuestThreadRunState.Running; - break; - } - } + _ = TryClaimReadyGuestThreadLocked(out thread); } if (thread == null) { @@ -2845,17 +3069,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I continue; } - var hostThread = new Thread(() => RunGuestThread(thread, reason)) - { - IsBackground = true, - Name = $"SharpEmu-{thread.Name}", - Priority = MapGuestThreadPriority(thread.Priority), - }; - using (LockGate("Pump.bind_host")) - { - thread.HostThread = hostThread; - } - hostThread.Start(); + ScheduleGuestThreadExecution(thread, reason); } } finally @@ -2964,7 +3178,49 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I thread.HasBlockedContinuation = true; thread.BlockWakeKey = wakeKey; thread.BlockWaiter = waiter; + thread.BlockResumeHandler = null; + thread.BlockWakeHandler = null; thread.BlockDeadlineTimestamp = blockDeadlineTimestamp; + TraceFocusedContinuation( + "register", + guestThreadHandle, + continuation, + wakeKey); + } + } + + private void RegisterBlockedGuestThreadContinuation( + ulong guestThreadHandle, + GuestCpuContinuation continuation, + string wakeKey, + Func? resumeHandler, + Func? wakeHandler, + long blockDeadlineTimestamp) + { + if (guestThreadHandle == 0 || continuation.Rip < 65536 || continuation.Rsp == 0) + { + return; + } + + using (LockGate("RegisterBlockedContinuation")) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out var thread)) + { + return; + } + + thread.BlockedContinuation = continuation; + thread.HasBlockedContinuation = true; + thread.BlockWakeKey = wakeKey; + thread.BlockWaiter = null; + thread.BlockResumeHandler = resumeHandler; + thread.BlockWakeHandler = wakeHandler; + thread.BlockDeadlineTimestamp = blockDeadlineTimestamp; + TraceFocusedContinuation( + "register", + guestThreadHandle, + continuation, + wakeKey); } } @@ -3079,6 +3335,32 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I string reason, out string? error) { + return TryCallGuestFunction( + callerContext, + entryPoint, + arg0, + arg1, + 0, + stackAddress, + stackSize, + reason, + out _, + out error); + } + + public bool TryCallGuestFunction( + CpuContext callerContext, + ulong entryPoint, + ulong arg0, + ulong arg1, + ulong arg2, + ulong stackAddress, + ulong stackSize, + string reason, + out ulong returnValue, + out string? error) + { + returnValue = 0; error = null; if (entryPoint < 65536) { @@ -3093,6 +3375,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ulong callbackStackBase; ulong callbackStackSize; + var usesCachedCallbackStack = false; if (stackAddress != 0 && stackSize >= 0x100) { callbackStackBase = stackAddress; @@ -3100,10 +3383,37 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } else { - if (!TryMapGuestThreadRegion(virtualMemory, GuestThreadStackBaseAddress, GuestThreadStackSize, ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, out callbackStackBase, out error)) + var callbackDepth = _nestedGuestCallbackDepth; + _nestedGuestCallbackStacks ??= []; + if (callbackDepth < _nestedGuestCallbackStacks.Count && + ReferenceEquals(_nestedGuestCallbackStacks[callbackDepth].Memory, virtualMemory)) { - return false; + callbackStackBase = _nestedGuestCallbackStacks[callbackDepth].Base; } + else + { + if (!TryMapGuestThreadRegion( + virtualMemory, + GuestThreadStackBaseAddress, + GuestThreadStackSize, + ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, + out callbackStackBase, + out error)) + { + return false; + } + + if (callbackDepth < _nestedGuestCallbackStacks.Count) + { + _nestedGuestCallbackStacks[callbackDepth] = (virtualMemory, callbackStackBase); + } + else + { + _nestedGuestCallbackStacks.Add((virtualMemory, callbackStackBase)); + } + } + + usesCachedCallbackStack = true; callbackStackSize = GuestThreadStackSize; } @@ -3119,7 +3429,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context[CpuRegister.Rsp] = AlignDown(callbackStackBase + callbackStackSize, 16) - sizeof(ulong); context[CpuRegister.Rdi] = arg0; context[CpuRegister.Rsi] = arg1; - context[CpuRegister.Rdx] = 0; + context[CpuRegister.Rdx] = arg2; context[CpuRegister.Rcx] = 0; context[CpuRegister.R8] = 0; context[CpuRegister.R9] = 0; @@ -3128,26 +3438,169 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I error = "failed to initialize guest callback stack"; return false; } + if (usesCachedCallbackStack) + { + _nestedGuestCallbackDepth++; + } var previousLastError = LastError; try { LastError = null; var exitReason = ExecuteGuestThreadEntry(context, entryPoint, reason, out var callbackReason); + if (exitReason == GuestNativeCallExitReason.Blocked && + !ResumeBlockedNestedGuestCallback(context, reason, ref exitReason, ref callbackReason)) + { + error = callbackReason ?? LastError ?? "guest callback could not resume after blocking"; + return false; + } if (exitReason is GuestNativeCallExitReason.Exception or GuestNativeCallExitReason.ForcedExit) { error = callbackReason ?? LastError ?? "guest callback failed"; return false; } + returnValue = context[CpuRegister.Rax]; return true; } finally { + if (usesCachedCallbackStack) + { + _nestedGuestCallbackDepth--; + } LastError = previousLastError; } } + /// + /// Completes a nested guest callback which blocked in an HLE import. The + /// outer guest entry is still executing managed HLE code, so returning a + /// successful callback result here would abandon the callback continuation + /// and let a noreturn operation such as pthread_exit unwind through live + /// libc cleanup state. Temporarily expose the owning guest thread as blocked, + /// let the normal scheduler wake it, and resume the callback continuation on + /// this executor until it either returns or fails. + /// + private bool ResumeBlockedNestedGuestCallback( + CpuContext callbackContext, + string reason, + ref GuestNativeCallExitReason exitReason, + ref string? callbackReason) + { + var guestThreadHandle = GuestThreadExecution.CurrentGuestThreadHandle; + if (guestThreadHandle == 0) + { + callbackReason = $"nested guest callback '{reason}' blocked without a schedulable guest thread"; + exitReason = GuestNativeCallExitReason.Exception; + return false; + } + + while (exitReason == GuestNativeCallExitReason.Blocked && !ActiveForcedGuestExit) + { + GuestThreadState? owner; + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out owner) || + !owner.HasBlockedContinuation) + { + callbackReason = + $"nested guest callback '{reason}' blocked without a captured continuation"; + exitReason = GuestNativeCallExitReason.Exception; + return false; + } + + owner.State = GuestThreadRunState.Blocked; + owner.BlockReason = callbackReason ?? reason; + if (owner.BlockWakeHandler is not null && owner.BlockWakeHandler()) + { + owner.State = GuestThreadRunState.Ready; + owner.BlockReason = null; + owner.BlockWakeHandler = null; + owner.BlockDeadlineTimestamp = 0; + } + } + if (_logGuestThreads) + { + Console.Error.WriteLine( + $"[LOADER][INFO] nested_callback.block name='{owner!.Name}' callback='{reason}' " + + $"wake={owner.BlockWakeKey ?? "none"} continuation=0x{owner.BlockedContinuation.Rip:X16}"); + } + + GuestCpuContinuation continuation = default; + Func? resumeHandler = null; + while (!ActiveForcedGuestExit) + { + WakeExpiredBlockedGuestThreads(); + var ready = false; + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out owner)) + { + callbackReason = + $"nested guest callback '{reason}' lost its owning guest thread"; + exitReason = GuestNativeCallExitReason.Exception; + return false; + } + + if (owner.State == GuestThreadRunState.Ready && owner.HasBlockedContinuation) + { + continuation = owner.BlockedContinuation; + owner.BlockedContinuation = default; + owner.HasBlockedContinuation = false; + owner.BlockWakeKey = null; + resumeHandler = owner.BlockResumeHandler; + owner.BlockResumeHandler = null; + owner.BlockWakeHandler = null; + owner.BlockDeadlineTimestamp = 0; + owner.BlockReason = null; + owner.State = GuestThreadRunState.Running; + ready = true; + } + } + + if (ready) + { + break; + } + + Thread.Sleep(1); + } + + if (ActiveForcedGuestExit) + { + callbackReason = LastError ?? $"nested guest callback '{reason}' was forced to exit"; + exitReason = GuestNativeCallExitReason.ForcedExit; + return false; + } + + if (resumeHandler is not null) + { + continuation = continuation with { Rax = unchecked((ulong)(long)resumeHandler()) }; + } + if (_logGuestThreads) + { + Console.Error.WriteLine( + $"[LOADER][INFO] nested_callback.resume thread=0x{guestThreadHandle:X16} callback='{reason}' " + + $"continuation=0x{continuation.Rip:X16}"); + } + + exitReason = ExecuteBlockedGuestThreadContinuation( + callbackContext, + continuation, + reason, + out callbackReason); + } + + if (exitReason == GuestNativeCallExitReason.Blocked && ActiveForcedGuestExit) + { + callbackReason = LastError ?? $"nested guest callback '{reason}' was forced to exit"; + exitReason = GuestNativeCallExitReason.ForcedExit; + } + + return exitReason == GuestNativeCallExitReason.Returned; + } + public bool TryCallGuestContinuation( CpuContext callerContext, GuestCpuContinuation continuation, @@ -3174,6 +3627,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Rflags = continuation.Rflags == 0 ? 0x202UL : continuation.Rflags, FsBase = callerContext.FsBase != 0 ? callerContext.FsBase : (continuation.FsBase != 0 ? continuation.FsBase : fallbackTlsBase), GsBase = callerContext.GsBase != 0 ? callerContext.GsBase : (continuation.GsBase != 0 ? continuation.GsBase : fallbackTlsBase), + FpuControlWord = continuation.FpuControlWord == 0 ? (ushort)0x037F : continuation.FpuControlWord, + Mxcsr = continuation.Mxcsr == 0 ? 0x1F80u : continuation.Mxcsr, }; context[CpuRegister.Rax] = continuation.Rax; @@ -3185,6 +3640,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context[CpuRegister.Rdi] = continuation.Rdi; context[CpuRegister.R8] = continuation.R8; context[CpuRegister.R9] = continuation.R9; + context[CpuRegister.R10] = continuation.R10; + context[CpuRegister.R11] = continuation.R11; context[CpuRegister.R12] = continuation.R12; context[CpuRegister.R13] = continuation.R13; context[CpuRegister.R14] = continuation.R14; @@ -3302,6 +3759,512 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return true; } + public bool TryRaiseGuestException( + CpuContext callerContext, + ulong threadHandle, + ulong handler, + int exceptionType, + out string? error) + { + error = null; + var logGuestExceptions = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal); + if (threadHandle == 0 || handler < 65536 || exceptionType is < 0 or >= 128) + { + error = "invalid guest exception delivery request"; + return false; + } + + GuestThreadState target; + GuestThreadRunState savedState; + bool savedExecutorActive; + string? savedBlockReason; + bool savedHasBlockedContinuation; + GuestCpuContinuation savedBlockedContinuation; + string? savedBlockWakeKey; + Func? savedBlockResumeHandler; + Func? savedBlockWakeHandler; + long savedBlockDeadlineTimestamp; + ulong exceptionStackBase; + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(threadHandle, out target!)) + { + if (!_externalGuestThreads.TryGetValue(threadHandle, out var external)) + { + error = $"unknown guest exception target 0x{threadHandle:X16}"; + return false; + } + + if (external.ExceptionStackBase == 0) + { + string? mapError = null; + if (!TryGetVirtualMemory(external.Context, out var virtualMemory) || + !TryMapGuestThreadRegion( + virtualMemory, + GuestThreadStackBaseAddress, + GuestThreadStackSize, + ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, + out var stackBase, + out mapError)) + { + error = mapError ?? "external guest context has no virtual memory"; + return false; + } + + external.ExceptionStackBase = stackBase; + } + + if (_pendingGuestExceptions.ContainsKey(threadHandle)) + { + return true; + } + if (_activeGuestExceptionDeliveries.Contains(threadHandle)) + { + // Preserve one signal raised while the previous handler is + // unwinding. Unity can begin its next stop-the-world cycle in + // that window; treating the new raise as part of the old delivery + // strands the collector waiting for an acknowledgement. + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + external.ExceptionStackBase); + return true; + } + + // A primary/external executor is already running guest code on its + // own host thread. Running its signal handler concurrently on a new + // managed thread corrupts the worker's control state. Queue the + // request and let that exact executor consume it at its next HLE + // boundary, where the original guest thread is safely paused. + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + external.ExceptionStackBase); + if (logGuestExceptions) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.queued " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} mode=external"); + } + return true; + } + + if (target.State is GuestThreadRunState.Exited or GuestThreadRunState.Faulted) + { + error = $"guest exception target 0x{threadHandle:X16} is no longer running"; + return false; + } + + if (target.ExceptionStackBase == 0) + { + string? mapError = null; + if (!TryGetVirtualMemory(target.Context, out var virtualMemory) || + !TryMapGuestThreadRegion( + virtualMemory, + GuestThreadStackBaseAddress, + GuestThreadStackSize, + ProgramHeaderFlags.Read | ProgramHeaderFlags.Write, + out var mappedExceptionStack, + out mapError)) + { + error = mapError ?? "guest thread context has no virtual memory"; + return false; + } + + target.ExceptionStackBase = mappedExceptionStack; + } + exceptionStackBase = target.ExceptionStackBase; + + if (target.State != GuestThreadRunState.Blocked || target.ExecutorActive) + { + if (_pendingGuestExceptions.ContainsKey(threadHandle) || + _activeGuestExceptionDeliveries.Contains(threadHandle)) + { + return true; + } + if (target.ExceptionDeliveryActive) + { + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + exceptionStackBase); + return true; + } + + _pendingGuestExceptions[threadHandle] = new PendingGuestException( + handler, + exceptionType, + exceptionStackBase); + if (logGuestExceptions) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.queued " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} " + + $"mode=scheduled state={target.State} executor={target.ExecutorActive}"); + } + return true; + } + + // A parked cooperative thread has no active executor, so its saved + // continuation can be handled immediately on a temporary executor. + if (target.ExceptionDeliveryActive) + { + return true; + } + + savedState = target.State; + savedExecutorActive = target.ExecutorActive; + savedBlockReason = target.BlockReason; + savedHasBlockedContinuation = target.HasBlockedContinuation; + savedBlockedContinuation = target.BlockedContinuation; + savedBlockWakeKey = target.BlockWakeKey; + savedBlockResumeHandler = target.BlockResumeHandler; + savedBlockWakeHandler = target.BlockWakeHandler; + savedBlockDeadlineTimestamp = target.BlockDeadlineTimestamp; + + target.State = GuestThreadRunState.Running; + target.ExecutorActive = true; + target.ExceptionDeliveryActive = true; + target.BlockReason = null; + target.HasBlockedContinuation = false; + target.BlockedContinuation = default; + target.BlockWakeKey = null; + target.BlockResumeHandler = null; + target.BlockWakeHandler = null; + target.BlockDeadlineTimestamp = 0; + } + + const ulong exceptionContextSize = 0x500; + const ulong callbackStackOffset = 0x1000; + const ulong callbackStackSize = 0xF000; + var exceptionContextAddress = exceptionStackBase + 0x100; + var guestExceptionCallback = 0UL; + if (handler >= 0x210) + { + _ = target.Context.TryReadUInt64(handler - 0x210 + 0xC020, out guestExceptionCallback); + } + if (!TryWriteGuestExceptionContext( + target.Context, + exceptionContextAddress, + savedHasBlockedContinuation ? savedBlockedContinuation : default, + exceptionContextSize)) + { + lock (_guestThreadGate) + { + RestoreInterruptedGuestThread(); + } + error = "failed to write guest exception context"; + return false; + } + + if (logGuestExceptions) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.delivery_enter " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} mode=parked " + + $"rip=0x{savedBlockedContinuation.Rip:X16} rsp=0x{savedBlockedContinuation.Rsp:X16} " + + $"rbp=0x{savedBlockedContinuation.Rbp:X16} rbx=0x{savedBlockedContinuation.Rbx:X16} " + + $"r12=0x{savedBlockedContinuation.R12:X16} r13=0x{savedBlockedContinuation.R13:X16} " + + $"r14=0x{savedBlockedContinuation.R14:X16} r15=0x{savedBlockedContinuation.R15:X16} " + + $"stack=0x{exceptionStackBase:X16} callback=0x{guestExceptionCallback:X16}"); + } + + void RestoreInterruptedGuestThread() + { + target.State = savedState; + target.ExecutorActive = savedExecutorActive; + target.ExceptionDeliveryActive = false; + target.BlockReason = savedBlockReason; + target.HasBlockedContinuation = savedHasBlockedContinuation; + target.BlockedContinuation = savedBlockedContinuation; + target.BlockWakeKey = savedBlockWakeKey; + target.BlockResumeHandler = savedBlockResumeHandler; + target.BlockWakeHandler = savedBlockWakeHandler; + target.BlockDeadlineTimestamp = savedBlockDeadlineTimestamp; + + // A condition/event wake can arrive while the parked thread is + // temporarily marked Running for signal delivery. WakeBlockedThreads + // cannot claim it in that state, so re-check the restored wait before + // releasing scheduler ownership. Without this handoff a completed + // pthread wait remains parked forever after a GC suspension races it. + if (target.State == GuestThreadRunState.Blocked && + target.HasBlockedContinuation && + target.BlockWakeHandler is not null && + target.BlockWakeHandler()) + { + target.State = GuestThreadRunState.Ready; + target.BlockReason = null; + target.BlockWakeHandler = null; + target.BlockDeadlineTimestamp = 0; + _readyGuestThreads.Enqueue(target); + Interlocked.Increment(ref _readyGuestThreadCount); + } + } + + void DeliverException() + { + var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(threadHandle); + var previousGuestThreadState = _activeGuestThreadState; + _activeGuestThreadState = target; + var deliverySucceeded = false; + string? deliveryError = null; + var deliveryStarted = Stopwatch.GetTimestamp(); + try + { + deliverySucceeded = TryCallGuestFunction( + target.Context, + handler, + unchecked((ulong)exceptionType), + exceptionContextAddress, + exceptionStackBase + callbackStackOffset, + callbackStackSize, + $"kernel exception 0x{exceptionType:X2}", + out deliveryError); + if (!deliverySucceeded) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception delivery failed: " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} " + + $"error={deliveryError ?? "unknown"}"); + } + } + finally + { + PendingGuestException? followUp = null; + if (logGuestExceptions) + { + var recordAddress = FindGuestExceptionThreadRecord( + target.Context, + guestExceptionCallback, + threadHandle); + var recordedStack = 0UL; + var registeredStackBound = 0UL; + if (recordAddress != 0) + { + _ = target.Context.TryReadUInt64(recordAddress + 0x100, out registeredStackBound); + _ = target.Context.TryReadUInt64(recordAddress + 0x18, out recordedStack); + } + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.delivery_exit " + + $"target=0x{threadHandle:X16} type=0x{exceptionType:X2} " + + $"success={deliverySucceeded} error={deliveryError ?? "none"} " + + $"elapsed_ms={Stopwatch.GetElapsedTime(deliveryStarted).TotalMilliseconds:F3} " + + $"record=0x{recordAddress:X16} stack_bound=0x{registeredStackBound:X16} " + + $"recorded_rsp=0x{recordedStack:X16}"); + } + _activeGuestThreadState = previousGuestThreadState; + GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); + lock (_guestThreadGate) + { + RestoreInterruptedGuestThread(); + if (target.State == GuestThreadRunState.Blocked && + !target.ExecutorActive && + _pendingGuestExceptions.Remove(threadHandle, out var queued)) + { + followUp = queued; + } + } + if (followUp is { } pendingFollowUp && + !TryRaiseGuestException( + target.Context, + threadHandle, + pendingFollowUp.Handler, + pendingFollowUp.ExceptionType, + out var followUpError)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception follow-up delivery failed: " + + $"target=0x{threadHandle:X16} type=0x{pendingFollowUp.ExceptionType:X2} " + + $"error={followUpError ?? "unknown"}"); + } + } + } + + // A real sceKernelRaiseException interrupts the target pthread and runs + // its signal handler on that same native thread. Parked cooperative + // guest threads have no active call frame to interrupt, but their + // persistent execution runner is idle and preserves the native-thread + // identity/TLS that Unity's stop-the-world collector registered. Using + // an unrelated temporary host thread makes the suspension acknowledge + // appear valid while publishing roots from the wrong native execution + // context, which lets live IL2CPP delegates be reclaimed. + GuestExecutionRunner deliveryRunner; + lock (_guestThreadGate) + { + deliveryRunner = target.ExecutionRunner ??= new GuestExecutionRunner( + target.ThreadHandle, + target.Name, + MapGuestThreadPriority(target.Priority)); + } + deliveryRunner.Schedule(DeliverException); + return true; + } + + private static ulong FindGuestExceptionThreadRecord( + CpuContext context, + ulong callback, + ulong threadHandle) + { + if (callback < 65536) + { + return 0; + } + + // Unity's suspend callback uses a 256-bucket table at this fixed + // image-relative offset from the callback entry. Each node stores the + // pthread handle at +0x08 and the next pointer at +0x00. + var tableAddress = callback + 0x102E8B0; + for (var bucket = 0; bucket < 256; bucket++) + { + if (!context.TryReadUInt64(tableAddress + unchecked((ulong)bucket * 8), out var node)) + { + return 0; + } + + for (var depth = 0; node >= 65536 && depth < 1024; depth++) + { + if (!context.TryReadUInt64(node + 0x08, out var registeredThread)) + { + break; + } + if (registeredThread == threadHandle) + { + return node; + } + if (!context.TryReadUInt64(node, out node)) + { + break; + } + } + } + + return 0; + } + + private void DeliverPendingGuestExceptionAtSafePoint( + CpuContext currentContext, + GuestCpuContinuation interruptedContinuation) + { + var threadHandle = GuestThreadExecution.CurrentGuestThreadHandle; + if (threadHandle == 0) + { + threadHandle = _currentExternalGuestThreadHandle; + } + PendingGuestException pending; + lock (_guestThreadGate) + { + if (threadHandle == 0) + { + return; + } + + if (!_pendingGuestExceptions.Remove(threadHandle, out pending)) + { + return; + } + + _activeGuestExceptionDeliveries.Add(threadHandle); + } + + const ulong exceptionContextSize = 0x500; + const ulong callbackStackOffset = 0x1000; + const ulong callbackStackSize = 0xF000; + var exceptionContextAddress = pending.ExceptionStackBase + 0x100; + try + { + if (!TryWriteGuestExceptionContext( + currentContext, + exceptionContextAddress, + interruptedContinuation, + exceptionContextSize)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception safe-point context write failed: " + + $"target=0x{threadHandle:X16} type=0x{pending.ExceptionType:X2}"); + return; + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.safe_point_enter " + + $"target=0x{threadHandle:X16} type=0x{pending.ExceptionType:X2} " + + $"rip=0x{interruptedContinuation.Rip:X16}"); + } + + if (!TryCallGuestFunction( + currentContext, + pending.Handler, + unchecked((ulong)pending.ExceptionType), + exceptionContextAddress, + pending.ExceptionStackBase + callbackStackOffset, + callbackStackSize, + $"kernel exception 0x{pending.ExceptionType:X2} safe point", + out var callbackError)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] Guest exception safe-point delivery failed: " + + $"target=0x{threadHandle:X16} type=0x{pending.ExceptionType:X2} " + + $"error={callbackError ?? "unknown"}"); + } + } + finally + { + lock (_guestThreadGate) + { + _activeGuestExceptionDeliveries.Remove(threadHandle); + } + } + } + + private static bool TryWriteGuestExceptionContext( + CpuContext context, + ulong address, + GuestCpuContinuation continuation, + ulong size) + { + var bytes = new byte[checked((int)size)]; + void Write64(int offset, ulong value) => + BinaryPrimitives.WriteUInt64LittleEndian(bytes.AsSpan(offset, sizeof(ulong)), value); + + var hasContinuation = continuation.Rip >= 65536 && continuation.Rsp != 0; + // Orbis ucontext_t has a 0x10-byte signal mask and 0x30 bytes of + // private fields before its amd64 mcontext. These offsets match the + // platform ABI used by libScePs5Util and Unity's Boehm GC. Supplying a + // bare mcontext here makes the collector miss live register roots. + const int mcontext = 0x40; + Write64(mcontext + 0x08, hasContinuation ? continuation.Rdi : context[CpuRegister.Rdi]); + Write64(mcontext + 0x10, hasContinuation ? continuation.Rsi : context[CpuRegister.Rsi]); + Write64(mcontext + 0x18, hasContinuation ? continuation.Rdx : context[CpuRegister.Rdx]); + Write64(mcontext + 0x20, hasContinuation ? continuation.Rcx : context[CpuRegister.Rcx]); + Write64(mcontext + 0x28, hasContinuation ? continuation.R8 : context[CpuRegister.R8]); + Write64(mcontext + 0x30, hasContinuation ? continuation.R9 : context[CpuRegister.R9]); + Write64(mcontext + 0x38, hasContinuation ? continuation.Rax : context[CpuRegister.Rax]); + Write64(mcontext + 0x40, hasContinuation ? continuation.Rbx : context[CpuRegister.Rbx]); + Write64(mcontext + 0x48, hasContinuation ? continuation.Rbp : context[CpuRegister.Rbp]); + Write64(mcontext + 0x50, hasContinuation ? continuation.R10 : context[CpuRegister.R10]); + Write64(mcontext + 0x58, hasContinuation ? continuation.R11 : context[CpuRegister.R11]); + Write64(mcontext + 0x60, hasContinuation ? continuation.R12 : context[CpuRegister.R12]); + Write64(mcontext + 0x68, hasContinuation ? continuation.R13 : context[CpuRegister.R13]); + Write64(mcontext + 0x70, hasContinuation ? continuation.R14 : context[CpuRegister.R14]); + Write64(mcontext + 0x78, hasContinuation ? continuation.R15 : context[CpuRegister.R15]); + var rip = hasContinuation ? continuation.Rip : context.Rip; + var rsp = hasContinuation ? continuation.Rsp : context[CpuRegister.Rsp]; + Write64(mcontext + 0xA0, rip); + Write64(mcontext + 0xB0, hasContinuation ? continuation.Rflags : 0); + Write64(mcontext + 0xB8, rsp); + Write64(mcontext + 0xC8, 0x480); // sizeof(Orbis mcontext_t) + Write64(mcontext + 0x440, hasContinuation ? continuation.FsBase : context.FsBase); + Write64(mcontext + 0x448, hasContinuation ? continuation.GsBase : context.GsBase); + return context.Memory.TryWrite(address, bytes); + } + private void TraceGuestContext(string message) { if (_logGuestContext) @@ -3333,61 +4296,35 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I continuationThread.Join(); } - /// - /// Parks every guest worker thread before the caller starts freeing executable - /// memory. Workers unwind to the host at their next import dispatch (see the - /// teardown check in DispatchImport); this waits for their host threads to - /// finish within . Returns false when at least one - /// worker is still running — the caller must then leak its executable stubs - /// rather than free memory a live thread may still execute. - /// - private bool RequestGuestThreadTeardown(int timeoutMs) - { - _guestTeardownRequested = true; - Thread[] hostThreads; - using (LockGate("RequestGuestThreadTeardown")) - { - _readyGuestThreads.Clear(); - Interlocked.Exchange(ref _readyGuestThreadCount, 0); - hostThreads = _guestThreads.Values - .Select(static thread => thread.HostThread) - .Where(static host => host is not null && host != Thread.CurrentThread && host.IsAlive) - .Cast() - .ToArray(); - } - - var deadline = Environment.TickCount64 + timeoutMs; - var allStopped = true; - foreach (var host in hostThreads) - { - var remaining = (int)Math.Max(1L, deadline - Environment.TickCount64); - if (!host.Join(remaining) && host.IsAlive) - { - allStopped = false; - Console.Error.WriteLine( - $"[LOADER][WARN] Guest worker host thread '{host.Name}' still running after teardown wait."); - } - } - - return allStopped; - } - private void ClearGuestThreads() { - GuestContinuationRunner[] runners; - using (LockGate("ClearGuestThreads")) + GuestContinuationRunner[] continuationRunners; + GuestExecutionRunner[] executionRunners; + lock (_guestThreadGate) { - runners = _guestThreads.Values + continuationRunners = _guestThreads.Values .Select(static thread => thread.ContinuationRunner) .Where(static runner => runner is not null) .Cast() .ToArray(); + executionRunners = _guestThreads.Values + .Select(static thread => thread.ExecutionRunner) + .Where(static runner => runner is not null) + .Cast() + .ToArray(); _readyGuestThreads.Clear(); Interlocked.Exchange(ref _readyGuestThreadCount, 0); _guestThreads.Clear(); + _externalGuestThreads.Clear(); + _pendingGuestExceptions.Clear(); + _activeGuestExceptionDeliveries.Clear(); } - foreach (var runner in runners) + foreach (var runner in continuationRunners) + { + runner.Dispose(); + } + foreach (var runner in executionRunners) { runner.Dispose(); } @@ -3440,6 +4377,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Priority = request.Priority, AffinityMask = request.AffinityMask, Context = context, + StackBase = stackBase, + StackSize = GuestThreadStackSize, State = GuestThreadRunState.Ready, }; error = null; @@ -3471,7 +4410,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I out ulong mappedBase, out string? error) { - for (int i = 0; i < GuestThreadRegionSlotCount; i++) + for (int i = 0; i < GuestThreadRegionSlots; i++) { var candidateBase = baseAddress - ((ulong)i * GuestThreadRegionStride); if (!IsGuestThreadRegionFree(virtualMemory, candidateBase, size)) @@ -3505,7 +4444,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I out ulong tlsBase, out string? error) { - for (int i = 0; i < GuestThreadRegionSlotCount; i++) + for (int i = 0; i < GuestThreadRegionSlots; i++) { var candidateBase = GuestThreadTlsBaseAddress - ((ulong)i * GuestThreadRegionStride); var mappedBase = candidateBase - GuestThreadTlsPrefixSize; @@ -3571,11 +4510,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I private static bool InitializeGuestThreadTls(CpuContext context, ulong tlsBase, ulong threadHandle) { - return context.TryWriteUInt64(tlsBase - 0xF0, 0) && - context.TryWriteUInt64(tlsBase + 0x00, tlsBase) && - context.TryWriteUInt64(tlsBase + 0x10, threadHandle) && - context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBABEUL) && - context.TryWriteUInt64(tlsBase + 0x60, tlsBase); + if (!context.TryWriteUInt64(tlsBase - 0xF0, 0) || + !context.TryWriteUInt64(tlsBase + 0x00, tlsBase) || + !context.TryWriteUInt64(tlsBase + 0x10, threadHandle) || + !context.TryWriteUInt64(tlsBase + 0x28, 0xC0DEC0DECAFEBA00UL) || + !context.TryWriteUInt64(tlsBase + 0x60, tlsBase)) + { + return false; + } + + // Seed initialized thread-locals into the static TLS block below the + // thread pointer so per-thread TLS matches the main thread. + SharpEmu.HLE.GuestTlsTemplate.SeedThreadBlock(context, tlsBase); + return true; } private static ThreadPriority MapGuestThreadPriority(int priority) @@ -3600,7 +4547,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return; } - if (!_hostThreading.TrySetCurrentThreadAffinity((nuint)hostAffinityMask) && _logGuestThreads) + if (SetThreadAffinityMask(GetCurrentThread(), (nuint)hostAffinityMask) == 0 && _logGuestThreads) { Console.Error.WriteLine( $"[LOADER][WARN] Failed to set guest thread affinity guest=0x{guestAffinityMask:X} " + @@ -3643,21 +4590,70 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return hostAffinityMask; } + public bool TrySetGuestThreadPriority(ulong guestThreadHandle, int guestPriority) + { + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out var thread)) + { + return false; + } + + thread.Priority = guestPriority; + var host = thread.HostThread; + if (host is not null && host.IsAlive) + { + try + { + host.Priority = MapGuestThreadPriority(guestPriority); + } + catch (Exception exception) when (exception is ThreadStateException or InvalidOperationException) + { + // The thread may have exited between the alive check and + // the assignment; the stored priority still takes effect + // if it is ever restarted. + } + } + + return true; + } + } + + public bool TrySetGuestThreadAffinity(ulong guestThreadHandle, ulong affinityMask) + { + lock (_guestThreadGate) + { + if (!_guestThreads.TryGetValue(guestThreadHandle, out var thread)) + { + return false; + } + + thread.AffinityMask = affinityMask; + // A running thread applies its own affinity via + // ApplyGuestThreadAffinity; cross-thread affinity is not portable, + // so the new mask takes effect on the thread's next scheduling. + return true; + } + } + private void RunGuestThread(GuestThreadState thread, string reason) { + lock (_guestThreadGate) + { + if (!thread.ExecutorActive) + { + throw new InvalidOperationException( + $"Guest thread '{thread.Name}' started without scheduler executor ownership."); + } + + thread.HostThread = Thread.CurrentThread; + } var previousLastError = LastError; var previousGuestThreadHandle = GuestThreadExecution.EnterGuestThread(thread.ThreadHandle); var previousGuestThreadState = _activeGuestThreadState; ApplyGuestThreadAffinity(thread.AffinityMask); - Volatile.Write(ref thread.HostThreadId, unchecked((int)_hostThreading.CurrentThreadId)); + Volatile.Write(ref thread.HostThreadId, unchecked((int)GetCurrentThreadId())); _activeGuestThreadState = thread; - if (LogThreadMode) - { - _threadModeCycleId = Interlocked.Increment(ref _threadModeCycleCounter); - TraceThreadMode( - $"cycle_start name='{thread.Name}' guest=0x{thread.ThreadHandle:X16} reason={reason} " + - $"rsp_slot=0x{(ulong)_hostThreading.GetTlsValue(_hostRspSlotTlsIndex):X}"); - } try { LastError = null; @@ -3701,6 +4697,12 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I case GuestNativeCallExitReason.Returned: thread.ExitValue = thread.Context[CpuRegister.Rax]; thread.State = GuestThreadRunState.Exited; + if (_logGuestThreads) + Console.Error.WriteLine( + $"[LOADER][INFO] Guest thread exited: name='{thread.Name}' " + + $"exitValue=0x{thread.ExitValue:X16} imports={Interlocked.Read(ref thread.ImportCount)} " + + $"lastNid={Volatile.Read(ref thread.LastImportNid) ?? "none"} " + + $"entry=0x{thread.EntryPoint:X16} ret=0x{Volatile.Read(ref thread.LastReturnRip):X16}"); break; case GuestNativeCallExitReason.Blocked: thread.State = GuestThreadRunState.Blocked; @@ -3730,17 +4732,18 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - if (LogThreadMode) - { - TraceThreadMode( - $"cycle_end name='{thread.Name}' state={thread.State} " + - $"imports={Interlocked.Read(ref thread.ImportCount)} " + - $"rsp_slot=0x{(ulong)_hostThreading.GetTlsValue(_hostRspSlotTlsIndex):X}"); - } _activeGuestThreadState = previousGuestThreadState; Volatile.Write(ref thread.HostThreadId, 0); GuestThreadExecution.RestoreGuestThread(previousGuestThreadHandle); LastError = previousLastError; + lock (_guestThreadGate) + { + if (ReferenceEquals(thread.HostThread, Thread.CurrentThread)) + { + thread.HostThread = null; + } + thread.ExecutorActive = false; + } } } @@ -3750,6 +4753,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I string name, out string? reason) { + TraceFocusedContinuation( + "execute", + GuestThreadExecution.CurrentGuestThreadHandle, + continuation, + name); ApplyGuestContinuation(context, continuation); return ExecuteGuestContinuationEntry( context, @@ -3759,6 +4767,30 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I out reason); } + private static void TraceFocusedContinuation( + string operation, + ulong threadHandle, + GuestCpuContinuation continuation, + string detail) + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_FOCUSED_CONTINUATION"), + "1", + StringComparison.Ordinal) || + continuation.Rsp < 0x00006FFFAC000000UL || + continuation.Rsp >= 0x00006FFFAC200000UL) + { + return; + } + + Console.Error.WriteLine( + $"[LOADER][TRACE] focused_continuation.{operation} " + + $"thread=0x{threadHandle:X16} rip=0x{continuation.Rip:X16} " + + $"rsp=0x{continuation.Rsp:X16} slot=0x{continuation.ReturnSlotAddress:X16} " + + $"rbp=0x{continuation.Rbp:X16} rbx=0x{continuation.Rbx:X16} " + + $"detail={detail}"); + } + private static void ApplyGuestContinuation(CpuContext context, GuestCpuContinuation continuation) { context.Rip = continuation.Rip; @@ -3781,11 +4813,17 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I context[CpuRegister.Rdi] = continuation.Rdi; context[CpuRegister.R8] = continuation.R8; context[CpuRegister.R9] = continuation.R9; + context[CpuRegister.R10] = continuation.R10; + context[CpuRegister.R11] = continuation.R11; context[CpuRegister.R12] = continuation.R12; context[CpuRegister.R13] = continuation.R13; context[CpuRegister.R14] = continuation.R14; context[CpuRegister.R15] = continuation.R15; context[CpuRegister.Rsp] = continuation.Rsp; + context.FpuControlWord = continuation.FpuControlWord == 0 + ? (ushort)0x037F + : continuation.FpuControlWord; + context.Mxcsr = continuation.Mxcsr == 0 ? 0x1F80u : continuation.Mxcsr; } private unsafe GuestNativeCallExitReason ExecuteGuestThreadEntry(CpuContext context, ulong entryPoint, string name, out string? reason) @@ -3797,12 +4835,19 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return GuestNativeCallExitReason.Exception; } const uint stubSize = 512u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) { reason = "failed to allocate executable memory for guest thread stub"; return GuestNativeCallExitReason.Exception; } + void* hostRspStorage = NativeMemory.Alloc((nuint)sizeof(ulong)); + if (hostRspStorage == null) + { + VirtualFree(ptr, 0u, 32768u); + reason = "failed to allocate writable host-RSP storage for guest thread stub"; + return GuestNativeCallExitReason.Exception; + } var previousActiveBackend = _activeExecutionBackend; var previousActiveContext = _activeCpuContext; var previousSentinel = _activeEntryReturnSentinelRip; @@ -3810,285 +4855,144 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I var previousForcedExit = _activeForcedGuestExit; var previousYieldRequested = _activeGuestThreadYieldRequested; var previousYieldReason = _activeGuestThreadYieldReason; - nint previousHostRspSlotValue = _hostThreading.GetTlsValue(_hostRspSlotTlsIndex); - if (LogThreadMode) - { - TraceThreadMode( - $"entry_setup name='{name}' entry=0x{entryPoint:X16} stub=0x{(ulong)ptr:X16} " + - $"guest_rsp=0x{context[CpuRegister.Rsp]:X16} rsp_slot_prev=0x{(ulong)previousHostRspSlotValue:X}"); - } - try - { - _activeExecutionBackend = this; - _activeCpuContext = context; - _activeEntryReturnSentinelRip = 0; - _activeGuestReturnSlotAddress = 0; - _activeForcedGuestExit = false; - _activeGuestThreadYieldRequested = false; - _activeGuestThreadYieldReason = null; - BindTlsBase(context); - byte* ptr2 = (byte*)ptr; - ulong hostRspSlot = (ulong)ptr + stubSize - 16uL; - int offset = 0; - ptr2[offset++] = 83; - ptr2[offset++] = 85; - ptr2[offset++] = 87; - ptr2[offset++] = 86; - ptr2[offset++] = 65; - ptr2[offset++] = 84; - ptr2[offset++] = 65; - ptr2[offset++] = 85; - ptr2[offset++] = 65; - ptr2[offset++] = 86; - ptr2[offset++] = 65; - ptr2[offset++] = 87; - EmitHostNonvolatileXmmSave(ptr2, ref offset); - ptr2[offset++] = 73; - ptr2[offset++] = 186; - *(ulong*)(ptr2 + offset) = hostRspSlot; - offset += 8; - ptr2[offset++] = 73; - ptr2[offset++] = 137; - ptr2[offset++] = 34; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsp]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 196; - ptr2[offset++] = 72; - ptr2[offset++] = 131; - ptr2[offset++] = 236; - ptr2[offset++] = 8; - ptr2[offset++] = 72; - ptr2[offset++] = 189; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rbp]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdi]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 199; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsi]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 198; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdx]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 194; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = context[CpuRegister.Rcx]; - offset += 8; - ptr2[offset++] = 72; - ptr2[offset++] = 137; - ptr2[offset++] = 193; - ptr2[offset++] = 72; - ptr2[offset++] = 184; - *(ulong*)(ptr2 + offset) = entryPoint; - offset += 8; - ptr2[offset++] = byte.MaxValue; - ptr2[offset++] = 208; - int sentinelOffset = offset + 4; - ptr2[offset++] = 72; - ptr2[offset++] = 131; - ptr2[offset++] = 196; - ptr2[offset++] = 8; - ptr2[offset++] = 73; - ptr2[offset++] = 186; - *(ulong*)(ptr2 + offset) = hostRspSlot; - offset += 8; - ptr2[offset++] = 73; - ptr2[offset++] = 139; - ptr2[offset++] = 34; - EmitHostNonvolatileXmmRestore(ptr2, ref offset); - ptr2[offset++] = 65; - ptr2[offset++] = 95; - ptr2[offset++] = 65; - ptr2[offset++] = 94; - ptr2[offset++] = 65; - ptr2[offset++] = 93; - ptr2[offset++] = 65; - ptr2[offset++] = 92; - ptr2[offset++] = 94; - ptr2[offset++] = 95; - ptr2[offset++] = 93; - ptr2[offset++] = 91; - ptr2[offset++] = 195; - ulong sentinel = (ulong)ptr + (ulong)sentinelOffset; - ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; - _activeGuestReturnSlotAddress = context[CpuRegister.Rsp] - 16uL; - if (!context.TryWriteUInt64(context[CpuRegister.Rsp], sentinel)) - { - reason = $"failed to patch guest thread return sentinel at 0x{context[CpuRegister.Rsp]:X16}"; - return GuestNativeCallExitReason.Exception; - } - uint oldProtect = default(uint); - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadWriteExecute, out oldProtect)) - { - reason = $"VirtualProtect failed for guest thread entry stub at 0x{(nint)ptr:X16}"; - return GuestNativeCallExitReason.Exception; - } - _hostMemory.FlushInstructionCache((ulong)ptr, stubSize); - ActiveGuestThreadYieldRequested = false; - ActiveGuestThreadYieldReason = null; - var nativeReturn = RunGuestEntryStub(ptr, hostRspSlot); - if (ActiveGuestThreadYieldRequested) - { - reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; - return GuestNativeCallExitReason.Blocked; - } - if (ActiveForcedGuestExit) - { - reason = LastError ?? "guest thread forced exit"; - return GuestNativeCallExitReason.ForcedExit; - } - reason = $"returned 0x{nativeReturn:X8}"; - return GuestNativeCallExitReason.Returned; - } - catch (AccessViolationException ex) - { - reason = "access violation: " + ex.Message; - return GuestNativeCallExitReason.Exception; - } - catch (Exception ex) - { - reason = ex.GetType().Name + ": " + ex.Message; - return GuestNativeCallExitReason.Exception; - } - finally - { - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); - RestoreActiveExecutionThread( - previousActiveBackend, - previousActiveContext, - previousSentinel, - previousReturnSlotAddress, - previousForcedExit, - previousYieldRequested, - previousYieldReason); - _hostMemory.Free((ulong)ptr); - } -} - - private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntry( - CpuContext context, - ulong entryPoint, - ulong returnSlotAddress, - string name, - out string? reason) - { - reason = null; - if (context[CpuRegister.Rsp] == 0) - { - reason = "guest thread stack pointer is zero"; - return GuestNativeCallExitReason.Exception; - } - const uint stubSize = 512u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); - if (ptr == null) - { - reason = "failed to allocate executable memory for guest thread stub"; - return GuestNativeCallExitReason.Exception; - } - var previousActiveBackend = _activeExecutionBackend; - var previousActiveContext = _activeCpuContext; - var previousSentinel = _activeEntryReturnSentinelRip; - var previousReturnSlotAddress = _activeGuestReturnSlotAddress; - var previousForcedExit = _activeForcedGuestExit; - var previousYieldRequested = _activeGuestThreadYieldRequested; - var previousYieldReason = _activeGuestThreadYieldReason; - nint previousHostRspSlotValue = _hostThreading.GetTlsValue(_hostRspSlotTlsIndex); - if (LogThreadMode) - { - TraceThreadMode( - $"continuation_setup name='{name}' resume=0x{entryPoint:X16} stub=0x{(ulong)ptr:X16} " + - $"guest_rsp=0x{context[CpuRegister.Rsp]:X16} rsp_slot_prev=0x{(ulong)previousHostRspSlotValue:X}"); - } + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); try { _activeExecutionBackend = this; _activeCpuContext = context; _activeEntryReturnSentinelRip = 0; - _activeGuestReturnSlotAddress = returnSlotAddress; + _activeGuestReturnSlotAddress = 0; _activeForcedGuestExit = false; _activeGuestThreadYieldRequested = false; _activeGuestThreadYieldReason = null; BindTlsBase(context); byte* ptr2 = (byte*)ptr; - ulong hostRspSlot = (ulong)ptr + stubSize - 16uL; + // Rosetta does not reliably permit a generated x86 thunk to write data + // in the same page from which it is currently executing, even when the + // mapping reports PAGE_EXECUTE_READWRITE. Keep mutable transition state + // in a separate writable allocation. + ulong hostRspSlot = (ulong)hostRspStorage; int offset = 0; - - void Emit(byte value) => ptr2[offset++] = value; - void EmitU64(ulong value) - { - *(ulong*)(ptr2 + offset) = value; - offset += sizeof(ulong); - } - void EmitMovR64Imm(byte rex, byte opcode, ulong value) - { - Emit(rex); - Emit(opcode); - EmitU64(value); - } - - Emit(0x53); // push rbx - Emit(0x55); // push rbp - Emit(0x57); // push rdi - Emit(0x56); // push rsi - Emit(0x41); Emit(0x54); // push r12 - Emit(0x41); Emit(0x55); // push r13 - Emit(0x41); Emit(0x56); // push r14 - Emit(0x41); Emit(0x57); // push r15 + ptr2[offset++] = 83; + ptr2[offset++] = 85; + ptr2[offset++] = 87; + ptr2[offset++] = 86; + ptr2[offset++] = 65; + ptr2[offset++] = 84; + ptr2[offset++] = 65; + ptr2[offset++] = 85; + ptr2[offset++] = 65; + ptr2[offset++] = 86; + ptr2[offset++] = 65; + ptr2[offset++] = 87; EmitHostNonvolatileXmmSave(ptr2, ref offset); - EmitMovR64Imm(0x49, 0xBA, hostRspSlot); // mov r10, hostRspSlot - Emit(0x49); Emit(0x89); Emit(0x22); // mov [r10], rsp - EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rsp]); // mov rax, guest rsp - Emit(0x48); Emit(0x89); Emit(0xC4); // mov rsp, rax - EmitMovR64Imm(0x48, 0xBB, context[CpuRegister.Rbx]); // mov rbx, imm64 - EmitMovR64Imm(0x48, 0xBD, context[CpuRegister.Rbp]); // mov rbp, imm64 - EmitMovR64Imm(0x48, 0xBF, context[CpuRegister.Rdi]); // mov rdi, imm64 - EmitMovR64Imm(0x48, 0xBE, context[CpuRegister.Rsi]); // mov rsi, imm64 - EmitMovR64Imm(0x48, 0xBA, context[CpuRegister.Rdx]); // mov rdx, imm64 - EmitMovR64Imm(0x48, 0xB9, context[CpuRegister.Rcx]); // mov rcx, imm64 - EmitMovR64Imm(0x49, 0xB8, context[CpuRegister.R8]); // mov r8, imm64 - EmitMovR64Imm(0x49, 0xB9, context[CpuRegister.R9]); // mov r9, imm64 - EmitMovR64Imm(0x49, 0xBC, context[CpuRegister.R12]); // mov r12, imm64 - EmitMovR64Imm(0x49, 0xBD, context[CpuRegister.R13]); // mov r13, imm64 - EmitMovR64Imm(0x49, 0xBE, context[CpuRegister.R14]); // mov r14, imm64 - EmitMovR64Imm(0x49, 0xBF, context[CpuRegister.R15]); // mov r15, imm64 - EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rax]); // mov rax, imm64 - EmitMovR64Imm(0x49, 0xBB, entryPoint); // mov r11, entryPoint - Emit(0x41); Emit(0xFF); Emit(0xE3); // jmp r11 + ptr2[offset++] = 73; + ptr2[offset++] = 186; + *(ulong*)(ptr2 + offset) = hostRspSlot; + offset += 8; + ptr2[offset++] = 73; + ptr2[offset++] = 137; + ptr2[offset++] = 34; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsp]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 196; + ptr2[offset++] = 72; + ptr2[offset++] = 131; + ptr2[offset++] = 236; + ptr2[offset++] = 8; + ptr2[offset++] = 72; + ptr2[offset++] = 189; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rbp]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdi]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 199; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rsi]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 198; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rdx]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 194; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = context[CpuRegister.Rcx]; + offset += 8; + ptr2[offset++] = 72; + ptr2[offset++] = 137; + ptr2[offset++] = 193; + ptr2[offset++] = 72; + ptr2[offset++] = 184; + *(ulong*)(ptr2 + offset) = entryPoint; + offset += 8; + ptr2[offset++] = byte.MaxValue; + ptr2[offset++] = 208; + int sentinelOffset = offset + 4; + ptr2[offset++] = 72; + ptr2[offset++] = 131; + ptr2[offset++] = 196; + ptr2[offset++] = 8; + ptr2[offset++] = 73; + ptr2[offset++] = 186; + *(ulong*)(ptr2 + offset) = hostRspSlot; + offset += 8; + ptr2[offset++] = 73; + ptr2[offset++] = 139; + ptr2[offset++] = 34; + EmitHostNonvolatileXmmRestore(ptr2, ref offset); + ptr2[offset++] = 65; + ptr2[offset++] = 95; + ptr2[offset++] = 65; + ptr2[offset++] = 94; + ptr2[offset++] = 65; + ptr2[offset++] = 93; + ptr2[offset++] = 65; + ptr2[offset++] = 92; + ptr2[offset++] = 94; + ptr2[offset++] = 95; + ptr2[offset++] = 93; + ptr2[offset++] = 91; + ptr2[offset++] = 195; + ulong sentinel = (ulong)ptr + (ulong)sentinelOffset; ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; - if (returnSlotAddress == 0 || !context.TryWriteUInt64(returnSlotAddress, (ulong)_guestReturnStub)) + _activeGuestReturnSlotAddress = context[CpuRegister.Rsp] - 16uL; + if (!context.TryWriteUInt64(context[CpuRegister.Rsp], sentinel)) { - reason = $"failed to patch guest continuation return slot at 0x{returnSlotAddress:X16}"; + reason = $"failed to patch guest thread return sentinel at 0x{context[CpuRegister.Rsp]:X16}"; return GuestNativeCallExitReason.Exception; } uint oldProtect = default(uint); - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadWriteExecute, out oldProtect)) + if (!VirtualProtect(ptr, stubSize, 32u, &oldProtect)) { - reason = $"VirtualProtect failed for guest continuation stub at 0x{(nint)ptr:X16}"; + reason = "failed to seal guest thread stub execute-read"; + return GuestNativeCallExitReason.Exception; + } + FlushInstructionCache(GetCurrentProcess(), ptr, stubSize); + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot)) + { + reason = "failed to bind host-RSP storage for guest thread stub"; return GuestNativeCallExitReason.Exception; } - _hostMemory.FlushInstructionCache((ulong)ptr, stubSize); ActiveGuestThreadYieldRequested = false; ActiveGuestThreadYieldReason = null; - try { - var nativeReturn = RunGuestEntryStub(ptr, hostRspSlot); + var nativeReturn = CallNativeEntry(ptr); if (ActiveGuestThreadYieldRequested) { reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; @@ -4115,7 +5019,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } finally { - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); RestoreActiveExecutionThread( previousActiveBackend, previousActiveContext, @@ -4124,7 +5028,176 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I previousForcedExit, previousYieldRequested, previousYieldReason); - _hostMemory.Free((ulong)ptr); + NativeMemory.Free(hostRspStorage); + VirtualFree(ptr, 0u, 32768u); + } + } + + private unsafe GuestNativeCallExitReason ExecuteGuestContinuationEntry( + CpuContext context, + ulong entryPoint, + ulong returnSlotAddress, + string name, + out string? reason) + { + reason = null; + if (context[CpuRegister.Rsp] == 0) + { + reason = "guest thread stack pointer is zero"; + return GuestNativeCallExitReason.Exception; + } + const uint stubSize = 512u; + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); + if (ptr == null) + { + reason = "failed to allocate executable memory for guest thread stub"; + return GuestNativeCallExitReason.Exception; + } + void* hostRspStorage = NativeMemory.Alloc((nuint)sizeof(ulong)); + if (hostRspStorage == null) + { + VirtualFree(ptr, 0u, 32768u); + reason = "failed to allocate writable host-RSP storage for guest continuation stub"; + return GuestNativeCallExitReason.Exception; + } + var previousActiveBackend = _activeExecutionBackend; + var previousActiveContext = _activeCpuContext; + var previousSentinel = _activeEntryReturnSentinelRip; + var previousReturnSlotAddress = _activeGuestReturnSlotAddress; + var previousForcedExit = _activeForcedGuestExit; + var previousYieldRequested = _activeGuestThreadYieldRequested; + var previousYieldReason = _activeGuestThreadYieldReason; + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); + try + { + _activeExecutionBackend = this; + _activeCpuContext = context; + _activeEntryReturnSentinelRip = 0; + _activeGuestReturnSlotAddress = returnSlotAddress; + _activeForcedGuestExit = false; + _activeGuestThreadYieldRequested = false; + _activeGuestThreadYieldReason = null; + BindTlsBase(context); + byte* ptr2 = (byte*)ptr; + ulong hostRspSlot = (ulong)hostRspStorage; + int offset = 0; + + void Emit(byte value) => ptr2[offset++] = value; + void EmitU64(ulong value) + { + *(ulong*)(ptr2 + offset) = value; + offset += sizeof(ulong); + } + void EmitMovR64Imm(byte rex, byte opcode, ulong value) + { + Emit(rex); + Emit(opcode); + EmitU64(value); + } + + Emit(0x53); // push rbx + Emit(0x55); // push rbp + Emit(0x57); // push rdi + Emit(0x56); // push rsi + Emit(0x41); Emit(0x54); // push r12 + Emit(0x41); Emit(0x55); // push r13 + Emit(0x41); Emit(0x56); // push r14 + Emit(0x41); Emit(0x57); // push r15 + EmitHostNonvolatileXmmSave(ptr2, ref offset); + // Restore the fiber's floating-point control environment before + // abandoning the host stack. This path is used when a blocked guest + // continuation migrates to another managed worker. + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // sub rsp,8 + Emit(0xC7); Emit(0x04); Emit(0x24); // mov dword [rsp],imm32 + *(uint*)(ptr2 + offset) = context.Mxcsr; offset += sizeof(uint); + Emit(0x0F); Emit(0xAE); Emit(0x14); Emit(0x24); // ldmxcsr [rsp] + Emit(0x66); Emit(0xC7); Emit(0x04); Emit(0x24); // mov word [rsp],imm16 + *(ushort*)(ptr2 + offset) = context.FpuControlWord; offset += sizeof(ushort); + Emit(0xD9); Emit(0x2C); Emit(0x24); // fldcw [rsp] + Emit(0x48); Emit(0x83); Emit(0xC4); Emit(0x08); // add rsp,8 + EmitMovR64Imm(0x49, 0xBA, hostRspSlot); // mov r10, hostRspSlot + Emit(0x49); Emit(0x89); Emit(0x22); // mov [r10], rsp + EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rsp]); // mov rax, guest rsp + Emit(0x48); Emit(0x89); Emit(0xC4); // mov rsp, rax + Emit(0x48); Emit(0x83); Emit(0xEC); Emit(0x08); // reserve transfer slot + EmitMovR64Imm(0x48, 0xB8, entryPoint); // mov rax, entryPoint + Emit(0x48); Emit(0x89); Emit(0x04); Emit(0x24); // mov [rsp],rax + EmitMovR64Imm(0x48, 0xBB, context[CpuRegister.Rbx]); // mov rbx, imm64 + EmitMovR64Imm(0x48, 0xBD, context[CpuRegister.Rbp]); // mov rbp, imm64 + EmitMovR64Imm(0x48, 0xBF, context[CpuRegister.Rdi]); // mov rdi, imm64 + EmitMovR64Imm(0x48, 0xBE, context[CpuRegister.Rsi]); // mov rsi, imm64 + EmitMovR64Imm(0x48, 0xBA, context[CpuRegister.Rdx]); // mov rdx, imm64 + EmitMovR64Imm(0x48, 0xB9, context[CpuRegister.Rcx]); // mov rcx, imm64 + EmitMovR64Imm(0x49, 0xB8, context[CpuRegister.R8]); // mov r8, imm64 + EmitMovR64Imm(0x49, 0xB9, context[CpuRegister.R9]); // mov r9, imm64 + EmitMovR64Imm(0x49, 0xBA, context[CpuRegister.R10]); // mov r10, imm64 + EmitMovR64Imm(0x49, 0xBC, context[CpuRegister.R12]); // mov r12, imm64 + EmitMovR64Imm(0x49, 0xBD, context[CpuRegister.R13]); // mov r13, imm64 + EmitMovR64Imm(0x49, 0xBE, context[CpuRegister.R14]); // mov r14, imm64 + EmitMovR64Imm(0x49, 0xBF, context[CpuRegister.R15]); // mov r15, imm64 + EmitMovR64Imm(0x49, 0xBB, context[CpuRegister.R11]); // mov r11, imm64 + EmitMovR64Imm(0x48, 0xB8, context[CpuRegister.Rax]); // mov rax, imm64 + Emit(0xC3); // ret through the synthetic transfer slot + ActiveEntryReturnSentinelRip = (ulong)_guestReturnStub; + if (returnSlotAddress == 0 || !context.TryWriteUInt64(returnSlotAddress, (ulong)_guestReturnStub)) + { + reason = $"failed to patch guest continuation return slot at 0x{returnSlotAddress:X16}"; + return GuestNativeCallExitReason.Exception; + } + uint oldProtect = default(uint); + if (!VirtualProtect(ptr, stubSize, 32u, &oldProtect)) + { + reason = "failed to seal guest continuation stub execute-read"; + return GuestNativeCallExitReason.Exception; + } + FlushInstructionCache(GetCurrentProcess(), ptr, stubSize); + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)hostRspSlot)) + { + reason = "failed to bind host-RSP storage for guest continuation stub"; + return GuestNativeCallExitReason.Exception; + } + ActiveGuestThreadYieldRequested = false; + ActiveGuestThreadYieldReason = null; + try + { + var nativeReturn = CallNativeEntry(ptr); + if (ActiveGuestThreadYieldRequested) + { + reason = ActiveGuestThreadYieldReason ?? "guest thread blocked"; + return GuestNativeCallExitReason.Blocked; + } + if (ActiveForcedGuestExit) + { + reason = LastError ?? "guest thread forced exit"; + return GuestNativeCallExitReason.ForcedExit; + } + reason = $"returned 0x{nativeReturn:X8}"; + return GuestNativeCallExitReason.Returned; + } + catch (AccessViolationException ex) + { + reason = "access violation: " + ex.Message; + return GuestNativeCallExitReason.Exception; + } + catch (Exception ex) + { + reason = ex.GetType().Name + ": " + ex.Message; + return GuestNativeCallExitReason.Exception; + } + } + finally + { + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + RestoreActiveExecutionThread( + previousActiveBackend, + previousActiveContext, + previousSentinel, + previousReturnSlotAddress, + previousForcedExit, + previousYieldRequested, + previousYieldReason); + NativeMemory.Free(hostRspStorage); + VirtualFree(ptr, 0u, 32768u); } } @@ -4208,13 +5281,21 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } Console.Error.WriteLine($"[LOADER][INFO] StackTop: 0x{num:X16}"); const uint stubSize = 512u; - void* ptr = (void*)_hostMemory.Allocate(0, stubSize, HostPageProtection.ReadWriteExecute); + void* ptr = VirtualAlloc(null, stubSize, 12288u, 4u); if (ptr == null) { LastError = "Failed to allocate executable memory for stub"; result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; return false; } + void* hostRspStorage = NativeMemory.Alloc((nuint)sizeof(ulong)); + if (hostRspStorage == null) + { + VirtualFree(ptr, 0u, 32768u); + LastError = "Failed to allocate writable host-RSP storage for stub"; + result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + return false; + } var previousActiveBackend = _activeExecutionBackend; var previousActiveContext = _activeCpuContext; var previousSentinel = _activeEntryReturnSentinelRip; @@ -4222,7 +5303,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I var previousForcedExit = _activeForcedGuestExit; var previousYieldRequested = _activeGuestThreadYieldRequested; var previousYieldReason = _activeGuestThreadYieldReason; - nint previousHostRspSlotValue = _hostThreading.GetTlsValue(_hostRspSlotTlsIndex); + nint previousHostRspSlotValue = TlsGetValue(_hostRspSlotTlsIndex); try { _activeExecutionBackend = this; @@ -4234,7 +5315,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _activeGuestThreadYieldReason = null; BindTlsBase(context); byte* ptr2 = (byte*)ptr; - ulong num2 = (ulong)ptr + stubSize - 16uL; + ulong num2 = (ulong)hostRspStorage; int num3 = 0; ptr2[num3++] = 83; ptr2[num3++] = 85; @@ -4341,18 +5422,23 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return false; } uint num5 = default(uint); - if (!_hostMemory.Protect((ulong)ptr, stubSize, HostPageProtection.ReadWriteExecute, out num5)) + if (!VirtualProtect(ptr, stubSize, 32u, &num5)) { - LastError = $"VirtualProtect failed for guest entry stub at 0x{(nint)ptr:X16}"; + LastError = "Failed to seal native entry stub execute-read"; result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; return false; } - _hostMemory.FlushInstructionCache((ulong)ptr, stubSize); + FlushInstructionCache(GetCurrentProcess(), ptr, stubSize); if (_hostRspSlotStorage != 0) { *(ulong*)_hostRspSlotStorage = num2; } - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, (nint)num2); + if (!TlsSetValue(_hostRspSlotTlsIndex, (nint)num2)) + { + LastError = "Failed to bind host-RSP storage for native entry stub"; + result = OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + return false; + } if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_SENTINEL_PROBE"), "1", StringComparison.Ordinal)) { Console.Error.WriteLine("[LOADER][INFO] Running unresolved sentinel probe..."); @@ -4361,10 +5447,11 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } Console.Error.WriteLine("[LOADER][INFO] Calling guest entry..."); StartStallWatchdog(); + StartReadyThreadDispatcher(); int num6 = -1; try { - num6 = RunGuestEntryStub(ptr, num2); + num6 = CallNativeEntry(ptr); Console.Error.WriteLine($"[LOADER][INFO] Guest returned: {num6}"); PumpUntilGuestThreadsIdle(context, "entry_return"); } @@ -4388,12 +5475,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I result = OrbisGen2Result.ORBIS_GEN2_ERROR_CPU_TRAP; if (string.IsNullOrEmpty(LastError)) { - LastError = _hostShutdownRequested - ? "Host shutdown requested." - : "Detected repeating import loop and forced guest unwind to host."; + LastError = "Detected repeating import loop and forced guest unwind to host."; } Console.Error.WriteLine("[LOADER][ERROR] " + LastError); - RequestGuestThreadTeardown(3000); return false; } if (num6 == 0) @@ -4408,14 +5492,14 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I LastError = $"Guest entry point returned non-zero: {num6}"; } Console.Error.WriteLine("[LOADER][ERROR] " + LastError); - RequestGuestThreadTeardown(3000); return false; } finally { + StopReadyThreadDispatcher(); StopStallWatchdog(); ActiveEntryReturnSentinelRip = 0uL; - _hostThreading.SetTlsValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); + TlsSetValue(_hostRspSlotTlsIndex, previousHostRspSlotValue); if (_hostRspSlotStorage != 0) { *(long*)_hostRspSlotStorage = 0L; @@ -4428,7 +5512,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I previousForcedExit, previousYieldRequested, previousYieldReason); - _hostMemory.Free((ulong)ptr); + NativeMemory.Free(hostRspStorage); + VirtualFree(ptr, 0u, 32768u); } } @@ -4447,17 +5532,6 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I return 20; } - // SHARPEMU_PERIODIC_SNAPSHOT_SECONDS=N: dump the stall snapshot every N - // seconds regardless of progress, for diagnosing soft stalls where imports - // keep flowing but the game stops advancing. - private static int GetPeriodicSnapshotSeconds() - { - if (int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_PERIODIC_SNAPSHOT_SECONDS"), out var result)) - { - return Math.Max(0, result); - } - return 0; - } private void StartStallWatchdog() { @@ -4488,8 +5562,12 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I dispatcherThread.Start(); long num = (long)((double)stallWatchdogSeconds * Stopwatch.Frequency); - var periodicSnapshotTicks = (long)((double)GetPeriodicSnapshotSeconds() * Stopwatch.Frequency); - var lastPeriodicSnapshot = Stopwatch.GetTimestamp(); + int periodicSnapshotSeconds = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_PERIODIC_SNAPSHOT_SECONDS"), out var pss) + ? Math.Max(0, pss) + : 0; + long periodicSnapshotTicks = (long)((double)periodicSnapshotSeconds * Stopwatch.Frequency); + long lastPeriodicSnapshot = Stopwatch.GetTimestamp(); _stallWatchdogThread = new Thread(new ThreadStart(delegate { while (!_stallWatchdogStop) @@ -4503,53 +5581,9 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I Stopwatch.GetTimestamp() - lastPeriodicSnapshot >= periodicSnapshotTicks) { lastPeriodicSnapshot = Stopwatch.GetTimestamp(); - var gateOwnerSite = _gateOwnerSite; - var gateOwnerTid = Volatile.Read(ref _gateOwnerManagedThreadId); - var gateHeldMs = gateOwnerSite is null - ? 0.0 - : Stopwatch.GetElapsedTime(Volatile.Read(ref _gateAcquireTimestamp)).TotalMilliseconds; - var snapshotText = new System.Text.StringBuilder(); - snapshotText.AppendLine( - $"[LOADER][DIAG] Periodic snapshot: gate_owner={gateOwnerSite ?? "none"} " + - $"gate_tid={gateOwnerTid} gate_held_ms={gateHeldMs:0}"); - // Never touch the gate here: the periodic snapshot must keep - // reporting even (especially) when the gate is wedged. - // Dump guest threads without the lock; tolerate torn reads. - try - { - foreach (var thread in _guestThreads.Values) - { - snapshotText.AppendLine( - $"[LOADER][DIAG] gateless guest-thread: handle=0x{thread.ThreadHandle:X16} name='{thread.Name}' " + - $"state={thread.State} imports={Interlocked.Read(ref thread.ImportCount)} " + - $"nid={Volatile.Read(ref thread.LastImportNid) ?? "none"} ret=0x{Volatile.Read(ref thread.LastReturnRip):X16} " + - $"block={thread.BlockReason ?? "none"} wake={thread.BlockWakeKey ?? "none"}"); - } - } - catch (Exception snapshotError) - { - snapshotText.AppendLine($"[LOADER][DIAG] gateless snapshot failed: {snapshotError.Message}"); - } - - // Console can be wedged by whatever is being diagnosed, so - // write to a side file when one is configured and fall back - // to stderr otherwise. - var snapshotPath = Environment.GetEnvironmentVariable("SHARPEMU_PERIODIC_SNAPSHOT_FILE"); - if (!string.IsNullOrWhiteSpace(snapshotPath)) - { - try - { - System.IO.File.AppendAllText(snapshotPath, snapshotText.ToString()); - } - catch - { - } - } - else - { - Console.Error.Write(snapshotText.ToString()); - Console.Error.Flush(); - } + Console.Error.WriteLine("[LOADER][ERROR] --- periodic snapshot ---"); + LogStallWatchdogSnapshot(); + Console.Error.Flush(); } long num2 = Stopwatch.GetTimestamp() - Volatile.Read(ref _lastProgressTimestamp); if (num2 < num) @@ -4558,12 +5592,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } if (HasReadyGuestThread()) { - if (_cpuContext is { } watchdogContext) - { - Pump(watchdogContext, "watchdog"); - } Console.Error.WriteLine( - $"[LOADER][WARN] No import progress for {stallWatchdogSeconds}s, but a guest thread is ready; continuing."); + $"[LOADER][WARN] No import progress for {stallWatchdogSeconds}s, but a guest thread is ready; dispatcher will resume it."); LogStallWatchdogSnapshot(); Console.Error.Flush(); MarkExecutionProgress(); @@ -4666,6 +5696,178 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _stallWatchdogThread = null; } + // A guest thread only gets dispatched to a native thread when some running + // guest thread calls Pump (which happens inside blocking HLE primitives: + // waits, usleep, pthread_create, entry_return). That leaves a starvation + // hole: a guest thread that spins on a non-blocking HLE call (e.g. + // sceAudioOutOutput) never pumps, so any thread that was made Ready — for + // example a job worker woken by sceKernelSetEventFlag — sits in the ready + // queue forever. Import progress keeps advancing (the spin), so the stall + // watchdog never fires either, and the whole game deadlocks with 0 draws. + // + // This background dispatcher closes the hole: it drains the ready queue on + // a short interval regardless of whether any guest thread pumps. It is + // deliberately self-contained (it does not touch Pump or the pump-depth + // guard) so it cannot alter the existing cooperative dispatch path. + private void StartReadyThreadDispatcher() + { + if (_readyDispatchThread != null) + { + return; + } + var logSnapshots = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_THREAD_SNAPSHOTS"), + "1", + StringComparison.Ordinal); + var nextSnapshotTimestamp = Stopwatch.GetTimestamp() + Stopwatch.Frequency; + _readyDispatchStop = false; + _readyDispatchThread = new Thread(new ThreadStart(delegate + { + while (!_readyDispatchStop) + { + Thread.Sleep(1); + if (_readyDispatchStop) + { + break; + } + // The count is a fast diagnostic hint, while the queue/state pair under + // _guestThreadGate is authoritative. Always attempt a locked drain so a + // stale hint cannot strand a runnable continuation. + DispatchReadyGuestThreads(); + if (logSnapshots && Stopwatch.GetTimestamp() >= nextSnapshotTimestamp) + { + lock (_guestThreadGate) + { + foreach (var thread in _guestThreads.Values) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_thread.snapshot " + + $"handle=0x{thread.ThreadHandle:X16} name='{thread.Name}' " + + $"state={thread.State} executor={thread.ExecutorActive} " + + $"imports={Interlocked.Read(ref thread.ImportCount)} " + + $"nid={Volatile.Read(ref thread.LastImportNid) ?? "none"} " + + $"ret=0x{Volatile.Read(ref thread.LastReturnRip):X16} " + + $"block={thread.BlockReason ?? "none"} " + + $"wake={thread.BlockWakeKey ?? "none"} " + + $"host_managed={thread.HostThread?.ManagedThreadId ?? 0} " + + $"host_tid={Volatile.Read(ref thread.HostThreadId)}"); + } + } + nextSnapshotTimestamp = Stopwatch.GetTimestamp() + Stopwatch.Frequency; + } + } + })) + { + IsBackground = true, + Name = "SharpEmu-ReadyDispatch", + }; + _readyDispatchThread.Start(); + } + + private void StopReadyThreadDispatcher() + { + _readyDispatchStop = true; + Thread? readyDispatchThread = _readyDispatchThread; + if (readyDispatchThread == null) + { + return; + } + if (!ReferenceEquals(Thread.CurrentThread, readyDispatchThread)) + { + try + { + readyDispatchThread.Join(300); + } + catch + { + } + } + _readyDispatchThread = null; + } + + // Dequeue every currently-ready guest thread and start a native thread for + // each, mirroring Pump's dispatch step. Dequeue and the Ready->Running + // transition happen under _guestThreadGate, so this races safely with a + // concurrent Pump: each ready thread is claimed once (the State check skips + // any that another dispatcher already took). + private void DispatchReadyGuestThreads() + { + while (true) + { + GuestThreadState? thread = null; + lock (_guestThreadGate) + { + _ = TryClaimReadyGuestThreadLocked(out thread); + } + + if (thread == null) + { + return; + } + + ScheduleGuestThreadExecution(thread, "ready-dispatch"); + } + } + + private void ScheduleGuestThreadExecution(GuestThreadState thread, string reason) + { + GuestExecutionRunner runner; + lock (_guestThreadGate) + { + runner = thread.ExecutionRunner ??= new GuestExecutionRunner( + thread.ThreadHandle, + thread.Name, + MapGuestThreadPriority(thread.Priority)); + } + runner.Schedule(() => RunGuestThread(thread, reason)); + } + + // Caller must hold _guestThreadGate. A guest wait can be satisfied before + // RunGuestThread has finished restoring its host/TLS state, so Ready alone + // is not sufficient to authorize another executor. ExecutorActive is the + // scheduler's authoritative single-owner token and covers both asynchronous + // host threads and synchronous Pump("entry_return") execution. + private bool TryClaimReadyGuestThreadLocked(out GuestThreadState? thread) + { + thread = null; + var candidatesToInspect = _readyGuestThreads.Count; + for (var index = 0; index < candidatesToInspect; index++) + { + var candidate = _readyGuestThreads.Dequeue(); + Interlocked.Decrement(ref _readyGuestThreadCount); + if (candidate.State != GuestThreadRunState.Ready) + { + continue; + } + + if (candidate.ExecutorActive) + { + _readyGuestThreads.Enqueue(candidate); + Interlocked.Increment(ref _readyGuestThreadCount); + candidate.ExecutorClaimDeferrals++; + if (_logGuestThreads && + (candidate.ExecutorClaimDeferrals <= 4 || + (candidate.ExecutorClaimDeferrals & (candidate.ExecutorClaimDeferrals - 1)) == 0)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_threads.defer_active_executor " + + $"handle=0x{candidate.ThreadHandle:X16} name='{candidate.Name}' " + + $"host_managed={candidate.HostThread?.ManagedThreadId ?? 0} " + + $"host_tid={Volatile.Read(ref candidate.HostThreadId)} " + + $"deferrals={candidate.ExecutorClaimDeferrals}"); + } + continue; + } + + candidate.ExecutorActive = true; + candidate.State = GuestThreadRunState.Running; + thread = candidate; + return true; + } + + return false; + } + private void LogStallWatchdogSnapshot() { try @@ -4678,14 +5880,13 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ulong rsp = cpuContext[CpuRegister.Rsp]; Console.Error.WriteLine($"[LOADER][ERROR] Stall snapshot: rip=0x{cpuContext.Rip:X16} rsp=0x{rsp:X16} rbp=0x{cpuContext[CpuRegister.Rbp]:X16} rax=0x{cpuContext[CpuRegister.Rax]:X16} rbx=0x{cpuContext[CpuRegister.Rbx]:X16} rcx=0x{cpuContext[CpuRegister.Rcx]:X16} rdx=0x{cpuContext[CpuRegister.Rdx]:X16} rsi=0x{cpuContext[CpuRegister.Rsi]:X16} rdi=0x{cpuContext[CpuRegister.Rdi]:X16}"); ulong num = cpuContext.Rip & 0xFFFFFFFFFFFFFFF0uL; - var importEntries = _importEntries; - for (int i = 0; i < importEntries.Length; i++) + for (int i = 0; i < _importEntries.Length; i++) { - if (importEntries[i].Address != num) + if (_importEntries[i].Address != num) { continue; } - string text = importEntries[i].Nid; + string text = _importEntries[i].Nid; if (_moduleManager.TryGetExport(text, out ExportedFunction export)) { Console.Error.WriteLine($"[LOADER][ERROR] Stall import-stub: rip=0x{num:X16} nid={text} -> {export.LibraryName}:{export.Name}"); @@ -4734,7 +5935,8 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I $"[LOADER][ERROR] Stall guest-thread: handle=0x{thread.ThreadHandle:X16} name='{thread.Name}' " + $"state={thread.State} imports={Interlocked.Read(ref thread.ImportCount)} " + $"nid={Volatile.Read(ref thread.LastImportNid) ?? "none"} ret=0x{Volatile.Read(ref thread.LastReturnRip):X16} " + - $"block={thread.BlockReason ?? "none"}{hostContextText}"); + $"rdi=0x{Volatile.Read(ref thread.LastImportRdi):X16} rsi=0x{Volatile.Read(ref thread.LastImportRsi):X16} " + + $"rdx=0x{Volatile.Read(ref thread.LastImportRdx):X16} block={thread.BlockReason ?? "none"}{hostContextText}"); logged++; if (logged >= 48 && threads.Length > logged) { @@ -4749,46 +5951,163 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } } - private bool TryCaptureHostThreadContext(int hostThreadId, out HostThreadContextSnapshot snapshot) + private unsafe static bool TryCaptureHostThreadContext(int hostThreadId, out HostThreadContextSnapshot snapshot) { snapshot = default; - if (hostThreadId == 0 || unchecked((uint)hostThreadId) == _hostThreading.CurrentThreadId) + if (hostThreadId == 0 || unchecked((uint)hostThreadId) == GetCurrentThreadId()) { return false; } - if (!_hostThreading.TryCaptureThreadRegisters(unchecked((uint)hostThreadId), out var registers)) + var threadHandle = OpenThread(ThreadGetContext | ThreadSuspendResume, false, unchecked((uint)hostThreadId)); + if (threadHandle == 0) { return false; } - snapshot = new HostThreadContextSnapshot( - true, - registers.Rip, - registers.Rsp, - registers.Rbp, - registers.Rax, - registers.Rbx, - registers.Rcx, - registers.Rdx); - return true; + void* contextRecord = null; + var suspended = false; + try + { + if (SuspendThread(threadHandle) == uint.MaxValue) + { + return false; + } + + suspended = true; + contextRecord = NativeMemory.AllocZeroed((nuint)Win64ContextSize); + WriteCtxU32(contextRecord, Win64ContextFlagsOffset, ContextAmd64ControlInteger); + if (!GetThreadContext(threadHandle, contextRecord)) + { + return false; + } + + snapshot = new HostThreadContextSnapshot( + true, + ReadCtxU64(contextRecord, 248), + ReadCtxU64(contextRecord, 152), + ReadCtxU64(contextRecord, 160), + ReadCtxU64(contextRecord, 120), + ReadCtxU64(contextRecord, 144), + ReadCtxU64(contextRecord, 128), + ReadCtxU64(contextRecord, 136)); + return true; + } + finally + { + if (contextRecord != null) + { + NativeMemory.Free(contextRecord); + } + if (suspended) + { + _ = ResumeThread(threadHandle); + } + _ = CloseHandle(threadHandle); + } } + + private static uint TlsAlloc() => + OperatingSystem.IsWindows() ? Win32TlsAlloc() : PosixHostStubs.TlsAlloc(); + + private static bool TlsFree(uint dwTlsIndex) => + OperatingSystem.IsWindows() ? Win32TlsFree(dwTlsIndex) : PosixHostStubs.TlsFree(dwTlsIndex); + + private static bool TlsSetValue(uint dwTlsIndex, nint lpTlsValue) => + OperatingSystem.IsWindows() ? Win32TlsSetValue(dwTlsIndex, lpTlsValue) : PosixHostStubs.TlsSetValue(dwTlsIndex, lpTlsValue); + + private static nint TlsGetValue(uint dwTlsIndex) => + OperatingSystem.IsWindows() ? Win32TlsGetValue(dwTlsIndex) : PosixHostStubs.TlsGetValue(dwTlsIndex); + + private unsafe static void* AddVectoredExceptionHandler(uint first, IntPtr handler) => + OperatingSystem.IsWindows() ? Win32AddVectoredExceptionHandler(first, handler) : null; + + private unsafe static uint RemoveVectoredExceptionHandler(void* handle) => + OperatingSystem.IsWindows() ? Win32RemoveVectoredExceptionHandler(handle) : 0u; + + private static IntPtr SetUnhandledExceptionFilter(IntPtr lpTopLevelExceptionFilter) => + OperatingSystem.IsWindows() ? Win32SetUnhandledExceptionFilter(lpTopLevelExceptionFilter) : IntPtr.Zero; + + private static uint GetCurrentThreadId() => + OperatingSystem.IsWindows() ? Win32GetCurrentThreadId() : PosixHostStubs.GetCurrentThreadId(); + + private static nint GetCurrentThread() => + OperatingSystem.IsWindows() ? Win32GetCurrentThread() : 0; + + private static nuint SetThreadAffinityMask(nint hThread, nuint dwThreadAffinityMask) => + OperatingSystem.IsWindows() ? Win32SetThreadAffinityMask(hThread, dwThreadAffinityMask) : 1; + + private static nint OpenThread(uint dwDesiredAccess, bool bInheritHandle, uint dwThreadId) => + OperatingSystem.IsWindows() ? Win32OpenThread(dwDesiredAccess, bInheritHandle, dwThreadId) : 0; + + private static uint SuspendThread(nint hThread) => + OperatingSystem.IsWindows() ? Win32SuspendThread(hThread) : uint.MaxValue; + + private static uint ResumeThread(nint hThread) => + OperatingSystem.IsWindows() ? Win32ResumeThread(hThread) : uint.MaxValue; + + private unsafe static bool GetThreadContext(nint hThread, void* lpContext) => + OperatingSystem.IsWindows() && Win32GetThreadContext(hThread, lpContext); + + private static bool CloseHandle(nint hObject) => + OperatingSystem.IsWindows() && Win32CloseHandle(hObject); + + [DllImport("kernel32.dll", EntryPoint = "TlsAlloc")] + private static extern uint Win32TlsAlloc(); + + [DllImport("kernel32.dll", EntryPoint = "TlsFree")] + private static extern bool Win32TlsFree(uint dwTlsIndex); + + [DllImport("kernel32.dll", EntryPoint = "TlsSetValue")] + private static extern bool Win32TlsSetValue(uint dwTlsIndex, nint lpTlsValue); + + [DllImport("kernel32.dll", EntryPoint = "TlsGetValue")] + private static extern nint Win32TlsGetValue(uint dwTlsIndex); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + private static extern nint GetModuleHandle(string lpModuleName); + + [DllImport("kernel32.dll", CharSet = CharSet.Ansi)] + private static extern nint GetProcAddress(nint hModule, string procName); + + [DllImport("kernel32.dll", EntryPoint = "AddVectoredExceptionHandler")] + private unsafe static extern void* Win32AddVectoredExceptionHandler(uint first, IntPtr handler); + + [DllImport("kernel32.dll", EntryPoint = "RemoveVectoredExceptionHandler")] + private unsafe static extern uint Win32RemoveVectoredExceptionHandler(void* handle); + + [DllImport("kernel32.dll", EntryPoint = "SetUnhandledExceptionFilter")] + private static extern IntPtr Win32SetUnhandledExceptionFilter(IntPtr lpTopLevelExceptionFilter); + + [DllImport("kernel32.dll", EntryPoint = "GetCurrentThreadId")] + private static extern uint Win32GetCurrentThreadId(); + + [DllImport("kernel32.dll", EntryPoint = "GetCurrentThread")] + private static extern nint Win32GetCurrentThread(); + + [DllImport("kernel32.dll", EntryPoint = "SetThreadAffinityMask", SetLastError = true)] + private static extern nuint Win32SetThreadAffinityMask(nint hThread, nuint dwThreadAffinityMask); + + [DllImport("kernel32.dll", EntryPoint = "OpenThread", SetLastError = true)] + private static extern nint Win32OpenThread(uint dwDesiredAccess, [MarshalAs(UnmanagedType.Bool)] bool bInheritHandle, uint dwThreadId); + + [DllImport("kernel32.dll", EntryPoint = "SuspendThread", SetLastError = true)] + private static extern uint Win32SuspendThread(nint hThread); + + [DllImport("kernel32.dll", EntryPoint = "ResumeThread", SetLastError = true)] + private static extern uint Win32ResumeThread(nint hThread); + + [DllImport("kernel32.dll", EntryPoint = "GetThreadContext", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private unsafe static extern bool Win32GetThreadContext(nint hThread, void* lpContext); + + [DllImport("kernel32.dll", EntryPoint = "CloseHandle", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32CloseHandle(nint hObject); + public unsafe void Dispose() { - if (!RequestGuestThreadTeardown(2000)) - { - // A guest worker is still executing native code; freeing the trampolines, - // exception-handler stubs, or GC handles under it turns process exit into - // an execute-AV / CLR fatal. Leak them — the process is going away anyway. - Console.Error.WriteLine( - "[LOADER][WARN] Skipping executable stub teardown: guest worker threads are still running."); - return; - } - // Native guest workers park idle once every guest thread has unwound; stop - // them before any executable stub or TLS index they reference is freed. - DisposeNativeGuestExecutors(); - if (ReferenceEquals(_posixSignalBackend, this)) { // The signal handlers stay installed (they chain to the previous @@ -4799,33 +6118,32 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I ClearImportHandlerTrampolines(); _importEntries = Array.Empty(); _runtimeSymbolsByName.Clear(); - ResetLazyDlsymStubState(); - _importNidHashCache.Clear(); + StopReadyThreadDispatcher(); StopStallWatchdog(); if (_exceptionHandler != 0) { - _faultHandling.RemoveHandler(_exceptionHandler); + RemoveVectoredExceptionHandler((void*)_exceptionHandler); _exceptionHandler = 0; } if (_rawExceptionHandler != 0) { - _faultHandling.RemoveHandler(_rawExceptionHandler); + RemoveVectoredExceptionHandler((void*)_rawExceptionHandler); _rawExceptionHandler = 0; } if (_rawExceptionHandlerStub != 0) { - _faultHandling.FreeThunk(_rawExceptionHandlerStub); + VirtualFree((void*)_rawExceptionHandlerStub, 0u, 32768u); _rawExceptionHandlerStub = 0; } if (_exceptionHandlerStub != 0) { - _faultHandling.FreeThunk(_exceptionHandlerStub); + VirtualFree((void*)_exceptionHandlerStub, 0u, 32768u); _exceptionHandlerStub = 0; } if (_unhandledFilterStub != 0) { - _faultHandling.SetUnhandledFilter(0); - _faultHandling.FreeThunk(_unhandledFilterStub); + SetUnhandledExceptionFilter(0); + VirtualFree((void*)_unhandledFilterStub, 0u, 32768u); _unhandledFilterStub = 0; } if (_handlerHandle.IsAllocated) @@ -4843,7 +6161,7 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I } if (_ownedTlsBaseAddress != 0) { - _hostMemory.Free((ulong)_ownedTlsBaseAddress); + VirtualFree((void*)_ownedTlsBaseAddress, 0u, 32768u); _ownedTlsBaseAddress = 0; } _tlsBaseAddress = 0; @@ -4854,44 +6172,44 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I { if (num3 != 0) { - _hostMemory.Free((ulong)num3); + VirtualFree((void*)num3, 0u, 32768u); } } _tlsModuleBases.Clear(); } if (_tlsHandlerAddress != 0) { - _hostMemory.Free((ulong)_tlsHandlerAddress); + VirtualFree((void*)_tlsHandlerAddress, 0u, 32768u); _tlsHandlerAddress = 0; } if (_hostRspSlotStorage != 0) { - _hostMemory.Free((ulong)_hostRspSlotStorage); + VirtualFree((void*)_hostRspSlotStorage, 0u, 32768u); _hostRspSlotStorage = 0; } if (_guestTlsBaseTlsIndex != uint.MaxValue) { - _hostThreading.FreeTlsSlot(_guestTlsBaseTlsIndex); + TlsFree(_guestTlsBaseTlsIndex); _guestTlsBaseTlsIndex = uint.MaxValue; } if (_hostRspSlotTlsIndex != uint.MaxValue) { - _hostThreading.FreeTlsSlot(_hostRspSlotTlsIndex); + TlsFree(_hostRspSlotTlsIndex); _hostRspSlotTlsIndex = uint.MaxValue; } if (_unresolvedReturnStub != 0) { - _hostMemory.Free((ulong)_unresolvedReturnStub); + VirtualFree((void*)_unresolvedReturnStub, 0u, 32768u); _unresolvedReturnStub = 0; } if (_guestReturnStub != 0) { - _hostMemory.Free((ulong)_guestReturnStub); + VirtualFree((void*)_guestReturnStub, 0u, 32768u); _guestReturnStub = 0; } if (_guestContextTransferStub != 0) { - _hostMemory.Free((ulong)_guestContextTransferStub); + VirtualFree((void*)_guestContextTransferStub, 0u, 32768u); _guestContextTransferStub = 0; } foreach (var frame in _guestContextTransferFrames.Values) @@ -4904,19 +6222,59 @@ public sealed unsafe partial class DirectExecutionBackend : INativeCpuBackend, I _guestContextTransferFrames.Dispose(); if (_lowIndexedTableScratch != 0) { - _hostMemory.Free((ulong)_lowIndexedTableScratch); + VirtualFree((void*)_lowIndexedTableScratch, 0u, 32768u); _lowIndexedTableScratch = 0; } if (_stackGuardCompareScratch != 0) { - _hostMemory.Free((ulong)_stackGuardCompareScratch); + VirtualFree((void*)_stackGuardCompareScratch, 0u, 32768u); _stackGuardCompareScratch = 0; } if (_nullObjectStoreScratch != 0) { - _hostMemory.Free((ulong)_nullObjectStoreScratch); + VirtualFree((void*)_nullObjectStoreScratch, 0u, 32768u); _nullObjectStoreScratch = 0; } Volatile.Write(ref _globalUnresolvedReturnStub, 0uL); } + + private unsafe static void* VirtualAlloc(void* lpAddress, nuint dwSize, uint flAllocationType, uint flProtect) => + HostMemory.Alloc(lpAddress, dwSize, flAllocationType, flProtect); + + private unsafe static bool VirtualFree(void* lpAddress, nuint dwSize, uint dwFreeType) => + HostMemory.Free(lpAddress, dwSize, dwFreeType); + + private unsafe static bool VirtualProtect(void* lpAddress, nuint dwSize, uint flNewProtect, uint* lpflOldProtect) + { + var success = HostMemory.Protect(lpAddress, dwSize, flNewProtect, out var oldProtect); + if (lpflOldProtect != null) + { + *lpflOldProtect = oldProtect; + } + + return success; + } + + private unsafe static void* GetCurrentProcess() => null; + + private unsafe static bool FlushInstructionCache(void* hProcess, void* lpBaseAddress, nuint dwSize) + { + _ = hProcess; + HostMemory.FlushInstructionCache(lpBaseAddress, dwSize); + return true; + } + + private unsafe static nuint VirtualQuery(void* lpAddress, out MEMORY_BASIC_INFORMATION64 lpBuffer, nuint dwLength) + { + _ = dwLength; + var result = HostMemory.Query(lpAddress, out var info); + lpBuffer = default; + lpBuffer.BaseAddress = info.BaseAddress; + lpBuffer.AllocationBase = info.AllocationBase; + lpBuffer.AllocationProtect = info.AllocationProtect; + lpBuffer.RegionSize = info.RegionSize; + lpBuffer.State = info.State; + lpBuffer.Protect = info.Protect; + return result; + } } diff --git a/src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs b/src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs new file mode 100644 index 0000000..43219b9 --- /dev/null +++ b/src/SharpEmu.Core/Cpu/Native/PosixHostStubs.cs @@ -0,0 +1,425 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using SharpEmu.HLE; + +namespace SharpEmu.Core.Cpu.Native; + +/// +/// POSIX replacements for the kernel32 helpers the native backend embeds in +/// emitted x86-64 code. Every stub exposed here follows the Win64 calling +/// convention the emitted call sites were written for (first argument in +/// ECX, result in RAX, Win64 non-volatile registers preserved), so the +/// emission code stays identical across platforms. +/// +internal static unsafe class PosixHostStubs +{ + private static readonly object Gate = new(); + private static bool _initialized; + private static nint _tlsGetValueStub; + private static nint _queryPerformanceCounterStub; + private static nint _switchToThreadStub; + private static nint _sleepStub; + + public static nint TlsGetValueStubAddress + { + get { EnsureInitialized(); return _tlsGetValueStub; } + } + + public static nint QueryPerformanceCounterStubAddress + { + get { EnsureInitialized(); return _queryPerformanceCounterStub; } + } + + public static nint SwitchToThreadStubAddress + { + get { EnsureInitialized(); return _switchToThreadStub; } + } + + public static nint SleepStubAddress + { + get { EnsureInitialized(); return _sleepStub; } + } + + public static nint CreateWorkerEvent() + { + if (OperatingSystem.IsMacOS()) + { + return dispatch_semaphore_create(0); + } + + var semaphore = Marshal.AllocHGlobal(64); + if (sem_init(semaphore, 0, 0) != 0) + { + Marshal.FreeHGlobal(semaphore); + return 0; + } + + return semaphore; + } + + public static bool SignalWorkerEvent(nint handle) + { + if (OperatingSystem.IsMacOS()) + { + _ = dispatch_semaphore_signal(handle); + return true; + } + + return sem_post(handle) == 0; + } + + public static bool WaitWorkerEvent(nint handle, int timeoutMilliseconds) + { + if (OperatingSystem.IsMacOS()) + { + if (timeoutMilliseconds < 0) + { + return dispatch_semaphore_wait(handle, ulong.MaxValue) == 0; + } + + var deadline = dispatch_time(0, timeoutMilliseconds * 1_000_000L); + return dispatch_semaphore_wait(handle, deadline) == 0; + } + + if (timeoutMilliseconds < 0) + { + while (sem_wait(handle) != 0) + { + } + + return true; + } + + var deadlineTicks = Environment.TickCount64 + timeoutMilliseconds; + while (sem_trywait(handle) != 0) + { + if (Environment.TickCount64 >= deadlineTicks) + { + return false; + } + + Thread.Sleep(1); + } + + return true; + } + + public static void DestroyWorkerEvent(nint handle) + { + if (handle == 0) + { + return; + } + + if (OperatingSystem.IsMacOS()) + { + dispatch_release(handle); + return; + } + + _ = sem_destroy(handle); + Marshal.FreeHGlobal(handle); + } + + /// Allocates a pthread TLS key, mirroring kernel32!TlsAlloc. + public static uint TlsAlloc() + { + if (OperatingSystem.IsMacOS()) + { + nuint key; + return pthread_key_create_mac(&key, 0) == 0 ? (uint)key : uint.MaxValue; + } + + uint key32; + return pthread_key_create_linux(&key32, 0) == 0 ? key32 : uint.MaxValue; + } + + public static bool TlsFree(uint key) + { + return OperatingSystem.IsMacOS() + ? pthread_key_delete_mac((nuint)key) == 0 + : pthread_key_delete_linux(key) == 0; + } + + public static bool TlsSetValue(uint key, nint value) + { + return OperatingSystem.IsMacOS() + ? pthread_setspecific_mac((nuint)key, value) == 0 + : pthread_setspecific_linux(key, value) == 0; + } + + public static nint TlsGetValue(uint key) + { + return OperatingSystem.IsMacOS() + ? pthread_getspecific_mac((nuint)key) + : pthread_getspecific_linux(key); + } + + /// Stable numeric id of the calling thread (kernel32!GetCurrentThreadId). + public static uint GetCurrentThreadId() + { + if (OperatingSystem.IsMacOS()) + { + ulong tid; + return pthread_threadid_np(0, &tid) == 0 ? unchecked((uint)tid) : 0u; + } + + return unchecked((uint)gettid()); + } + + /// + /// Wraps a managed callback (compiled for the SysV ABI on POSIX .NET) in a + /// thunk that accepts up to four integer arguments in the Win64 ABI the + /// emitted x86-64 call sites use. Win64 passes args in rcx/rdx/r8/r9 and + /// treats rdi/rsi as non-volatile; SysV expects rdi/rsi/rdx/rcx and + /// clobbers them, so the thunk saves rdi/rsi, shuffles the registers, keeps + /// the stack 16-byte aligned for the call, and forwards the rax result. + /// + public static nint CreateWin64ToSysVThunk(nint sysvTarget) + { + var page = (byte*)HostMemory.Alloc( + null, + 4096, + HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, + HostMemory.PAGE_EXECUTE_READWRITE); + if (page == null) + { + throw new OutOfMemoryException("Failed to allocate Win64->SysV thunk page"); + } + + var offset = 0; + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x48, 0x89, 0xCF); // mov rdi, rcx + Emit(page, ref offset, 0x48, 0x89, 0xD6); // mov rsi, rdx + Emit(page, ref offset, 0x4C, 0x89, 0xC2); // mov rdx, r8 + Emit(page, ref offset, 0x4C, 0x89, 0xC9); // mov rcx, r9 + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 (realign to 16) + EmitMovRaxImm64(page, ref offset, sysvTarget); // mov rax, target + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0xC3); // ret + + if (!HostMemory.Protect(page, 4096, HostMemory.PAGE_EXECUTE_READ, out _)) + { + throw new InvalidOperationException("Failed to protect Win64->SysV thunk page"); + } + + HostMemory.FlushInstructionCache(page, (nuint)offset); + return (nint)page; + } + + private static void EnsureInitialized() + { + if (_initialized) + { + return; + } + + lock (Gate) + { + if (_initialized) + { + return; + } + + BuildStubs(); + _initialized = true; + } + } + + private static void BuildStubs() + { + var page = (byte*)HostMemory.Alloc( + null, + 4096, + HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE, + HostMemory.PAGE_EXECUTE_READWRITE); + if (page == null) + { + throw new OutOfMemoryException("Failed to allocate POSIX host helper stub page"); + } + + var offset = 0; + _tlsGetValueStub = EmitTlsGetValue(page, ref offset); + _queryPerformanceCounterStub = EmitQueryPerformanceCounter(page, ref offset); + _switchToThreadStub = EmitSwitchToThread(page, ref offset); + _sleepStub = EmitSleep(page, ref offset); + + if (!HostMemory.Protect(page, 4096, HostMemory.PAGE_EXECUTE_READ, out _)) + { + throw new InvalidOperationException("Failed to protect POSIX host helper stub page"); + } + + HostMemory.FlushInstructionCache(page, (nuint)offset); + } + + private static nint EmitTlsGetValue(byte* page, ref int offset) + { + var start = (nint)(page + offset); + if (OperatingSystem.IsMacOS()) + { + // On macOS x86-64 pthread keys index the gs-based thread specific + // data array directly, so TlsGetValue(index in ecx) collapses to a + // single load that clobbers nothing but RAX. + Emit(page, ref offset, 0x89, 0xC8); // mov eax, ecx + Emit(page, ref offset, 0x65, 0x48, 0x8B, 0x04, 0xC5, 0, 0, 0, 0); // mov rax, gs:[rax*8] + Emit(page, ref offset, 0xC3); // ret + return start; + } + + // Linux: call pthread_getspecific, preserving the registers that are + // volatile in SysV but non-volatile in Win64 (rsi, rdi). + var pthreadGetSpecific = ResolveLibcExport("pthread_getspecific"); + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 + Emit(page, ref offset, 0x89, 0xCF); // mov edi, ecx + EmitMovRaxImm64(page, ref offset, pthreadGetSpecific); // mov rax, imm64 + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint EmitQueryPerformanceCounter(byte* page, ref int offset) + { + // BOOL QueryPerformanceCounter(LARGE_INTEGER* out in rcx): the emitted + // consumers only need a monotonically increasing counter, which rdtsc + // provides without leaving Win64-safe registers. + var start = (nint)(page + offset); + Emit(page, ref offset, 0x0F, 0x31); // rdtsc + Emit(page, ref offset, 0x48, 0xC1, 0xE2, 0x20); // shl rdx, 32 + Emit(page, ref offset, 0x48, 0x09, 0xD0); // or rax, rdx + Emit(page, ref offset, 0x48, 0x89, 0x01); // mov [rcx], rax + Emit(page, ref offset, 0xB8, 0x01, 0x00, 0x00, 0x00); // mov eax, 1 + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint EmitSwitchToThread(byte* page, ref int offset) + { + var schedYield = ResolveLibcExport("sched_yield"); + var start = (nint)(page + offset); + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 + EmitMovRaxImm64(page, ref offset, schedYield); // mov rax, imm64 + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0xB8, 0x01, 0x00, 0x00, 0x00); // mov eax, 1 + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint EmitSleep(byte* page, ref int offset) + { + // void Sleep(DWORD milliseconds in ecx) -> usleep(microseconds in edi). + var usleep = ResolveLibcExport("usleep"); + var start = (nint)(page + offset); + Emit(page, ref offset, 0x56); // push rsi + Emit(page, ref offset, 0x57); // push rdi + Emit(page, ref offset, 0x48, 0x83, 0xEC, 0x08); // sub rsp, 8 + Emit(page, ref offset, 0x89, 0xCF); // mov edi, ecx + Emit(page, ref offset, 0x81, 0xFF, 0xFF, 0x0F, 0x00, 0x00); // cmp edi, 0xFFF + Emit(page, ref offset, 0x76, 0x05); // jbe +5 + Emit(page, ref offset, 0xBF, 0xFF, 0x0F, 0x00, 0x00); // mov edi, 0xFFF (cap at ~4s) + Emit(page, ref offset, 0x69, 0xFF, 0xE8, 0x03, 0x00, 0x00); // imul edi, edi, 1000 + EmitMovRaxImm64(page, ref offset, usleep); // mov rax, imm64 + Emit(page, ref offset, 0xFF, 0xD0); // call rax + Emit(page, ref offset, 0x48, 0x83, 0xC4, 0x08); // add rsp, 8 + Emit(page, ref offset, 0x5F); // pop rdi + Emit(page, ref offset, 0x5E); // pop rsi + Emit(page, ref offset, 0xC3); // ret + return start; + } + + private static nint ResolveLibcExport(string name) + { + var libc = NativeLibrary.Load(OperatingSystem.IsMacOS() ? "libSystem.dylib" : "libc.so.6"); + return NativeLibrary.GetExport(libc, name); + } + + private static void Emit(byte* page, ref int offset, params byte[] bytes) + { + foreach (var value in bytes) + { + page[offset++] = value; + } + } + + private static void EmitMovRaxImm64(byte* page, ref int offset, nint value) + { + Emit(page, ref offset, 0x48, 0xB8); + *(long*)(page + offset) = value; + offset += sizeof(long); + } + + [DllImport("libc", EntryPoint = "pthread_key_create", SetLastError = true)] + private static extern int pthread_key_create_mac(nuint* key, nint destructor); + + [DllImport("libc", EntryPoint = "pthread_key_create", SetLastError = true)] + private static extern int pthread_key_create_linux(uint* key, nint destructor); + + [DllImport("libc", EntryPoint = "pthread_key_delete")] + private static extern int pthread_key_delete_mac(nuint key); + + [DllImport("libc", EntryPoint = "pthread_key_delete")] + private static extern int pthread_key_delete_linux(uint key); + + [DllImport("libc", EntryPoint = "pthread_setspecific")] + private static extern int pthread_setspecific_mac(nuint key, nint value); + + [DllImport("libc", EntryPoint = "pthread_setspecific")] + private static extern int pthread_setspecific_linux(uint key, nint value); + + [DllImport("libc", EntryPoint = "pthread_getspecific")] + private static extern nint pthread_getspecific_mac(nuint key); + + [DllImport("libc", EntryPoint = "pthread_getspecific")] + private static extern nint pthread_getspecific_linux(uint key); + + [DllImport("libc")] + private static extern int pthread_threadid_np(nint thread, ulong* threadId); + + [DllImport("libc")] + private static extern int gettid(); + + [DllImport("libc")] + private static extern nint dispatch_semaphore_create(long value); + + [DllImport("libc")] + private static extern nint dispatch_semaphore_signal(nint semaphore); + + [DllImport("libc")] + private static extern nint dispatch_semaphore_wait(nint semaphore, ulong timeout); + + [DllImport("libc")] + private static extern ulong dispatch_time(ulong when, long deltaNanoseconds); + + [DllImport("libc")] + private static extern void dispatch_release(nint handle); + + [DllImport("libc")] + private static extern int sem_init(nint semaphore, int shared, uint value); + + [DllImport("libc")] + private static extern int sem_post(nint semaphore); + + [DllImport("libc")] + private static extern int sem_wait(nint semaphore); + + [DllImport("libc")] + private static extern int sem_trywait(nint semaphore); + + [DllImport("libc")] + private static extern int sem_destroy(nint semaphore); +} diff --git a/src/SharpEmu.Core/Cpu/Native/StubManager.cs b/src/SharpEmu.Core/Cpu/Native/StubManager.cs index 37a2453..ffe3b79 100644 --- a/src/SharpEmu.Core/Cpu/Native/StubManager.cs +++ b/src/SharpEmu.Core/Cpu/Native/StubManager.cs @@ -3,7 +3,6 @@ using System.Runtime.InteropServices; using SharpEmu.HLE; -using SharpEmu.HLE.Host; namespace SharpEmu.Core.Cpu.Native; @@ -12,15 +11,17 @@ public sealed unsafe class StubManager : IDisposable private readonly List _allocatedStubs = new(); private readonly Dictionary _importHandlers = new(); private readonly Dictionary _stubAddresses = new(); - private readonly IHostMemory _hostMemory; private byte* _pltMemory; private int _pltOffset; private const int PltMemorySize = 1024 * 1024; // 1MB for stubs - public StubManager(IHostMemory? hostMemory = null) + public StubManager() { - _hostMemory = hostMemory ?? HostPlatform.Current.Memory; - _pltMemory = (byte*)_hostMemory.Allocate(0, PltMemorySize, HostPageProtection.ReadWriteExecute); + _pltMemory = (byte*)VirtualAlloc( + null, + (nuint)PltMemorySize, + AllocationType.Reserve | AllocationType.Commit, + MemoryProtection.ExecuteReadWrite); if (_pltMemory == null) { @@ -184,7 +185,7 @@ public sealed unsafe class StubManager : IDisposable { if (_pltMemory != null) { - _hostMemory.Free((ulong)_pltMemory); + VirtualFree(_pltMemory, 0, FreeType.Release); _pltMemory = null; } @@ -193,5 +194,29 @@ public sealed unsafe class StubManager : IDisposable _stubAddresses.Clear(); } + private static void* VirtualAlloc(void* lpAddress, nuint dwSize, AllocationType flAllocationType, MemoryProtection flProtect) => + HostMemory.Alloc(lpAddress, dwSize, (uint)flAllocationType, (uint)flProtect); + + private static bool VirtualFree(void* lpAddress, nuint dwSize, FreeType dwFreeType) => + HostMemory.Free(lpAddress, dwSize, (uint)dwFreeType); + + [Flags] + private enum AllocationType : uint + { + Commit = 0x1000, + Reserve = 0x2000, + } + + [Flags] + private enum MemoryProtection : uint + { + ExecuteReadWrite = 0x40, + } + + private enum FreeType : uint + { + Release = 0x8000, + } + public delegate void ImportHandler(CpuContext context); } diff --git a/src/SharpEmu.Core/Loader/ProgramHeader.cs b/src/SharpEmu.Core/Loader/ProgramHeader.cs index aa03d1c..400da70 100644 --- a/src/SharpEmu.Core/Loader/ProgramHeader.cs +++ b/src/SharpEmu.Core/Loader/ProgramHeader.cs @@ -48,6 +48,8 @@ public enum ProgramHeaderType : uint Tls = 7, + GnuEhFrame = 0x6474E550, + SceRela = 0x60000000, SceProcParam = 0x61000001, diff --git a/src/SharpEmu.Core/Loader/SelfImage.cs b/src/SharpEmu.Core/Loader/SelfImage.cs index 2f86c53..0919835 100644 --- a/src/SharpEmu.Core/Loader/SelfImage.cs +++ b/src/SharpEmu.Core/Loader/SelfImage.cs @@ -24,7 +24,10 @@ public sealed class SelfImage ulong procParamAddress = 0, string? title = null, string? titleId = null, - string? version = null) + string? version = null, + uint tlsModuleId = 0, + ulong tlsMemorySize = 0, + ulong tlsStaticOffset = 0) { ArgumentNullException.ThrowIfNull(programHeaders); ArgumentNullException.ThrowIfNull(mappedRegions); @@ -44,6 +47,9 @@ public sealed class SelfImage Title = title; TitleId = titleId; Version = version; + TlsModuleId = tlsModuleId; + TlsMemorySize = tlsMemorySize; + TlsStaticOffset = tlsStaticOffset; } public bool IsSelf { get; } @@ -75,4 +81,11 @@ public sealed class SelfImage public string? TitleId { get; } public string? Version { get; } + + public uint TlsModuleId { get; } + + public ulong TlsMemorySize { get; } + + /// Variant II distance from the thread pointer to this module's static TLS base. + public ulong TlsStaticOffset { get; } } diff --git a/src/SharpEmu.Core/Loader/SelfLoader.cs b/src/SharpEmu.Core/Loader/SelfLoader.cs index 7d30245..e804dc9 100644 --- a/src/SharpEmu.Core/Loader/SelfLoader.cs +++ b/src/SharpEmu.Core/Loader/SelfLoader.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System.Buffers.Binary; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -10,16 +11,12 @@ using SharpEmu.Core; using SharpEmu.Core.Cpu; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.Logging; namespace SharpEmu.Core.Loader; public sealed class SelfLoader : ISelfLoader { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Loader"); - private const uint ElfMagic = 0x7F454C46; - private const uint Ps4SelfMagic = 0x4F153D1D; - private const uint Ps5SelfMagic = 0x5414F5EE; + private const uint SelfMagic = 0x4F153D1D; private const ulong SelfSegmentFlag = 0x800; private const int PageSize = 0x1000; private const ulong ImportStubBaseAddress = 0x0000_7000_0000_0000UL; @@ -58,11 +55,22 @@ public sealed class SelfLoader : ISelfLoader private const long DtSceSymTab = 0x61000039; private const long DtSceSymTabSize = 0x6100003F; + private const uint RelocationTypeNone = 0; private const uint RelocationTypeAbsolute64 = 1; + private const uint RelocationTypePc32 = 2; + private const uint RelocationTypePlt32 = 4; private const uint RelocationTypeGlobalData = 6; private const uint RelocationTypeJumpSlot = 7; private const uint RelocationTypeRelative = 8; - private const uint RelocationTypeTlsModuleId = 16; + private const uint RelocationTypeUnsigned32 = 10; + private const uint RelocationTypeSigned32 = 11; + private const uint RelocationTypeTlsModuleId = 16; // R_X86_64_DTPMOD64 + private const uint RelocationTypeTlsDtpOff64 = 17; // R_X86_64_DTPOFF64 + private const uint RelocationTypeTlsTpOff64 = 18; // R_X86_64_TPOFF64 + private const uint RelocationTypePc64 = 24; + private const uint RelocationTypeSize32 = 32; + private const uint RelocationTypeSize64 = 33; + private const uint RelocationTypeRelative64 = 38; private const ulong Ps5MainImageBase = 0x0000000800000000UL; private const ulong Ps4MainImageBase = 0x0000000000400000UL; private const ulong Ps5ModuleSearchStart = 0x0000000804000000UL; @@ -88,6 +96,11 @@ public sealed class SelfLoader : ISelfLoader private static readonly int SelfSegmentSize = Unsafe.SizeOf(); private static readonly int ProgramHeaderSize = Unsafe.SizeOf(); + static SelfLoader() + { + RunRelocationSelfChecks(); + } + public SelfImage Load(ReadOnlySpan imageData, IVirtualMemory virtualMemory) { return Load(imageData, virtualMemory, fs: null, mountRoot: null); @@ -156,16 +169,22 @@ public sealed class SelfLoader : ISelfLoader { virtualMemory.Clear(); _nextTlsModuleId = 1; + GuestTlsTemplate.Reset(); } - var tlsModuleId = _nextTlsModuleId == 0 ? 1u : _nextTlsModuleId; - Log.Debug($"TLS load_start clear={clearVirtualMemory} next={_nextTlsModuleId} assigned={tlsModuleId}"); - var loadContext = ParseLayout(imageData); var elfHeader = ReadUnmanaged(imageData, loadContext.ElfOffset); ValidateElfHeader(elfHeader); var programHeaders = ParseProgramHeaders(imageData, loadContext, elfHeader); + var hasTlsSegment = TryGetProgramHeader(programHeaders, ProgramHeaderType.Tls, out var processTlsHeader, out _) && + processTlsHeader.MemorySize != 0; + var tlsModuleId = hasTlsSegment + ? (_nextTlsModuleId == 0 ? 1u : _nextTlsModuleId) + : 0u; + Console.Error.WriteLine( + $"[LOADER][TLS] load_start clear={clearVirtualMemory} next={_nextTlsModuleId} " + + $"assigned={tlsModuleId} has_pt_tls={hasTlsSegment}"); var totalImageSize = CalculateTotalImageSize(programHeaders); Console.WriteLine($"Total image size needed: 0x{totalImageSize:X} ({totalImageSize} bytes)"); @@ -198,6 +217,13 @@ public sealed class SelfLoader : ISelfLoader } MapLoadSegments(imageData, loadContext, programHeaders, virtualMemory, imageBase); + // Register every module before relocations so DTPMOD/DTPOFF/TPOFF use + // the module's real PT_TLS identity and Variant II static offset. + var tlsInfo = RegisterModuleTlsTemplate( + programHeaders, + virtualMemory, + imageBase, + tlsModuleId); var importStubs = ResolveAndPatchImportStubs( imageData, loadContext, @@ -255,11 +281,11 @@ public sealed class SelfLoader : ISelfLoader Console.WriteLine($"[LOADER] PH[{i}]: type={ph.HeaderType}, vaddr=0x{ph.VirtualAddress:X16} -> 0x{ph.VirtualAddress + imageBase:X16}, memsz=0x{ph.MemorySize:X}"); } - if (_nextTlsModuleId == tlsModuleId && _nextTlsModuleId < uint.MaxValue) + if (tlsModuleId != 0 && _nextTlsModuleId == tlsModuleId && _nextTlsModuleId < uint.MaxValue) { _nextTlsModuleId++; } - Log.Debug($"TLS load_done assigned={tlsModuleId} next={_nextTlsModuleId}"); + Console.Error.WriteLine($"[LOADER][TLS] load_done assigned={tlsModuleId} next={_nextTlsModuleId}"); return new SelfImage( loadContext.IsSelf, @@ -276,7 +302,10 @@ public sealed class SelfLoader : ISelfLoader procParamAddress, applicationInfo.Title, applicationInfo.TitleId, - applicationInfo.Version); + applicationInfo.Version, + tlsModuleId, + tlsInfo.MemorySize, + tlsInfo.StaticOffset); } private static (string? Title, string? TitleId, string? Version) TryLoadParamJson( @@ -325,8 +354,7 @@ public sealed class SelfLoader : ISelfLoader throw new InvalidDataException("Input image is too small to contain an ELF header."); } - var magic = BinaryPrimitives.ReadUInt32BigEndian(imageData[..sizeof(uint)]); - if (magic is Ps4SelfMagic or Ps5SelfMagic) + if (imageData.Length >= sizeof(uint) && BinaryPrimitives.ReadUInt32BigEndian(imageData[..sizeof(uint)]) == SelfMagic) { var selfHeader = ReadUnmanaged(imageData, 0); if (!selfHeader.HasKnownLayout || selfHeader.Unknown != 0x22) @@ -348,10 +376,19 @@ public sealed class SelfLoader : ISelfLoader return new LoadContext(IsSelf: true, elfOffset, selfHeader.FileSize, segments); } - if (magic != ElfMagic) + // Not a recognized (fake-signed) SELF. Only a bare, decrypted ELF is + // acceptable here; anything else — most commonly a still-encrypted + // retail eboot — must be reported clearly rather than failing later + // with an opaque "not a valid ELF header" message. + const uint ElfMagicBigEndian = 0x7F454C46; // "\x7fELF" + var leadingWord = BinaryPrimitives.ReadUInt32BigEndian(imageData[..sizeof(uint)]); + if (leadingWord != ElfMagicBigEndian) { throw new InvalidDataException( - $"Unsupported executable signature 0x{magic:X8}"); + $"Image is neither a decrypted ELF nor a recognized fake-signed SELF " + + $"(leading bytes 0x{leadingWord:X8}). This is almost certainly a still-encrypted " + + $"retail eboot — SharpEmu has no decryption keys and requires a decrypted / " + + $"fake-signed (fSELF) image."); } return new LoadContext(IsSelf: false, ElfOffset: 0, SelfFileSize: 0, Array.Empty()); @@ -414,15 +451,15 @@ public sealed class SelfLoader : ISelfLoader var virtualAddress = header.VirtualAddress + imageBase; - Log.Debug($"Segment {index}: VAddr=0x{virtualAddress:X16}, FileSize=0x{header.FileSize:X}, MemSize=0x{header.MemorySize:X}, Align=0x{header.Alignment:X}"); + Console.Error.WriteLine($"[LOADER] Segment {index}: VAddr=0x{virtualAddress:X16}, FileSize=0x{header.FileSize:X}, MemSize=0x{header.MemorySize:X}, Align=0x{header.Alignment:X}"); if (header.Alignment > 1) { var vaddrMod = virtualAddress % header.Alignment; var offsetMod = header.Offset % header.Alignment; if (vaddrMod != offsetMod) { - Log.Warning( - $"Segment {index} ELF alignment mismatch! " + + Console.Error.WriteLine( + $"[LOADER] WARNING: Segment {index} ELF alignment mismatch! " + $"VAddr=0x{virtualAddress:X}, Offset=0x{header.Offset:X}, Align=0x{header.Alignment:X}, " + $"VAddr%Align=0x{vaddrMod:X}, Offset%Align=0x{offsetMod:X}"); } @@ -465,6 +502,43 @@ public sealed class SelfLoader : ISelfLoader return 0; } + private static ModuleTlsInfo RegisterModuleTlsTemplate( + IReadOnlyList programHeaders, + IVirtualMemory virtualMemory, + ulong imageBase, + uint tlsModuleId) + { + if (!TryGetProgramHeader(programHeaders, ProgramHeaderType.Tls, out var tlsHeader, out _) || + tlsHeader.MemorySize == 0) + { + return default; + } + + // tdata (initialized) bytes come from the mapped segment; tbss is the + // implicitly-zero remainder up to MemorySize. + var fileSize = (int)Math.Min(tlsHeader.FileSize, tlsHeader.MemorySize); + var initImage = fileSize > 0 ? new byte[fileSize] : []; + if (fileSize > 0 && + !virtualMemory.TryRead(imageBase + tlsHeader.VirtualAddress, initImage)) + { + Console.Error.WriteLine( + $"[LOADER][TLS] Failed to read TLS init image at 0x{imageBase + tlsHeader.VirtualAddress:X}; seeding zeros."); + initImage = []; + } + + var staticOffset = GuestTlsTemplate.RegisterModule( + tlsModuleId, + initImage, + tlsHeader.MemorySize, + tlsHeader.Alignment, + tlsHeader.VirtualAddress); + Console.Error.WriteLine( + $"[LOADER][TLS] Module {tlsModuleId} TLS template: memsz=0x{tlsHeader.MemorySize:X} " + + $"filesz=0x{tlsHeader.FileSize:X} align=0x{tlsHeader.Alignment:X} " + + $"static_offset=0x{staticOffset:X} total_static=0x{GuestTlsTemplate.StaticTlsSize:X}"); + return new ModuleTlsInfo(tlsHeader.MemorySize, staticOffset); + } + private static IReadOnlyDictionary ResolveAndPatchImportStubs( ReadOnlySpan imageData, LoadContext loadContext, @@ -544,12 +618,17 @@ public sealed class SelfLoader : ISelfLoader uint maxSymbolIndex = 0; foreach (var relocation in relocations) { + if (relocation.Type == RelocationTypeNone) + { + continue; + } + if (!IsSupportedRelocationType(relocation.Type)) { continue; } - if (relocation.Type is RelocationTypeRelative or RelocationTypeTlsModuleId) + if (relocation.Type is RelocationTypeNone or RelocationTypeRelative or RelocationTypeRelative64 or RelocationTypeTlsModuleId) { continue; } @@ -633,13 +712,16 @@ public sealed class SelfLoader : ISelfLoader importedRelocations = BuildImportedRelocations(descriptors); - var stubsByAddress = CreateImportStubMapping(virtualMemory, orderedImportNids); + var stubImportNids = orderedImportNids + .Where(nid => ShouldCreateImportStub(nid, descriptors, moduleManager)) + .ToArray(); + var stubsByAddress = CreateImportStubMapping(virtualMemory, stubImportNids); Console.WriteLine($"[LOADER] Created {stubsByAddress.Count} import stubs"); - int printCount = Math.Min(10, orderedImportNids.Count); + int printCount = Math.Min(10, stubImportNids.Length); for (int i = 0; i < printCount; i++) { - var nid = orderedImportNids[i]; + var nid = stubImportNids[i]; var addr = stubsByAddress.First(x => x.Value == nid).Key; } @@ -663,45 +745,65 @@ public sealed class SelfLoader : ISelfLoader foreach (var descriptor in descriptors) { - ulong targetValue; + ulong symbolValue; if (descriptor.ImportNid is null) { - targetValue = AddSigned(descriptor.SymbolValue, descriptor.Addend); + symbolValue = descriptor.SymbolValue; } else { - if (!addressesByNid.TryGetValue(descriptor.ImportNid, out var stubAddress)) + if (addressesByNid.TryGetValue(descriptor.ImportNid, out var stubAddress)) { - throw new InvalidOperationException($"Import stub not found for NID '{descriptor.ImportNid}'."); + symbolValue = stubAddress; } - - targetValue = AddSigned(stubAddress, descriptor.Addend); - } - - if (targetValue < 0x1000) - { - if (descriptor.ValueKind == RelocationValueKind.TlsModuleId) + else if (descriptor.IsWeak) { - Log.Debug( - $"Patching DTPMOD64 at 0x{descriptor.TargetAddress:X} with module id 0x{targetValue:X}"); + // ELF unresolved weak definitions use S=0. They must not + // receive a trap import stub, which would turn a permitted + // null test into a call to an unresolved-import handler. + symbolValue = 0; } else { - Log.Warning($"!!! CRITICAL !!! Patching address 0x{descriptor.TargetAddress:X} with INVALID value 0x{targetValue:X} for NID {descriptor.ImportNid ?? "(null)"}"); - Log.Warning($" SymbolValue=0x{descriptor.SymbolValue:X}, Addend=0x{descriptor.Addend:X}, StubAddress=0x{(addressesByNid.TryGetValue(descriptor.ImportNid ?? "", out var sa) ? sa : 0):X}"); + throw new InvalidOperationException($"Import stub not found for NID '{descriptor.ImportNid}'."); } } - if (!TryWriteUInt64(virtualMemory, descriptor.TargetAddress, targetValue)) + var targetValue = ComputeRelocationValue(descriptor, symbolValue); + + if (targetValue < 0x1000 && descriptor.ValueKind is + RelocationValueKind.TlsOffset or + RelocationValueKind.PcRelative or + RelocationValueKind.SymbolSize) { - throw new InvalidDataException($"Failed to patch relocation at 0x{descriptor.TargetAddress:X16}."); + // A TLS offset (TPOFF64/DTPOFF64) is a signed displacement, not a + // mapped address, so a small or negative value here is expected. + } + else if (targetValue < 0x1000 && !descriptor.IsWeak) + { + if (descriptor.ValueKind == RelocationValueKind.TlsModuleId) + { + Console.Error.WriteLine( + $"[LOADER][TLS] Patching DTPMOD64 at 0x{descriptor.TargetAddress:X} with module id 0x{targetValue:X}"); + } + else + { + Console.Error.WriteLine($"[LOADER] !!! CRITICAL !!! Patching address 0x{descriptor.TargetAddress:X} with INVALID value 0x{targetValue:X} for NID {descriptor.ImportNid ?? "(null)"}"); + Console.Error.WriteLine($"[LOADER] SymbolValue=0x{descriptor.SymbolValue:X}, Addend=0x{descriptor.Addend:X}, StubAddress=0x{(addressesByNid.TryGetValue(descriptor.ImportNid ?? "", out var sa) ? sa : 0):X}"); + } + } + + if (!TryWriteRelocationValue(virtualMemory, descriptor, targetValue, out var writeError)) + { + throw new InvalidDataException( + $"Failed to patch relocation at 0x{descriptor.TargetAddress:X16}: {writeError}"); } if (descriptor.TargetAddress >= 0x00000008030FC300UL && descriptor.TargetAddress <= 0x00000008030FC3F0UL) { - Log.Debug( - $"[RELOC] target=0x{descriptor.TargetAddress:X16} value=0x{targetValue:X16} addend=0x{descriptor.Addend:X} nid={(descriptor.ImportNid ?? "")}"); + Console.Error.WriteLine( + $"[LOADER][RELOC] target=0x{descriptor.TargetAddress:X16} value=0x{targetValue:X16} addend=0x{descriptor.Addend:X} nid={(descriptor.ImportNid ?? "")}"); } } @@ -794,29 +896,36 @@ public sealed class SelfLoader : ISelfLoader { if (IsFocusRelocationOffset(relocation.Offset, imageBase)) { - Log.Debug( - $"[FOCUS][SCAN] off=0x{relocation.Offset:X16} type={relocation.Type} sym={relocation.SymbolIndex} addend=0x{relocation.Addend:X}"); + Console.Error.WriteLine( + $"[LOADER][FOCUS][SCAN] off=0x{relocation.Offset:X16} type={relocation.Type} sym={relocation.SymbolIndex} addend=0x{relocation.Addend:X}"); } if (!IsSupportedRelocationType(relocation.Type)) { - if (IsFocusRelocationOffset(relocation.Offset, imageBase)) + // Surface unsupported relocation types loudly instead of + // skipping silently: TLS-relative (17/18), COPY (5), and + // IRELATIVE/ifunc (37) leave their targets unrelocated, which + // manifests later as reads of zero or calls into address 0. + ReportUnsupportedRelocation(relocation.Type, relocation.Offset, imageBase); + if (relocation.Type is 5 or 37) { - Log.Debug($"[FOCUS][SKIP] unsupported type={relocation.Type}"); + throw new NotSupportedException( + $"Relocation type {relocation.Type} requires deferred runtime-linker processing."); } continue; } - if (!TryResolveMappedAddress(virtualMemory, relocation.Offset, imageBase, sizeof(ulong), out var targetAddress)) + var relocationWriteSize = GetRelocationWriteSize(relocation.Type); + if (!TryResolveMappedAddress(virtualMemory, relocation.Offset, imageBase, relocationWriteSize, out var targetAddress)) { if (IsFocusRelocationOffset(relocation.Offset, imageBase)) { - Log.Debug("[FOCUS][SKIP] target address not mapped"); + Console.Error.WriteLine("[LOADER][FOCUS][SKIP] target address not mapped"); } continue; } - if (relocation.Type == RelocationTypeRelative) + if (relocation.Type is RelocationTypeRelative or RelocationTypeRelative64) { descriptors.Add(new RelocationDescriptor( targetAddress, @@ -830,7 +939,12 @@ public sealed class SelfLoader : ISelfLoader if (relocation.Type == RelocationTypeTlsModuleId) { - var dtpmodValue = tlsModuleId == 0 ? 1u : tlsModuleId; + if (tlsModuleId == 0) + { + throw new InvalidDataException( + $"R_X86_64_DTPMOD64 at 0x{targetAddress:X16} references a module without PT_TLS."); + } + var dtpmodValue = tlsModuleId; descriptors.Add(new RelocationDescriptor( targetAddress, 0, @@ -851,50 +965,100 @@ public sealed class SelfLoader : ISelfLoader { if (targetAddress >= FocusRelocGuestStart && targetAddress <= FocusRelocGuestEnd) { - Log.Debug($"[FOCUS][SKIP] symbol read failed index={symbolIndex}"); + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] symbol read failed index={symbolIndex}"); } continue; } - var addend = relocation.Type is RelocationTypeGlobalData or RelocationTypeJumpSlot ? 0 : relocation.Addend; - var symbolBind = GetSymbolBind(symbol.Info); - if (symbolBind == SymbolBindLocal) + if (relocation.Type is RelocationTypeTlsDtpOff64 or RelocationTypeTlsTpOff64) { - var symbolAddress = ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); - if (symbolAddress == 0) + // Variant II static TLS: the module block sits at + // [tp - blockSize, tp). DTPOFF64 is the module-relative offset + // (st_value + addend); TPOFF64 is that offset expressed relative + // to the thread pointer, i.e. minus the aligned block size. + var tlsSymbolOffset = AddSigned(symbol.Value, relocation.Addend); + if (!GuestTlsTemplate.TryGetStaticOffset(tlsModuleId, out var moduleStaticOffset)) { - Log.Warning( - $"Skipping local relocation with invalid symbol value 0x{symbol.Value:X} " + - $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + Console.Error.WriteLine( + $"[LOADER][TLS] Missing PT_TLS registration for module {tlsModuleId}; " + + $"cannot apply relocation {relocation.Type} at 0x{targetAddress:X16}."); + continue; } - + var tlsValue = relocation.Type == RelocationTypeTlsTpOff64 + ? unchecked(tlsSymbolOffset - moduleStaticOffset) + : tlsSymbolOffset; descriptors.Add(new RelocationDescriptor( targetAddress, - addend, + 0, null, - symbolAddress, - RelocationValueKind.Pointer, + tlsValue, + RelocationValueKind.TlsOffset, IsDataImport: false)); continue; } + var symbolBind = GetSymbolBind(symbol.Info); + if (symbolIndex == 0) + { + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress: 0, + importNid: null, + isWeak: false)); + continue; + } + + if (symbolBind == SymbolBindLocal) + { + var symbolAddress = relocation.Type is RelocationTypeSize32 or RelocationTypeSize64 + ? 0 + : ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); + if (symbolAddress == 0) + { + if (relocation.Type is not (RelocationTypeSize32 or RelocationTypeSize64)) + { + Console.Error.WriteLine( + $"[LOADER] Skipping local relocation with invalid symbol value 0x{symbol.Value:X} " + + $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + continue; + } + } + + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress, + importNid: null, + isWeak: false)); + continue; + } + if (symbol.Value != 0) { - var symbolAddress = ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); + var symbolAddress = relocation.Type is RelocationTypeSize32 or RelocationTypeSize64 + ? 0 + : ResolveMappedAddressOrFallback(virtualMemory, symbol.Value, imageBase); if (symbolAddress == 0) { - Log.Warning( - $"Skipping relocation with invalid symbol value 0x{symbol.Value:X} " + - $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + if (relocation.Type is not (RelocationTypeSize32 or RelocationTypeSize64)) + { + Console.Error.WriteLine( + $"[LOADER] Skipping relocation with invalid symbol value 0x{symbol.Value:X} " + + $"at target 0x{targetAddress:X16}, type={relocation.Type}, sym={symbolIndex}"); + continue; + } } - descriptors.Add(new RelocationDescriptor( + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, targetAddress, - addend, - null, + symbol, symbolAddress, - RelocationValueKind.Pointer, - IsDataImport: false)); + importNid: null, + isWeak: false)); continue; } @@ -902,16 +1066,26 @@ public sealed class SelfLoader : ISelfLoader { if (targetAddress >= FocusRelocGuestStart && targetAddress <= FocusRelocGuestEnd) { - Log.Debug($"[FOCUS][SKIP] bind={symbolBind} not importable"); + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] bind={symbolBind} not importable"); } continue; } if (!TryReadNullTerminatedAscii(stringTable, symbol.NameOffset, out var symbolName)) { + if (symbolBind == SymbolBindWeak) + { + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress: 0, + importNid: null, + isWeak: true)); + } if (targetAddress >= FocusRelocGuestStart && targetAddress <= FocusRelocGuestEnd) { - Log.Debug($"[FOCUS][SKIP] symbol name read failed offset={symbol.NameOffset}"); + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] symbol name read failed offset={symbol.NameOffset}"); } continue; } @@ -919,6 +1093,16 @@ public sealed class SelfLoader : ISelfLoader var nid = ExtractNid(symbolName); if (string.IsNullOrWhiteSpace(nid)) { + if (symbolBind == SymbolBindWeak) + { + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, + targetAddress, + symbol, + symbolAddress: 0, + importNid: null, + isWeak: true)); + } continue; } @@ -927,16 +1111,74 @@ public sealed class SelfLoader : ISelfLoader orderedImportNids.Add(nid); } - descriptors.Add(new RelocationDescriptor( + descriptors.Add(CreateSymbolRelocationDescriptor( + relocation, targetAddress, - addend, - nid, - 0, - RelocationValueKind.Pointer, - IsDataImport: GetSymbolType(symbol.Info) == SymbolTypeObject)); + symbol, + symbolAddress: 0, + importNid: nid, + isWeak: symbolBind == SymbolBindWeak)); } } + private static RelocationDescriptor CreateSymbolRelocationDescriptor( + ElfRelocation relocation, + ulong targetAddress, + ElfSymbol symbol, + ulong symbolAddress, + string? importNid, + bool isWeak) + { + var valueKind = relocation.Type switch + { + RelocationTypePc32 or RelocationTypePlt32 or RelocationTypePc64 => RelocationValueKind.PcRelative, + RelocationTypeSize32 or RelocationTypeSize64 => RelocationValueKind.SymbolSize, + _ => RelocationValueKind.Pointer, + }; + var writeKind = relocation.Type switch + { + RelocationTypePc32 or RelocationTypePlt32 or RelocationTypeSigned32 => RelocationWriteKind.Int32, + RelocationTypeUnsigned32 or RelocationTypeSize32 => RelocationWriteKind.UInt32, + _ => RelocationWriteKind.UInt64, + }; + var symbolValue = valueKind == RelocationValueKind.SymbolSize + ? symbol.Size + : symbolAddress; + var addend = relocation.Type is RelocationTypeGlobalData or RelocationTypeJumpSlot + ? 0 + : relocation.Addend; + return new RelocationDescriptor( + targetAddress, + addend, + importNid, + symbolValue, + valueKind, + IsDataImport: GetSymbolType(symbol.Info) == SymbolTypeObject, + writeKind, + isWeak); + } + + private static bool ShouldCreateImportStub( + string nid, + IReadOnlyList descriptors, + IModuleManager? moduleManager) + { + for (var i = 0; i < descriptors.Count; i++) + { + var descriptor = descriptors[i]; + if (!string.Equals(descriptor.ImportNid, nid, StringComparison.Ordinal)) + { + continue; + } + if (!descriptor.IsWeak || moduleManager?.TryGetExport(nid, out _) == true) + { + return true; + } + } + + return false; + } + private static void RegisterRuntimeSymbolsAndHooks( ReadOnlySpan imageData, LoadContext loadContext, @@ -959,8 +1201,8 @@ public sealed class SelfLoader : ISelfLoader if (sectionSymbols > 0 || dynamicSymbols > 0) { - Log.Info( - $"Runtime symbol index populated: section={sectionSymbols}, dynamic={dynamicSymbols}, total={runtimeSymbols.Count}"); + Console.Error.WriteLine( + $"[LOADER] Runtime symbol index populated: section={sectionSymbols}, dynamic={dynamicSymbols}, total={runtimeSymbols.Count}"); } } @@ -1034,8 +1276,8 @@ public sealed class SelfLoader : ISelfLoader if (preInitializers.Count != 0 || initializers.Count != 0) { - Log.Info( - $"Initializers discovered: preinit={preInitializers.Count}, init={initializers.Count}"); + Console.Error.WriteLine( + $"[LOADER] Initializers discovered: preinit={preInitializers.Count}, init={initializers.Count}"); } } @@ -1418,8 +1660,8 @@ public sealed class SelfLoader : ISelfLoader var requiredBytes = checked((ulong)orderedImportNids.Count * ImportStubSlotSize); var mapSize = AlignUp(Math.Max(requiredBytes, (ulong)PageSize), PageSize); - Log.Debug( - $"CreateImportStubMapping: nids={orderedImportNids.Count}, required=0x{requiredBytes:X}, map_size=0x{mapSize:X}"); + Console.Error.WriteLine( + $"[LOADER] CreateImportStubMapping: nids={orderedImportNids.Count}, required=0x{requiredBytes:X}, map_size=0x{mapSize:X}"); if (mapSize > int.MaxValue) { throw new NotSupportedException("Import stub mapping exceeds 2 GB and is not supported."); @@ -1691,11 +1933,51 @@ public sealed class SelfLoader : ISelfLoader private static bool IsSupportedRelocationType(uint relocationType) { return relocationType is + RelocationTypeNone or RelocationTypeAbsolute64 or + RelocationTypePc32 or + RelocationTypePlt32 or RelocationTypeGlobalData or RelocationTypeJumpSlot or RelocationTypeRelative or - RelocationTypeTlsModuleId; + RelocationTypeUnsigned32 or + RelocationTypeSigned32 or + RelocationTypeTlsModuleId or + RelocationTypeTlsDtpOff64 or + RelocationTypeTlsTpOff64 or + RelocationTypePc64 or + RelocationTypeSize32 or + RelocationTypeSize64 or + RelocationTypeRelative64; + } + + private static readonly HashSet _reportedUnsupportedRelocationTypes = new(); + + private static void ReportUnsupportedRelocation(uint relocationType, ulong offset, ulong imageBase) + { + // Report each distinct unsupported type once to keep the log useful. + lock (_reportedUnsupportedRelocationTypes) + { + if (!_reportedUnsupportedRelocationTypes.Add(relocationType)) + { + if (IsFocusRelocationOffset(offset, imageBase)) + { + Console.Error.WriteLine($"[LOADER][FOCUS][SKIP] unsupported type={relocationType}"); + } + + return; + } + } + + var name = relocationType switch + { + 5 => "R_X86_64_COPY", + 37 => "R_X86_64_IRELATIVE (ifunc)", + _ => "unknown", + }; + Console.Error.WriteLine( + $"[LOADER][ERROR] Unsupported relocation type {relocationType} ({name}) rejected " + + $"(first at off=0x{offset:X16}); COPY requires dependency symbol storage and IRELATIVE requires resolver execution."); } private static ulong DetermineRequestedImageBase( @@ -1851,16 +2133,16 @@ public sealed class SelfLoader : ISelfLoader tableBytes = GC.AllocateUninitializedArray((int)size); var guestAddr = location + imageBase; - Log.Debug($"TryLoadTableBytes: trying guest address 0x{guestAddr:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: trying guest address 0x{guestAddr:X}"); if (virtualMemory.TryRead(guestAddr, tableBytes)) { - Log.Debug($"TryLoadTableBytes: loaded from guest memory at 0x{guestAddr:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: loaded from guest memory at 0x{guestAddr:X}"); return true; } if (virtualMemory.TryRead(location, tableBytes)) { - Log.Debug($"TryLoadTableBytes: loaded from absolute guest address 0x{location:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: loaded from absolute guest address 0x{location:X}"); return true; } @@ -1868,11 +2150,11 @@ public sealed class SelfLoader : ISelfLoader { var slice = elfData.Slice((int)location, (int)size); tableBytes = slice.ToArray(); - Log.Debug($"TryLoadTableBytes: loaded from elfData as file offset at 0x{location:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: loaded from elfData as file offset at 0x{location:X}"); return true; } - Log.Warning($"TryLoadTableBytes: FAILED for location 0x{location:X}"); + Console.Error.WriteLine($"[LOADER] TryLoadTableBytes: FAILED for location 0x{location:X}"); tableBytes = Array.Empty(); return false; } @@ -1929,17 +2211,17 @@ public sealed class SelfLoader : ISelfLoader private static ulong ResolveMappedAddressOrFallback(IVirtualMemory virtualMemory, ulong address, ulong imageBase) { - Log.Debug($"ResolveMappedAddressOrFallback addr=0x{address:X} imageBase=0x{imageBase:X16}"); + Console.Error.WriteLine($"[LOADER][TEST] ResolveMappedAddressOrFallback addr=0x{address:X} imageBase=0x{imageBase:X16}"); if (address == 0) { - Log.Debug("-> return 0 (null)"); + Console.Error.WriteLine("[LOADER][TEST] -> return 0 (null)"); return 0; } if (TryResolveMappedAddress(virtualMemory, address, imageBase, 1, out var resolved)) { - Log.Debug($"-> resolved raw 0x{resolved:X16}"); + Console.Error.WriteLine($"[LOADER][TEST] -> resolved raw 0x{resolved:X16}"); return resolved; } @@ -1948,18 +2230,18 @@ public sealed class SelfLoader : ISelfLoader var rebased = address + imageBase; if (TryResolveMappedAddress(virtualMemory, rebased, imageBase, 1, out var resolvedRebased)) { - Log.Debug($"-> resolved rebased 0x{resolvedRebased:X16}"); + Console.Error.WriteLine($"[LOADER][TEST] -> resolved rebased 0x{resolvedRebased:X16}"); return resolvedRebased; } } if (address < 0x10000) { - Log.Debug($"-> reject small 0x{address:X}"); + Console.Error.WriteLine($"[LOADER][TEST] -> reject small 0x{address:X}"); return 0; } - Log.Debug($"-> fallback raw 0x{address:X}"); + Console.Error.WriteLine($"[LOADER][TEST] -> fallback raw 0x{address:X}"); return address; } @@ -2049,6 +2331,106 @@ public sealed class SelfLoader : ISelfLoader return virtualMemory.TryWrite(address, buffer); } + private static ulong ComputeRelocationValue(RelocationDescriptor descriptor, ulong resolvedSymbolValue) + { + var baseValue = descriptor.ValueKind == RelocationValueKind.SymbolSize + ? descriptor.SymbolValue + : resolvedSymbolValue; + var value = AddSigned(baseValue, descriptor.Addend); + return descriptor.ValueKind == RelocationValueKind.PcRelative + ? unchecked(value - descriptor.TargetAddress) + : value; + } + + private static bool TryWriteRelocationValue( + IVirtualMemory virtualMemory, + RelocationDescriptor descriptor, + ulong value, + out string? error) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + var length = sizeof(ulong); + switch (descriptor.WriteKind) + { + case RelocationWriteKind.UInt64: + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + break; + + case RelocationWriteKind.UInt32: + if (value > uint.MaxValue) + { + error = $"value 0x{value:X16} overflows an unsigned 32-bit relocation"; + return false; + } + BinaryPrimitives.WriteUInt32LittleEndian(buffer, (uint)value); + length = sizeof(uint); + break; + + case RelocationWriteKind.Int32: + var signedValue = unchecked((long)value); + if (signedValue is < int.MinValue or > int.MaxValue) + { + error = $"value {signedValue} overflows a signed 32-bit relocation"; + return false; + } + BinaryPrimitives.WriteInt32LittleEndian(buffer, (int)signedValue); + length = sizeof(int); + break; + + default: + error = $"unknown relocation write kind {descriptor.WriteKind}"; + return false; + } + + error = virtualMemory.TryWrite(descriptor.TargetAddress, buffer[..length]) + ? null + : "target memory is not writable"; + return error is null; + } + + private static int GetRelocationWriteSize(uint relocationType) + { + return relocationType is + RelocationTypePc32 or + RelocationTypePlt32 or + RelocationTypeUnsigned32 or + RelocationTypeSigned32 or + RelocationTypeSize32 + ? sizeof(uint) + : sizeof(ulong); + } + + [Conditional("DEBUG")] + private static void RunRelocationSelfChecks() + { + var pc32 = new RelocationDescriptor( + TargetAddress: 0x1000, + Addend: -4, + ImportNid: null, + SymbolValue: 0x1800, + RelocationValueKind.PcRelative, + IsDataImport: false, + RelocationWriteKind.Int32); + Debug.Assert( + unchecked((long)ComputeRelocationValue(pc32, pc32.SymbolValue)) == 0x7FC, + "R_X86_64_PC32 did not apply S + A - P."); + + var weak = new RelocationDescriptor( + TargetAddress: 0x2000, + Addend: 7, + ImportNid: "weak", + SymbolValue: 0, + RelocationValueKind.Pointer, + IsDataImport: false, + IsWeak: true); + Debug.Assert( + ComputeRelocationValue(weak, resolvedSymbolValue: 0) == 7, + "An unresolved weak relocation did not use S=0."); + Debug.Assert( + !ShouldCreateImportStub("weak", [weak], moduleManager: null), + "An unresolved weak symbol incorrectly received a trap import stub."); + } + private static ulong AlignUp(ulong value, ulong alignment) { var mask = alignment - 1; @@ -2257,6 +2639,16 @@ public sealed class SelfLoader : ISelfLoader { throw new InvalidDataException($"Unsupported ELF program header entry size: {header.ProgramHeaderEntrySize}."); } + + // The CPU backend executes guest instructions natively, so a non + // x86-64 image can only fail deep in execution with an opaque SIGILL. + // Catch it here with an actionable message. (EM_X86_64 == 62.) + const ushort ElfMachineX86_64 = 62; + if (header.Machine != ElfMachineX86_64) + { + throw new InvalidDataException( + $"Unsupported ELF machine type 0x{header.Machine:X4}; SharpEmu only runs x86-64 (EM_X86_64) images."); + } } private static unsafe T ReadUnmanaged(ReadOnlySpan source, int offset) @@ -2337,10 +2729,25 @@ public sealed class SelfLoader : ISelfLoader public uint Type => (uint)(Info & uint.MaxValue); } + private readonly record struct ModuleTlsInfo(ulong MemorySize, ulong StaticOffset); + private enum RelocationValueKind : byte { Pointer = 0, - TlsModuleId = 1 + TlsModuleId = 1, + // A pre-computed TLS offset written verbatim (TPOFF64/DTPOFF64). Unlike + // Pointer it is a signed displacement, not a mapped address, so it is + // patched as-is without the low-address validity warning. + TlsOffset = 2, + PcRelative = 3, + SymbolSize = 4, + } + + private enum RelocationWriteKind : byte + { + UInt64 = 0, + UInt32 = 1, + Int32 = 2, } private readonly record struct RelocationDescriptor( @@ -2349,7 +2756,9 @@ public sealed class SelfLoader : ISelfLoader string? ImportNid, ulong SymbolValue, RelocationValueKind ValueKind, - bool IsDataImport); + bool IsDataImport, + RelocationWriteKind WriteKind = RelocationWriteKind.UInt64, + bool IsWeak = false); private enum SelfSegmentResolveStatus { @@ -2389,14 +2798,10 @@ public sealed class SelfLoader : ISelfLoader public ulong FileSize => _fileSize; public bool HasKnownLayout => - ((_ident0 == 0x4F && - _ident1 == 0x15 && - _ident2 == 0x3D && - _ident3 == 0x1D) || - (_ident0 == 0x54 && - _ident1 == 0x14 && - _ident2 == 0xF5 && - _ident3 == 0xEE)) && + _ident0 == 0x4F && + _ident1 == 0x15 && + _ident2 == 0x3D && + _ident3 == 0x1D && _ident4 == 0x00 && _ident5 == 0x01 && _ident6 == 0x01 && diff --git a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs index 7add040..f069d8f 100644 --- a/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs +++ b/src/SharpEmu.Core/Memory/PhysicalVirtualMemory.cs @@ -48,7 +48,107 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA public PhysicalVirtualMemory(IHostMemory? hostMemory = null) { - _hostMemory = hostMemory ?? HostPlatform.Current.Memory; + _hostMemory = hostMemory ?? CrossPlatformHostMemory.Instance; + } + + private sealed class CrossPlatformHostMemory : IHostMemory + { + public static readonly CrossPlatformHostMemory Instance = new(); + + public ulong Allocate(ulong desiredAddress, ulong size, HostPageProtection protection) => + unchecked((ulong)HostMemory.Alloc( + (void*)desiredAddress, + (nuint)size, + HostMemory.MEM_RESERVE | HostMemory.MEM_COMMIT, + ToRawProtection(protection))); + + public ulong Reserve(ulong desiredAddress, ulong size, HostPageProtection protection) => + unchecked((ulong)HostMemory.Alloc( + (void*)desiredAddress, + (nuint)size, + HostMemory.MEM_RESERVE, + ToRawProtection(protection))); + + public bool Commit(ulong address, ulong size, HostPageProtection protection) => + HostMemory.Alloc( + (void*)address, + (nuint)size, + HostMemory.MEM_COMMIT, + ToRawProtection(protection)) != null; + + public bool Free(ulong address) => + HostMemory.Free((void*)address, 0, HostMemory.MEM_RELEASE); + + public bool Protect( + ulong address, + ulong size, + HostPageProtection protection, + out uint rawOldProtection) => + HostMemory.Protect( + (void*)address, + (nuint)size, + ToRawProtection(protection), + out rawOldProtection); + + public bool ProtectRaw( + ulong address, + ulong size, + uint rawProtection, + out uint rawOldProtection) => + HostMemory.Protect((void*)address, (nuint)size, rawProtection, out rawOldProtection); + + public bool Query(ulong address, out HostRegionInfo info) + { + if (HostMemory.Query((void*)address, out var raw) == 0) + { + info = default; + return false; + } + + var state = raw.State switch + { + HostMemory.MEM_FREE_STATE => HostRegionState.Free, + HostMemory.MEM_RESERVE => HostRegionState.Reserved, + _ => HostRegionState.Committed, + }; + + info = new HostRegionInfo( + raw.BaseAddress, + raw.AllocationBase, + raw.RegionSize, + state, + raw.State, + FromRawProtection(raw.Protect), + raw.Protect, + raw.AllocationProtect); + return true; + } + + public void FlushInstructionCache(ulong address, ulong size) => + HostMemory.FlushInstructionCache((void*)address, (nuint)size); + + private static uint ToRawProtection(HostPageProtection protection) => protection switch + { + HostPageProtection.NoAccess => HostMemory.PAGE_NOACCESS, + HostPageProtection.ReadOnly => HostMemory.PAGE_READONLY, + HostPageProtection.ReadWrite => HostMemory.PAGE_READWRITE, + HostPageProtection.Execute => HostMemory.PAGE_EXECUTE, + HostPageProtection.ReadExecute => HostMemory.PAGE_EXECUTE_READ, + HostPageProtection.ReadWriteExecute => HostMemory.PAGE_EXECUTE_READWRITE, + HostPageProtection.ExecuteWriteCopy => 0x80, + _ => HostMemory.PAGE_NOACCESS, + }; + + private static HostPageProtection FromRawProtection(uint protection) => protection switch + { + HostMemory.PAGE_READONLY => HostPageProtection.ReadOnly, + HostMemory.PAGE_READWRITE => HostPageProtection.ReadWrite, + HostMemory.PAGE_EXECUTE => HostPageProtection.Execute, + HostMemory.PAGE_EXECUTE_READ => HostPageProtection.ReadExecute, + HostMemory.PAGE_EXECUTE_READWRITE => HostPageProtection.ReadWriteExecute, + 0x80 => HostPageProtection.ExecuteWriteCopy, + _ => HostPageProtection.NoAccess, + }; } public bool TryAllocateAtExact(ulong desiredAddress, ulong size, bool executable, out ulong actualAddress) @@ -249,42 +349,11 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA var requestedCursor = AlignUp(desiredAddress, effectiveAlignment); var cursor = GetAllocationSearchCursor(desiredAddress, requestedCursor, effectiveAlignment, executable); - // Under Rosetta 2 the kernel can ignore placement hints for whole - // windows, so page-stepped exact probes are pathological on macOS. - // Linux must keep using the exact-address search below: PS5 resource - // descriptors cannot represent ordinary 0x7F... host mappings. Linux - // HostMemory uses MAP_FIXED_NOREPLACE, making those low-address probes - // safe without clobbering existing host mappings. - if (OperatingSystem.IsMacOS()) + // POSIX treats the requested address as a hint, and Rosetta may + // relocate whole guest windows. Over-allocate once so the returned + // host range always contains an aligned guest-visible start. + if (!OperatingSystem.IsWindows()) { - // Prefer the requested low address. Besides matching the guest - // address model, this keeps the allocation representable by every - // PS5 GPU descriptor (the strictest ones carry 40 address bits). - try - { - var exactAddress = AllocateAt( - cursor, - alignedSize, - executable, - allowAlternative: false); - if (exactAddress == cursor) - { - actualAddress = exactAddress; - UpdateAllocationSearchCursor( - desiredAddress, - effectiveAlignment, - executable, - exactAddress + alignedSize); - return true; - } - } - catch - { - } - - // Over-allocate by the alignment so a kernel-chosen placement - // always contains an aligned start; the unused head/tail stays - // part of the tracked region and is simply never handed out. var reserveSize = effectiveAlignment > PageSize ? alignedSize + effectiveAlignment : alignedSize; @@ -294,10 +363,7 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA if (posixAddress != 0) { var alignedBase = AlignUp(posixAddress, effectiveAlignment); - const ulong gpuAddressLimit = 1UL << 40; - if (alignedBase < gpuAddressLimit && - alignedSize <= gpuAddressLimit - alignedBase && - alignedBase + alignedSize <= posixAddress + reserveSize) + if (alignedBase + alignedSize <= posixAddress + reserveSize) { actualAddress = alignedBase; UpdateAllocationSearchCursor(desiredAddress, effectiveAlignment, executable, alignedBase + alignedSize); @@ -327,19 +393,10 @@ public sealed unsafe class PhysicalVirtualMemory : IVirtualMemory, IGuestMemoryA continue; } - try - { - actualAddress = AllocateAt(cursor, alignedSize, executable, allowAlternative: false); - if (actualAddress == cursor) - { - UpdateAllocationSearchCursor(desiredAddress, effectiveAlignment, executable, actualAddress + alignedSize); - return true; - } - - actualAddress = 0; - } - catch + if (TryAllocateAtExact(cursor, alignedSize, executable, out actualAddress)) { + UpdateAllocationSearchCursor(desiredAddress, effectiveAlignment, executable, actualAddress + alignedSize); + return true; } cursor = AlignUp(cursor + effectiveAlignment, effectiveAlignment); diff --git a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs index beeaf7f..b929248 100644 --- a/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs +++ b/src/SharpEmu.Core/Runtime/SharpEmuRuntime.cs @@ -7,25 +7,21 @@ using SharpEmu.Core.Cpu.Disasm; using SharpEmu.Core.Loader; using SharpEmu.Core.Memory; using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Libs.VideoOut; using SharpEmu.Libs.Kernel; using SharpEmu.Libs.AppContent; using SharpEmu.Libs.SaveData; -using SharpEmu.Logging; +using SharpEmu.Libs.Fiber; using System.Buffers.Binary; using System.Collections.Generic; using System.Linq; using System.Text; -using System.IO; namespace SharpEmu.Core.Runtime; public sealed class SharpEmuRuntime : ISharpEmuRuntime { - private static readonly SharpEmuLogger Log = SharpEmuLog.For("Runtime"); - - private readonly record struct LoadedModuleImage(string Path, SelfImage Image); + private readonly record struct LoadedModuleImage(string Path, SelfImage Image, int Handle, bool StartAtBoot); private static readonly HashSet PreloadSkipModules = new(StringComparer.OrdinalIgnoreCase) { @@ -89,19 +85,14 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime moduleManager.RegisterExports(SharpEmu.Generated.SysAbiExportRegistry.CreateExports(Generation.Gen4 | Generation.Gen5)); moduleManager.Freeze(); - // Resolve the host platform once at the composition root; on unsupported - // OSes this throws PlatformNotSupportedException with a clear message - // instead of failing on the first native call. - var hostPlatform = HostPlatform.Current; - - var virtualMemory = new PhysicalVirtualMemory(hostPlatform.Memory); + var virtualMemory = new PhysicalVirtualMemory(); var fileSystem = new PhysicalFileSystem(); return new SharpEmuRuntime( new SelfLoader(), virtualMemory, - new CpuDispatcher(virtualMemory, moduleManager, hostPlatform: hostPlatform), + new CpuDispatcher(virtualMemory, moduleManager), moduleManager, Aerolib.Instance, cpuExecutionOptions, @@ -139,20 +130,20 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { var normalizedEbootPath = Path.GetFullPath(ebootPath); using var app0Binding = BindApp0Root(normalizedEbootPath); - Log.Info($"Loading: {ebootPath}"); + Console.Error.WriteLine($"[RUNTIME] Loading: {ebootPath}"); LastExecutionDiagnostics = null; LastExecutionTrace = null; LastSessionSummary = null; LastBasicBlockTrace = null; LastMilestoneLog = null; + FiberExports.ResetRuntimeState(); KernelModuleRegistry.Reset(); var image = LoadImage(normalizedEbootPath); - VideoOutExports.ConfigureApplicationInfo(image.Title, image.TitleId, image.Version, BuildInfo.CommitSha); + VideoOutExports.ConfigureApplicationInfo(image.Title, image.TitleId, image.Version); SaveDataExports.ConfigureApplicationInfo(image.TitleId); - LogAppBundleInfo(normalizedEbootPath, image); - RegisterLoadedModule(normalizedEbootPath, image, isMain: true, isSystemModule: false); + _ = RegisterLoadedModule(normalizedEbootPath, image, isMain: true, isSystemModule: false); KernelRuntimeCompatExports.ConfigureProcessProcParamAddress(image.ProcParamAddress); - Log.Info($"Entry: 0x{image.EntryPoint:X16}"); + Console.Error.WriteLine($"[RUNTIME] Entry: 0x{image.EntryPoint:X16}"); var generation = image.ElfHeader.AbiVersion == 2 ? Generation.Gen5 : Generation.Gen4; var activeImportStubs = new Dictionary(image.ImportStubs); var activeRuntimeSymbols = new Dictionary(image.RuntimeSymbols, StringComparer.Ordinal); @@ -175,7 +166,7 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime processImageName); if (initializerResult is { } failedInitializerResult) { - Log.Error($"Initializer dispatch failed: {failedInitializerResult}"); + Console.Error.WriteLine($"[RUNTIME] Initializer dispatch failed: {failedInitializerResult}"); LastExecutionTrace = _cpuDispatcher.LastImportResolutionTrace; LastMilestoneLog = _cpuDispatcher.LastMilestoneLog; LastSessionSummary = BuildSessionSummary(_cpuDispatcher.LastSessionSummary); @@ -183,8 +174,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return failedInitializerResult; } - Log.Info($"Dispatching, gen: {generation}"); - Log.Debug($"About to call DispatchEntry with entryPoint=0x{image.EntryPoint:X16}"); + Console.Error.WriteLine($"[RUNTIME] Dispatching, gen: {generation}"); + Console.Error.WriteLine($"[RUNTIME] About to call DispatchEntry with entryPoint=0x{image.EntryPoint:X16}"); var result = _cpuDispatcher.DispatchEntry( image.EntryPoint, @@ -194,8 +185,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime processImageName, _cpuExecutionOptions); - Log.Info($"DispatchEntry returned: {result}"); - Log.Info($"Dispatch result: {result}"); + Console.Error.WriteLine($"[RUNTIME] DispatchEntry returned: {result}"); + Console.Error.WriteLine($"[RUNTIME] Dispatch result: {result}"); LastExecutionTrace = _cpuDispatcher.LastImportResolutionTrace; LastMilestoneLog = _cpuDispatcher.LastMilestoneLog; LastSessionSummary = BuildSessionSummary(_cpuDispatcher.LastSessionSummary); @@ -366,66 +357,6 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return result; } - private static void LogAppBundleInfo(string ebootPath, SelfImage image) - { - var executableName = Path.GetFileName(ebootPath); - if (string.IsNullOrWhiteSpace(executableName)) - { - executableName = "eboot.bin"; - } - - var displayName = string.IsNullOrWhiteSpace(image.Title) ? "(unknown)" : image.Title!.Trim(); - var titleId = string.IsNullOrWhiteSpace(image.TitleId) ? "(unknown)" : image.TitleId!.Trim(); - var version = string.IsNullOrWhiteSpace(image.Version) ? "(unknown)" : image.Version!.Trim(); - var contentId = ResolveContentId(Path.GetDirectoryName(ebootPath)) ?? "(unknown)"; - - var builder = new StringBuilder(); - builder.AppendLine("App bundle info:"); - builder.AppendLine($"- Display name: {displayName}"); - builder.AppendLine($"- Version: {version}"); - builder.AppendLine($"- Title ID: {titleId}"); - builder.AppendLine($"- Content ID: {contentId}"); - builder.AppendLine($"- Executable: {executableName}"); - builder.Append("- Platform: PlayStation 5"); - Log.Info(builder.ToString()); - } - - private static string? ResolveContentId(string? bundleRoot) - { - if (string.IsNullOrEmpty(bundleRoot)) - { - return null; - } - - foreach (var candidate in new[] - { - Path.Combine(bundleRoot, "sce_sys", "param.json"), - Path.Combine(bundleRoot, "param.json"), - }) - { - if (!File.Exists(candidate)) - { - continue; - } - - try - { - using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllBytes(candidate)); - if (doc.RootElement.TryGetProperty("contentId", out var contentId)) - { - return contentId.GetString(); - } - } - catch (Exception ex) when (ex is IOException or System.Text.Json.JsonException) - { - } - - break; - } - - return null; - } - private static App0BindingScope? BindApp0Root(string normalizedEbootPath) { const string app0VariableName = "SHARPEMU_APP0_DIR"; @@ -440,6 +371,20 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return null; } + // Dump tools commonly place decrypted executables in an app0/decrypted + // sidecar while leaving Unity data in the parent app0 directory. Keep + // loading code/modules beside the decrypted eboot, but mount /app0 at + // the content-bearing parent so boot.config, metadata and assets resolve. + if (string.Equals(Path.GetFileName(app0Root), "decrypted", StringComparison.OrdinalIgnoreCase)) + { + var parentRoot = Path.GetDirectoryName(app0Root); + if (!string.IsNullOrWhiteSpace(parentRoot) && + Directory.Exists(Path.Combine(parentRoot, "Media"))) + { + app0Root = parentRoot; + } + } + Environment.SetEnvironmentVariable(app0VariableName, app0Root); return new App0BindingScope(app0VariableName); } @@ -485,6 +430,56 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return null; } + private bool TryGetEhFrameInfo( + SelfImage image, + ulong imageSize, + out ulong ehFrameHeaderAddress, + out ulong ehFrameAddress, + out ulong ehFrameSize) + { + ehFrameHeaderAddress = 0; + ehFrameAddress = 0; + ehFrameSize = 0; + var imageBase = image.EntryPoint >= image.ElfHeader.EntryPoint + ? image.EntryPoint - image.ElfHeader.EntryPoint + : 0UL; + for (var i = 0; i < image.ProgramHeaders.Count; i++) + { + var header = image.ProgramHeaders[i]; + if (header.HeaderType != ProgramHeaderType.GnuEhFrame || header.MemorySize < 8) + { + continue; + } + + var headerAddress = imageBase + header.VirtualAddress; + Span ehHeader = stackalloc byte[8]; + if (!_virtualMemory.TryRead(headerAddress, ehHeader) || + ehHeader[0] != 1 || + ehHeader[1] != 0x1B) + { + continue; + } + + var relativeOffset = BinaryPrimitives.ReadInt32LittleEndian(ehHeader[4..]); + ehFrameHeaderAddress = headerAddress; + ehFrameAddress = unchecked((ulong)((long)headerAddress + 4 + relativeOffset)); + if (ehFrameAddress < 0x10000) + { + continue; + } + + var relativeEhFrameAddress = ehFrameAddress - imageBase; + ehFrameSize = header.VirtualAddress > relativeEhFrameAddress + ? header.VirtualAddress - relativeEhFrameAddress + : imageSize > header.VirtualAddress + ? imageSize - header.VirtualAddress + : 0; + return true; + } + + return false; + } + private OrbisGen2Result? RunPreloadedModuleInitializers( IReadOnlyList loadedModuleImages, Generation generation, @@ -494,20 +489,30 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime for (var i = 0; i < loadedModuleImages.Count; i++) { var loadedModule = loadedModuleImages[i]; + if (!loadedModule.StartAtBoot) + { + continue; + } + var initEntryPoint = loadedModule.Image.InitFunctionEntryPoint; if (initEntryPoint < 0x10000) { continue; } + if (!KernelModuleRegistry.TryBeginModuleStart(loadedModule.Handle, out _)) + { + continue; + } + var moduleName = Path.GetFileName(loadedModule.Path); if (string.IsNullOrWhiteSpace(moduleName)) { moduleName = $"module#{i}"; } - Log.Info( - $"Starting module {moduleName}: dt_init=0x{initEntryPoint:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Starting module {moduleName}: dt_init=0x{initEntryPoint:X16}"); var result = _cpuDispatcher.DispatchModuleInitializer( initEntryPoint, @@ -516,10 +521,13 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime activeRuntimeSymbols, moduleName, _cpuExecutionOptions); + KernelModuleRegistry.CompleteModuleStart( + loadedModule.Handle, + result == OrbisGen2Result.ORBIS_GEN2_OK); if (result != OrbisGen2Result.ORBIS_GEN2_OK) { - Log.Error( - $"Module start failed: {moduleName} -> {result}"); + Console.Error.WriteLine( + $"[RUNTIME] Module start failed: {moduleName} -> {result}"); return result; } } @@ -540,8 +548,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return null; } - Log.Info( - $"Running initializers for {label}: preinit={image.PreInitializerFunctions.Count}, init={image.InitializerFunctions.Count}"); + Console.Error.WriteLine( + $"[RUNTIME] Running initializers for {label}: preinit={image.PreInitializerFunctions.Count}, init={image.InitializerFunctions.Count}"); var result = RunInitializerList( $"{label}:preinit", @@ -580,8 +588,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime continue; } - Log.Debug( - $" Initializer {label}[{i}] -> 0x{initializerAddress:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Initializer {label}[{i}] -> 0x{initializerAddress:X16}"); var result = _cpuDispatcher.DispatchEntry( initializerAddress, @@ -613,12 +621,17 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime var moduleDirectories = new[] { - Path.Combine(ebootDirectory, "sce_module"), - Path.Combine(ebootDirectory, "sce_modules"), - Path.Combine(ebootDirectory, "Media", "Modules"), + (Path: Path.Combine(ebootDirectory, "sce_module"), StartAtBoot: true), + (Path: Path.Combine(ebootDirectory, "sce_modules"), StartAtBoot: true), + (Path: Path.Combine(ebootDirectory, "Media", "Modules"), StartAtBoot: true), + // Unity native plugins are loaded later through sceKernelLoadStartModule. Map + // them up front so the HLE loader can return a real module handle and dlsym + // can resolve their exports, but defer DT_INIT until the guest requests them. + (Path: Path.Combine(ebootDirectory, "Media", "Plugins"), StartAtBoot: false), } - .Distinct(StringComparer.OrdinalIgnoreCase) - .Where(Directory.Exists) + .GroupBy(entry => entry.Path, StringComparer.OrdinalIgnoreCase) + .Select(group => group.First()) + .Where(entry => Directory.Exists(entry.Path)) .ToArray(); if (moduleDirectories.Length == 0) @@ -628,28 +641,30 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime var allModulePaths = moduleDirectories .SelectMany(directory => Directory - .EnumerateFiles(directory) - .OrderBy(path => path, StringComparer.OrdinalIgnoreCase)) - .Where(path => + .EnumerateFiles(directory.Path) + .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) + .Select(path => (Path: path, directory.StartAtBoot))) + .Where(entry => { - var extension = Path.GetExtension(path); + var extension = Path.GetExtension(entry.Path); return string.Equals(extension, ".prx", StringComparison.OrdinalIgnoreCase) || string.Equals(extension, ".sprx", StringComparison.OrdinalIgnoreCase); }) - .Distinct(StringComparer.OrdinalIgnoreCase) + .GroupBy(entry => entry.Path, StringComparer.OrdinalIgnoreCase) + .Select(group => group.First()) .ToArray(); var modulePaths = allModulePaths - .Where(ShouldPreloadModule) + .Where(entry => ShouldPreloadModule(entry.Path)) .ToArray(); var skippedModules = allModulePaths - .Where(path => !ShouldPreloadModule(path)) - .Select(Path.GetFileName) + .Where(entry => !ShouldPreloadModule(entry.Path)) + .Select(entry => Path.GetFileName(entry.Path)) .Where(name => !string.IsNullOrWhiteSpace(name)) .ToArray(); if (skippedModules.Length > 0) { - Log.Info($"Skipping {skippedModules.Length} core module(s): {string.Join(", ", skippedModules)}"); + Console.Error.WriteLine($"[RUNTIME] Skipping {skippedModules.Length} core module(s): {string.Join(", ", skippedModules)}"); } if (modulePaths.Length == 0) @@ -657,14 +672,15 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return loadedImages; } - Log.Debug($"Module search directories: {string.Join(", ", moduleDirectories)}"); - Log.Info($"Loading {modulePaths.Length} module(s)..."); + Console.Error.WriteLine($"[RUNTIME] Module search directories: {string.Join(", ", moduleDirectories.Select(entry => entry.Path))}"); + Console.Error.WriteLine($"[RUNTIME] Loading {modulePaths.Length} module(s)..."); var loadedModules = 0; var failedModules = 0; var mergedImportCount = 0; var mergedSymbolCount = 0; - foreach (var modulePath in modulePaths) + foreach (var moduleEntry in modulePaths) { + var modulePath = moduleEntry.Path; try { var fileInfo = new FileInfo(modulePath); @@ -689,25 +705,52 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime mergedImportCount += MergeImportStubs(importStubs, moduleImage.ImportStubs, modulePath); mergedSymbolCount += MergeRuntimeSymbols(runtimeSymbols, moduleImage.RuntimeSymbols); - RegisterLoadedModule(modulePath, moduleImage, isMain: false, isSystemModule: false); - loadedImages.Add(new LoadedModuleImage(modulePath, moduleImage)); + InstallNativePluginCompatibilityHooks(importStubs, moduleImage, modulePath); + var moduleHandle = RegisterLoadedModule(modulePath, moduleImage, isMain: false, isSystemModule: false); + var moduleName = Path.GetFileName(modulePath); + var startAtBoot = moduleEntry.StartAtBoot || + string.Equals(moduleName, "libfmod.prx", StringComparison.OrdinalIgnoreCase) || + string.Equals(moduleName, "libfmodstudio.prx", StringComparison.OrdinalIgnoreCase); + loadedImages.Add(new LoadedModuleImage(modulePath, moduleImage, moduleHandle, startAtBoot)); loadedModules++; - Log.Info( - $"Loaded module {Path.GetFileName(modulePath)}: entry=0x{moduleImage.EntryPoint:X16}, imports={moduleImage.ImportStubs.Count}, symbols={moduleImage.RuntimeSymbols.Count}"); + Console.Error.WriteLine( + $"[RUNTIME] Loaded module {Path.GetFileName(modulePath)}: entry=0x{moduleImage.EntryPoint:X16}, imports={moduleImage.ImportStubs.Count}, symbols={moduleImage.RuntimeSymbols.Count}"); } catch (Exception ex) { failedModules++; - Log.Error($"Module load failed: {modulePath} ({ex.GetType().Name}: {ex.Message})", ex); + Console.Error.WriteLine($"[RUNTIME] Module load failed: {modulePath} ({ex.GetType().Name}: {ex.Message})"); } } - Log.Info( - $"Module preload summary: loaded={loadedModules}, failed={failedModules}, merged_imports={mergedImportCount}, merged_symbols={mergedSymbolCount}"); + Console.Error.WriteLine( + $"[RUNTIME] Module preload summary: loaded={loadedModules}, failed={failedModules}, merged_imports={mergedImportCount}, merged_symbols={mergedSymbolCount}"); return loadedImages; } + private static void InstallNativePluginCompatibilityHooks( + IDictionary importStubs, + SelfImage moduleImage, + string modulePath) + { + if (!string.Equals(Path.GetFileName(modulePath), "libfmod.prx", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + ReadOnlySpan compatibilityNids = ["uPLTdl3psGk"]; + foreach (var nid in compatibilityNids) + { + if (moduleImage.RuntimeSymbols.TryGetValue(nid, out var address) && address >= 0x10000) + { + importStubs[address] = nid; + Console.Error.WriteLine( + $"[RUNTIME] Installed FMOD compatibility hook: {nid} -> 0x{address:X16}"); + } + } + } + private void RebindImportedDataSymbols( SelfImage mainImage, IReadOnlyList loadedModuleImages, @@ -724,8 +767,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime if (rebound != 0 || unresolved != 0) { - Log.Info( - $"Imported data rebind: rebound={rebound}, unresolved={unresolved}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data rebind: rebound={rebound}, unresolved={unresolved}"); } } @@ -757,8 +800,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { if (logRebind) { - Log.Warning( - $"Imported data unresolved: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} addend=0x{unchecked((ulong)relocation.Addend):X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data unresolved: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} addend=0x{unchecked((ulong)relocation.Addend):X16}"); } unresolved++; @@ -770,8 +813,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { if (logRebind) { - Log.Error( - $"Imported data write-failed: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data write-failed: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); } unresolved++; @@ -780,8 +823,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime if (logRebind) { - Log.Debug( - $"Imported data rebound: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); + Console.Error.WriteLine( + $"[RUNTIME] Imported data rebound: nid={relocation.Nid} target=0x{relocation.TargetAddress:X16} value=0x{reboundValue:X16}"); } rebound++; @@ -817,8 +860,8 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime { if (!string.Equals(existingNid, nid, StringComparison.Ordinal)) { - Log.Warning( - $"Import stub conflict at 0x{address:X16}: keep={existingNid}, skip={nid} ({Path.GetFileName(modulePath)})"); + Console.Error.WriteLine( + $"[RUNTIME] Import stub conflict at 0x{address:X16}: keep={existingNid}, skip={nid} ({Path.GetFileName(modulePath)})"); } continue; @@ -912,7 +955,7 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime return !PreloadSkipModules.Contains(fileName); } - private static void RegisterLoadedModule(string modulePath, SelfImage image, bool isMain, bool isSystemModule) + private int RegisterLoadedModule(string modulePath, SelfImage image, bool isMain, bool isSystemModule) { if (!TryComputeImageRange(image, out var baseAddress, out var size)) { @@ -920,15 +963,27 @@ public sealed class SharpEmuRuntime : ISharpEmuRuntime size = 0; } + _ = TryGetEhFrameInfo( + image, + size, + out var ehFrameHeaderAddress, + out var ehFrameAddress, + out var ehFrameSize); var handle = KernelModuleRegistry.RegisterModule( modulePath, baseAddress, size, image.EntryPoint, + image.InitFunctionEntryPoint, + ehFrameHeaderAddress, + ehFrameAddress, + ehFrameSize, isMain, isSystemModule); - Log.Info( - $"Registered module handle={handle} name={Path.GetFileName(modulePath)} base=0x{baseAddress:X16} size=0x{size:X16}"); + KernelModuleRegistry.RegisterModuleSymbols(handle, image.RuntimeSymbols); + Console.Error.WriteLine( + $"[RUNTIME] Registered module handle={handle} name={Path.GetFileName(modulePath)} base=0x{baseAddress:X16} size=0x{size:X16}"); + return handle; } private static bool TryComputeImageRange(SelfImage image, out ulong baseAddress, out ulong size) diff --git a/src/SharpEmu.HLE/CpuContext.cs b/src/SharpEmu.HLE/CpuContext.cs index adb3768..66c24d6 100644 --- a/src/SharpEmu.HLE/CpuContext.cs +++ b/src/SharpEmu.HLE/CpuContext.cs @@ -26,6 +26,12 @@ public sealed class CpuContext(ICpuMemory memory, Generation generation) public ulong GsBase { get; set; } + /// x87 control word observed at the current guest boundary. + public ushort FpuControlWord { get; set; } = 0x037F; + + /// MXCSR observed at the current guest boundary. + public uint Mxcsr { get; set; } = 0x1F80; + public ulong this[CpuRegister register] { get => _registers[(int)register]; diff --git a/src/SharpEmu.HLE/GuestImageWriteTracker.cs b/src/SharpEmu.HLE/GuestImageWriteTracker.cs new file mode 100644 index 0000000..079bd09 --- /dev/null +++ b/src/SharpEmu.HLE/GuestImageWriteTracker.cs @@ -0,0 +1,613 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Globalization; +using System.Runtime.InteropServices; + +namespace SharpEmu.HLE; + +/// +/// Detects guest CPU writes into memory that backs a host GPU image. On PS5 +/// render targets alias unified memory, so games freely mix CPU writes and GPU +/// draws on the same surface (Chowdren titles memset their fog layers every +/// frame). Host GPU images are separate storage, so the video backend needs to +/// know when the guest CPU rewrote a surface to re-upload it. Ranges are +/// write-protected; the first write faults, the fault handler restores write +/// access and marks the range dirty, and the video backend consumes the dirty +/// flag once per flip and re-arms protection after re-uploading. +/// +public static unsafe class GuestImageWriteTracker +{ + private const int ProtRead = 0x1; + private const int ProtWrite = 0x2; + private const int ClockMonotonicRaw = 4; + + private sealed class TrackedRange + { + public ulong Address; + public ulong ByteCount; + public ulong Start; + public ulong End; + public int Dirty; + public int Armed; + public int FirstCpuWriteSeen; + public int PendingFirstCpuWrite; + public bool TraceLifetime; + public long SourceSequence; + public long FirstCpuWriteTraceSequence; + public long FirstCpuWriteTimestampNanoseconds; + public ulong FirstCpuWriteAddress; + public ulong FirstCpuWritePage; + public string Source = "unspecified"; + } + + [StructLayout(LayoutKind.Sequential)] + private struct Timespec + { + public long Seconds; + public long Nanoseconds; + } + + private static readonly object _gate = new(); + private static readonly Dictionary _rangesByAddress = new(); + + // Snapshot array read lock-free from the signal handler; rebuilt on every + // mutation under the gate. Signal handlers must not take managed locks. + private static TrackedRange[] _rangeSnapshot = []; + + private static readonly bool _enabled = !OperatingSystem.IsWindows() && + Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_CPU_SYNC") != "0"; + private static readonly (bool Wildcard, ulong[] Addresses) _lifetimeTraceFilter = + ParseAddressList(Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_ADDRS")); + private static readonly (bool Wildcard, string[] Sources) _lifetimeSourceTraceFilter = + ParseSourceList(Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_MEMORY_LIFETIME")); + private static readonly bool _lifetimeTraceEnabled = + _lifetimeTraceFilter.Wildcard || + _lifetimeTraceFilter.Addresses.Length != 0 || + _lifetimeSourceTraceFilter.Wildcard || + _lifetimeSourceTraceFilter.Sources.Length != 0; + private static readonly long _lifetimeTraceEpochNanoseconds = + _enabled && _lifetimeTraceEnabled ? GetMonotonicNanoseconds() : 0; + private static long _lifetimeTraceSequence; + + [DllImport("libc", EntryPoint = "mprotect", SetLastError = true)] + private static extern int Mprotect(nint address, nuint length, int protection); + + [DllImport("libc", EntryPoint = "clock_gettime", SetLastError = false)] + private static extern int ClockGetTime(int clockId, Timespec* time); + + public static bool Enabled => _enabled; + + /// + /// Exercises the fault-handling path once outside signal context so every + /// branch is JIT-compiled (and, under Rosetta 2, translated) before a real + /// fault arrives — a cold signal path is silently never entered there. + /// + public static void WarmUp() + { + if (!_enabled) + { + return; + } + + var scratch = NativeMemory.AllocZeroed(4096); + try + { + // Warm the timestamp P/Invoke used by the signal-safe scalar + // capture path before a real protected-page write reaches it. + _ = GetMonotonicNanoseconds(); + var address = (ulong)scratch; + Track(address, 4096); + _ = TryHandleWriteFault(address); + _ = ConsumeDirty(address); + Untrack(address); + } + finally + { + NativeMemory.Free(scratch); + } + } + + /// Registers a range and arms write protection on it. + public static void Track( + ulong address, + ulong byteCount, + long sourceSequence = 0, + string source = "unspecified") + { + if (!_enabled || address == 0 || byteCount == 0) + { + return; + } + + var (start, length) = PageAlign(address, byteCount); + lock (_gate) + { + _rangesByAddress.TryGetValue(address, out var range); + if (range is not null && + (range.Start != start || + range.End != start + length || + range.ByteCount != byteCount)) + { + // Never resize an object that is still reachable from the + // signal handler's lock-free snapshot. Retire it and publish + // a fresh immutable range. + DisarmLocked(range, "replace-range"); + _rangesByAddress.Remove(address); + range = null; + } + + if (range is null) + { + range = new TrackedRange + { + Address = address, + ByteCount = byteCount, + Start = start, + End = start + length, + TraceLifetime = + ShouldTraceRange(start, start + length) || ShouldTraceSource(source), + SourceSequence = sourceSequence, + Source = source, + }; + _rangesByAddress[address] = range; + RebuildSnapshotLocked(); + } + else + { + FlushPendingFirstCpuWrite(range); + } + + range.SourceSequence = sourceSequence; + range.Source = source; + range.TraceLifetime = + ShouldTraceRange(range.Start, range.End) || ShouldTraceSource(source); + ArmLocked(range, "arm"); + } + } + + public static void Untrack(ulong address) + { + if (!_enabled) + { + return; + } + + lock (_gate) + { + if (_rangesByAddress.TryGetValue(address, out var range)) + { + DisarmLocked(range, "untrack"); + _rangesByAddress.Remove(address); + RebuildSnapshotLocked(); + } + } + } + + /// + /// Returns true when the guest CPU wrote the range since the last call, + /// clearing the flag. The caller re-arms via after it + /// finished reading the guest bytes. + /// + public static bool ConsumeDirty(ulong address) + { + if (!_enabled) + { + return false; + } + + lock (_gate) + { + if (!_rangesByAddress.TryGetValue(address, out var range)) + { + return false; + } + + FlushPendingFirstCpuWrite(range); + return Interlocked.Exchange(ref range.Dirty, 0) != 0; + } + } + + /// + /// Non-consuming variant of : reports whether + /// the range has been written since it was last re-armed, leaving the + /// flag for the owner that evicts and re-uploads. + /// + public static bool PeekDirty(ulong address) + { + if (!_enabled) + { + return false; + } + + lock (_gate) + { + if (!_rangesByAddress.TryGetValue(address, out var range)) + { + return false; + } + + FlushPendingFirstCpuWrite(range); + return Volatile.Read(ref range.Dirty) != 0; + } + } + + public static void Rearm(ulong address) + { + if (!_enabled) + { + return; + } + + lock (_gate) + { + if (_rangesByAddress.TryGetValue(address, out var range)) + { + ArmLocked(range, "rearm"); + } + } + } + + /// + /// Prepares pages touched by a managed HLE memory write. Native guest + /// stores fault and enter through the + /// POSIX signal bridge, but a managed Buffer.MemoryCopy into a protected + /// page is surfaced by the runtime as a fatal AccessViolation instead of + /// a resumable guest fault. Visit every page in the write span up front so + /// all overlapping texture owners are dirtied and made writable. + /// + public static void NotifyManagedWrite(ulong address, ulong byteCount) + { + if (!_enabled || address == 0 || byteCount == 0) + { + return; + } + + var end = address > ulong.MaxValue - byteCount + ? ulong.MaxValue + : address + byteCount; + var candidate = address; + while (candidate < end) + { + _ = TryHandleWriteFault(candidate); + var nextPage = (candidate & ~0xFFFUL) + 0x1000UL; + if (nextPage <= candidate) + { + break; + } + candidate = nextPage; + } + } + + /// + /// Flushes scalar first-write records captured by the POSIX signal handler. + /// Call only from ordinary managed execution, never from signal context. + /// + public static void FlushPendingDiagnostics() + { + if (!_enabled || !_lifetimeTraceEnabled) + { + return; + } + + lock (_gate) + { + foreach (var range in _rangesByAddress.Values) + { + FlushPendingFirstCpuWrite(range); + } + } + } + + /// + /// Signal-handler entry: if the fault address lies in a tracked, armed + /// range, restore write access, mark the range dirty, and return true so + /// the faulting write can be retried. Must not allocate or lock. + /// + public static bool TryHandleWriteFault(ulong faultAddress) + { + if (!_enabled || faultAddress == 0) + { + return false; + } + + var ranges = Volatile.Read(ref _rangeSnapshot); + var writableStart = ulong.MaxValue; + var writableEnd = 0UL; + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (faultAddress < range.Start || faultAddress >= range.End) + { + continue; + } + + writableStart = Math.Min(writableStart, range.Start); + writableEnd = Math.Max(writableEnd, range.End); + } + + if (writableStart == ulong.MaxValue) + { + return false; + } + + // Ranges are page-aligned and may overlap (font atlases and other + // suballocations commonly share pages). Unprotecting one range also + // makes every overlapping tracked page writable. Expand to the full + // transitive overlap and dirty/disarm every owner, otherwise only the + // first dictionary entry observes the write and the others retain a + // stale cached texture indefinitely. + var expanded = true; + while (expanded) + { + expanded = false; + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (range.Start >= writableEnd || range.End <= writableStart) + { + continue; + } + + var start = Math.Min(writableStart, range.Start); + var end = Math.Max(writableEnd, range.End); + if (start != writableStart || end != writableEnd) + { + writableStart = start; + writableEnd = end; + expanded = true; + } + } + } + + var needsUnprotect = false; + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (range.Start < writableEnd && range.End > writableStart && + Volatile.Read(ref range.Armed) != 0) + { + needsUnprotect = true; + break; + } + } + + if (needsUnprotect && + Mprotect( + (nint)writableStart, + (nuint)(writableEnd - writableStart), + ProtRead | ProtWrite) != 0) + { + return false; + } + + for (var index = 0; index < ranges.Length; index++) + { + var range = ranges[index]; + if (range.Start >= writableEnd || range.End <= writableStart) + { + continue; + } + + var wasArmed = Interlocked.Exchange(ref range.Armed, 0) != 0; + if (wasArmed && + range.TraceLifetime && + Interlocked.CompareExchange(ref range.FirstCpuWriteSeen, 1, 0) == 0) + { + // Signal context: capture preallocated scalar fields only. + // Formatting and I/O are deferred to a locked safe path. + range.FirstCpuWriteTraceSequence = + Interlocked.Increment(ref _lifetimeTraceSequence); + range.FirstCpuWriteTimestampNanoseconds = GetMonotonicNanoseconds(); + range.FirstCpuWriteAddress = faultAddress; + range.FirstCpuWritePage = faultAddress & ~0xFFFUL; + Volatile.Write(ref range.PendingFirstCpuWrite, 1); + Volatile.Write(ref range.FirstCpuWriteSeen, 2); + } + + Volatile.Write(ref range.Dirty, 1); + } + + return true; + } + + private static void ArmLocked(TrackedRange range, string operation) + { + FlushPendingFirstCpuWrite(range); + if (Interlocked.Exchange(ref range.Armed, 1) == 1) + { + return; + } + + // A new publication/rearm starts a new first-write lifetime. + Volatile.Write(ref range.FirstCpuWriteSeen, 0); + var failed = Mprotect( + (nint)range.Start, + (nuint)(range.End - range.Start), + ProtRead) != 0; + if (failed) + { + Volatile.Write(ref range.Armed, 0); + } + + if (range.TraceLifetime) + { + TraceLifetime( + range, + failed ? $"{operation}-failed-errno-{Marshal.GetLastPInvokeError()}" : operation); + } + } + + private static void DisarmLocked(TrackedRange range, string operation) + { + FlushPendingFirstCpuWrite(range); + var wasArmed = Interlocked.Exchange(ref range.Armed, 0) == 1; + if (wasArmed) + { + _ = Mprotect( + (nint)range.Start, + (nuint)(range.End - range.Start), + ProtRead | ProtWrite); + } + + if (range.TraceLifetime) + { + TraceLifetime(range, wasArmed ? operation : $"{operation}-already-disarmed"); + } + } + + private static void RebuildSnapshotLocked() + { + _rangeSnapshot = _rangesByAddress.Values.ToArray(); + } + + private static (ulong Start, ulong Length) PageAlign(ulong address, ulong byteCount) + { + const ulong pageMask = 0xFFFUL; + var start = address & ~pageMask; + var end = (address + byteCount + pageMask) & ~pageMask; + return (start, end - start); + } + + private static bool ShouldTraceRange(ulong start, ulong end) + { + if (_lifetimeTraceFilter.Wildcard) + { + return true; + } + + var addresses = _lifetimeTraceFilter.Addresses; + for (var index = 0; index < addresses.Length; index++) + { + if (addresses[index] >= start && addresses[index] < end) + { + return true; + } + } + + return false; + } + + private static (bool Wildcard, ulong[] Addresses) ParseAddressList(string? addresses) + { + if (string.IsNullOrWhiteSpace(addresses)) + { + return (false, []); + } + + var parsedAddresses = new List(); + foreach (var token in addresses.Split( + [',', ';', ' ', '\t'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (token == "*") + { + return (true, []); + } + + var span = token.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + if (ulong.TryParse(span, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) + { + parsedAddresses.Add(parsed); + } + } + + return (false, parsedAddresses.ToArray()); + } + + private static bool ShouldTraceSource(string source) + { + if (_lifetimeSourceTraceFilter.Wildcard) + { + return true; + } + + return Array.IndexOf(_lifetimeSourceTraceFilter.Sources, source) >= 0; + } + + private static (bool Wildcard, string[] Sources) ParseSourceList(string? sources) + { + if (string.IsNullOrWhiteSpace(sources)) + { + return (false, []); + } + + var parsedSources = new List(); + foreach (var token in sources.Split( + [',', ';'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (token == "*") + { + return (true, []); + } + + parsedSources.Add(token); + } + + return (false, parsedSources.ToArray()); + } + + private static void FlushPendingFirstCpuWrite(TrackedRange range) + { + var spin = new SpinWait(); + while (Volatile.Read(ref range.FirstCpuWriteSeen) == 1) + { + spin.SpinOnce(); + } + + if (!range.TraceLifetime || Interlocked.Exchange(ref range.PendingFirstCpuWrite, 0) == 0) + { + return; + } + + TraceLifetime( + range, + "first-cpu-write-disarm", + range.FirstCpuWriteAddress, + range.FirstCpuWritePage, + range.FirstCpuWriteTraceSequence, + range.FirstCpuWriteTimestampNanoseconds); + } + + private static void TraceLifetime( + TrackedRange range, + string operation, + ulong faultAddress = 0, + ulong faultPage = 0, + long traceSequence = 0, + long timestampNanoseconds = 0) + { + if (traceSequence == 0) + { + traceSequence = Interlocked.Increment(ref _lifetimeTraceSequence); + } + + if (timestampNanoseconds == 0) + { + timestampNanoseconds = GetMonotonicNanoseconds(); + } + + var elapsedMilliseconds = + (timestampNanoseconds - _lifetimeTraceEpochNanoseconds) / 1_000_000.0; + Console.Error.WriteLine( + $"[WT][LIFETIME] seq={traceSequence} t_ms={elapsedMilliseconds:F3} " + + $"event={operation} source_seq={range.SourceSequence} source='{range.Source}' " + + $"requested=0x{range.Address:X16}+0x{range.ByteCount:X} " + + $"range=0x{range.Start:X16}..0x{range.End:X16} " + + $"fault=0x{faultAddress:X16} page=0x{faultPage:X16}"); + } + + private static long GetMonotonicNanoseconds() + { + Timespec time; + return ClockGetTime(ClockMonotonicRaw, &time) == 0 + ? unchecked((time.Seconds * 1_000_000_000L) + time.Nanoseconds) + : 0; + } +} diff --git a/src/SharpEmu.HLE/GuestThreadExecution.cs b/src/SharpEmu.HLE/GuestThreadExecution.cs index 7a0c5ab..90d6f08 100644 --- a/src/SharpEmu.HLE/GuestThreadExecution.cs +++ b/src/SharpEmu.HLE/GuestThreadExecution.cs @@ -41,6 +41,14 @@ public interface IGuestThreadScheduler { bool SupportsGuestContextTransfer { get; } + /// + /// Associates a pthread identity created on the primary guest executor + /// with its live CPU context. Primary execution does not pass through + /// TryStartThread, but kernel exception delivery must still be able to + /// target it. + /// + void RegisterGuestThreadContext(ulong threadHandle, CpuContext context); + bool TryStartThread(CpuContext creatorContext, GuestThreadStartRequest request, out string? error); bool TryJoinThread( @@ -53,6 +61,19 @@ public interface IGuestThreadScheduler int WakeBlockedThreads(string wakeKey, int maxCount = int.MaxValue); + /// + /// Applies a new guest scheduling priority to a live thread, mapping it + /// onto the host thread if one is running. Returns false when the thread + /// handle is unknown. + /// + bool TrySetGuestThreadPriority(ulong guestThreadHandle, int guestPriority); + + /// + /// Records a new affinity mask for a guest thread and re-applies it to + /// the host thread where the platform supports it. + /// + bool TrySetGuestThreadAffinity(ulong guestThreadHandle, ulong affinityMask); + IReadOnlyList SnapshotThreads(); bool TryCallGuestFunction( @@ -65,11 +86,36 @@ public interface IGuestThreadScheduler string reason, out string? error); + bool TryCallGuestFunction( + CpuContext callerContext, + ulong entryPoint, + ulong arg0, + ulong arg1, + ulong arg2, + ulong stackAddress, + ulong stackSize, + string reason, + out ulong returnValue, + out string? error); + bool TryCallGuestContinuation( CpuContext callerContext, GuestCpuContinuation continuation, string reason, out string? error); + + /// + /// Asynchronously invokes an installed kernel exception handler as the + /// target guest thread. This is used by IL2CPP's stop-the-world collector: + /// the handler acknowledges suspension and may remain blocked until the + /// collecting thread resumes it. + /// + bool TryRaiseGuestException( + CpuContext callerContext, + ulong threadHandle, + ulong handler, + int exceptionType, + out string? error); } public readonly record struct GuestImportCallFrame( @@ -94,13 +140,34 @@ public readonly record struct GuestCpuContinuation( ulong Rdi, ulong R8, ulong R9, + ulong R10, + ulong R11, ulong R12, ulong R13, ulong R14, - ulong R15); + ulong R15, + ushort FpuControlWord, + uint Mxcsr, + bool RestoreFullFpuState); public static class GuestThreadExecution { + private sealed class DelegateGuestThreadBlockWaiter : IGuestThreadBlockWaiter + { + private readonly Func _resume; + private readonly Func _tryWake; + + public DelegateGuestThreadBlockWaiter(Func resume, Func tryWake) + { + _resume = resume; + _tryWake = tryWake; + } + + public int Resume() => _resume(); + + public bool TryWake() => _tryWake(); + } + [ThreadStatic] private static ulong _currentGuestThreadHandle; @@ -246,6 +313,23 @@ public static class GuestThreadExecution return true; } + // Compatibility bridge for exports that still describe blocked work as a + // resume/wake delegate pair. New hot paths should provide an + // IGuestThreadBlockWaiter directly to avoid allocating closures. + public static bool RequestCurrentThreadBlock( + CpuContext? context, + string reason, + string? wakeKey, + Func resumeHandler, + Func wakeHandler, + long blockDeadlineTimestamp = 0) => + RequestCurrentThreadBlock( + context, + reason, + wakeKey, + new DelegateGuestThreadBlockWaiter(resumeHandler, wakeHandler), + blockDeadlineTimestamp); + public static bool TryConsumeCurrentThreadBlock(out string reason) { return TryConsumeCurrentThreadBlock(out reason, out _, out _); @@ -314,6 +398,27 @@ public static class GuestThreadExecution return true; } + public static bool TryConsumeCurrentThreadBlock( + out string reason, + out GuestCpuContinuation continuation, + out bool hasContinuation, + out string wakeKey, + out Func? resumeHandler, + out Func? wakeHandler, + out long blockDeadlineTimestamp) + { + var consumed = TryConsumeCurrentThreadBlock( + out reason, + out continuation, + out hasContinuation, + out wakeKey, + out var waiter, + out blockDeadlineTimestamp); + resumeHandler = waiter is null ? null : waiter.Resume; + wakeHandler = waiter is null ? null : waiter.TryWake; + return consumed; + } + public static long ComputeDeadlineTimestamp(TimeSpan timeout) { if (timeout <= TimeSpan.Zero) @@ -360,10 +465,15 @@ public static class GuestThreadExecution context[CpuRegister.Rdi], context[CpuRegister.R8], context[CpuRegister.R9], + context[CpuRegister.R10], + context[CpuRegister.R11], context[CpuRegister.R12], context[CpuRegister.R13], context[CpuRegister.R14], - context[CpuRegister.R15]); + context[CpuRegister.R15], + context.FpuControlWord, + context.Mxcsr, + RestoreFullFpuState: false); return true; } diff --git a/src/SharpEmu.HLE/GuestTlsTemplate.cs b/src/SharpEmu.HLE/GuestTlsTemplate.cs new file mode 100644 index 0000000..65cf835 --- /dev/null +++ b/src/SharpEmu.HLE/GuestTlsTemplate.cs @@ -0,0 +1,442 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using System.Diagnostics; + +namespace SharpEmu.HLE; + +/// +/// Process-wide registry of ELF PT_TLS templates and per-thread dynamic thread +/// vectors. AMD64 uses TLS Variant II: startup/static module blocks precede the +/// thread pointer, while modules introduced after a thread was initialized are +/// represented by separately allocated DTV entries. +/// +public static class GuestTlsTemplate +{ + // Must match CpuDispatcher/DirectExecutionBackend's mapped prefix. PS5 + // modules can require more than one host page of Variant II static TLS; + // Dreaming Sarah's startup image, for example, reaches 0x1870 bytes. + public const ulong StartupStaticTlsReservation = 0x10000UL; + private static readonly object _gate = new(); + private static readonly SortedDictionary _modules = new(); + private static readonly Dictionary _threadDtvs = new(); + private static ulong _staticTlsSize; + private static ulong _maximumAlignment = 1; + private static ulong _generation; + + static GuestTlsTemplate() + { + RunTlsLayoutSelfChecks(); + } + + private sealed class ModuleTemplate + { + public required ulong ModuleId { get; init; } + public required byte[] InitImage { get; init; } + public required ulong MemorySize { get; init; } + public required ulong Alignment { get; init; } + public required ulong AlignmentBias { get; init; } + public required ulong StaticOffset { get; init; } + } + + private sealed class ThreadDtv + { + public ulong Generation { get; set; } + public Dictionary Entries { get; } = new(); + public nint DtvAllocationBase { get; set; } + public ulong DtvAddress { get; set; } + } + + private sealed class DtvEntry + { + public required ulong Address { get; init; } + public nint AllocationBase { get; init; } + } + + /// Main executable's initialized PT_TLS bytes. + public static byte[] InitImage + { + get + { + lock (_gate) + { + return _modules.TryGetValue(1, out var module) + ? (byte[])module.InitImage.Clone() + : []; + } + } + } + + /// Aligned size of the main executable's static TLS block. + public static ulong BlockSize + { + get + { + lock (_gate) + { + return _modules.TryGetValue(1, out var module) + ? module.StaticOffset + : 0; + } + } + } + + /// Main executable's PT_TLS alignment. + public static ulong Alignment + { + get + { + lock (_gate) + { + return _modules.TryGetValue(1, out var module) + ? module.Alignment + : 1; + } + } + } + + /// Total Variant II static TLS span required below the TCB. + public static ulong StaticTlsSize + { + get { lock (_gate) { return _staticTlsSize; } } + } + + /// Largest PT_TLS alignment required by the registered modules. + public static ulong MaximumAlignment + { + get { lock (_gate) { return _maximumAlignment; } } + } + + /// Current DTV generation, incremented for every registered module. + public static ulong Generation + { + get { lock (_gate) { return _generation; } } + } + + /// + /// Backwards-compatible main-module registration entry point. + /// + public static void Set(ReadOnlySpan initImage, ulong memorySize, ulong alignment) + { + Reset(); + RegisterModule(1, initImage, memorySize, alignment, alignmentBias: 0); + } + + /// + /// Registers one module's PT_TLS template and allocates its Variant II + /// static offset. Modules must be registered in loader module-ID order. + /// + public static ulong RegisterModule( + ulong moduleId, + ReadOnlySpan initImage, + ulong memorySize, + ulong alignment, + ulong alignmentBias = 0) + { + if (moduleId == 0) + { + throw new ArgumentOutOfRangeException(nameof(moduleId)); + } + + var normalizedAlignment = alignment == 0 ? 1 : alignment; + if ((normalizedAlignment & (normalizedAlignment - 1)) != 0) + { + throw new InvalidDataException($"PT_TLS alignment 0x{alignment:X} is not a power of two."); + } + if (memorySize > int.MaxValue || (ulong)initImage.Length > memorySize) + { + throw new InvalidDataException("PT_TLS template size is invalid or exceeds the supported process limit."); + } + + lock (_gate) + { + if (_modules.TryGetValue(moduleId, out var existing)) + { + if (existing.MemorySize != memorySize || + existing.Alignment != normalizedAlignment || + existing.AlignmentBias != alignmentBias || + !existing.InitImage.AsSpan().SequenceEqual(initImage)) + { + throw new InvalidOperationException($"TLS module {moduleId} was registered with a different template."); + } + + return existing.StaticOffset; + } + + // FreeBSD/AMD64 Variant II: choose the smallest offset satisfying + // offset - previous >= size and (-offset) % align == p_vaddr % align. + var staticOffset = CalculateStaticOffset( + _staticTlsSize, + memorySize, + normalizedAlignment, + alignmentBias); + if (staticOffset > StartupStaticTlsReservation) + { + throw new InvalidOperationException( + $"Static TLS requires 0x{staticOffset:X} bytes, but startup maps only " + + $"0x{StartupStaticTlsReservation:X} bytes below the thread pointer."); + } + _modules.Add(moduleId, new ModuleTemplate + { + ModuleId = moduleId, + InitImage = initImage.ToArray(), + MemorySize = memorySize, + Alignment = normalizedAlignment, + AlignmentBias = alignmentBias, + StaticOffset = staticOffset, + }); + _staticTlsSize = staticOffset; + _maximumAlignment = Math.Max(_maximumAlignment, normalizedAlignment); + _generation++; + return staticOffset; + } + } + + /// Returns the static TP-relative block offset for a module. + public static bool TryGetStaticOffset(ulong moduleId, out ulong staticOffset) + { + lock (_gate) + { + if (_modules.TryGetValue(moduleId, out var module)) + { + staticOffset = module.StaticOffset; + return true; + } + } + + staticOffset = 0; + return false; + } + + /// Clears all module templates and frees dynamic DTV blocks. + public static void Reset() + { + lock (_gate) + { + foreach (var dtv in _threadDtvs.Values) + { + FreeDynamicEntries(dtv); + } + + _threadDtvs.Clear(); + _modules.Clear(); + _staticTlsSize = 0; + _maximumAlignment = 1; + _generation++; + } + } + + /// + /// Initializes every currently registered startup TLS block for a thread + /// and records their addresses in that thread's DTV. + /// + public static void SeedThreadBlock(CpuContext context, ulong threadPointer) + { + ArgumentNullException.ThrowIfNull(context); + if (threadPointer == 0) + { + return; + } + + lock (_gate) + { + if (_threadDtvs.Remove(threadPointer, out var oldDtv)) + { + FreeDynamicEntries(oldDtv); + } + + var dtv = new ThreadDtv(); + _threadDtvs.Add(threadPointer, dtv); + foreach (var module in _modules.Values) + { + dtv.Entries[module.ModuleId] = CreateStaticOrDynamicEntry(context, threadPointer, module); + } + RebuildGuestDtv(context, threadPointer, dtv); + } + } + + /// + /// Implements the DTV lookup used by __tls_get_addr. Unknown module + /// IDs and offsets outside the module's TLS image are rejected with zero. + /// + public static ulong ResolveAddress(CpuContext context, ulong moduleId, ulong offset) + { + ArgumentNullException.ThrowIfNull(context); + if (context.FsBase == 0 || moduleId == 0) + { + return 0; + } + + lock (_gate) + { + if (!_modules.TryGetValue(moduleId, out var module) || offset >= module.MemorySize) + { + return 0; + } + + if (!_threadDtvs.TryGetValue(context.FsBase, out var dtv)) + { + dtv = new ThreadDtv(); + _threadDtvs.Add(context.FsBase, dtv); + foreach (var startupModule in _modules.Values) + { + dtv.Entries[startupModule.ModuleId] = CreateStaticOrDynamicEntry(context, context.FsBase, startupModule); + } + RebuildGuestDtv(context, context.FsBase, dtv); + } + + var entryAdded = false; + if (!dtv.Entries.TryGetValue(moduleId, out var entry)) + { + // This module appeared after the thread's startup TLS layout was + // seeded. Model rtld's lazy DTV allocation instead of assuming + // unused space exists below the already-live thread pointer. + entry = CreateDynamicEntry(module); + dtv.Entries.Add(moduleId, entry); + entryAdded = true; + } + + if (entryAdded || dtv.Generation != _generation) + { + RebuildGuestDtv(context, context.FsBase, dtv); + } + + return checked(entry.Address + offset); + } + } + + private static DtvEntry CreateStaticOrDynamicEntry( + CpuContext context, + ulong threadPointer, + ModuleTemplate module) + { + if (threadPointer >= module.StaticOffset) + { + var address = threadPointer - module.StaticOffset; + var zeroImage = module.MemorySize == 0 ? [] : new byte[(int)module.MemorySize]; + if ((zeroImage.Length == 0 || context.Memory.TryWrite(address, zeroImage)) && + (module.InitImage.Length == 0 || context.Memory.TryWrite(address, module.InitImage))) + { + return new DtvEntry { Address = address }; + } + } + + return CreateDynamicEntry(module); + } + + private static DtvEntry CreateDynamicEntry(ModuleTemplate module) + { + var size = Math.Max(1UL, module.MemorySize); + var allocationSize = checked(size + module.Alignment - 1); + if (allocationSize > int.MaxValue) + { + throw new OutOfMemoryException("TLS module allocation exceeds the supported host allocation size."); + } + + var allocationBase = Marshal.AllocHGlobal((int)allocationSize); + var alignedAddress = AlignUp(unchecked((ulong)allocationBase), module.Alignment); + Marshal.Copy(new byte[(int)size], 0, unchecked((nint)alignedAddress), (int)size); + if (module.InitImage.Length != 0) + { + Marshal.Copy(module.InitImage, 0, unchecked((nint)alignedAddress), module.InitImage.Length); + } + + return new DtvEntry + { + Address = alignedAddress, + AllocationBase = allocationBase, + }; + } + + private static void FreeDynamicEntries(ThreadDtv dtv) + { + foreach (var entry in dtv.Entries.Values) + { + if (entry.AllocationBase != 0) + { + Marshal.FreeHGlobal(entry.AllocationBase); + } + } + if (dtv.DtvAllocationBase != 0) + { + Marshal.FreeHGlobal(dtv.DtvAllocationBase); + dtv.DtvAllocationBase = 0; + dtv.DtvAddress = 0; + } + } + + private static void RebuildGuestDtv(CpuContext context, ulong threadPointer, ThreadDtv dtv) + { + var maximumModuleId = _modules.Count == 0 ? 0UL : _modules.Keys.Max(); + var byteSize = checked(2UL * sizeof(ulong) + maximumModuleId * sizeof(ulong)); + if (byteSize > int.MaxValue) + { + throw new OutOfMemoryException("Guest DTV exceeds the supported host allocation size."); + } + + var newAllocation = Marshal.AllocHGlobal((int)Math.Max((ulong)sizeof(ulong) * 2, byteSize)); + var newAddress = unchecked((ulong)newAllocation); + Marshal.Copy(new byte[(int)Math.Max((ulong)sizeof(ulong) * 2, byteSize)], 0, newAllocation, (int)Math.Max((ulong)sizeof(ulong) * 2, byteSize)); + Marshal.WriteInt64(newAllocation, 0, unchecked((long)_generation)); + Marshal.WriteInt64(newAllocation, sizeof(ulong), unchecked((long)maximumModuleId)); + foreach (var (moduleId, entry) in dtv.Entries) + { + var slotOffset = checked((int)(2UL * sizeof(ulong) + (moduleId - 1) * sizeof(ulong))); + Marshal.WriteInt64(newAllocation, slotOffset, unchecked((long)entry.Address)); + } + + if (!context.TryWriteUInt64(threadPointer + sizeof(ulong), newAddress)) + { + Marshal.FreeHGlobal(newAllocation); + throw new InvalidOperationException("Failed to install the guest DTV pointer in the thread control block."); + } + + if (dtv.DtvAllocationBase != 0) + { + Marshal.FreeHGlobal(dtv.DtvAllocationBase); + } + dtv.DtvAllocationBase = newAllocation; + dtv.DtvAddress = newAddress; + dtv.Generation = _generation; + } + + private static ulong AlignUp(ulong value, ulong alignment) + { + var mask = alignment - 1; + return checked((value + mask) & ~mask); + } + + private static ulong CalculateStaticOffset( + ulong previousOffset, + ulong size, + ulong alignment, + ulong alignmentBias) + { + var result = checked(previousOffset + size + alignment - 1); + return result - ((result + alignmentBias) & (alignment - 1)); + } + + [Conditional("DEBUG")] + private static void RunTlsLayoutSelfChecks() + { + var firstOffset = RegisterModule(1, [0x11, 0x22], 0x20, 0x10, 0); + var secondOffset = RegisterModule(2, [0x7A], 0x18, 0x20, 8); + Debug.Assert(firstOffset == 0x20, "First Variant II TLS offset is incorrect."); + Debug.Assert(secondOffset >= firstOffset + 0x18, "TLS modules overlap in the static layout."); + Debug.Assert((unchecked(0UL - secondOffset) & 0x1F) == 8, "PT_TLS virtual-address congruence was not preserved."); + + var lateEntry = CreateDynamicEntry(_modules[2]); + try + { + Debug.Assert(lateEntry.AllocationBase != 0, "A late TLS module did not receive a dynamic DTV block."); + Debug.Assert(Marshal.ReadByte(unchecked((nint)lateEntry.Address)) == 0x7A, "Late-module tdata was not initialized."); + Debug.Assert(Marshal.ReadByte(unchecked((nint)lateEntry.Address + 1)) == 0, "Late-module tbss was not zero initialized."); + } + finally + { + Marshal.FreeHGlobal(lateEntry.AllocationBase); + Reset(); + } + } +} diff --git a/src/SharpEmu.HLE/HleDataSymbols.cs b/src/SharpEmu.HLE/HleDataSymbols.cs index aa0bad0..12d1e3f 100644 --- a/src/SharpEmu.HLE/HleDataSymbols.cs +++ b/src/SharpEmu.HLE/HleDataSymbols.cs @@ -13,7 +13,9 @@ public static class HleDataSymbols private const string LibcNeedFlagNid = "P330P3dFF68"; private const string LibcInternalNeedFlagNid = "ZT4ODD2Ts9o"; private const int ProgNameMaxBytes = 511; - private const ulong StackChkGuardValue = 0xC0DEC0DECAFEBABEUL; + // Terminator canaries reserve the low byte as NUL. Keep the process data + // symbol and every per-thread TLS copy byte-for-byte identical. + private const ulong StackChkGuardValue = 0xC0DEC0DECAFEBA00UL; private static readonly object _gate = new(); private static readonly nint _stackChkGuardAddress = Allocate(sizeof(ulong) * 2); diff --git a/src/SharpEmu.HLE/HostMainThread.cs b/src/SharpEmu.HLE/HostMainThread.cs index 78f27ca..bcf94fd 100644 --- a/src/SharpEmu.HLE/HostMainThread.cs +++ b/src/SharpEmu.HLE/HostMainThread.cs @@ -6,12 +6,11 @@ using System.Collections.Concurrent; namespace SharpEmu.HLE; /// -/// Runs work on the real process main thread. GLFW windowing must live on -/// that thread on macOS (AppKit) and Linux (X11's single event queue), so the -/// CLI moves emulation onto a worker thread, parks the main thread in -/// , and the video presenter posts its window loop here. On -/// Windows stays false and the window keeps its own -/// thread. +/// Runs work on the real process main thread. macOS only allows AppKit (and +/// therefore GLFW windowing) on that thread, so the CLI moves emulation onto +/// a worker thread, parks the main thread in , and the +/// video presenter posts its window loop here. On other platforms +/// stays false and nothing changes. /// public static class HostMainThread { diff --git a/src/SharpEmu.HLE/HostMemory.cs b/src/SharpEmu.HLE/HostMemory.cs new file mode 100644 index 0000000..6049852 --- /dev/null +++ b/src/SharpEmu.HLE/HostMemory.cs @@ -0,0 +1,498 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; + +namespace SharpEmu.HLE; + +/// +/// Cross-platform host virtual memory API with Win32 semantics. +/// On Windows this forwards directly to kernel32. On POSIX systems it is +/// implemented over mmap/mprotect/munmap with a shadow region table that +/// answers VirtualQuery-style questions and tracks page protections. +/// POSIX anonymous mappings are demand-paged by the kernel, so Win32 +/// "reserve-only" regions are mapped as committed memory directly and +/// commit requests become protection changes. +/// +public static unsafe class HostMemory +{ + public const uint MEM_COMMIT = 0x1000; + public const uint MEM_RESERVE = 0x2000; + public const uint MEM_RELEASE = 0x8000; + public const uint MEM_FREE_STATE = 0x10000; + public const uint MEM_PRIVATE = 0x20000; + + public const uint PAGE_NOACCESS = 0x01; + public const uint PAGE_READONLY = 0x02; + public const uint PAGE_READWRITE = 0x04; + public const uint PAGE_EXECUTE = 0x10; + public const uint PAGE_EXECUTE_READ = 0x20; + public const uint PAGE_EXECUTE_READWRITE = 0x40; + + private const ulong PageSize = 0x1000; + + /// Win32 MEMORY_BASIC_INFORMATION (64-bit) layout. + public struct BasicInfo + { + public ulong BaseAddress; + public ulong AllocationBase; + public uint AllocationProtect; + public uint Alignment1; + public ulong RegionSize; + public uint State; + public uint Protect; + public uint Type; + public uint Alignment2; + } + + public static void* Alloc(void* address, nuint size, uint allocationType, uint protect) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualAlloc(address, size, allocationType, protect); + } + + return Posix.Alloc(address, size, allocationType, protect); + } + + public static bool Free(void* address, nuint size, uint freeType) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualFree(address, size, freeType); + } + + return Posix.Free(address, size, freeType); + } + + public static bool Protect(void* address, nuint size, uint newProtect, out uint oldProtect) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualProtect(address, size, newProtect, out oldProtect); + } + + return Posix.Protect(address, size, newProtect, out oldProtect); + } + + public static nuint Query(void* address, out BasicInfo info) + { + if (OperatingSystem.IsWindows()) + { + return Win32VirtualQuery(address, out info, (nuint)sizeof(BasicInfo)); + } + + return Posix.Query(address, out info); + } + + public static void FlushInstructionCache(void* address, nuint size) + { + if (OperatingSystem.IsWindows()) + { + Win32FlushInstructionCache(Win32GetCurrentProcess(), address, size); + return; + } + + // The emulator only executes x86-64 guest code, so a non-Windows host + // is either x86-64 (including Rosetta 2 translation) with a coherent + // instruction cache, or would need sys_icache_invalidate for a future + // arm64 recompiler. Nothing to do today. + } + + [DllImport("kernel32.dll", EntryPoint = "VirtualAlloc", SetLastError = true)] + private static extern void* Win32VirtualAlloc(void* lpAddress, nuint dwSize, uint flAllocationType, uint flProtect); + + [DllImport("kernel32.dll", EntryPoint = "VirtualFree", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32VirtualFree(void* lpAddress, nuint dwSize, uint dwFreeType); + + [DllImport("kernel32.dll", EntryPoint = "VirtualProtect", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32VirtualProtect(void* lpAddress, nuint dwSize, uint flNewProtect, out uint lpflOldProtect); + + [DllImport("kernel32.dll", EntryPoint = "VirtualQuery")] + private static extern nuint Win32VirtualQuery(void* lpAddress, out BasicInfo lpBuffer, nuint dwLength); + + [DllImport("kernel32.dll", EntryPoint = "GetCurrentProcess")] + private static extern void* Win32GetCurrentProcess(); + + [DllImport("kernel32.dll", EntryPoint = "FlushInstructionCache")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool Win32FlushInstructionCache(void* hProcess, void* lpBaseAddress, nuint dwSize); + + private static class Posix + { + private const int PROT_NONE = 0x0; + private const int PROT_READ = 0x1; + private const int PROT_WRITE = 0x2; + private const int PROT_EXEC = 0x4; + + private const int MAP_PRIVATE = 0x02; + private const int MAP_FIXED = 0x10; + private static readonly int MAP_ANON = OperatingSystem.IsMacOS() ? 0x1000 : 0x20; + private static readonly int MAP_NORESERVE = OperatingSystem.IsMacOS() ? 0 : 0x4000; + + // Linux-only: fail instead of clobbering an existing mapping. + private const int MAP_FIXED_NOREPLACE = 0x100000; + + private static readonly nint MAP_FAILED = -1; + + private static readonly object Gate = new(); + private static readonly SortedList Regions = new(); + + private sealed class Region + { + public ulong Base; + public ulong Size; + public uint DefaultProtect; + public Dictionary? PageProtects; + + public ulong End => Base + Size; + + public uint ProtectAt(ulong pageAddress) + { + if (PageProtects is not null && PageProtects.TryGetValue(pageAddress, out var overriden)) + { + return overriden; + } + + return DefaultProtect; + } + } + + public static void* Alloc(void* address, nuint size, uint allocationType, uint protect) + { + if (size == 0) + { + return null; + } + + var alignedSize = AlignUp((ulong)size, PageSize); + + lock (Gate) + { + if (allocationType == MEM_COMMIT && address != null && + TryFindRegionLocked((ulong)address, out var existing)) + { + // Note: MEM_RESERVE requests that overlap an existing + // region must fail like Win32 does; only a pure commit + // may target pages inside a tracked mapping. + // Commit inside an existing mapping: the pages are already + // backed (demand paged), so only apply the protection. + var start = AlignDown((ulong)address, PageSize); + var end = Math.Min(existing.End, AlignUp((ulong)address + alignedSize, PageSize)); + if (end <= start) + { + return null; + } + + if (mprotect((nint)start, (nuint)(end - start), ToPosixProtect(protect)) != 0) + { + return null; + } + + SetProtectRangeLocked(existing, start, end - start, protect); + return address; + } + + if ((allocationType & MEM_RESERVE) == 0) + { + // MEM_COMMIT alone outside any known region is invalid here. + return null; + } + + var posixProtect = ToPosixProtect(protect); + var flags = MAP_PRIVATE | MAP_ANON; + if ((allocationType & MEM_COMMIT) == 0) + { + // Reserve-only: keep the requested protection so the region + // is usable without a separate commit step, but tell the + // kernel not to account swap for it where supported. + flags |= MAP_NORESERVE; + } + + nint result; + if (address != null) + { + // Win32 maps at exactly the requested address or fails + // without touching existing mappings. Fail up front on + // any overlap we track, then place the mapping: Linux + // gets MAP_FIXED_NOREPLACE (fails cleanly on host + // mappings too). Darwin lacks NOREPLACE and plain + // MAP_FIXED would silently clobber untracked host + // memory (dyld, the runtime's JIT heap, Rosetta), so + // pass the address as a hint instead -- the kernel + // honors it when the range is free and relocates the + // mapping otherwise, which we treat as failure. + if (OverlapsTrackedRegionLocked((ulong)address, alignedSize)) + { + Trace($"exact overlap: addr=0x{(ulong)address:X16} size=0x{alignedSize:X}"); + return null; + } + + var exactFlags = OperatingSystem.IsMacOS() ? flags : flags | MAP_FIXED_NOREPLACE; + result = mmap((nint)address, (nuint)alignedSize, posixProtect, exactFlags, -1, 0); + if (result == MAP_FAILED || (ulong)result != (ulong)address) + { + Trace($"exact mmap failed: addr=0x{(ulong)address:X16} got=0x{(ulong)result:X16} size=0x{alignedSize:X} errno={Marshal.GetLastPInvokeError()}"); + if (result != MAP_FAILED) + { + munmap(result, (nuint)alignedSize); + } + + return null; + } + } + else + { + result = mmap(0, (nuint)alignedSize, posixProtect, flags, -1, 0); + if (result == MAP_FAILED) + { + Trace($"mmap failed: size=0x{alignedSize:X} errno={Marshal.GetLastPInvokeError()}"); + return null; + } + } + + Regions[(ulong)result] = new Region + { + Base = (ulong)result, + Size = alignedSize, + DefaultProtect = protect + }; + + return (void*)result; + } + } + + public static bool Free(void* address, nuint size, uint freeType) + { + _ = size; + _ = freeType; + + lock (Gate) + { + if (!Regions.TryGetValue((ulong)address, out var region)) + { + return false; + } + + Regions.Remove((ulong)address); + return munmap((nint)address, (nuint)region.Size) == 0; + } + } + + public static bool Protect(void* address, nuint size, uint newProtect, out uint oldProtect) + { + oldProtect = PAGE_NOACCESS; + if (size == 0) + { + return false; + } + + var start = AlignDown((ulong)address, PageSize); + var end = AlignUp((ulong)address + size, PageSize); + + lock (Gate) + { + if (!TryFindRegionLocked(start, out var region) || end > region.End) + { + return false; + } + + oldProtect = region.ProtectAt(start); + if (mprotect((nint)start, (nuint)(end - start), ToPosixProtect(newProtect)) != 0) + { + return false; + } + + SetProtectRangeLocked(region, start, end - start, newProtect); + return true; + } + } + + public static nuint Query(void* address, out BasicInfo info) + { + info = default; + var pageAddress = AlignDown((ulong)address, PageSize); + + lock (Gate) + { + if (TryFindRegionLocked(pageAddress, out var region)) + { + // Win32 VirtualQuery reports a run of pages sharing the + // same protection, so stop the run where it changes. Do + // not walk the whole mapping page-by-page here: PS5 GPU + // apertures can span hundreds of GiB, while protection + // overrides are sparse. A tiny read inside such a mapping + // otherwise performs tens of millions of dictionary + // lookups before it can return. + var pageProtects = region.PageProtects; + uint protect; + ulong runEnd; + if (pageProtects is null || pageProtects.Count == 0) + { + protect = region.DefaultProtect; + runEnd = region.End; + } + else if (pageProtects.TryGetValue(pageAddress, out protect)) + { + runEnd = pageAddress + PageSize; + while (runEnd < region.End && + pageProtects.TryGetValue(runEnd, out var nextProtect) && + nextProtect == protect) + { + runEnd += PageSize; + } + } + else + { + protect = region.DefaultProtect; + runEnd = region.End; + foreach (var overrideAddress in pageProtects.Keys) + { + if (overrideAddress > pageAddress && overrideAddress < runEnd) + { + runEnd = overrideAddress; + } + } + } + + info.BaseAddress = pageAddress; + info.AllocationBase = region.Base; + info.AllocationProtect = region.DefaultProtect; + info.RegionSize = runEnd - pageAddress; + info.State = MEM_COMMIT; + info.Protect = protect; + info.Type = MEM_PRIVATE; + return (nuint)sizeof(BasicInfo); + } + + // Untracked host memory (runtime heaps, stacks, libraries) is + // reported as a free block reaching to the next tracked region + // so scanning callers keep advancing. + var nextBase = ulong.MaxValue; + foreach (var regionBase in Regions.Keys) + { + if (regionBase > pageAddress) + { + nextBase = regionBase; + break; + } + } + + info.BaseAddress = pageAddress; + info.AllocationBase = 0; + info.AllocationProtect = PAGE_NOACCESS; + info.RegionSize = (nextBase == ulong.MaxValue ? pageAddress + PageSize : nextBase) - pageAddress; + info.State = MEM_FREE_STATE; + info.Protect = PAGE_NOACCESS; + info.Type = 0; + return (nuint)sizeof(BasicInfo); + } + } + + private static bool OverlapsTrackedRegionLocked(ulong start, ulong size) + { + var end = start + size; + foreach (var region in Regions.Values) + { + if (region.Base < end && start < region.End) + { + return true; + } + } + + return false; + } + + private static bool TryFindRegionLocked(ulong address, out Region region) + { + region = null!; + var keys = Regions.Keys; + var low = 0; + var high = keys.Count - 1; + Region? candidate = null; + while (low <= high) + { + var middle = low + ((high - low) >> 1); + var entry = Regions.Values[middle]; + if (entry.Base <= address) + { + candidate = entry; + low = middle + 1; + } + else + { + high = middle - 1; + } + } + + if (candidate is null || address >= candidate.End) + { + return false; + } + + region = candidate; + return true; + } + + private static void SetProtectRangeLocked(Region region, ulong start, ulong size, uint protect) + { + if (start == region.Base && size >= region.Size) + { + region.DefaultProtect = protect; + region.PageProtects = null; + return; + } + + region.PageProtects ??= new Dictionary(); + var end = start + size; + for (var pageAddress = start; pageAddress < end; pageAddress += PageSize) + { + if (protect == region.DefaultProtect) + { + region.PageProtects.Remove(pageAddress); + } + else + { + region.PageProtects[pageAddress] = protect; + } + } + } + + private static int ToPosixProtect(uint win32Protect) + { + return win32Protect switch + { + PAGE_NOACCESS => PROT_NONE, + PAGE_READONLY => PROT_READ, + PAGE_READWRITE => PROT_READ | PROT_WRITE, + PAGE_EXECUTE => PROT_READ | PROT_EXEC, + PAGE_EXECUTE_READ => PROT_READ | PROT_EXEC, + PAGE_EXECUTE_READWRITE => PROT_READ | PROT_WRITE | PROT_EXEC, + _ => PROT_READ | PROT_WRITE + }; + } + + private static void Trace(string message) + { + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_VMEM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine($"[HOSTMEM] {message}"); + } + } + + private static ulong AlignDown(ulong value, ulong alignment) => value & ~(alignment - 1); + + private static ulong AlignUp(ulong value, ulong alignment) => checked((value + alignment - 1) & ~(alignment - 1)); + + [DllImport("libc", SetLastError = true)] + private static extern nint mmap(nint addr, nuint length, int prot, int flags, int fd, long offset); + + [DllImport("libc", SetLastError = true)] + private static extern int munmap(nint addr, nuint length); + + [DllImport("libc", SetLastError = true)] + private static extern int mprotect(nint addr, nuint length, int prot); + } +} diff --git a/src/SharpEmu.Libs/Agc/AgcExports.cs b/src/SharpEmu.Libs/Agc/AgcExports.cs index 853bf26..c7c8fc7 100644 --- a/src/SharpEmu.Libs/Agc/AgcExports.cs +++ b/src/SharpEmu.Libs/Agc/AgcExports.cs @@ -12,8 +12,19 @@ using System.Runtime.CompilerServices; namespace SharpEmu.Libs.Agc; -public static class AgcExports +public static partial class AgcExports { +#if DEBUG + static AgcExports() + { + ValidateWriteDataControlDecoders(); + ValidateDispatchInitiators(); + ValidateSubmittedQueueAndReleaseMemDecoders(); + ValidateAcquireMemAndQueueResetDecoders(); + ValidateDepthTargetDecoder(); + } +#endif + private const uint ShaderFileHeader = 0x34333231; private const uint ShaderVersion = 0x18; private const uint ItNop = 0x10; @@ -34,6 +45,7 @@ public static class AgcExports private const uint ItWaitRegMem = 0x3C; private const uint ItIndirectBuffer = 0x3F; private const uint ItEventWrite = 0x46; + private const uint ItReleaseMem = 0x49; private const uint ItDmaData = 0x50; private const uint ItSetContextReg = 0x69; private const uint ItSetShReg = 0x76; @@ -56,6 +68,8 @@ public static class AgcExports private const uint RFlip = 0x17; private const uint RReleaseMem = 0x18; private const uint RDmaData = 0x19; + private const uint RIndexBase = 0x1B; + private const uint RIndexCount = 0x1C; private const uint SpiShaderPgmLoPs = 0x8; private const uint SpiShaderPgmHiPs = 0x9; private const uint SpiShaderPgmLoEs = 0xC8; @@ -69,6 +83,9 @@ public static class AgcExports private const uint ComputePgmLo = 0x20C; private const uint ComputePgmHi = 0x20D; private const uint ComputePgmRsrc2 = 0x213; + private const uint ComputeStartX = 0x204; + private const uint ComputeStartY = 0x205; + private const uint ComputeStartZ = 0x206; private const uint ComputeNumThreadX = 0x207; private const uint ComputeNumThreadY = 0x208; private const uint ComputeNumThreadZ = 0x209; @@ -90,11 +107,11 @@ public static class AgcExports private const uint PaClVportYOffset = 0x112; private const uint PaScVportZMin0 = 0xB4; private const uint PaScVportZMax0 = 0xB5; + private const uint CbColorControl = 0x202; private const uint CbBlendRed = 0x105; private const uint CbBlendGreen = 0x106; private const uint CbBlendBlue = 0x107; private const uint CbBlendAlpha = 0x108; - private const uint CbColorControl = 0x202; private const uint CbColor0Base = 0x318; private const uint CbColorRegisterStride = 15; private const uint CbColor0Info = 0x31C; @@ -103,6 +120,15 @@ public static class AgcExports private const uint CbColor0Attrib3 = 0x3B8; private const uint CbBlend0Control = 0x1E0; private const uint PaScModeCntl0 = 0x292; + // GFX10 DB context registers (register byte address minus 0x28000, / 4). + private const uint DbDepthView = 0x002; + private const uint DbDepthSizeXy = 0x007; + private const uint DbDepthClear = 0x00B; + private const uint DbZInfo = 0x010; + private const uint DbZReadBase = 0x012; + private const uint DbZWriteBase = 0x014; + private const uint DbZReadBaseHi = 0x01A; + private const uint DbZWriteBaseHi = 0x01C; private const int ColorTargetCount = 8; private const uint PsTextureUserDataRegister = 0xC; private const uint VsUserDataRegister = 0x4C; @@ -110,23 +136,27 @@ public static class AgcExports private const uint EsUserDataRegister = 0xCC; private const uint ComputeUserDataRegister = 0x240; private const uint NggUserDataScalarRegisterBase = 8; - private const uint Gen5TextureFormatR8G8B8A8Unorm = 56; - private const uint Gen5TextureFormatR16G16B16A16Float = 71; + private const uint Gen5TextureFormatR8G8B8A8Unorm = 10; + private const uint Gen5TextureFormatR16G16B16A16Float = 12; + private const uint Gen5TextureType1D = 8; private const uint Gen5TextureType2D = 9; private const ulong MaxPresentedTextureBytes = 128UL * 1024UL * 1024UL; private const ulong VideoOutPixelFormatA8R8G8B8Srgb = 0x80000000; private const ulong VideoOutPixelFormatA8B8G8R8Srgb = 0x80002200; - private const ulong VideoOutPixelFormatB8G8R8A8Unorm = 0x8100000000000000; - private const ulong VideoOutPixelFormatR8G8B8A8Unorm = 0x8100000022000000; + private const ulong VideoOutPixelFormat2R8G8B8A8Srgb = 0x8000000022000000; + private const ulong VideoOutPixelFormat2B8G8R8A8Srgb = 0x8000000000000000; + private const ulong VideoOutPixelFormat2R10G10B10A2 = 0x8100000622000000; + private const ulong VideoOutPixelFormat2B10G10R10A2 = 0x8100000600000000; + private const ulong VideoOutPixelFormat2R10G10B10A2Srgb = 0x8100000022000000; + private const ulong VideoOutPixelFormat2B10G10R10A2Srgb = 0x8100000000000000; + private const ulong VideoOutPixelFormat2R10G10B10A2Bt2100Pq = 0x8100070422000000; + private const ulong VideoOutPixelFormat2B10G10R10A2Bt2100Pq = 0x8100070400000000; private const uint RegisterDefaultsVersion7 = 7; private const uint RegisterDefaultsVersion8 = 8; private const uint RegisterDefaultsVersion10 = 10; private const uint RegisterDefaultsVersion13 = 13; private const int RegisterDefaultsSize = 0x40; private const int RegisterDefaultBlockSize = 16 * 8; - private const ulong ResourceRegistrationBytesPerResource = 0x118; - private const ulong ResourceRegistrationBytesPerOwner = 0x1E0; - private const int ResourceRegistrationMaxNameLength = 256; private const ulong ShaderUserDataOffset = 0x08; private const ulong ShaderCodeOffset = 0x10; @@ -135,6 +165,9 @@ public static class AgcExports private const ulong ShaderSpecialsOffset = 0x28; private const ulong ShaderInputSemanticsOffset = 0x30; private const ulong ShaderOutputSemanticsOffset = 0x38; + private const ulong ResourceRegistrationBytesPerResource = 0x118; + private const ulong ResourceRegistrationBytesPerOwner = 0x1E0; + private const int ResourceRegistrationMaxNameLength = 256; private const ulong ShaderNumInputSemanticsOffset = 0x50; private const ulong ShaderNumOutputSemanticsOffset = 0x56; private const ulong ShaderTypeOffset = 0x5A; @@ -142,6 +175,7 @@ public static class AgcExports private const ulong CommandBufferCursorUpOffset = 0x10; private const ulong CommandBufferCursorDownOffset = 0x18; private const ulong CommandBufferCallbackOffset = 0x20; + private const ulong CommandBufferUserDataOffset = 0x28; private const ulong CommandBufferReservedDwOffset = 0x30; private const ulong ShaderSpecialGeCntlOffset = 0x00; private const ulong ShaderSpecialVgtShaderStagesEnOffset = 0x08; @@ -149,7 +183,6 @@ public static class AgcExports private const ulong ShaderSpecialGeUserVgprEnOffset = 0x28; private const uint CbSetShRegisterRangeMarker = 0x6875000D; private static readonly object _submitTraceGate = new(); - private static readonly object _textureHashTraceGate = new(); private static readonly HashSet _tracedDcbSizes = new(); private static readonly HashSet<(ulong Es, ulong Ps, GuestDrawKind Kind)> _tracedShaderTranslations = new(); private static readonly HashSet<(ulong Es, ulong Ps)> _tracedShaderDecodePairs = new(); @@ -157,39 +190,88 @@ public static class AgcExports private static readonly HashSet<(ulong Ps, string Error)> _tracedShaderFailures = new(); private static readonly HashSet<(int Handle, int Index, ulong Address, string Path)> _tracedDisplayBuffers = new(); private static readonly HashSet _tracedComputeShaders = new(); - private static readonly Dictionary<(ulong Address, uint Width, uint Height), ulong> _tracedTextureHashes = []; + private static readonly HashSet<(ulong Address, uint X, uint Y, uint Z)> + _tracedDispatchArguments = new(); + private static readonly HashSet<(ulong Address, uint Initiator, string Reason)> + _rejectedDispatchArguments = new(); private static readonly HashSet _tracedSubmittedDrawOpcodes = new(); // Concurrent so the per-draw/per-dispatch hit path is lock-free (and no longer // shares _submitTraceGate with tracing). private static readonly ConcurrentDictionary< - (ulong Es, ulong EsState, ulong Ps, ulong PsState, ulong OutputLayout, uint OutputCount, uint Attributes), + (ulong Es, ulong EsState, ulong Ps, ulong PsState, ulong OutputLayout, + uint OutputCount, uint Attributes, uint PsInputEna, uint PsInputAddr, + ulong AliasAlignment), (IGuestCompiledShader Vertex, IGuestCompiledShader Pixel)> _graphicsShaderCache = new(); private static readonly ConcurrentDictionary< - (ulong Cs, ulong State, uint LocalX, uint LocalY, uint LocalZ), + (ulong Cs, ulong State, uint LocalX, uint LocalY, uint LocalZ, + uint WaveLanes, ulong AliasAlignment), IGuestCompiledShader> _computeShaderCache = new(); + private static readonly ConcurrentDictionary< + (ulong Es, ulong State, ulong AliasAlignment), + IGuestCompiledShader> _depthOnlyVertexShaderCache = new(); private static readonly Dictionary _shaderHeadersByCode = new(); private static readonly bool _traceAgc = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal); + // Drop a draw on an undecodable texture descriptor instead of substituting + // a 1x1 fallback binding. Off by default so a garbage descriptor degrades + // the pass rather than dropping it (Demon's Souls composite feeders). + private static readonly bool _strictShaderDescriptors = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_STRICT_SHADER_DESCRIPTORS"), + "1", + StringComparison.Ordinal); private static readonly bool _traceAgcShader = _traceAgc || string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC_SHADER"), "1", StringComparison.Ordinal); - private static readonly bool _traceTextureHashes = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TEXTURE_HASHES"), + private static readonly ulong? _traceComputeShaderAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_COMPUTE_SHADER_ADDRESS")); + private static readonly ulong? _tracePixelShaderAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_PIXEL_SHADER_ADDRESS")); + private static readonly ulong? _traceRenderTargetAddress = ParseOptionalHexAddress( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_RENDER_TARGET_ADDRESS")); + private static readonly bool _traceDraws = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_DRAWS"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceFramePackets = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_FRAME_PACKETS"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceVertexRanges = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_VERTEX_RANGES"), + "1", + StringComparison.Ordinal); + private static readonly bool _compatibilitySubmitCompletionEvent = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_AGC_SUBMIT_COMPLETION_EVENT"), + "1", + StringComparison.Ordinal); + // Escape hatch for the cached-texture copy skip (per-draw texel copies + // are re-enabled unconditionally when set), for A/B-ing rendering issues. + private static readonly bool _textureCopySkipDisabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_NO_TEXTURE_SKIP"), "1", StringComparison.Ordinal); private static long _dcbWriteDataTraceCount; + private static int _tracedVertexRangeCount; private static long _dcbWaitRegMemTraceCount; private static long _createShaderTraceCount; private static long _packetPayloadTraceCount; private static bool _tracedMissingPixelShaderBindings; + private static long _unsatisfiedWaitTraceCount; + private static long _labelProducerSequence; + private static readonly object _labelProducerGate = new(); + private static readonly List _labelProducers = []; + private static readonly HashSet<(object Memory, ulong Address, ulong SubmissionId)> + _tracedProducerlessWaits = new(); private static long _shaderTranslationMissTraceCount; private static long _translatedDrawTraceCount; private static long _standardDmaTraceCount; + private static long _packetParseFailureTraceCount; + private static int _textureFallbackTraceCount; private static readonly object _softwarePresenterGate = new(); private static readonly Dictionary<(ulong Source, ulong Destination), ulong> _softwarePresenterFingerprints = new(); private static readonly Dictionary<(ulong Shader, ulong Source, ulong Destination), ulong> _softwareComputeBlitFingerprints = new(); @@ -198,72 +280,7 @@ public static class AgcExports private static readonly ConditionalWeakTable _submittedGpuStates = new(); private static readonly RegisterDefaultGroup[] PrimaryRegisterDefaults = - [ - new(0, 0, 0xE24F806D, [new(CbColorControl, 0x00CC0010)]), - new(0, 3, 0x0BC65DA4, [new(0x08F, 0)]), - new(0, 4, 0x9E5AD592, [new(0x08E, 0)]), - new(0, 12, 0x6DE4C312, [new(0x203, 0)]), - new(0, 28, 0x1EB8D73A, [new(PaScModeCntl0, 0x00000002)]), - new(0, 31, 0xA20EFC70, [new(PaScWindowOffset, 0)]), - new(0, 58, 0x43FBD769, - [ - new(CbBlendRed, 0), - new(CbBlendBlue, 0), - new(CbBlendGreen, 0), - new(CbBlendAlpha, 0), - ]), - new(0, 59, 0xEF550356, [new(CbBlend0Control, 0x20010001)]), - new(0, 67, 0x918106BB, - [ - new(PaScGenericScissorTl, 0x80000000), - new(PaScGenericScissorBr, 0x40004000), - ]), - new(0, 72, 0x38E92C91, - [ - new(0x318, 0), - new(0x31B, 0), - new(0x31C, 0), - new(0x31D, 0), - new(0x31E, 0x48), - new(0x31F, 0), - new(0x321, 0), - new(0x323, 0), - new(0x324, 0), - new(0x325, 0), - new(0x390, 0), - new(0x398, 0), - new(0x3A0, 0), - new(0x3A8, 0), - new(0x3B0, 0), - new(0x3B8, 0x0006C000), - ]), - new(0, 73, 0x0B177B43, [new(0x00C, 0), new(0x00D, 0x40004000)]), - new(0, 74, 0x48531062, [new(0x191, 0)]), - new(0, 76, 0x7690AF6F, - [ - new(0x10F, 0x4E7E0000), - new(0x111, 0x4E7E0000), - new(0x113, 0x4E7E0000), - new(0x110, 0), - new(0x112, 0), - new(0x114, 0), - new(0x094, 0x80000000), - new(0x095, 0x40004000), - new(0x0B4, 0), - new(0x0B5, 0), - ]), - new(0, 77, 0x078D7060, - [ - new(PaScWindowScissorTl, 0x80000000), - new(PaScWindowScissorBr, 0x40004000), - ]), - new(1, 13, 0xC918DF3E, [new(0x20C, 0), new(0x20D, 0)]), - new(1, 14, 0xC9751C9C, [new(0x0C8, 0), new(0x0C9, 0)]), - new(1, 18, 0xC9E01B31, [new(0x008, 0), new(0x009, 0)]), - new(2, 3, 0x105971C2, [new(0x25B, 0)]), - new(2, 7, 0x40D49AD1, [new(0x262, 0)]), - new(2, 12, 0x9EBFAB10, [new(0x242, 0)]), - ]; + CreatePrimaryRegisterDefaults(); private static readonly RegisterDefaultGroup[] InternalRegisterDefaults = [ @@ -302,26 +319,79 @@ public static class AgcExports uint BaseLevel, uint LastLevel, uint Pitch, - uint DstSelect) + uint DstSelect, + uint Depth = 1, + uint BaseArray = 0, + uint ArrayPitch = 0, + uint MaxMip = 0, + uint MinLod = 0, + uint MinLodWarn = 0, + uint BcSwizzle = 0, + ulong MetadataAddress = 0, + uint DescriptorFlags = 0, + bool HasExtendedDescriptor = false) { + public uint ResourceMipLevels + { + get + { + // RDNA2 table 45 explicitly distinguishes MAX_MIP (the + // resource allocation) from BASE_LEVEL/LAST_LEVEL (the + // resource view). Do not size a Vulkan image from a view: + // another descriptor for the same allocation may expose a + // different subset of its mip chain. + var maximumMipLevels = GetMaximumMipLevels(); + var resourceMipLevels = HasExtendedDescriptor + ? MaxMip + 1 + : maximumMipLevels; + return Math.Min(Math.Max(resourceMipLevels, 1u), maximumMipLevels); + } + } + public uint MipLevels { get { - var largestDimension = Math.Max(Width, Height); - uint maximumMipLevels = 1; - while (largestDimension > 1) - { - largestDimension >>= 1; - maximumMipLevels++; - } - - var descriptorMipLevels = LastLevel >= BaseLevel - ? LastLevel - BaseLevel + 1 + var descriptorMipLevels = LastLevel >= ViewBaseLevel + ? LastLevel - ViewBaseLevel + 1 : 1; - return Math.Min(descriptorMipLevels, maximumMipLevels); + return Math.Min( + descriptorMipLevels, + ResourceMipLevels - ViewBaseLevel); } } + + public uint ViewBaseLevel + { + get + { + // Some single-mip Gen5 descriptors use the reserved/inverted + // 15-0 range as a mip-disabled sentinel. The resource still + // has exactly one addressable level (MAX_MIP=0). Treating 15 + // literally makes Vulkan reject an otherwise compatible GPU + // image and falls back to stale guest-memory pixels. For any + // malformed range, keep BASE_LEVEL's meaning and clamp it to + // the allocation's last addressable mip. In particular, the + // common 15-0/MAX_MIP=0 sentinel resolves to mip 0 without + // making LAST_LEVEL the base of unrelated inverted views. + return Math.Min(BaseLevel, ResourceMipLevels - 1); + } + } + + private uint GetMaximumMipLevels() + { + var largestDimension = Type == 10 + ? Math.Max(Math.Max(Width, Height), Depth) + : Math.Max(Width, Height); + uint maximumMipLevels = 1; + while (largestDimension > 1) + { + largestDimension >>= 1; + maximumMipLevels++; + } + + return maximumMipLevels; + } } private readonly record struct RenderTargetDescriptor( @@ -347,10 +417,15 @@ public static class AgcExports IReadOnlyList GlobalMemoryBindings, IReadOnlyList VertexInputs, IReadOnlyList RenderTargets, - // The seam-shaped view of RenderTargets, built once here so the per-frame - // submit path does not rebuild it for every draw of a cached translation. + GuestDepthTarget? DepthTarget, + // Seam-shaped color targets are built once with the cached translation. IReadOnlyList GuestTargets, - GuestRenderState RenderState); + GuestRenderState RenderState, + IReadOnlyList PixelUserData, + uint RawBlendControl, + uint RawColorInfo, + IReadOnlyList PixelInitialScalars, + IReadOnlyList VertexInitialScalars); private sealed record TranslatedImageBinding( TextureDescriptor Descriptor, @@ -373,16 +448,80 @@ public static class AgcExports private readonly record struct ComputeDispatch( uint GroupCountX, uint GroupCountY, - uint GroupCountZ); + uint GroupCountZ, + uint BaseGroupX, + uint BaseGroupY, + uint BaseGroupZ, + uint WaveLaneCount, + bool IsIndirect, + uint ThreadCountX, + uint ThreadCountY, + uint ThreadCountZ); + + private readonly record struct SubmittedAcquireMem( + uint Engine, + uint CbDbControl, + ulong BaseAddress, + ulong SizeBytes, + uint PollInterval, + uint GcrControl) + { + // GFX10 GCR_CNTL invalidation controls. The host has no separate GLI, + // GLM, GLK, GLV, GL1 and GL2 caches; they all converge on the guest + // memory snapshots used to build Vulkan resources. + private const uint GliInvalidateMask = 0x3u; + private const int Gl1RangeShift = 2; + private const uint Gl1RangeMask = 0x3u; + private const uint GlmInvalidate = 1u << 5; + private const uint GlkInvalidate = 1u << 7; + private const uint GlvInvalidate = 1u << 8; + private const uint Gl1Invalidate = 1u << 9; + private const uint Gl2Discard = 1u << 13; + private const uint Gl2Invalidate = 1u << 14; + private const int Gl2RangeShift = 11; + private const uint Gl2RangeMask = 0x3u; + + public bool InvalidatesGuestResources => + (GcrControl & (GliInvalidateMask | + GlmInvalidate | + GlkInvalidate | + GlvInvalidate | + Gl1Invalidate | + Gl2Discard | + Gl2Invalidate)) != 0; + + // sceAgc encodes its all-memory sentinel with a zero COHER_SIZE. GFX10 + // can also request ALL independently in GLI_INV, GL1_RANGE or + // GL2_RANGE; in the host's unified resource cache, any invalidated + // domain with ALL scope expands the operation to all tracked images. + public bool CoversAllGuestMemory => + SizeBytes == 0 || + (GcrControl & GliInvalidateMask) == 1u || + ((GcrControl & (GlmInvalidate | + GlkInvalidate | + GlvInvalidate | + Gl1Invalidate)) != 0 && + ((GcrControl >> Gl1RangeShift) & Gl1RangeMask) == 0) || + ((GcrControl & (Gl2Discard | Gl2Invalidate)) != 0 && + ((GcrControl >> Gl2RangeShift) & Gl2RangeMask) == 0); + } private sealed class SubmittedDcbState { + public readonly record struct PendingSubmission( + ulong CommandAddress, + uint DwordCount, + ulong SubmissionId, + bool TracePackets); + public Dictionary CxRegisters { get; } = new(); public Dictionary ShRegisters { get; } = new(); public Dictionary UcRegisters { get; } = new(); public TextureDescriptor? PresenterTexture { get; set; } public GuestDrawKind GuestDrawKind { get; set; } public TranslatedGuestDraw? TranslatedDraw { get; set; } + public TranslatedGuestDraw? PendingTargetlessDraw { get; set; } + public Dictionary KnownRenderTargets { get; } = new(); public Dictionary RenderTargetWriters { get; } = new(); public ulong IndirectArgsAddress { get; set; } public bool SawIndexedDraw { get; set; } @@ -391,6 +530,17 @@ public static class AgcExports public uint IndexSize { get; set; } public uint InstanceCount { get; set; } = 1; public uint DrawIndexOffset { get; set; } + public string QueueName { get; set; } = "graphics"; + public ulong ActiveSubmissionId { get; set; } + public Queue PendingSubmissions { get; } = new(); + public bool HasActiveSubmission { get; set; } + public bool IsSuspended { get; set; } + public ulong CompletionEventNotifiedSubmissionId { get; set; } + public Dictionary<(uint Op, uint Register), uint> FramePacketCounts { get; } = new(); + public uint FramePacketCount { get; set; } + public uint FrameDrawCount { get; set; } + public uint FrameDispatchCount { get; set; } + public ulong FlipCount { get; set; } } private sealed class SubmittedGpuState @@ -409,6 +559,8 @@ public static class AgcExports public uint NextOwner { get; set; } = 1; public uint NextResource { get; set; } = 1; public ulong WorkSequence { get; set; } + public ulong SubmissionSequence { get; set; } + public bool WaitMonitorRunning { get; set; } } private readonly record struct RegisteredAgcResource( @@ -419,6 +571,19 @@ public static class AgcExports uint Type, uint Flags); + private sealed class LabelProducerTrace + { + public long Sequence; + public required object Memory; + public ulong Address; + public ulong Length; + public ulong PacketAddress; + public ulong SubmissionId; + public required string QueueName; + public required string DebugName; + public bool Completed; + } + private readonly record struct RegisterDefaultValue(uint Offset, uint Value); private readonly record struct RegisterDefaultGroup( @@ -442,11 +607,11 @@ public static class AgcExports var version = (uint)ctx[CpuRegister.Rsi]; if (stateAddress == 0 || !IsSupportedRegisterDefaultsVersion(version)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } TraceAgc($"agc.init state=0x{stateAddress:X16} version={version}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } #pragma warning restore SHEM004 @@ -478,19 +643,19 @@ public static class AgcExports var codeAddress = ctx[CpuRegister.Rdx]; if (headerAddress == 0 || codeAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadUInt32(headerAddress, out var fileHeader) || - !ctx.TryReadUInt32(headerAddress + sizeof(uint), out var version)) + if (!TryReadUInt32(ctx, headerAddress, out var fileHeader) || + !TryReadUInt32(ctx, headerAddress + sizeof(uint), out var version)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (fileHeader != ShaderFileHeader || version != ShaderVersion) { TraceCreateShader(destinationAddress, headerAddress, codeAddress, $"invalid-header file=0x{fileHeader:X8} version=0x{version:X8}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } if (!RelocatePointerField(ctx, headerAddress + ShaderCxRegistersOffset) || @@ -501,12 +666,12 @@ public static class AgcExports !RelocatePointerField(ctx, headerAddress + ShaderOutputSemanticsOffset) || !ctx.TryWriteUInt64(headerAddress + ShaderCodeOffset, codeAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - if (!ctx.TryReadUInt64(headerAddress + ShaderUserDataOffset, out var userDataAddress)) + if (!TryReadUInt64(ctx, headerAddress + ShaderUserDataOffset, out var userDataAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (userDataAddress != 0 && @@ -516,18 +681,18 @@ public static class AgcExports !RelocatePointerField(ctx, userDataAddress + 0x18) || !RelocatePointerField(ctx, userDataAddress + 0x20))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (!PatchShaderProgramRegisters(ctx, headerAddress, codeAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } if (destinationAddress != 0 && !ctx.TryWriteUInt64(destinationAddress, headerAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (_submitTraceGate) @@ -603,24 +768,24 @@ public static class AgcExports if (cxRegistersAddress == 0 || ucRegistersAddress == 0 || hullShaderAddress != 0 || geometryShaderAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadByte(geometryShaderAddress + ShaderTypeOffset, out var shaderType) || !IsEsGeometryShaderType(shaderType) || - !ctx.TryReadUInt64(geometryShaderAddress + ShaderSpecialsOffset, out var specialsAddress) || + if (!TryReadByte(ctx, geometryShaderAddress + ShaderTypeOffset, out var shaderType) || !IsEsGeometryShaderType(shaderType) || + !TryReadUInt64(ctx, geometryShaderAddress + ShaderSpecialsOffset, out var specialsAddress) || specialsAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } if (!CopyShaderRegister(ctx, specialsAddress + ShaderSpecialVgtShaderStagesEnOffset, cxRegistersAddress) || !CopyShaderRegister(ctx, specialsAddress + ShaderSpecialVgtGsOutPrimTypeOffset, cxRegistersAddress + 8) || !CopyShaderRegister(ctx, specialsAddress + ShaderSpecialGeCntlOffset, ucRegistersAddress) || !CopyShaderRegister(ctx, specialsAddress + ShaderSpecialGeUserVgprEnOffset, ucRegistersAddress + 8) || - !ctx.TryWriteUInt32(ucRegistersAddress + 16, VgtPrimitiveType) || - !ctx.TryWriteUInt32(ucRegistersAddress + 20, primitiveType)) + !TryWriteUInt32(ctx, ucRegistersAddress + 16, VgtPrimitiveType) || + !TryWriteUInt32(ctx, ucRegistersAddress + 20, primitiveType)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceAgc($"agc.create_prim_state cx=0x{cxRegistersAddress:X16} uc=0x{ucRegistersAddress:X16} gs=0x{geometryShaderAddress:X16} type={shaderType} prim=0x{primitiveType:X8}"); @@ -643,21 +808,21 @@ public static class AgcExports if (registersAddress == 0 || geometryShaderAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadUInt64(geometryShaderAddress + ShaderOutputSemanticsOffset, out var outputSemanticsAddress) || - !ctx.TryReadUInt32(geometryShaderAddress + ShaderNumOutputSemanticsOffset, out var outputSemanticsCount)) + if (!TryReadUInt64(ctx, geometryShaderAddress + ShaderOutputSemanticsOffset, out var outputSemanticsAddress) || + !TryReadUInt32(ctx, geometryShaderAddress + ShaderNumOutputSemanticsOffset, out var outputSemanticsCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } ulong inputSemanticsAddress = 0; if (pixelShaderAddress != 0 && - (!ctx.TryReadUInt64(pixelShaderAddress + ShaderInputSemanticsOffset, out inputSemanticsAddress) || - !ctx.TryReadUInt32(pixelShaderAddress + ShaderNumInputSemanticsOffset, out _))) + (!TryReadUInt64(ctx, pixelShaderAddress + ShaderInputSemanticsOffset, out inputSemanticsAddress) || + !TryReadUInt32(ctx, pixelShaderAddress + ShaderNumInputSemanticsOffset, out _))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } for (uint i = 0; i < 32; i++) @@ -667,7 +832,7 @@ public static class AgcExports { var flat = false; if (pixelShaderAddress != 0 && inputSemanticsAddress != 0 && - ctx.TryReadUInt32(inputSemanticsAddress + (i * sizeof(uint)), out var inputSemantic)) + TryReadUInt32(ctx, inputSemanticsAddress + (i * sizeof(uint)), out var inputSemantic)) { flat = ((inputSemantic >> 22) & 0x1) != 0; } @@ -676,10 +841,10 @@ public static class AgcExports } var destination = registersAddress + (i * 8); - if (!ctx.TryWriteUInt32(destination, SpiPsInputCntl0 + i) || - !ctx.TryWriteUInt32(destination + sizeof(uint), value)) + if (!TryWriteUInt32(ctx, destination, SpiPsInputCntl0 + i) || + !TryWriteUInt32(ctx, destination + sizeof(uint), value)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } @@ -703,15 +868,15 @@ public static class AgcExports var type = (int)ctx[CpuRegister.Rdx]; if (outputAddress == 0 || commandAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var payloadAddress = commandAddress + 8; if (type == 0) { - if (!ctx.TryReadUInt32(commandAddress, out var header)) + if (!TryReadUInt32(ctx, commandAddress, out var header)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } payloadAddress = (header & 0x3FFF_0000u) == 0x3FFF_0000u @@ -721,7 +886,7 @@ public static class AgcExports if (!ctx.TryWriteUInt64(outputAddress, payloadAddress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (ShouldTraceHotPath(ref _packetPayloadTraceCount)) @@ -747,20 +912,20 @@ public static class AgcExports var dwordCount = (uint)ctx[CpuRegister.Rsi]; if (commandBufferAddress == 0 || dwordCount < 2 || dwordCount > 0x4001) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return ReturnPointer(ctx, 0); } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, dwordCount, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(dwordCount, ItNop, RZero))) + !TryWriteUInt32(ctx, commandAddress, Pm4(dwordCount, ItNop, RZero))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return ReturnPointer(ctx, 0); } for (uint index = 1; index < dwordCount; index++) { - if (!ctx.TryWriteUInt32(commandAddress + ((ulong)index * sizeof(uint)), 0)) + if (!TryWriteUInt32(ctx, commandAddress + ((ulong)index * sizeof(uint)), 0)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return ReturnPointer(ctx, 0); } } @@ -781,11 +946,11 @@ public static class AgcExports var modifier = (uint)ctx[CpuRegister.R8]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(5, ItDispatchDirect, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, groupCountX) || - !ctx.TryWriteUInt32(commandAddress + 8, groupCountY) || - !ctx.TryWriteUInt32(commandAddress + 12, groupCountZ) || - !ctx.TryWriteUInt32(commandAddress + 16, (modifier & 0xA038u) | 0x41u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDispatchDirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, groupCountX) || + !TryWriteUInt32(ctx, commandAddress + 8, groupCountY) || + !TryWriteUInt32(ctx, commandAddress + 12, groupCountZ) || + !TryWriteUInt32(ctx, commandAddress + 16, DirectDispatchInitiator(modifier))) { return ReturnPointer(ctx, 0); } @@ -793,6 +958,14 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + private static uint DirectDispatchInitiator(uint modifier) => + // AGC's direct API takes workgroup counts by default. Preserve the + // caller's USE_THREAD_DIMENSIONS bit when explicitly requested; do not + // force it. Demon's Souls' 0xF00100 dispatch is paired with a + // 0x3C004000 element bound (exactly 64 lanes per group), proving the + // default packet is group-dimensional. + (modifier & 0xA038u) | 0x41u; + [SysAbiExport( Nid = "UZbQjYAwwXM", ExportName = "sceAgcCbSetShRegistersDirect", @@ -817,8 +990,8 @@ public static class AgcExports for (uint index = 0; index < registerCount; index++) { var entryAddress = registersAddress + ((ulong)index * 8); - if (!ctx.TryReadUInt32(entryAddress, out var offset) || - !ctx.TryReadUInt32(entryAddress + sizeof(uint), out var value)) + if (!TryReadUInt32(ctx, entryAddress, out var offset) || + !TryReadUInt32(ctx, entryAddress + sizeof(uint), out var value)) { return ReturnPointer(ctx, 0); } @@ -841,8 +1014,8 @@ public static class AgcExports var valueCount = (uint)(endIndex - startIndex); var packetDwords = valueCount + 2; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItSetShReg, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, registers[startIndex].Offset & 0xFFFFu)) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItSetShReg, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, registers[startIndex].Offset & 0xFFFFu)) { return ReturnPointer(ctx, 0); } @@ -850,7 +1023,8 @@ public static class AgcExports firstCommandAddress = firstCommandAddress == 0 ? commandAddress : firstCommandAddress; for (var index = startIndex; index < endIndex; index++) { - if (!ctx.TryWriteUInt32( + if (!TryWriteUInt32( + ctx, commandAddress + 8 + ((ulong)(index - startIndex) * sizeof(uint)), registers[index].Value)) { @@ -874,8 +1048,8 @@ public static class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNop, RAcbReset)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RAcbReset)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) { return ReturnPointer(ctx, 0); } @@ -901,15 +1075,15 @@ public static class AgcExports var hasAddress = (eventType & ~1u) == 0x38; var packetDwords = hasAddress ? 4u : 2u; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItEventWrite, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, hasAddress ? eventType | 0x100u : eventType & 0x3Fu)) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItEventWrite, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, hasAddress ? eventType | 0x100u : eventType & 0x3Fu)) { return ReturnPointer(ctx, 0); } if (hasAddress && - (!ctx.TryWriteUInt32(commandAddress + 8, (uint)eventAddress & ~7u) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(eventAddress >> 32)))) + (!TryWriteUInt32(ctx, commandAddress + 8, (uint)eventAddress & ~7u) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(eventAddress >> 32)))) { return ReturnPointer(ctx, 0); } @@ -940,14 +1114,14 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0x8000_0000u) || - !ctx.TryWriteUInt32(commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(baseAddress >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, pollCycles / 40) || - !ctx.TryWriteUInt32(commandAddress + 28, gcrControl)) + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0x8000_0000u) || + !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || + !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -969,8 +1143,8 @@ public static class AgcExports var address = ctx[CpuRegister.R8]; var reference = ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var mask) || - !ctx.TryReadUInt32(stackAddress + (2 * sizeof(ulong)), out var pollCycles) || + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var mask) || + !TryReadUInt32(ctx, stackAddress + (2 * sizeof(ulong)), out var pollCycles) || commandBufferAddress == 0 || size > 1 || compareFunction > 7 || @@ -982,27 +1156,27 @@ public static class AgcExports var packetDwords = size == 0 ? 6u : 9u; var packetRegister = size == 0 ? RWaitMem32 : RWaitMem64; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)address) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(address >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)mask)) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)address) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(address >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)mask)) { return ReturnPointer(ctx, 0); } if (size == 0) { - if (!ctx.TryWriteUInt32(commandAddress + 16, compareFunction) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference)) + if (!TryWriteUInt32(ctx, commandAddress + 16, compareFunction) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference)) { return ReturnPointer(ctx, 0); } } - else if (!ctx.TryWriteUInt32(commandAddress + 16, (uint)(mask >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference) || - !ctx.TryWriteUInt32(commandAddress + 24, (uint)(reference >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 28, compareFunction) || - !ctx.TryWriteUInt32(commandAddress + 32, pollCycles / 40)) + else if (!TryWriteUInt32(ctx, commandAddress + 16, (uint)(mask >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) || + !TryWriteUInt32(ctx, commandAddress + 24, (uint)(reference >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 28, compareFunction) || + !TryWriteUInt32(ctx, commandAddress + 32, pollCycles / 40)) { return ReturnPointer(ctx, 0); } @@ -1030,10 +1204,10 @@ public static class AgcExports var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(4, ItDispatchIndirect, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)argumentsAddress) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(argumentsAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, (modifier & 0xA038u) | 0x41u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItDispatchIndirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)argumentsAddress) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(argumentsAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -1058,11 +1232,11 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var markerAddress) || - !ctx.TryWriteUInt32(markerAddress, Pm4(2, ItNop, RZero)) || - !ctx.TryWriteUInt32(markerAddress + 4, CbSetShRegisterRangeMarker) || + !TryWriteUInt32(ctx, markerAddress, Pm4(2, ItNop, RZero)) || + !TryWriteUInt32(ctx, markerAddress + 4, CbSetShRegisterRangeMarker) || !TryAllocateCommandDwords(ctx, commandBufferAddress, valueCount + 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(valueCount + 2, ItSetShReg, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, offset)) + !TryWriteUInt32(ctx, commandAddress, Pm4(valueCount + 2, ItSetShReg, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, offset)) { return ReturnPointer(ctx, 0); } @@ -1071,12 +1245,12 @@ public static class AgcExports { var value = 0u; if (valuesAddress != 0 && - !ctx.TryReadUInt32(valuesAddress + (i * sizeof(uint)), out value)) + !TryReadUInt32(ctx, valuesAddress + (i * sizeof(uint)), out value)) { return ReturnPointer(ctx, 0); } - if (!ctx.TryWriteUInt32(commandAddress + 8 + (i * sizeof(uint)), value)) + if (!TryWriteUInt32(ctx, commandAddress + 8 + (i * sizeof(uint)), value)) { return ReturnPointer(ctx, 0); } @@ -1100,12 +1274,12 @@ public static class AgcExports var cachePolicy = (uint)(ctx[CpuRegister.R8] & 0xFF); var destinationAddress = ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + 8, out var dataSelectionRaw) || - !ctx.TryReadUInt64(stackAddress + 16, out var data) || - !ctx.TryReadUInt64(stackAddress + 24, out var gdsOffsetRaw) || - !ctx.TryReadUInt64(stackAddress + 32, out var gdsSizeRaw) || - !ctx.TryReadUInt64(stackAddress + 40, out var interruptRaw) || - !ctx.TryReadUInt64(stackAddress + 48, out var interruptContextIdRaw)) + if (!TryReadUInt64(ctx, stackAddress + 8, out var dataSelectionRaw) || + !TryReadUInt64(ctx, stackAddress + 16, out var data) || + !TryReadUInt64(ctx, stackAddress + 24, out var gdsOffsetRaw) || + !TryReadUInt64(ctx, stackAddress + 32, out var gdsSizeRaw) || + !TryReadUInt64(ctx, stackAddress + 40, out var interruptRaw) || + !TryReadUInt64(ctx, stackAddress + 48, out var interruptContextIdRaw)) { return ReturnPointer(ctx, 0); } @@ -1126,16 +1300,17 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RReleaseMem)) || - !ctx.TryWriteUInt32(commandAddress + 4, action | (cachePolicy << 8)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RReleaseMem)) || + !TryWriteUInt32(ctx, commandAddress + 4, action | (cachePolicy << 8)) || + !TryWriteUInt32( + ctx, commandAddress + 8, gcrControl | (dataSelection << 16) | (interrupt << 24)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)destinationAddress) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(destinationAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)data) || - !ctx.TryWriteUInt32(commandAddress + 24, (uint)(data >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 28, interruptContextId)) + !TryWriteUInt32(ctx, commandAddress + 12, (uint)destinationAddress) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(destinationAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)data) || + !TryWriteUInt32(ctx, commandAddress + 24, (uint)(data >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 28, interruptContextId)) { return ReturnPointer(ctx, 0); } @@ -1162,8 +1337,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNop, RDrawReset)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RDrawReset)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) { return ReturnPointer(ctx, 0); } @@ -1212,8 +1387,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItIndexType, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, indexSize)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItIndexType, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexSize)) { return ReturnPointer(ctx, 0); } @@ -1222,6 +1397,37 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "8N2tmT3jmC8", + ExportName = "sceAgcDcbSetIndexCount", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbSetIndexCount(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var indexCount = (uint)ctx[CpuRegister.Rsi]; + if (commandBufferAddress == 0 || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RIndexCount)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexCount)) + { + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "mljzuGDZRQ4", + ExportName = "sceAgcDcbSetIndexCountGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbSetIndexCountGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 7u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "tSBxhAPyytQ", ExportName = "sceAgcDcbSetNumInstances", @@ -1237,8 +1443,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNumInstances, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, instanceCount)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNumInstances, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, instanceCount)) { return ReturnPointer(ctx, 0); } @@ -1265,21 +1471,27 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var baseCommand) || - !ctx.TryWriteUInt32(baseCommand, Pm4(3, ItIndexBase, 0)) || - !ctx.TryWriteUInt32(baseCommand + 4, (uint)indexAddress) || - !ctx.TryWriteUInt32(baseCommand + 8, (uint)(indexAddress >> 32)) || - !ctx.TryWriteUInt32(baseCommand + 12, Pm4(2, ItIndexBufferSize, 0)) || - !ctx.TryWriteUInt32(baseCommand + 16, indexCount)) + !TryWriteUInt32(ctx, baseCommand, Pm4(3, ItIndexBase, 0)) || + !TryWriteUInt32(ctx, baseCommand + 4, (uint)indexAddress) || + !TryWriteUInt32(ctx, baseCommand + 8, (uint)(indexAddress >> 32)) || + !TryWriteUInt32(ctx, baseCommand + 12, Pm4(2, ItIndexBufferSize, 0)) || + !TryWriteUInt32(ctx, baseCommand + 16, indexCount)) { return ReturnPointer(ctx, 0); } - if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var drawCommand) || - !ctx.TryWriteUInt32(drawCommand, Pm4(5, ItDrawIndex2, 0)) || - !ctx.TryWriteUInt32(drawCommand + 4, indexCount) || - !ctx.TryWriteUInt32(drawCommand + 8, 0) || - !ctx.TryWriteUInt32(drawCommand + 12, 0) || - !ctx.TryWriteUInt32(drawCommand + 16, 0)) + // DRAW_INDEX_2 is six dwords: header, maximum index count, the + // 64-bit index-buffer base, the draw count and the initiator. The + // former five-dword packet omitted both the real base and the count + // field, so every call made by Unity looked like a zero-count draw to + // the submitted-command parser and the complete scene was discarded. + if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var drawCommand) || + !TryWriteUInt32(ctx, drawCommand, Pm4(6, ItDrawIndex2, 0)) || + !TryWriteUInt32(ctx, drawCommand + 4, indexCount) || + !TryWriteUInt32(ctx, drawCommand + 8, (uint)indexAddress) || + !TryWriteUInt32(ctx, drawCommand + 12, (uint)(indexAddress >> 32)) || + !TryWriteUInt32(ctx, drawCommand + 16, indexCount) || + !TryWriteUInt32(ctx, drawCommand + 20, 0)) { return ReturnPointer(ctx, 0); } @@ -1308,13 +1520,13 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(7, ItNop, RDrawIndexAuto)) || - !ctx.TryWriteUInt32(commandAddress + 4, indexCount) || - !ctx.TryWriteUInt32(commandAddress + 8, 0) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, 0) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RDrawIndexAuto)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexCount) || + !TryWriteUInt32(ctx, commandAddress + 8, 0) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, 0) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, 0)) { return ReturnPointer(ctx, 0); } @@ -1323,6 +1535,44 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "t1vNu082-jM", + ExportName = "sceAgcDcbDrawIndexIndirect", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbDrawIndexIndirect(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var dataOffset = (uint)ctx[CpuRegister.Rsi]; + var modifier = (uint)ctx[CpuRegister.Rdx]; + if (commandBufferAddress == 0 || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDrawIndexIndirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, dataOffset) || + !TryWriteUInt32(ctx, commandAddress + 8, 0) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, modifier)) + { + return ReturnPointer(ctx, 0); + } + + TraceAgc( + $"agc.dcb_draw_index_indirect buf=0x{commandBufferAddress:X16} " + + $"cmd=0x{commandAddress:X16} offset=0x{dataOffset:X8} modifier=0x{modifier:X8}"); + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "mStuvI0zOtc", + ExportName = "sceAgcDcbDrawIndexIndirectGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbDrawIndexIndirectGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 5u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "rUuVjyR+Rd4", ExportName = "sceAgcDcbGetLodStatsGetSize", @@ -1348,8 +1598,8 @@ public static class AgcExports var control = (uint)ctx[CpuRegister.Rcx]; var counterMask = (uint)ctx[CpuRegister.R8] & 0xFFu; var resetCounters = (uint)ctx[CpuRegister.R9] & 0x1u; - if (!ctx.TryReadUInt64(ctx[CpuRegister.Rsp] + sizeof(ulong), out var enableRaw) || - !ctx.TryReadUInt64(ctx[CpuRegister.Rsp] + (2 * sizeof(ulong)), out var counterSelectRaw) || + if (!TryReadUInt64(ctx, ctx[CpuRegister.Rsp] + sizeof(ulong), out var enableRaw) || + !TryReadUInt64(ctx, ctx[CpuRegister.Rsp] + (2 * sizeof(ulong)), out var counterSelectRaw) || commandBufferAddress == 0) { return ReturnPointer(ctx, 0); @@ -1364,11 +1614,11 @@ public static class AgcExports (counterMask << 10) | (counterSelect << 2); if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(5, ItGetLodStats, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, control) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)destinationAddress & ~0x3Fu) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(destinationAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 16, packetControl)) + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItGetLodStats, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, control) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)destinationAddress & ~0x3Fu) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(destinationAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 16, packetControl)) { return ReturnPointer(ctx, 0); } @@ -1395,8 +1645,8 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItEventWrite, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, eventType)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItEventWrite, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, eventType)) { return ReturnPointer(ctx, 0); } @@ -1418,7 +1668,7 @@ public static class AgcExports var gcrControl = (uint)ctx[CpuRegister.Rcx]; var baseAddress = ctx[CpuRegister.R8]; var sizeBytes = ctx[CpuRegister.R9]; - if (!ctx.TryReadUInt32(ctx[CpuRegister.Rsp] + sizeof(ulong), out var pollCycles)) + if (!TryReadUInt32(ctx, ctx[CpuRegister.Rsp] + sizeof(ulong), out var pollCycles)) { return ReturnPointer(ctx, 0); } @@ -1435,14 +1685,14 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RAcquireMem)) || - !ctx.TryWriteUInt32(commandAddress + 4, (engine << 31) | cbDbOp) || - !ctx.TryWriteUInt32(commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(baseAddress >> 8)) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, pollCycles / 40) || - !ctx.TryWriteUInt32(commandAddress + 28, gcrControl)) + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RAcquireMem)) || + !TryWriteUInt32(ctx, commandAddress + 4, (engine << 31) | cbDbOp) || + !TryWriteUInt32(ctx, commandAddress + 8, noSize ? 0 : (uint)(sizeBytes >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(baseAddress >> 8)) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40) || + !TryWriteUInt32(ctx, commandAddress + 28, gcrControl)) { return ReturnPointer(ctx, 0); } @@ -1467,8 +1717,8 @@ public static class AgcExports var dataAddress = ctx[CpuRegister.R8]; var dwordCount = (uint)ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var incrementRaw) || - !ctx.TryReadUInt64(stackAddress + (2 * sizeof(ulong)), out var writeConfirmRaw)) + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var incrementRaw) || + !TryReadUInt64(ctx, stackAddress + (2 * sizeof(ulong)), out var writeConfirmRaw)) { return ReturnPointer(ctx, 0); } @@ -1485,20 +1735,21 @@ public static class AgcExports var packetDwords = dwordCount + 4; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, RWriteData)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, RWriteData)) || + !TryWriteUInt32( + ctx, commandAddress + 4, destination | (cachePolicy << 8) | (increment << 16) | (writeConfirm << 24)) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)destinationAddress) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(destinationAddress >> 32))) + !TryWriteUInt32(ctx, commandAddress + 8, (uint)destinationAddress) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(destinationAddress >> 32))) { return ReturnPointer(ctx, 0); } for (uint index = 0; index < dwordCount; index++) { - if (!ctx.TryReadUInt32(dataAddress + ((ulong)index * sizeof(uint)), out var value) || - !ctx.TryWriteUInt32(commandAddress + 16 + ((ulong)index * sizeof(uint)), value)) + if (!TryReadUInt32(ctx, dataAddress + ((ulong)index * sizeof(uint)), out var value) || + !TryWriteUInt32(ctx, commandAddress + 16 + ((ulong)index * sizeof(uint)), value)) { return ReturnPointer(ctx, 0); } @@ -1529,9 +1780,9 @@ public static class AgcExports var cachePolicy = (uint)(ctx[CpuRegister.R8] & 0xFF); var address = ctx[CpuRegister.R9]; var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var reference) || - !ctx.TryReadUInt64(stackAddress + (2 * sizeof(ulong)), out var mask) || - !ctx.TryReadUInt32(stackAddress + (3 * sizeof(ulong)), out var pollCycles)) + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var reference) || + !TryReadUInt64(ctx, stackAddress + (2 * sizeof(ulong)), out var mask) || + !TryReadUInt32(ctx, stackAddress + (3 * sizeof(ulong)), out var pollCycles)) { return ReturnPointer(ctx, 0); } @@ -1555,37 +1806,37 @@ public static class AgcExports if (standardWait) { - if (!ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItWaitRegMem, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, compareFunction | ((operation & 1) << 8)) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)address) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(address >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)reference) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)mask) || - !ctx.TryWriteUInt32(commandAddress + 24, pollCycles / 40)) + if (!TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItWaitRegMem, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, compareFunction | ((operation & 1) << 8)) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)address) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(address >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)reference) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)mask) || + !TryWriteUInt32(ctx, commandAddress + 24, pollCycles / 40)) { return ReturnPointer(ctx, 0); } } - else if (!ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)address) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(address >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)mask)) + else if (!TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, packetRegister)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)address) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(address >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)mask)) { return ReturnPointer(ctx, 0); } else if (size == 0) { - if (!ctx.TryWriteUInt32(commandAddress + 16, compareFunction | (operation << 8)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference)) + if (!TryWriteUInt32(ctx, commandAddress + 16, compareFunction | (operation << 8)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference)) { return ReturnPointer(ctx, 0); } } - else if (!ctx.TryWriteUInt32(commandAddress + 16, (uint)(mask >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)reference) || - !ctx.TryWriteUInt32(commandAddress + 24, (uint)(reference >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 28, compareFunction | (operation << 8)) || - !ctx.TryWriteUInt32(commandAddress + 32, pollCycles / 40)) + else if (!TryWriteUInt32(ctx, commandAddress + 16, (uint)(mask >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) || + !TryWriteUInt32(ctx, commandAddress + 24, (uint)(reference >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 28, compareFunction | (operation << 8)) || + !TryWriteUInt32(ctx, commandAddress + 32, pollCycles / 40)) { return ReturnPointer(ctx, 0); } @@ -1601,6 +1852,45 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "u2T2DiA5hRI", + ExportName = "sceAgcDcbStallCommandBufferParser", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbStallCommandBufferParser(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var size = (uint)(ctx[CpuRegister.Rsi] & 0xFF); + var address = ctx[CpuRegister.Rdx]; + var reference = ctx[CpuRegister.Rcx]; + if (commandBufferAddress == 0 || size > 1 || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RZero)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) + { + return ReturnPointer(ctx, 0); + } + + // Direct execution submits work synchronously, so there is no independent + // hardware command processor to stall. Keep a well-formed no-op in the DCB + // so packet addresses and the command-buffer cursor remain coherent. + TraceAgc( + $"agc.dcb_stall_parser buf=0x{commandBufferAddress:X16} cmd=0x{commandAddress:X16} " + + $"size={size} addr=0x{address:X16} reference=0x{reference:X16}"); + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "+u6dKSLWM2o", + ExportName = "sceAgcDcbStallCommandBufferParserGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbStallCommandBufferParserGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 2u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "WmAc2MEj6Io", ExportName = "sceAgcDcbDmaData", @@ -1615,12 +1905,12 @@ public static class AgcExports var destinationAddress = ctx[CpuRegister.R8]; var sourceCachePolicy = (uint)(ctx[CpuRegister.R9] & 0xFF); var stackAddress = ctx[CpuRegister.Rsp]; - if (!ctx.TryReadUInt64(stackAddress + sizeof(ulong), out var control4Raw) || - !ctx.TryReadUInt64(stackAddress + (2 * sizeof(ulong)), out var sourceAddress) || - !ctx.TryReadUInt32(stackAddress + (3 * sizeof(ulong)), out var byteCount) || - !ctx.TryReadUInt64(stackAddress + (4 * sizeof(ulong)), out var control7Raw) || - !ctx.TryReadUInt64(stackAddress + (5 * sizeof(ulong)), out var control8Raw) || - !ctx.TryReadUInt64(stackAddress + (6 * sizeof(ulong)), out var control9Raw)) + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var control4Raw) || + !TryReadUInt64(ctx, stackAddress + (2 * sizeof(ulong)), out var sourceAddress) || + !TryReadUInt32(ctx, stackAddress + (3 * sizeof(ulong)), out var byteCount) || + !TryReadUInt64(ctx, stackAddress + (4 * sizeof(ulong)), out var control7Raw) || + !TryReadUInt64(ctx, stackAddress + (5 * sizeof(ulong)), out var control8Raw) || + !TryReadUInt64(ctx, stackAddress + (6 * sizeof(ulong)), out var control9Raw)) { return ReturnPointer(ctx, 0); } @@ -1635,17 +1925,19 @@ public static class AgcExports var control8 = (uint)(control8Raw & 0xFF); var control9 = (uint)(control9Raw & 0xFF); if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 8, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(8, ItNop, RDmaData)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32(ctx, commandAddress, Pm4(8, ItNop, RDmaData)) || + !TryWriteUInt32( + ctx, commandAddress + 4, destination | (destinationCachePolicy << 8) | (source << 16) | (sourceCachePolicy << 24)) || - !ctx.TryWriteUInt32( + !TryWriteUInt32( + ctx, commandAddress + 8, control4 | (control7 << 8) | (control8 << 16) | (control9 << 24)) || - !ctx.TryWriteUInt32(commandAddress + 12, byteCount) || + !TryWriteUInt32(ctx, commandAddress + 12, byteCount) || !ctx.TryWriteUInt64(commandAddress + 16, destinationAddress) || !ctx.TryWriteUInt64(commandAddress + 24, sourceAddress)) { @@ -1659,6 +1951,61 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "2ccJz9LQI+w", + ExportName = "sceAgcDcbDmaDataGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbDmaDataGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 8u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + + [SysAbiExport( + Nid = "-RnpfpxIhec", + ExportName = "sceAgcAcbDmaData", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbDmaData(CpuContext ctx) + { + var commandBufferAddress = ctx[CpuRegister.Rdi]; + var sourceSelector = (uint)(ctx[CpuRegister.Rsi] & 0xFF); + var destinationSelector = (uint)(ctx[CpuRegister.Rdx] & 0xFF); + var destinationAddress = ctx[CpuRegister.Rcx]; + var stackAddress = ctx[CpuRegister.Rsp]; + if (!TryReadUInt64(ctx, stackAddress + sizeof(ulong), out var sourceOrImmediate) || + !TryReadUInt32(ctx, stackAddress + (2 * sizeof(ulong)), out var byteCount) || + commandBufferAddress == 0 || + byteCount == 0 || + byteCount > 256u * 1024u * 1024u || + !TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || + !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RDmaData)) || + !ctx.TryWriteUInt64(commandAddress + 4, destinationAddress) || + !ctx.TryWriteUInt64(commandAddress + 12, sourceOrImmediate) || + !TryWriteUInt32(ctx, commandAddress + 20, byteCount) || + !TryWriteUInt32( + ctx, + commandAddress + 24, + sourceSelector | (destinationSelector << 8))) + { + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, commandAddress); + } + + [SysAbiExport( + Nid = "M0ttm8h7SKA", + ExportName = "sceAgcAcbDmaDataGetSize", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbDmaDataGetSize(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 8u * sizeof(uint); + return (int)ctx[CpuRegister.Rax]; + } + [SysAbiExport( Nid = "RmaJwLtc8rY", ExportName = "sceAgcDcbSetBaseIndirectArgs", @@ -1671,10 +2018,10 @@ public static class AgcExports var address = ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1)) || - !ctx.TryWriteUInt32(commandAddress + 4, 1) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)address & ~7u) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(address >> 32))) + !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItSetBase, 0) | (baseIndex << 1)) || + !TryWriteUInt32(ctx, commandAddress + 4, 1) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)address & ~7u) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(address >> 32))) { return ReturnPointer(ctx, 0); } @@ -1694,9 +2041,9 @@ public static class AgcExports var modifier = (uint)ctx[CpuRegister.Rdx]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 3, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(3, ItDispatchIndirect, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, dataOffset) || - !ctx.TryWriteUInt32(commandAddress + 8, (modifier & 0xA038u) | 0x41u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItDispatchIndirect, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, dataOffset) || + !TryWriteUInt32(ctx, commandAddress + 8, (modifier & 0xA038u) | 0x41u)) { return ReturnPointer(ctx, 0); } @@ -1722,7 +2069,7 @@ public static class AgcExports var payloadDwords = Math.Max(((uint)marker.Length + 4) / 4, 1); var packetDwords = payloadDwords + 1; if (!TryAllocateCommandDwords(ctx, commandBufferAddress, packetDwords, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(packetDwords, ItNop, RPushMarker))) + !TryWriteUInt32(ctx, commandAddress, Pm4(packetDwords, ItNop, RPushMarker))) { return ReturnPointer(ctx, 0); } @@ -1739,7 +2086,7 @@ public static class AgcExports } } - if (!ctx.TryWriteUInt32(commandAddress + 4 + ((ulong)index * sizeof(uint)), value)) + if (!TryWriteUInt32(ctx, commandAddress + 4 + ((ulong)index * sizeof(uint)), value)) { return ReturnPointer(ctx, 0); } @@ -1748,6 +2095,13 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "cpCILPya5Zk", + ExportName = "sceAgcAcbPushMarker", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbPushMarker(CpuContext ctx) => DcbPushMarker(ctx); + [SysAbiExport( Nid = "H7uZqCoNuWk", ExportName = "sceAgcDcbPopMarker", @@ -1758,8 +2112,8 @@ public static class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 2, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(2, ItNop, RPopMarker)) || - !ctx.TryWriteUInt32(commandAddress + 4, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(2, ItNop, RPopMarker)) || + !TryWriteUInt32(ctx, commandAddress + 4, 0)) { return ReturnPointer(ctx, 0); } @@ -1767,6 +2121,13 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } + [SysAbiExport( + Nid = "6mFxkVqdmbQ", + ExportName = "sceAgcAcbPopMarker", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int AcbPopMarker(CpuContext ctx) => DcbPopMarker(ctx); + [SysAbiExport( Nid = "IxYiarKlXxM", ExportName = "sceAgcDmaDataPatchSetDstAddressOrOffset", @@ -1778,14 +2139,116 @@ public static class AgcExports var destinationAddress = ctx[CpuRegister.Rsi]; if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register) || op != ItNop || - register != RDmaData) + register != RDmaData || + !TryReadUInt32(ctx, commandAddress, out var header)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - return ctx.TryWriteUInt64(commandAddress + 16, destinationAddress) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + var packetLength = ((header >> 16) & 0x3FFFu) + 2; + var destinationOffset = packetLength == 7 ? 4UL : 16UL; + return ctx.TryWriteUInt64(commandAddress + destinationOffset, destinationAddress) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + // The SRC counterpart of sceAgcDmaDataPatchSetDstAddressOrOffset. Patches + // the source field (offset +24, matching the layout written by + // sceAgcDcbDmaData) of a NOP/RDmaData packet. Games patch this to point a + // GPU DMA at the data it should copy — commonly a completion/label write. + // When it is missing the source stays 0, ApplySubmittedDmaData skips the + // copy (copied=False), and whatever the guest waits on that label for never + // fires (observed: Void Terrarium's first draw batch presents a black frame + // then the render pipeline stalls with no further flips). + [SysAbiExport( + Nid = "cdDRpqcFGbU", + ExportName = "sceAgcDmaDataPatchSetSrcAddressOrOffsetOrImmediate", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DmaDataPatchSetSrcAddressOrOffsetOrImmediate(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var sourceValue = ctx[CpuRegister.Rsi]; + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register) || + op != ItNop || + register != RDmaData || + !TryReadUInt32(ctx, commandAddress, out var header)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var packetLength = ((header >> 16) & 0x3FFFu) + 2; + var sourceOffset = packetLength == 7 ? 12UL : 24UL; + return ctx.TryWriteUInt64(commandAddress + sourceOffset, sourceValue) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "eAy8eGNsCuU", + ExportName = "sceAgcWriteDataPatchSetCachePolicy", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WriteDataPatchSetCachePolicy(CpuContext ctx) => + PatchWriteDataControlByte(ctx, byteIndex: 1); + + [SysAbiExport( + Nid = "tmy-+rBpspY", + ExportName = "sceAgcWriteDataPatchSetDst", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WriteDataPatchSetDst(CpuContext ctx) => + PatchWriteDataControlByte(ctx, byteIndex: 0); + + [SysAbiExport( + Nid = "fPSCdQxgpSw", + ExportName = "sceAgcWriteDataPatchSetAddressOrOffset", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WriteDataPatchSetAddressOrOffset(CpuContext ctx) + { + // SDK revisions disagree on whether the packet or destination is the + // first argument. Astro passes (destination, packet), while older + // captures use (packet, destination), so identify the packet by its + // header instead of hard-coding one ordering. + var first = ctx[CpuRegister.Rdi]; + var second = ctx[CpuRegister.Rsi]; + ulong commandAddress; + ulong destinationAddress; + if (TryGetPacketIdentity(ctx, first, out var firstOp, out var firstRegister) && + firstOp == ItNop && firstRegister == RWriteData) + { + commandAddress = first; + destinationAddress = second; + } + else if (TryGetPacketIdentity(ctx, second, out var secondOp, out var secondRegister) && + secondOp == ItNop && secondRegister == RWriteData) + { + commandAddress = second; + destinationAddress = first; + } + else + { + // Astro's SDK 9 ABI passes (address-or-offset, pointer-to-field) + // rather than the whole packet. The field is already the packet's + // 64-bit address payload, so patch it directly. + if (second == 0 || !ctx.TryWriteUInt64(second, first)) + { + return SetReturn(ctx, second == 0 + ? OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT + : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceAgc( + $"agc.patch_write_data_field field=0x{second:X16} value=0x{first:X16}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + + TraceAgc( + $"agc.patch_write_data_addr cmd=0x{commandAddress:X16} dst=0x{destinationAddress:X16}"); + return ctx.TryWriteUInt64(commandAddress + 8, destinationAddress) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -1799,7 +2262,7 @@ public static class AgcExports var address = ctx[CpuRegister.Rsi]; if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var fieldOffset = op == ItWaitRegMem @@ -1809,12 +2272,98 @@ public static class AgcExports : 0; if (fieldOffset == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } return ctx.TryWriteUInt64(commandAddress + fieldOffset, address) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "n485EBnIWmk", + ExportName = "sceAgcWaitRegMemPatchCompareFunction", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WaitRegMemPatchCompareFunction(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var compareFunction = (uint)ctx[CpuRegister.Rsi]; + if (compareFunction > 7 || + !TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var fieldOffset = op == ItWaitRegMem + ? 4UL + : op == ItNop && register == RWaitMem32 + ? 16UL + : op == ItNop && register == RWaitMem64 + ? 28UL + : 0; + return fieldOffset != 0 && + TryPatchUInt32Bits(ctx, commandAddress + fieldOffset, 0x7u, compareFunction) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, fieldOffset == 0 + ? OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT + : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "7nOoijNPvEU", + ExportName = "sceAgcWaitRegMemPatchReference", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WaitRegMemPatchReference(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var reference = ctx[CpuRegister.Rsi]; + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var wrote = op == ItWaitRegMem + ? TryWriteUInt32(ctx, commandAddress + 16, (uint)reference) + : op == ItNop && register == RWaitMem32 + ? TryWriteUInt32(ctx, commandAddress + 20, (uint)reference) + : op == ItNop && register == RWaitMem64 && + ctx.TryWriteUInt64(commandAddress + 20, reference); + return wrote + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, op == ItWaitRegMem || + (op == ItNop && register is RWaitMem32 or RWaitMem64) + ? OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT + : OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + [SysAbiExport( + Nid = "hXAnLgDHCoI", + ExportName = "sceAgcWaitRegMemPatchMask", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int WaitRegMemPatchMask(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var mask = ctx[CpuRegister.Rsi]; + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var wrote = op == ItWaitRegMem + ? TryWriteUInt32(ctx, commandAddress + 20, (uint)mask) + : op == ItNop && register == RWaitMem32 + ? TryWriteUInt32(ctx, commandAddress + 12, (uint)mask) + : op == ItNop && register == RWaitMem64 && + ctx.TryWriteUInt64(commandAddress + 12, mask); + return wrote + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, op == ItWaitRegMem || + (op == ItNop && register is RWaitMem32 or RWaitMem64) + ? OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT + : OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } [SysAbiExport( @@ -1830,14 +2379,97 @@ public static class AgcExports op != ItNop || register != RReleaseMem) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } return ctx.TryWriteUInt64(commandAddress + 12, address) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "J8YCgfKAMQs", + ExportName = "sceAgcQueueEndOfPipeActionPatchGcrCntl", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int QueueEndOfPipeActionPatchGcrCntl(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + if (!IsAgcReleaseMemPacket(ctx, commandAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + return TryPatchUInt32Bits( + ctx, + commandAddress + 8, + 0x0000_FFFFu, + (uint)ctx[CpuRegister.Rsi] & 0xFFFFu) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "MlEw1feXcjg", + ExportName = "sceAgcQueueEndOfPipeActionPatchData", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int QueueEndOfPipeActionPatchData(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + if (!IsAgcReleaseMemPacket(ctx, commandAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + return ctx.TryWriteUInt64(commandAddress + 20, ctx[CpuRegister.Rsi]) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "T9fjQIINoeE", + ExportName = "sceAgcQueueEndOfPipeActionPatchType", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int QueueEndOfPipeActionPatchType(CpuContext ctx) + { + var commandAddress = ctx[CpuRegister.Rdi]; + var dataSelection = (uint)ctx[CpuRegister.Rsi]; + TraceAgc( + $"agc.eop_patch_type cmd=0x{commandAddress:X16} value=0x{dataSelection:X8}"); + if (dataSelection > 3 || !IsAgcReleaseMemPacket(ctx, commandAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + return TryPatchUInt32Bits( + ctx, + commandAddress + 8, + 0x00FF_0000u, + dataSelection << 16) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + private static bool IsAgcReleaseMemPacket(CpuContext ctx, ulong commandAddress) => + TryGetPacketIdentity(ctx, commandAddress, out var op, out var register) && + op == ItNop && + register == RReleaseMem; + + private static bool TryPatchUInt32Bits( + CpuContext ctx, + ulong address, + uint mask, + uint value) + { + return TryReadUInt32(ctx, address, out var current) && + TryWriteUInt32(ctx, address, PatchUInt32Bits(current, mask, value)); + } + + private static uint PatchUInt32Bits(uint current, uint mask, uint value) => + (current & ~mask) | (value & mask); + [SysAbiExport( Nid = "l4fM9K-Lyks", ExportName = "sceAgcDcbSetIndexBuffer", @@ -1854,11 +2486,11 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(3, ItIndexBase, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, (uint)(indexBufferAddress & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(indexBufferAddress >> 32)) || - !ctx.TryWriteUInt32(commandAddress + 12, Pm4(2, ItIndexBufferSize, 0)) || - !ctx.TryWriteUInt32(commandAddress + 16, indexCount)) + !TryWriteUInt32(ctx, commandAddress, Pm4(3, ItIndexBase, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, (uint)(indexBufferAddress & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(indexBufferAddress >> 32)) || + !TryWriteUInt32(ctx, commandAddress + 12, Pm4(2, ItIndexBufferSize, 0)) || + !TryWriteUInt32(ctx, commandAddress + 16, indexCount)) { return ReturnPointer(ctx, 0); } @@ -1884,11 +2516,11 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 5, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(5, ItDrawIndexOffset2, 0)) || - !ctx.TryWriteUInt32(commandAddress + 4, indexCount) || - !ctx.TryWriteUInt32(commandAddress + 8, indexOffset) || - !ctx.TryWriteUInt32(commandAddress + 12, indexCount) || - !ctx.TryWriteUInt32(commandAddress + 16, flags & 0xE000_0001u)) + !TryWriteUInt32(ctx, commandAddress, Pm4(5, ItDrawIndexOffset2, 0)) || + !TryWriteUInt32(ctx, commandAddress + 4, indexCount) || + !TryWriteUInt32(ctx, commandAddress + 8, indexOffset) || + !TryWriteUInt32(ctx, commandAddress + 12, indexCount) || + !TryWriteUInt32(ctx, commandAddress + 16, flags & 0xE000_0001u)) { return ReturnPointer(ctx, 0); } @@ -1913,13 +2545,13 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 7, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(7, ItNop, RWaitFlipDone)) || - !ctx.TryWriteUInt32(commandAddress + 4, videoOutHandle) || - !ctx.TryWriteUInt32(commandAddress + 8, displayBufferIndex) || - !ctx.TryWriteUInt32(commandAddress + 12, 0) || - !ctx.TryWriteUInt32(commandAddress + 16, 0) || - !ctx.TryWriteUInt32(commandAddress + 20, 0) || - !ctx.TryWriteUInt32(commandAddress + 24, 0)) + !TryWriteUInt32(ctx, commandAddress, Pm4(7, ItNop, RWaitFlipDone)) || + !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || + !TryWriteUInt32(ctx, commandAddress + 8, displayBufferIndex) || + !TryWriteUInt32(ctx, commandAddress + 12, 0) || + !TryWriteUInt32(ctx, commandAddress + 16, 0) || + !TryWriteUInt32(ctx, commandAddress + 20, 0) || + !TryWriteUInt32(ctx, commandAddress + 24, 0)) { return ReturnPointer(ctx, 0); } @@ -1946,12 +2578,12 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 6, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(6, ItNop, RFlip)) || - !ctx.TryWriteUInt32(commandAddress + 4, videoOutHandle) || - !ctx.TryWriteUInt32(commandAddress + 8, unchecked((uint)displayBufferIndex)) || - !ctx.TryWriteUInt32(commandAddress + 12, flipMode) || - !ctx.TryWriteUInt32(commandAddress + 16, (uint)(flipArg & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 20, (uint)(flipArg >> 32))) + !TryWriteUInt32(ctx, commandAddress, Pm4(6, ItNop, RFlip)) || + !TryWriteUInt32(ctx, commandAddress + 4, videoOutHandle) || + !TryWriteUInt32(ctx, commandAddress + 8, unchecked((uint)displayBufferIndex)) || + !TryWriteUInt32(ctx, commandAddress + 12, flipMode) || + !TryWriteUInt32(ctx, commandAddress + 16, (uint)(flipArg & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 20, (uint)(flipArg >> 32))) { return ReturnPointer(ctx, 0); } @@ -1960,96 +2592,6 @@ public static class AgcExports return ReturnPointer(ctx, commandAddress); } - // Guest draw-command builders: emit valid (skippable) packets and return a live - // command pointer so the guest's command-buffer build succeeds; full draw processing is TODO. - [SysAbiExport( - Nid = "8N2tmT3jmC8", - ExportName = "sceAgcDcbSetIndexCount", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DcbSetIndexCount(CpuContext ctx) - { - var dcb = ctx[CpuRegister.Rdi]; - var indexCount = (uint)ctx[CpuRegister.Rsi]; - if (dcb == 0) - { - return ReturnPointer(ctx, 0); - } - - if (!TryAllocateCommandDwords(ctx, dcb, 2, out var cmd) || - !ctx.TryWriteUInt32(cmd, Pm4(2, ItNop, RZero)) || - !ctx.TryWriteUInt32(cmd + 4, indexCount)) - { - return ReturnPointer(ctx, 0); - } - - return ReturnPointer(ctx, cmd); - } - - [SysAbiExport( - Nid = "xSAR0LTcRKM", - ExportName = "sceAgcDcbJump", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DcbJump(CpuContext ctx) - { - var dcb = ctx[CpuRegister.Rdi]; - var target = ctx[CpuRegister.Rsi]; - var sizeDwords = (uint)ctx[CpuRegister.Rdx]; - if (dcb == 0) - { - return ReturnPointer(ctx, 0); - } - - if (!TryAllocateCommandDwords(ctx, dcb, 4, out var cmd) || - !ctx.TryWriteUInt32(cmd, Pm4(4, ItIndirectBuffer, RZero)) || - !ctx.TryWriteUInt32(cmd + 4, (uint)(target & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(cmd + 8, (uint)((target >> 32) & 0xFFFFUL)) || - !ctx.TryWriteUInt32(cmd + 12, sizeDwords & 0xFFFFF)) - { - return ReturnPointer(ctx, 0); - } - - return ReturnPointer(ctx, cmd); - } - - [SysAbiExport( - Nid = "bbFueFP+J4k", - ExportName = "sceAgcDcbSetPredication", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DcbSetPredication(CpuContext ctx) - { - var dcb = ctx[CpuRegister.Rdi]; - var address = ctx[CpuRegister.Rsi]; - if (dcb == 0) - { - return ReturnPointer(ctx, 0); - } - - if (!TryAllocateCommandDwords(ctx, dcb, 3, out var cmd) || - !ctx.TryWriteUInt32(cmd, Pm4(3, ItNop, RZero)) || - !ctx.TryWriteUInt32(cmd + 4, (uint)(address & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(cmd + 8, (uint)(address >> 32))) - { - return ReturnPointer(ctx, 0); - } - - return ReturnPointer(ctx, cmd); - } - - [SysAbiExport( - Nid = "w6Dj1VJt5qY", - ExportName = "sceAgcSetPacketPredication", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int SetPacketPredication(CpuContext ctx) - { - // Global predication toggle on a packet; a no-op is safe for rendering. - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "w2rJhmD+dsE", ExportName = "sceAgcDriverAddEqEvent", @@ -2066,11 +2608,11 @@ public static class AgcExports KernelEventQueueCompatExports.KernelEventFilterGraphics, userData)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } TraceAgc($"agc.driver_add_eq_event eq=0x{equeue:X16} id=0x{eventId:X16} udata=0x{userData:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -2087,11 +2629,11 @@ public static class AgcExports eventId, KernelEventQueueCompatExports.KernelEventFilterGraphics)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } TraceAgc($"agc.driver_delete_eq_event eq=0x{equeue:X16} id=0x{eventId:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -2103,14 +2645,14 @@ public static class AgcExports { var packetAddress = ctx[CpuRegister.Rdi]; if (packetAddress == 0 || - !ctx.TryReadUInt64(packetAddress, out var commandAddress) || - !ctx.TryReadUInt32(packetAddress + 8, out var dwordCount)) + !TryReadUInt64(ctx, packetAddress, out var commandAddress) || + !TryReadUInt32(ctx, packetAddress + 8, out var dwordCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var tracePackets = false; - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal)) + if (_traceAgc) { lock (_submitTraceGate) { @@ -2123,77 +2665,20 @@ public static class AgcExports TraceAgc($"agc.driver_submit_dcb packet=0x{packetAddress:X16} addr=0x{commandAddress:X16} dwords={dwordCount}"); } + VulkanVideoPresenter.AttachGuestMemory(ctx.Memory); var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); lock (gpuState.Gate) { - Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); - try - { - ParseSubmittedDcb(ctx, gpuState, gpuState.Graphics, commandAddress, dwordCount, tracePackets); - DrainResumableDcbs(ctx, gpuState, tracePackets); - } - finally - { - Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); - } - } - - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - // ABI (reversed from Quake): rdi = array of DCB base addresses (u64 each), - // rsi = array of DCB sizes in dwords (u32 each), rdx = buffer count. - [SysAbiExport( - Nid = "6UzEidRZwkg", - ExportName = "sceAgcDriverSubmitMultiDcbs", - Target = Generation.Gen5, - LibraryName = "libSceAgcDriver")] - public static int DriverSubmitMultiDcbs(CpuContext ctx) - { - var addressArray = ctx[CpuRegister.Rdi]; - var sizeArray = ctx[CpuRegister.Rsi]; - var bufferCount = (uint)ctx[CpuRegister.Rdx]; - if (addressArray == 0 || sizeArray == 0 || bufferCount == 0 || bufferCount > 4096) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var tracePackets = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal); - - var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (gpuState.Gate) - { - Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); - try - { - for (uint i = 0; i < bufferCount; i++) - { - if (!ctx.TryReadUInt64(addressArray + i * 8, out var commandAddress) || - commandAddress == 0 || - !ctx.TryReadUInt32(sizeArray + i * 4, out var dwordCount) || - dwordCount == 0) - { - continue; - } - - if (tracePackets) - { - TraceAgc( - $"agc.driver_submit_multi_dcbs index={i}/{bufferCount} " + - $"addr=0x{commandAddress:X16} dwords={dwordCount}"); - } - - ParseSubmittedDcb(ctx, gpuState, gpuState.Graphics, commandAddress, dwordCount, tracePackets); - } - - DrainResumableDcbs(ctx, gpuState, tracePackets); - } - finally - { - Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); - } + gpuState.Graphics.QueueName = "dcb.graphics"; + EnqueueSubmittedDcb( + ctx, + gpuState, + gpuState.Graphics, + commandAddress, + dwordCount, + ++gpuState.SubmissionSequence, + tracePackets); + DrainResumableDcbs(ctx, gpuState, tracePackets); } ctx[CpuRegister.Rax] = 0; @@ -2210,14 +2695,14 @@ public static class AgcExports var ownerHandle = (uint)ctx[CpuRegister.Rdi]; var packetAddress = ctx[CpuRegister.Rsi]; if (packetAddress == 0 || - !ctx.TryReadUInt64(packetAddress, out var commandAddress) || - !ctx.TryReadUInt32(packetAddress + 8, out var dwordCount)) + !TryReadUInt64(ctx, packetAddress, out var commandAddress) || + !TryReadUInt32(ctx, packetAddress + 8, out var dwordCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var tracePackets = false; - if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal)) + if (_traceAgc) { lock (_submitTraceGate) { @@ -2232,6 +2717,7 @@ public static class AgcExports $"addr=0x{commandAddress:X16} dwords={dwordCount}"); } + VulkanVideoPresenter.AttachGuestMemory(ctx.Memory); var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); lock (gpuState.Gate) { @@ -2241,16 +2727,16 @@ public static class AgcExports gpuState.ComputeQueues.Add(ownerHandle, queueState); } - Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); - try - { - ParseSubmittedDcb(ctx, gpuState, queueState, commandAddress, dwordCount, tracePackets); - DrainResumableDcbs(ctx, gpuState, tracePackets); - } - finally - { - Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); - } + queueState.QueueName = $"acb.compute[{ownerHandle}]"; + EnqueueSubmittedDcb( + ctx, + gpuState, + queueState, + commandAddress, + dwordCount, + ++gpuState.SubmissionSequence, + tracePackets); + DrainResumableDcbs(ctx, gpuState, tracePackets); } ctx[CpuRegister.Rax] = 0; @@ -2268,91 +2754,16 @@ public static class AgcExports if (outAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryWriteUInt32(outAddress, ResourceRegistrationMaxNameLength)) + if (!TryWriteUInt32(ctx, outAddress, 256)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - TraceAgc( - $"agc.driver_get_resource_registration_max_name_length " + - $"out=0x{outAddress:X16} value={ResourceRegistrationMaxNameLength}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "AOLcoIkQDgM", - ExportName = "sceAgcDriverQueryResourceRegistrationUserMemoryRequirements", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverQueryResourceRegistrationUserMemoryRequirements(CpuContext ctx) - { - var sizeAddress = ctx[CpuRegister.Rdi]; - var resourceCount = ctx[CpuRegister.Rsi]; - var ownerCount = ctx[CpuRegister.Rdx]; - if (sizeAddress == 0 || resourceCount == 0 || ownerCount == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - ulong requiredSize; - try - { - requiredSize = checked( - resourceCount * ResourceRegistrationBytesPerResource + - ownerCount * ResourceRegistrationBytesPerOwner); - } - catch (OverflowException) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - if (!ctx.TryWriteUInt64(sizeAddress, requiredSize)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - TraceAgc( - $"agc.driver_query_resource_registration_memory resources={resourceCount} " + - $"owners={ownerCount} bytes=0x{requiredSize:X}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "F0Y42t-3e18", - ExportName = "sceAgcDriverInitResourceRegistration", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverInitResourceRegistration(CpuContext ctx) - { - var memoryAddress = ctx[CpuRegister.Rdi]; - var memorySize = ctx[CpuRegister.Rsi]; - var ownerCount = ctx[CpuRegister.Rdx]; - if (memoryAddress == 0 || memorySize == 0 || ownerCount == 0 || ownerCount > uint.MaxValue) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (state.Gate) - { - state.ResourceRegistrationInitialized = true; - state.ResourceRegistrationMemory = memoryAddress; - state.ResourceRegistrationMemorySize = memorySize; - state.ResourceRegistrationMaxOwners = (uint)ownerCount; - state.ResourceOwners.Clear(); - state.RegisteredResources.Clear(); - state.DefaultOwner = DefaultAgcOwner; - state.NextOwner = 1; - state.NextResource = 1; - } - - TraceAgc( - $"agc.driver_init_resource_registration memory=0x{memoryAddress:X16} " + - $"bytes=0x{memorySize:X} owners={ownerCount}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceAgc($"agc.driver_get_resource_registration_max_name_length out=0x{outAddress:X16} value=256"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } private const uint DefaultAgcOwner = 1; @@ -2367,101 +2778,16 @@ public static class AgcExports if (ownerAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - uint owner; - lock (state.Gate) + if (!TryWriteUInt32(ctx, ownerAddress, DefaultAgcOwner)) { - owner = state.DefaultOwner; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - if (!ctx.TryWriteUInt32(ownerAddress, owner)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - TraceAgc($"agc.driver_get_default_owner out=0x{ownerAddress:X16} owner={owner}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "U9ueyEhSkF4", - ExportName = "sceAgcDriverRegisterDefaultOwner", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverRegisterDefaultOwner(CpuContext ctx) - { - var owner = (uint)ctx[CpuRegister.Rdi]; - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (state.Gate) - { - state.DefaultOwner = owner; - } - - TraceAgc($"agc.driver_register_default_owner owner={owner}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "X-Nm5KLREeg", - ExportName = "sceAgcDriverRegisterOwner", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverRegisterOwner(CpuContext ctx) - { - var ownerAddress = ctx[CpuRegister.Rdi]; - var nameAddress = ctx[CpuRegister.Rsi]; - if (ownerAddress == 0 || nameAddress == 0 || - !TryReadGuestCString( - ctx, - nameAddress, - ResourceRegistrationMaxNameLength, - out var nameBytes)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - uint owner; - lock (state.Gate) - { - if (!state.ResourceRegistrationInitialized || - state.ResourceRegistrationMaxOwners != 0 && - state.ResourceOwners.Count >= state.ResourceRegistrationMaxOwners) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - owner = state.NextOwner; - while (owner == state.DefaultOwner || state.ResourceOwners.ContainsKey(owner)) - { - owner++; - if (owner == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - } - - state.NextOwner = owner + 1; - state.ResourceOwners.Add(owner, System.Text.Encoding.UTF8.GetString(nameBytes)); - } - - if (!ctx.TryWriteUInt32(ownerAddress, owner)) - { - lock (state.Gate) - { - state.ResourceOwners.Remove(owner); - } - - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - TraceAgc( - $"agc.driver_register_owner out=0x{ownerAddress:X16} owner={owner} " + - $"name={System.Text.Encoding.UTF8.GetString(nameBytes)}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceAgc($"agc.driver_get_default_owner out=0x{ownerAddress:X16} owner={DefaultAgcOwner}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -2471,88 +2797,17 @@ public static class AgcExports LibraryName = "libSceAgc")] public static int DriverRegisterResource(CpuContext ctx) { - var resourceHandleAddress = ctx[CpuRegister.Rdi]; + var resourceAddress = ctx[CpuRegister.Rdi]; var owner = (uint)ctx[CpuRegister.Rsi]; - var resourceAddress = ctx[CpuRegister.Rdx]; - var resourceSize = ctx[CpuRegister.Rcx]; - var nameAddress = ctx[CpuRegister.R8]; - var type = (uint)ctx[CpuRegister.R9]; - if (resourceHandleAddress == 0 || resourceAddress == 0 || resourceSize == 0 || - !ctx.TryReadUInt32(ctx[CpuRegister.Rsp] + sizeof(ulong), out var flags) || - !TryReadGuestCString( - ctx, - nameAddress, - ResourceRegistrationMaxNameLength, - out var nameBytes)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - uint resourceHandle; - lock (state.Gate) - { - if (!state.ResourceRegistrationInitialized || - owner != state.DefaultOwner && - !state.ResourceOwners.ContainsKey(owner)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - resourceHandle = state.NextResource++; - if (resourceHandle == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - state.RegisteredResources.Add( - resourceHandle, - new RegisteredAgcResource( - owner, - resourceAddress, - resourceSize, - System.Text.Encoding.UTF8.GetString(nameBytes), - type, - flags)); - } - - if (!ctx.TryWriteUInt32(resourceHandleAddress, resourceHandle)) - { - lock (state.Gate) - { - state.RegisteredResources.Remove(resourceHandle); - } - - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } + var nameAddress = ctx[CpuRegister.Rdx]; + var type = (uint)ctx[CpuRegister.R8]; + var flags = (uint)ctx[CpuRegister.R9]; TraceAgc( - $"agc.driver_register_resource handle={resourceHandle} owner={owner} " + - $"resource=0x{resourceAddress:X16} bytes=0x{resourceSize:X} " + - $"name={System.Text.Encoding.UTF8.GetString(nameBytes)} type={type} flags={flags}"); + $"agc.driver_register_resource resource=0x{resourceAddress:X16} owner={owner} " + + $"name=0x{nameAddress:X16} type={type} flags={flags}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - [SysAbiExport( - Nid = "pWLG7WOpVcw", - ExportName = "sceAgcDriverUnregisterResource", - Target = Generation.Gen5, - LibraryName = "libSceAgc")] - public static int DriverUnregisterResource(CpuContext ctx) - { - var resourceHandle = (uint)ctx[CpuRegister.Rdi]; - var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); - lock (state.Gate) - { - if (!state.RegisteredResources.Remove(resourceHandle)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - } - - TraceAgc($"agc.driver_unregister_resource handle={resourceHandle}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } // Synthetic label for an uncatalogued NID (the Unknown* convention); the NID is authoritative. @@ -2569,7 +2824,7 @@ public static class AgcExports $"rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} " + $"rcx=0x{ctx[CpuRegister.Rcx]:X16} r8=0x{ctx[CpuRegister.R8]:X16} r9=0x{ctx[CpuRegister.R9]:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } #pragma warning restore SHEM006 @@ -2597,7 +2852,7 @@ public static class AgcExports var commandBufferAddress = ctx[CpuRegister.Rdi]; if (commandBufferAddress == 0 || !TryAllocateCommandDwords(ctx, commandBufferAddress, 1, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, 0x8000_0000)) + !TryWriteUInt32(ctx, commandAddress, 0x8000_0000)) { return ReturnPointer(ctx, 0); } @@ -2609,64 +2864,99 @@ public static class AgcExports } #pragma warning restore SHEM006 - // WAIT_REG_MEM packets whose condition is not met suspend their DCB into - // GpuWaitRegistry. Each submit re-checks every suspended DCB against current guest - // memory (labels are advanced by ReleaseMem/WriteData/DmaData packets or by direct - // CPU writes) and resumes the ones whose condition is now satisfied. A resumed DCB - // can itself write labels that unblock others, so loop until a fixed point. - private static void DrainResumableDcbs( + private static void EnqueueSubmittedDcb( CpuContext ctx, SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong commandAddress, + uint dwordCount, + ulong submissionId, bool tracePackets) { - for (var pass = 0; pass < 256; pass++) - { - var woken = GpuWaitRegistry.CollectSatisfied((address, is64Bit) => - { - if (is64Bit) - { - return ctx.TryReadUInt64(address, out var value64) - ? value64 - : (ulong?)null; - } + state.PendingSubmissions.Enqueue(new SubmittedDcbState.PendingSubmission( + commandAddress, + dwordCount, + submissionId, + tracePackets)); + PumpSubmittedQueue(ctx, gpuState, state); + } - return ctx.TryReadUInt32(address, out var value32) - ? value32 - : (ulong?)null; - }); - if (woken is null) + private static void PumpSubmittedQueue( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state) + { + if (state.IsSuspended) + { + return; + } + + while (!state.HasActiveSubmission && + state.PendingSubmissions.TryDequeue(out var submission)) + { + state.HasActiveSubmission = true; + state.ActiveSubmissionId = submission.SubmissionId; + state.IsSuspended = ParseSubmittedDcb( + ctx, + gpuState, + state, + submission.CommandAddress, + submission.DwordCount, + submission.TracePackets); + if (state.IsSuspended) { return; } - foreach (var waiter in woken) - { - var remainingDwords = waiter.TotalDwords - waiter.ResumeOffset; - if (remainingDwords == 0) - { - continue; - } - - if (tracePackets) - { - TraceAgc( - $"agc.dcb.resumed addr=0x{waiter.WaitAddress:X16} " + - $"resume=0x{waiter.ResumeAddress:X16} dwords={remainingDwords}"); - } - - var state = waiter.State as SubmittedDcbState ?? gpuState.Graphics; - ParseSubmittedDcb( - ctx, - gpuState, - state, - waiter.ResumeAddress, - remainingDwords, - tracePackets); - } + state.HasActiveSubmission = false; + NotifySubmittedDcbCompleted(gpuState, state, submission.SubmissionId); } } - private static void ParseSubmittedDcb( + private static void NotifySubmittedDcbCompleted( + SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong submissionId) + { + if (!ReferenceEquals(state, gpuState.Graphics) || + state.CompletionEventNotifiedSubmissionId == submissionId) + { + return; + } + + state.CompletionEventNotifiedSubmissionId = submissionId; + void TriggerCompletionEvents() + { + var triggered = KernelEventQueueCompatExports.TriggerRegisteredEvents( + ident: 0, + KernelEventQueueCompatExports.KernelEventFilterGraphics, + data: 0); + if (_compatibilitySubmitCompletionEvent) + { + triggered += KernelEventQueueCompatExports.TriggerRegisteredEventsDistinct( + KernelEventQueueCompatExports.KernelEventFilterGraphics); + } + TraceAgc( + $"agc.driver_submit_dcb completion submission={submissionId} " + + $"queues={triggered}"); + } + + // A DCB is complete only after its translated Vulkan work and ordered + // guest-memory writes have finished. Put the notification on that same + // logical graphics queue instead of approximating completion with a + // timer, which can wake Unity while its upload data is still stale. + if (VulkanVideoPresenter.SubmitOrderedGuestAction( + TriggerCompletionEvents, + $"agc submit completion {submissionId}") == 0) + { + TriggerCompletionEvents(); + } + } + + // Returns true only when parsing stopped on an unsatisfied WAIT_REG_MEM. + // Malformed packets are dropped as completed so one bad submission cannot + // permanently wedge all later work on the same hardware queue. + private static bool ParseSubmittedDcb( CpuContext ctx, SubmittedGpuState gpuState, SubmittedDcbState state, @@ -2676,16 +2966,55 @@ public static class AgcExports { if (commandAddress == 0 || dwordCount == 0 || dwordCount > 1_000_000) { - return; + return false; } + using var guestQueueScope = VulkanVideoPresenter.EnterGuestQueue( + state.QueueName, + state.ActiveSubmissionId); + var windowByteCount = checked((int)(dwordCount * sizeof(uint))); + var rented = VulkanVideoPresenter.GuestDataPool.Rent(windowByteCount); + try + { + if (ctx.Memory.TryRead(commandAddress, rented.AsSpan(0, windowByteCount))) + { + _dcbWindowBuffer = rented; + _dcbWindowStart = commandAddress; + _dcbWindowByteLength = windowByteCount; + } + + return ParseSubmittedDcbCore( + ctx, + gpuState, + state, + commandAddress, + dwordCount, + tracePackets); + } + finally + { + _dcbWindowBuffer = null; + _dcbWindowByteLength = 0; + VulkanVideoPresenter.GuestDataPool.Return(rented); + } + } + + private static bool ParseSubmittedDcbCore( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong commandAddress, + uint dwordCount, + bool tracePackets) + { var offset = 0u; while (offset < dwordCount) { var currentAddress = commandAddress + ((ulong)offset * sizeof(uint)); - if (!ctx.TryReadUInt32(currentAddress, out var header)) + if (!TryReadUInt32(ctx, currentAddress, out var header)) { - return; + TracePacketParseFailure(state, currentAddress, offset, 0, "header-read"); + return false; } var packetType = header >> 30; @@ -2697,28 +3026,76 @@ public static class AgcExports $"agc.dcb.packet dw={offset} addr=0x{currentAddress:X16} " + $"header=0x{header:X8} len=1 type=2"); } + offset++; continue; } if (packetType != 3) { - return; + TracePacketParseFailure( + state, + currentAddress, + offset, + header, + $"packet-type-{packetType}"); + return false; } var length = Pm4Length(header); if (length == 0 || offset + length > dwordCount) { - return; + TracePacketParseFailure( + state, + currentAddress, + offset, + header, + $"length-{length}-remaining-{dwordCount - offset}"); + return false; } var op = (header >> 8) & 0xFFu; var register = (header >> 2) & 0x3Fu; + if (_traceFramePackets && ReferenceEquals(state, gpuState.Graphics)) + { + var packetKey = (op, op == ItNop ? register : uint.MaxValue); + state.FramePacketCounts[packetKey] = + state.FramePacketCounts.TryGetValue(packetKey, out var packetCount) + ? packetCount + 1 + : 1; + state.FramePacketCount++; + } if (tracePackets) { TraceSubmittedPacket(ctx, currentAddress, offset, header, length, op, register); } + if (_traceDraws) + { + CountSubmittedOpcode(op, register); + } + + if (op == ItNop && + register is RDrawReset or RAcbReset && + length >= 2) + { + ResetSubmittedParserState(state); + TraceAgc( + $"agc.queue_reset queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} " + + $"kind={(register == RDrawReset ? "draw" : "acb")} " + + $"packet=0x{currentAddress:X16}"); + } + + if (op == ItNop && register == RAcquireMem && length >= 8) + { + ApplySubmittedAcquireMem( + ctx, + state, + currentAddress, + tracePackets); + } + if (op == ItSetShReg && TryReadTextureDescriptor(ctx, currentAddress, length, out var texture)) { @@ -2729,61 +3106,95 @@ public static class AgcExports if (op == ItSetBase && length >= 4 && - ctx.TryReadUInt32(currentAddress + 4, out var baseSelector) && + TryReadUInt32(ctx, currentAddress + 4, out var baseSelector) && baseSelector == 1 && - ctx.TryReadUInt64(currentAddress + 8, out var indirectArgsAddress)) + TryReadUInt64(ctx, currentAddress + 8, out var indirectArgsAddress)) { state.IndirectArgsAddress = indirectArgsAddress; } if (op == ItEventWrite && length >= 2 && - ctx.TryReadUInt32(currentAddress + sizeof(uint), out var eventTypeRaw)) + TryReadUInt32(ctx, currentAddress + sizeof(uint), out var eventTypeRaw)) { var eventType = eventTypeRaw & 0x3Fu; - - // The guest registers AGC events with a full eventId, but the command buffer - // only carries a 6-bit EVENT_TYPE. Those two values are not the same numbering - // scheme, so exact ident matching never wakes anything. Trigger every graphics - // event registration instead (workaround for issue #173). - var triggered = KernelEventQueueCompatExports.TriggerRegisteredEventsByFilter( - KernelEventQueueCompatExports.KernelEventFilterGraphics, - eventType); - if (tracePackets) - { - TraceAgc($"agc.dcb.event type=0x{eventType:X2} queues={triggered}"); - } + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + var triggered = KernelEventQueueCompatExports.TriggerRegisteredEventsByFilter( + KernelEventQueueCompatExports.KernelEventFilterGraphics, + eventType); + if (tracePackets) + { + TraceAgc($"agc.dcb.event type=0x{eventType:X2} queues={triggered}"); + } + }, + $"event_write type=0x{eventType:X2}", + currentAddress); } if (op == ItNop && register == RReleaseMem && length >= 7) { - ApplySubmittedReleaseMem(ctx, currentAddress, tracePackets); + ApplySubmittedReleaseMem(ctx, gpuState, state, currentAddress, tracePackets); + } + + if (op == ItReleaseMem && length >= 8) + { + ApplySubmittedStandardReleaseMem( + ctx, + gpuState, + state, + currentAddress, + tracePackets); } if (op == ItNop && register == RWriteData && length >= 4) { - ApplySubmittedWriteData(ctx, currentAddress, length, tracePackets); + ApplySubmittedWriteData( + ctx, + gpuState, + state, + currentAddress, + length, + standardPacket: false, + tracePacket: tracePackets); } if (op == ItWriteData && length >= 4) { - ApplySubmittedWriteData(ctx, currentAddress, length, tracePackets); + ApplySubmittedWriteData( + ctx, + gpuState, + state, + currentAddress, + length, + standardPacket: true, + tracePacket: tracePackets); } - if (op == ItNop && register == RDmaData && length >= 8) + if (op == ItNop && register == RDmaData && length >= 7) { - ApplySubmittedDmaData(ctx, currentAddress, tracePackets); + ApplySubmittedDmaData( + ctx, + gpuState, + state, + currentAddress, + compactLayout: length == 7, + tracePacket: tracePackets); } if (op == ItDmaData && length >= 7) { - ApplySubmittedStandardDmaData(ctx, currentAddress); + ApplySubmittedStandardDmaData(ctx, gpuState, state, currentAddress); } if (op == ItIndexBase && length >= 3 && - ctx.TryReadUInt32(currentAddress + 4, out var indexBaseLo) && - ctx.TryReadUInt32(currentAddress + 8, out var indexBaseHi)) + TryReadUInt32(ctx, currentAddress + 4, out var indexBaseLo) && + TryReadUInt32(ctx, currentAddress + 8, out var indexBaseHi)) { state.IndexBufferAddress = indexBaseLo | ((ulong)indexBaseHi << 32); @@ -2791,109 +3202,53 @@ public static class AgcExports if (op == ItIndexBufferSize && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var indexBufferCount)) + TryReadUInt32(ctx, currentAddress + 4, out var indexBufferCount)) { state.IndexBufferCount = indexBufferCount; } + if (op == ItNop && + register == RIndexCount && + length >= 2 && + TryReadUInt32(ctx, currentAddress + 4, out var customIndexCount)) + { + state.IndexBufferCount = customIndexCount; + } + if (op == ItIndexType && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var indexSize)) + TryReadUInt32(ctx, currentAddress + 4, out var indexSize)) { state.IndexSize = indexSize & 0x3; } if (op == ItNumInstances && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var instanceCount)) + TryReadUInt32(ctx, currentAddress + 4, out var instanceCount)) { state.InstanceCount = Math.Max(instanceCount, 1); } - // WAIT_REG_MEM (AGC NOP-encapsulated 32/64-bit variants): when the condition is - // not yet satisfied, suspend this DCB; DrainResumableDcbs resumes it once the - // watched memory advances. - if (op == ItNop && register is RWaitMem32 or RWaitMem64 && + if (op == ItNop && + register is RWaitMem32 or RWaitMem64 && length >= (register == RWaitMem32 ? 6u : 9u)) { - if (TryParseWaitRegMem(ctx, currentAddress, register == RWaitMem64, - out var waitAddr, out var refVal, out var waitMask, out var cmpFunc)) + if (HandleSubmittedWaitRegMem( + ctx, state, commandAddress, currentAddress, offset, length, + dwordCount, is64Bit: register == RWaitMem64, isStandard: false, + tracePackets)) { - ulong curVal = 0; - bool hasCurVal; - if (register == RWaitMem64) - { - hasCurVal = ctx.TryReadUInt64(waitAddr, out curVal); - } - else if (ctx.TryReadUInt32(waitAddr, out var curVal32)) - { - curVal = curVal32; - hasCurVal = true; - } - else - { - hasCurVal = false; - } - - var waiter = new GpuWaitRegistry.WaitingDcb - { - CommandBufferAddress = commandAddress, - ResumeAddress = currentAddress + ((ulong)length * sizeof(uint)), - TotalDwords = dwordCount, - ResumeOffset = offset + length, - ReferenceValue = refVal, - Mask = waitMask, - CompareFunction = cmpFunc, - Is64Bit = register == RWaitMem64, - State = state, - }; - if (hasCurVal && !GpuWaitRegistry.Compare(waiter, curVal)) - { - GpuWaitRegistry.Register(waitAddr, waiter); - - if (tracePackets) - { - TraceAgc( - $"agc.dcb.suspended addr=0x{waitAddr:X16} ref=0x{refVal:X16} " + - $"mask=0x{waitMask:X16} cur=0x{curVal:X16} cmp={cmpFunc}"); - } - - return; // suspend parsing of this DCB - } + return true; // DCB suspended until the awaited label is written } } if (op == ItWaitRegMem && length >= 7) { - if (TryParseStandardWaitRegMem(ctx, currentAddress, - out var waitAddr, out var refVal, out var waitMask, out var cmpFunc) && - ctx.TryReadUInt32(waitAddr, out var curVal)) + if (HandleSubmittedWaitRegMem( + ctx, state, commandAddress, currentAddress, offset, length, + dwordCount, is64Bit: false, isStandard: true, tracePackets)) { - var waiter = new GpuWaitRegistry.WaitingDcb - { - CommandBufferAddress = commandAddress, - ResumeAddress = currentAddress + ((ulong)length * sizeof(uint)), - TotalDwords = dwordCount, - ResumeOffset = offset + length, - ReferenceValue = refVal, - Mask = waitMask, - CompareFunction = cmpFunc, - Is64Bit = false, - State = state, - }; - if (!GpuWaitRegistry.Compare(waiter, curVal)) - { - GpuWaitRegistry.Register(waitAddr, waiter); - - if (tracePackets) - { - TraceAgc( - $"agc.dcb.suspended_std addr=0x{waitAddr:X16} ref=0x{refVal:X16} " + - $"mask=0x{waitMask:X16} cur=0x{curVal:X16} cmp={cmpFunc}"); - } - - return; - } + return true; // DCB suspended until the awaited label is written } } @@ -2906,12 +3261,16 @@ public static class AgcExports out var indexCount) && indexCount != 0) { - lock (_submitTraceGate) + state.FrameDrawCount++; + if (_traceAgcShader) { - if (_tracedSubmittedDrawOpcodes.Add(op)) + lock (_submitTraceGate) { - TraceAgcShader( - $"agc.draw_packet op=0x{op:X2} count={indexCount}"); + if (_tracedSubmittedDrawOpcodes.Add(op)) + { + TraceAgcShader( + $"agc.draw_packet op=0x{op:X2} count={indexCount}"); + } } } @@ -2926,9 +3285,10 @@ public static class AgcExports if (op == ItNop && register == RDrawIndexAuto && length >= 2 && - ctx.TryReadUInt32(currentAddress + 4, out var autoIndexCount) && + TryReadUInt32(ctx, currentAddress + 4, out var autoIndexCount) && autoIndexCount != 0) { + state.FrameDrawCount++; TryTranslateGuestDraw( ctx, gpuState, @@ -2944,30 +3304,98 @@ public static class AgcExports currentAddress, length, op, - out var dispatch)) + out var dispatch)) { + state.FrameDispatchCount++; ObserveComputeDispatch(ctx, gpuState, state, dispatch); } + if (op == ItNop && + register == RWaitFlipDone && + length >= 3 && + TryReadUInt32(ctx, currentAddress + 4, out var waitVideoOutHandle) && + TryReadUInt32(ctx, currentAddress + 8, out var waitDisplayBufferIndex)) + { + var waitSequence = VulkanVideoPresenter.SubmitOrderedGuestFlipWait( + unchecked((int)waitVideoOutHandle), + unchecked((int)waitDisplayBufferIndex)); + TraceAgcShader( + $"agc.flip_wait_safe queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} " + + $"handle={waitVideoOutHandle} index={waitDisplayBufferIndex} " + + $"work_sequence={waitSequence}"); + } + if (op == ItNop && register == RFlip && length >= 6) { - if (!ctx.TryReadUInt32(currentAddress + 4, out var videoOutHandle) || - !ctx.TryReadUInt32(currentAddress + 8, out var displayBufferIndexRaw) || - !ctx.TryReadUInt32(currentAddress + 12, out var flipMode) || - !ctx.TryReadUInt32(currentAddress + 16, out var flipArgLo) || - !ctx.TryReadUInt32(currentAddress + 20, out var flipArgHi)) + TraceFramePacketSummary(state); + SyncCpuWrittenGuestImages(ctx); + if (!TryReadUInt32(ctx, currentAddress + 4, out var videoOutHandle) || + !TryReadUInt32(ctx, currentAddress + 8, out var displayBufferIndexRaw) || + !TryReadUInt32(ctx, currentAddress + 12, out var flipMode) || + !TryReadUInt32(ctx, currentAddress + 16, out var flipArgLo) || + !TryReadUInt32(ctx, currentAddress + 20, out var flipArgHi)) { - return; + return false; } var flipArg = unchecked((long)(((ulong)flipArgHi << 32) | flipArgLo)); var displayBufferIndex = unchecked((int)displayBufferIndexRaw); var handle = unchecked((int)videoOutHandle); + if (state.PendingTargetlessDraw is { } pendingComposite && + VideoOutExports.TryGetDisplayBufferInfo( + handle, + displayBufferIndex, + out var pendingDisplayBuffer) && + state.KnownRenderTargets.TryGetValue( + pendingDisplayBuffer.Address, + out var pendingDisplayTarget)) + { + var textures = CreateGuestDrawTextures( + ctx, + pendingComposite.Textures, + out _); + var globalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(pendingComposite); + var vertexBuffers = + CreateGuestVertexBuffers(pendingComposite.VertexInputs); + ProvideRenderTargetInitialData(ctx, pendingDisplayTarget); + GuestGpu.Current.SubmitOffscreenTranslatedDraw( + pendingComposite.PixelShader, + textures, + globalMemoryBuffers, + pendingComposite.AttributeCount, + [new GuestRenderTarget( + pendingDisplayTarget.Address, + pendingDisplayTarget.Width, + pendingDisplayTarget.Height, + pendingDisplayTarget.Format, + pendingDisplayTarget.NumberType)], + pendingComposite.VertexShader, + pendingComposite.VertexCount, + pendingComposite.InstanceCount, + pendingComposite.PrimitiveType, + pendingComposite.IndexBuffer, + vertexBuffers, + pendingComposite.RenderState, + pendingComposite.DepthTarget, + pendingComposite.PixelShaderAddress); + TraceAgcShader( + $"agc.deferred_composite ps=0x{pendingComposite.PixelShaderAddress:X16} " + + $"src=0x{pendingComposite.Textures.FirstOrDefault()?.Descriptor.Address ?? 0:X16} " + + $"dst=0x{pendingDisplayTarget.Address:X16} " + + $"size={pendingDisplayTarget.Width}x{pendingDisplayTarget.Height}"); + state.PendingTargetlessDraw = null; + state.TranslatedDraw = null; + } + if (VideoOutExports.TryGetDisplayBufferInfo( handle, displayBufferIndex, out var cachedDisplayBuffer) && - GuestGpu.Current.TrySubmitGuestImage( + GuestGpu.Current.TrySubmitOrderedGuestImageFlip( + handle, + displayBufferIndex, cachedDisplayBuffer.Address, cachedDisplayBuffer.Width, cachedDisplayBuffer.Height, @@ -2993,7 +3421,7 @@ public static class AgcExports "draw-fallback"); var textures = CreateGuestDrawTextures(ctx, translatedDraw.Textures, out var fallbackTextureCount); var globalMemoryBuffers = - CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); + CreateTranslatedDrawGlobalBuffersForPresent(ctx, translatedDraw); GuestGpu.Current.SubmitTranslatedDraw( translatedDraw.PixelShader, textures, @@ -3047,11 +3475,72 @@ public static class AgcExports _ = VideoOutExports.SubmitFlipFromAgc(ctx, handle, displayBufferIndex, unchecked((int)flipMode), flipArg); state.SawIndexedDraw = false; state.GuestDrawKind = GuestDrawKind.None; + if (state.PendingTargetlessDraw is { } unusedPendingDraw) + { + ReturnPooledDrawArrays( + unusedPendingDraw, + globals: true, + vertex: true, + index: true); + state.PendingTargetlessDraw = null; + } state.TranslatedDraw = null; } offset += length; } + + return false; + } + + private static void TraceFramePacketSummary(SubmittedDcbState state) + { + if (!_traceFramePackets) + { + return; + } + + var flip = ++state.FlipCount; + if (flip <= 8 || flip % 60 == 0 || state.FrameDrawCount == 0) + { + var opcodes = string.Join( + ',', + state.FramePacketCounts + .OrderByDescending(entry => entry.Value) + .ThenBy(entry => entry.Key.Op) + .Take(32) + .Select(entry => entry.Key.Register == uint.MaxValue + ? $"0x{entry.Key.Op:X2}:{entry.Value}" + : $"0x{entry.Key.Op:X2}/r{entry.Key.Register}:{entry.Value}")); + Console.Error.WriteLine( + $"[FRAMEPKT] flip={flip} submission={state.ActiveSubmissionId} " + + $"packets={state.FramePacketCount} draws={state.FrameDrawCount} " + + $"dispatches={state.FrameDispatchCount} opcodes=[{opcodes}]"); + } + + state.FramePacketCounts.Clear(); + state.FramePacketCount = 0; + state.FrameDrawCount = 0; + state.FrameDispatchCount = 0; + } + + private static void TracePacketParseFailure( + SubmittedDcbState state, + ulong address, + uint offset, + uint header, + string reason) + { + if (!_traceFramePackets || + Interlocked.Increment(ref _packetParseFailureTraceCount) > 128) + { + return; + } + + Console.Error.WriteLine( + $"[FRAMEPKT] parse-failure queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} offset={offset} " + + $"address=0x{address:X16} header=0x{header:X8} reason={reason}"); } private static void TraceDisplayBuffer( @@ -3077,44 +3566,623 @@ public static class AgcExports private static void ApplySubmittedDmaData( CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress, + bool compactLayout, bool tracePacket) { - if (!ctx.TryReadUInt32(packetAddress + 12, out var byteCount) || - !ctx.TryReadUInt64(packetAddress + 16, out var destinationAddress) || - !ctx.TryReadUInt64(packetAddress + 24, out var sourceAddress)) + var byteCountOffset = compactLayout ? 20UL : 12UL; + var destinationOffset = compactLayout ? 4UL : 16UL; + var sourceOffset = compactLayout ? 12UL : 24UL; + if (!TryReadUInt32(ctx, packetAddress + byteCountOffset, out var byteCount) || + !TryReadUInt64(ctx, packetAddress + destinationOffset, out var destinationAddress) || + !TryReadUInt64(ctx, packetAddress + sourceOffset, out var sourceAddress)) { return; } - var copied = - byteCount != 0 && - byteCount <= 256u * 1024u * 1024u && - destinationAddress != 0 && - sourceAddress != 0 && - TryCopyGuestMemory(ctx, sourceAddress, destinationAddress, byteCount); + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + InvalidateDcbWindowIfOverlaps(destinationAddress, byteCount); + var immediateFill = + compactLayout && + destinationAddress >= 0x10000 && + sourceAddress <= uint.MaxValue; + var copied = + byteCount != 0 && + byteCount <= 256u * 1024u * 1024u && + destinationAddress != 0 && + (immediateFill + ? TryFillGuestMemory(ctx, (uint)sourceAddress, destinationAddress, byteCount) + : sourceAddress != 0 && + TryCopyGuestMemory(ctx, sourceAddress, destinationAddress, byteCount)); + if (copied) + { + MirrorDmaWriteToGuestImage( + ctx, + destinationAddress, + byteCount, + immediateFill ? (uint)sourceAddress : null); + } + + if (tracePacket) + { + TraceAgc( + $"agc.dcb.dma_data dst=0x{destinationAddress:X16} " + + $"src=0x{sourceAddress:X16} bytes={byteCount} " + + $"fill={immediateFill} copied={copied}"); + } + }, + $"agc_dma_data dst=0x{destinationAddress:X16} bytes={byteCount}", + packetAddress, + destinationAddress, + byteCount); + } + + private static void SubmitOrderedGpuSideEffect( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, + Action action, + string debugName, + ulong packetAddress, + ulong producerAddress = 0, + ulong producerLength = 0) + { + var producer = RegisterLabelProducer( + ctx.Memory, + state, + packetAddress, + producerAddress, + producerLength, + debugName); + + void CompleteAndWake() + { + CompleteLabelProducer(producer); + if (GpuWaitRegistry.Count == 0) + { + return; + } + + // Resuming a DCB can enqueue another compute dispatch and wait for + // it. Never do that reentrantly on the Vulkan render thread. + ThreadPool.UnsafeQueueUserWorkItem( + static state => + { + var (resumeContext, resumeGpuState) = state; + lock (resumeGpuState.Gate) + { + DrainResumableDcbs( + resumeContext, + resumeGpuState, + tracePackets: _traceAgc); + } + }, + (ctx, gpuState), + preferLocal: false); + } + + void ApplyAndQueueCompletion() + { + action(); + // DMA side effects can enqueue a Vulkan image mirror while this + // ordered action is executing. Completing the label here would + // wake another queue before that mirror is visible. Queue a + // second same-queue ordered action after all immediate follow-up + // writes; it fences those writes before publishing the producer. + if (VulkanVideoPresenter.SubmitOrderedGuestAction( + CompleteAndWake, + $"{debugName} completion") == 0) + { + CompleteAndWake(); + } + } + + if (VulkanVideoPresenter.SubmitOrderedGuestAction( + ApplyAndQueueCompletion, + debugName) == 0) + { + // Headless/startup submissions have no Vulkan queue to order + // against, so retaining the previous immediate behavior is exact. + ApplyAndQueueCompletion(); + } + } + + private static LabelProducerTrace? RegisterLabelProducer( + object memory, + SubmittedDcbState state, + ulong packetAddress, + ulong address, + ulong length, + string debugName) + { + if (address == 0 || length == 0) + { + return null; + } + + var producer = new LabelProducerTrace + { + Sequence = Interlocked.Increment(ref _labelProducerSequence), + Memory = memory, + Address = address, + Length = length, + PacketAddress = packetAddress, + SubmissionId = state.ActiveSubmissionId, + QueueName = state.QueueName, + DebugName = debugName, + }; + lock (_labelProducerGate) + { + if (_labelProducers.Count >= 4096) + { + _labelProducers.RemoveRange(0, 1024); + } + + _labelProducers.Add(producer); + } + + foreach (var waiting in GpuWaitRegistry.SnapshotInRange(memory, address, length)) + { + TraceAgc( + $"agc.wait_producer_scheduled label=0x{waiting.Address:X16} " + + $"waiters={waiting.Count} producer_seq={producer.Sequence} " + + $"queue={producer.QueueName} submission={producer.SubmissionId} " + + $"packet=0x{packetAddress:X16} action='{debugName}'"); + } + + return producer; + } + + private static void CompleteLabelProducer(LabelProducerTrace? producer) + { + if (producer is null) + { + return; + } + + lock (_labelProducerGate) + { + producer.Completed = true; + } + + foreach (var waiting in GpuWaitRegistry.SnapshotInRange( + producer.Memory, + producer.Address, + producer.Length)) + { + TraceAgc( + $"agc.wait_producer_completed label=0x{waiting.Address:X16} " + + $"waiters={waiting.Count} producer_seq={producer.Sequence} " + + $"queue={producer.QueueName} submission={producer.SubmissionId} " + + $"action='{producer.DebugName}'"); + } + } + + private static void TraceWaitProducerState( + object memory, + in GpuWaitRegistry.WaitingDcb waiter, + ulong commandAddress, + ulong packetAddress, + bool stale, + ulong? currentValue = null) + { + LabelProducerTrace? producer = null; + lock (_labelProducerGate) + { + for (var index = _labelProducers.Count - 1; index >= 0; index--) + { + var candidate = _labelProducers[index]; + if (!ReferenceEquals(candidate.Memory, memory) || + !RangesOverlap( + candidate.Address, + candidate.Length, + waiter.WaitAddress, + waiter.Is64Bit ? (ulong)sizeof(ulong) : sizeof(uint))) + { + continue; + } + + producer = candidate; + break; + } + + if (_tracedProducerlessWaits.Count >= 4096) + { + _tracedProducerlessWaits.Clear(); + } + + if (!stale && producer is null && + !_tracedProducerlessWaits.Add( + (memory, waiter.WaitAddress, waiter.SubmissionId))) + { + return; + } + } + + var prefix = stale ? "agc.wait_stale" : "agc.wait_suspended"; + var current = currentValue.HasValue + ? $"0x{currentValue.Value:X16}" + : "unreadable"; + var condition = + $"value={current} mask=0x{waiter.Mask:X16} " + + $"ref=0x{waiter.ReferenceValue:X16} cmp={waiter.CompareFunction} " + + $"control=0x{waiter.ControlValue:X8} bits={(waiter.Is64Bit ? 64 : 32)} " + + $"form={(waiter.IsStandard ? "standard" : "agc-nop")}"; + if (producer is null) + { + Console.Error.WriteLine( + $"[LOADER][WARN] {prefix} label=0x{waiter.WaitAddress:X16} " + + $"queue={waiter.QueueName} submission={waiter.SubmissionId} " + + $"command=0x{commandAddress:X16} packet=0x{packetAddress:X16} " + + condition + " " + + "producer=none-observed; remaining-suspended"); + return; + } + + TraceAgc( + $"{prefix} label=0x{waiter.WaitAddress:X16} " + + $"queue={waiter.QueueName} submission={waiter.SubmissionId} " + + condition + " " + + $"producer_seq={producer.Sequence} producer_state=" + + $"{(producer.Completed ? "completed" : "queued")} " + + $"producer_queue={producer.QueueName} " + + $"producer_submission={producer.SubmissionId} " + + $"producer_packet=0x{producer.PacketAddress:X16} " + + $"action='{producer.DebugName}'"); + } + + private static void ApplySubmittedAcquireMem( + CpuContext ctx, + SubmittedDcbState state, + ulong packetAddress, + bool tracePacket) + { + if (!TryDecodeSubmittedAcquireMem(ctx, packetAddress, out var acquire)) + { + TraceAgc( + $"agc.acquire_mem_decode_failed queue={state.QueueName} " + + $"submission={state.ActiveSubmissionId} packet=0x{packetAddress:X16}"); + return; + } + + var queueName = state.QueueName; + var submissionId = state.ActiveSubmissionId; + var debugName = + $"acquire_mem base=0x{acquire.BaseAddress:X16} size=0x{acquire.SizeBytes:X16} " + + $"gcr=0x{acquire.GcrControl:X8}"; + void ApplyAcquire() + { + // ExecuteOrderedGuestAction first flushes and waits for this guest + // queue, then writes back dirty guest buffers. At that exact PM4 + // point, refresh only tracked guest images covered by the acquire + // range. Cached sampled textures use the same dirty tracker and + // are evicted by the presenter without throwing away clean cache + // entries (hardware invalidation does not imply changed bytes). + if (acquire.InvalidatesGuestResources) + { + SyncCpuWrittenGuestImages( + ctx, + acquire.BaseAddress, + acquire.CoversAllGuestMemory + ? ulong.MaxValue + : acquire.SizeBytes); + } + + if (tracePacket) + { + TraceAgc( + $"agc.acquire_mem_applied queue={queueName} " + + $"submission={submissionId} packet=0x{packetAddress:X16} " + + $"work_sequence={VulkanVideoPresenter.CurrentGuestWorkSequenceForDiagnostics}"); + } + } + + var sequence = VulkanVideoPresenter.SubmitOrderedGuestAction( + ApplyAcquire, + debugName); + if (sequence == 0) + { + // Headless startup has no host GPU queue, but the guest-memory + // cache model still needs the same invalidation semantics. + ApplyAcquire(); + } + + // The bulk PM4 read is itself a parser-side cache. Do not retain it + // across a guest cache-invalidation point; subsequent packets return + // to live guest memory while the host barrier remains ordered in the + // logical GPU queue. Submission stays asynchronous, matching hardware + // and avoiding a CPU stall for every ACQUIRE_MEM packet. + _dcbWindowBuffer = null; + _dcbWindowByteLength = 0; + if (tracePacket) { TraceAgc( - $"agc.dcb.dma_data dst=0x{destinationAddress:X16} src=0x{sourceAddress:X16} " + - $"bytes={byteCount} copied={copied}"); + $"agc.acquire_mem queue={queueName} " + + $"submission={submissionId} packet=0x{packetAddress:X16} " + + $"engine={acquire.Engine} cbdb=0x{acquire.CbDbControl:X8} " + + $"base=0x{acquire.BaseAddress:X16} size=0x{acquire.SizeBytes:X16} " + + $"scope={(acquire.CoversAllGuestMemory ? "all" : "range")} " + + $"poll={acquire.PollInterval} gcr=0x{acquire.GcrControl:X8} " + + $"resource_inv={acquire.InvalidatesGuestResources} " + + $"sequence={sequence} scheduled={(sequence != 0)}"); + } + } + + private static bool TryDecodeSubmittedAcquireMem( + CpuContext ctx, + ulong packetAddress, + out SubmittedAcquireMem acquire) + { + if (!TryReadUInt32(ctx, packetAddress + 4, out var coherControl) || + !TryReadUInt32(ctx, packetAddress + 8, out var sizeLow) || + !TryReadUInt32(ctx, packetAddress + 12, out var sizeHigh) || + !TryReadUInt32(ctx, packetAddress + 16, out var baseLow) || + !TryReadUInt32(ctx, packetAddress + 20, out var baseHigh) || + !TryReadUInt32(ctx, packetAddress + 24, out var pollInterval) || + !TryReadUInt32(ctx, packetAddress + 28, out var gcrControl)) + { + acquire = default; + return false; + } + + acquire = DecodeSubmittedAcquireMem( + coherControl, + sizeLow, + sizeHigh, + baseLow, + baseHigh, + pollInterval, + gcrControl); + return true; + } + + private static SubmittedAcquireMem DecodeSubmittedAcquireMem( + uint coherControl, + uint sizeLow, + uint sizeHigh, + uint baseLow, + uint baseHigh, + uint pollInterval, + uint gcrControl) + { + // GFX10 ACQUIRE_MEM expresses COHER_SIZE and COHER_BASE in 256-byte + // units. SIZE_HI is 8 bits and BASE_HI is 24 bits in the packet. + var sizeUnits = sizeLow | ((ulong)(sizeHigh & 0xFFu) << 32); + var baseUnits = baseLow | ((ulong)(baseHigh & 0x00FF_FFFFu) << 32); + return new SubmittedAcquireMem( + Engine: coherControl >> 31, + CbDbControl: coherControl & 0x7FFF_FFFFu, + BaseAddress: baseUnits << 8, + SizeBytes: sizeUnits << 8, + PollInterval: pollInterval & 0xFFFFu, + GcrControl: gcrControl & 0x7FFFFu); + } + + private static void ResetSubmittedParserState(SubmittedDcbState state) + { + // Queue ownership, pending submissions and suspension bookkeeping are + // deliberately retained. Work emitted before this packet already owns + // immutable snapshots; clearing these fields affects only commands + // translated after RESET at this precise packet position. + state.CxRegisters.Clear(); + state.ShRegisters.Clear(); + state.UcRegisters.Clear(); + state.PresenterTexture = null; + state.GuestDrawKind = GuestDrawKind.None; + state.TranslatedDraw = null; + state.RenderTargetWriters.Clear(); + state.IndirectArgsAddress = 0; + state.SawIndexedDraw = false; + state.IndexBufferAddress = 0; + state.IndexBufferCount = 0; + state.IndexSize = 0; + state.InstanceCount = 1; + state.DrawIndexOffset = 0; + } + + private static bool RangesOverlap( + ulong leftAddress, + ulong leftLength, + ulong rightAddress, + ulong rightLength) + { + var leftEnd = leftAddress > ulong.MaxValue - leftLength + ? ulong.MaxValue + : leftAddress + leftLength; + var rightEnd = rightAddress > ulong.MaxValue - rightLength + ? ulong.MaxValue + : rightAddress + rightLength; + return leftAddress < rightEnd && rightAddress < leftEnd; + } + + /// + /// PS5 render targets alias guest memory, so a CP DMA fill or copy that + /// lands on an RT is visible to later GPU reads. Our render targets live + /// in Vulkan images, so mirror DMA writes into them: fills become + /// vkCmdClearColorImage, copies re-upload the guest bytes. Without this, + /// per-frame DMA clears never reach the image (the fog layer in Dreaming + /// Sarah accumulates until it saturates, washing the scene out). + /// + /// + /// PS5 render targets alias unified memory, so the game's CPU can rewrite a + /// surface (Chowdren memsets its fog-noise layer every frame) and the GPU + /// observes it. Our Vulkan guest images are separate storage, so re-upload + /// CPU-authored surfaces once per flip. Surfaces only the GPU writes keep + /// all-zero guest memory and are skipped, preserving their GPU content. + /// + private static long _guestImageSyncTraceCount; + + private static void SyncCpuWrittenGuestImages( + CpuContext ctx, + ulong scopeAddress = 0, + ulong scopeByteCount = ulong.MaxValue) + { + if (!SharpEmu.HLE.GuestImageWriteTracker.Enabled || scopeByteCount == 0) + { + return; + } + + foreach (var (address, width, height, byteCount) in VulkanVideoPresenter.GetGuestImageExtents()) + { + if (scopeByteCount != ulong.MaxValue && + !RangesOverlap(address, byteCount, scopeAddress, scopeByteCount)) + { + continue; + } + + if (!SharpEmu.HLE.GuestImageWriteTracker.ConsumeDirty(address)) + { + continue; + } + + if (byteCount == 0 || byteCount > MaxPresentedTextureBytes) + { + continue; + } + + var pixels = new byte[byteCount]; + if (ctx.Memory.TryRead(address, pixels)) + { + VulkanVideoPresenter.SubmitGuestImageWrite(address, pixels); + if (Interlocked.Increment(ref _guestImageSyncTraceCount) <= 64) + { + Console.Error.WriteLine( + $"[SYNC] cpu-write addr=0x{address:X} {width}x{height}"); + } + } + + SharpEmu.HLE.GuestImageWriteTracker.Rearm(address); + } + } + + private static long _dmaMirrorTraceCount; + private static readonly Dictionary<(uint Op, uint Register), long> _submittedOpcodeCounts = new(); + private static long _submittedOpcodeTotal; + + private static void CountSubmittedOpcode(uint op, uint register) + { + var key = (op, op == ItNop ? register : uint.MaxValue); + lock (_submittedOpcodeCounts) + { + _submittedOpcodeCounts[key] = + _submittedOpcodeCounts.TryGetValue(key, out var count) ? count + 1 : 1; + if (++_submittedOpcodeTotal % 500_000 == 0) + { + var summary = string.Join( + ' ', + _submittedOpcodeCounts + .OrderByDescending(entry => entry.Value) + .Select(entry => entry.Key.Register == uint.MaxValue + ? $"0x{entry.Key.Op:X2}:{entry.Value}" + : $"0x{entry.Key.Op:X2}/r{entry.Key.Register}:{entry.Value}")); + Console.Error.WriteLine($"[PKT] total={_submittedOpcodeTotal} {summary}"); + } + } + } + + private static void MirrorDmaWriteToGuestImage( + CpuContext ctx, + ulong destinationAddress, + ulong byteCount, + uint? fillValue) + { + var hasImage = VulkanVideoPresenter.TryGetGuestImageExtent( + destinationAddress, + out var width, + out var height, + out var imageBytes); + if (_traceDraws && Interlocked.Increment(ref _dmaMirrorTraceCount) <= 400) + { + Console.Error.WriteLine( + $"[DMA] dst=0x{destinationAddress:X} bytes={byteCount} " + + $"fill={(fillValue is { } f ? $"0x{f:X8}" : "copy")} image={hasImage}"); + } + + if (!hasImage) + { + return; + } + + if (imageBytes == 0 || byteCount < imageBytes) + { + return; + } + + if (fillValue is { } fill) + { + VulkanVideoPresenter.SubmitGuestImageFill(destinationAddress, fill); + return; + } + + var pixels = new byte[imageBytes]; + if (ctx.Memory.TryRead(destinationAddress, pixels)) + { + VulkanVideoPresenter.SubmitGuestImageWrite(destinationAddress, pixels); } } private static void ApplySubmittedStandardDmaData( CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress) { - if (!ctx.TryReadUInt32(packetAddress + 4, out var control) || - !ctx.TryReadUInt32(packetAddress + 8, out var sourceLow) || - !ctx.TryReadUInt32(packetAddress + 12, out var sourceHigh) || - !ctx.TryReadUInt32(packetAddress + 16, out var destinationLow) || - !ctx.TryReadUInt32(packetAddress + 20, out var destinationHigh) || - !ctx.TryReadUInt32(packetAddress + 24, out var command)) + if (!TryReadUInt32(ctx, packetAddress + 4, out var control) || + !TryReadUInt32(ctx, packetAddress + 8, out var sourceLow) || + !TryReadUInt32(ctx, packetAddress + 12, out var sourceHigh) || + !TryReadUInt32(ctx, packetAddress + 16, out var destinationLow) || + !TryReadUInt32(ctx, packetAddress + 20, out var destinationHigh) || + !TryReadUInt32(ctx, packetAddress + 24, out var command)) { return; } + var byteCount = command & 0x1F_FFFFu; + var destinationSelect = (control >> 20) & 0x3u; + var destinationSwap = (command >> 24) & 0x3u; + var destinationAddressSpace = (command >> 27) & 0x1u; + var destinationAddress = destinationLow | ((ulong)destinationHigh << 32); + var writesGuestMemory = + byteCount != 0 && + destinationSwap == 0 && + destinationSelect is 0 or 3 && + (destinationSelect == 3 || destinationAddressSpace == 0); + + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => ApplySubmittedStandardDmaDataSnapshot( + ctx, + control, + sourceLow, + sourceHigh, + destinationLow, + destinationHigh, + command), + $"dma_data dst=0x{destinationHigh:X8}{destinationLow:X8} bytes={byteCount}", + packetAddress, + writesGuestMemory ? destinationAddress : 0, + writesGuestMemory ? byteCount : 0); + } + + private static void ApplySubmittedStandardDmaDataSnapshot( + CpuContext ctx, + uint control, + uint sourceLow, + uint sourceHigh, + uint destinationLow, + uint destinationHigh, + uint command) + { var byteCount = command & 0x1F_FFFFu; var sourceSelect = (control >> 29) & 0x3u; var destinationSelect = (control >> 20) & 0x3u; @@ -3132,6 +4200,7 @@ public static class AgcExports var destinationAddress = destinationLow | ((ulong)destinationHigh << 32); + InvalidateDcbWindowIfOverlaps(destinationAddress, byteCount); bool copied; ulong sourceAddress; if (sourceSelect is 0 or 3 && @@ -3141,12 +4210,16 @@ public static class AgcExports if (sourceAddressIncrement != 0) { copied = - ctx.TryReadUInt32(sourceAddress, out var fillValue) && + TryReadUInt32(ctx, sourceAddress, out var fillValue) && TryFillGuestMemory( ctx, fillValue, destinationAddress, byteCount); + if (copied) + { + MirrorDmaWriteToGuestImage(ctx, destinationAddress, byteCount, fillValue); + } } else { @@ -3155,6 +4228,10 @@ public static class AgcExports sourceAddress, destinationAddress, byteCount); + if (copied) + { + MirrorDmaWriteToGuestImage(ctx, destinationAddress, byteCount, fillValue: null); + } } } else if (sourceSelect == 2) @@ -3165,6 +4242,10 @@ public static class AgcExports sourceLow, destinationAddress, byteCount); + if (copied) + { + MirrorDmaWriteToGuestImage(ctx, destinationAddress, byteCount, sourceLow); + } } else { @@ -3183,48 +4264,792 @@ public static class AgcExports private static void ApplySubmittedWriteData( CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress, uint packetLength, + bool standardPacket, bool tracePacket) { - if (!ctx.TryReadUInt32(packetAddress + 4, out var control) || - !ctx.TryReadUInt64(packetAddress + 8, out var destinationAddress)) + if (!TryReadUInt32(ctx, packetAddress + 4, out var control) || + !TryReadUInt64(ctx, packetAddress + 8, out var destinationAddress)) { return; } - var destination = control & 0xFFu; - var increment = (control >> 16) & 0xFFu; + var (destination, incrementAddress, writeConfirm, cachePolicy) = standardPacket + ? DecodeStandardWriteDataControl(control) + : DecodeAgcWriteDataControl(control); var dwordCount = packetLength - 4; - var wroteData = destination is 1 or 2 or 4 or 5; - for (uint index = 0; wroteData && index < dwordCount; index++) + var values = new uint[dwordCount]; + for (uint index = 0; index < dwordCount; index++) { var sourceAddress = packetAddress + 16 + ((ulong)index * sizeof(uint)); - var targetAddress = destinationAddress + - (increment == 0 ? (ulong)index * sizeof(uint) : 0); - wroteData = - ctx.TryReadUInt32(sourceAddress, out var value) && - ctx.TryWriteUInt32(targetAddress, value); + if (!TryReadUInt32(ctx, sourceAddress, out values[index])) + { + return; + } } + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + InvalidateDcbWindowIfOverlaps( + destinationAddress, + incrementAddress ? (ulong)dwordCount * sizeof(uint) : sizeof(uint)); + var wroteData = destination is 1 or 2 or 4 or 5; + for (uint index = 0; wroteData && index < dwordCount; index++) + { + var targetAddress = destinationAddress + + (incrementAddress ? (ulong)index * sizeof(uint) : 0); + wroteData = TryWriteUInt32(ctx, targetAddress, values[index]); + } + + if (tracePacket) + { + TraceAgc( + $"agc.dcb.write_data dst={destination} " + + $"addr=0x{destinationAddress:X16} count={dwordCount} " + + $"increment={incrementAddress} confirm={writeConfirm} " + + $"cache={cachePolicy} standard={standardPacket} wrote={wroteData}"); + } + }, + $"write_data dst=0x{destinationAddress:X16} count={dwordCount}", + packetAddress, + destination is 1 or 2 or 4 or 5 ? destinationAddress : 0, + destination is 1 or 2 or 4 or 5 + ? incrementAddress ? (ulong)dwordCount * sizeof(uint) : sizeof(uint) + : 0); + } + + private static (uint Destination, bool IncrementAddress, bool WriteConfirm, uint CachePolicy) + DecodeStandardWriteDataControl(uint control) + { + // GFX10 PKT3_WRITE_DATA is not byte-packed like sceAgcDcbWriteData's + // NOP wrapper: DST_SEL is 11:8, ADDR_INCR is bit 16 (0 increments), + // WR_CONFIRM is bit 20, and CACHE_POLICY is 26:25. In particular, the + // low byte is reserved and must never be interpreted as DST_SEL. + return ( + Destination: (control >> 8) & 0xFu, + IncrementAddress: (control & (1u << 16)) == 0, + WriteConfirm: (control & (1u << 20)) != 0, + CachePolicy: (control >> 25) & 0x3u); + } + + private static (uint Destination, bool IncrementAddress, bool WriteConfirm, uint CachePolicy) + DecodeAgcWriteDataControl(uint control) => + ( + Destination: control & 0xFFu, + IncrementAddress: ((control >> 16) & 0xFFu) == 0, + WriteConfirm: ((control >> 24) & 0xFFu) != 0, + CachePolicy: (control >> 8) & 0xFFu); + +#if DEBUG + private static void ValidateWriteDataControlDecoders() + { + // Regression vector: reserved low-byte noise previously decoded 0xA5 + // as DST_SEL, causing a valid standard memory write to be discarded. + const uint standardControl = 0xA5u | (5u << 8) | (1u << 16) | (1u << 20) | (2u << 25); + var standard = DecodeStandardWriteDataControl(standardControl); + System.Diagnostics.Debug.Assert(standard.Destination == 5u); + System.Diagnostics.Debug.Assert(!standard.IncrementAddress); + System.Diagnostics.Debug.Assert(standard.WriteConfirm); + System.Diagnostics.Debug.Assert(standard.CachePolicy == 2u); + + const uint agcControl = 4u | (3u << 8) | (1u << 24); + var agc = DecodeAgcWriteDataControl(agcControl); + System.Diagnostics.Debug.Assert(agc.Destination == 4u); + System.Diagnostics.Debug.Assert(agc.IncrementAddress); + System.Diagnostics.Debug.Assert(agc.WriteConfirm); + System.Diagnostics.Debug.Assert(agc.CachePolicy == 3u); + } + + private static void ValidateDispatchInitiators() + { + const uint threadCount = 0x00F0_0100u; + const uint localSize = 64u; + var initiator = DirectDispatchInitiator(0); + System.Diagnostics.Debug.Assert((initiator & (1u << 5)) == 0); + System.Diagnostics.Debug.Assert((initiator & (1u << 6)) != 0); + System.Diagnostics.Debug.Assert(threadCount * localSize == 0x3C00_4000u); + System.Diagnostics.Debug.Assert(CeilDivide(20, 8) == 3); + System.Diagnostics.Debug.Assert(CeilDivide(12, 8) == 2); + } + + private static void ValidateSubmittedQueueAndReleaseMemDecoders() + { + var nggRegisters = new Dictionary + { + [GsUserDataRegister - 1] = 3u << 1, + }; + System.Diagnostics.Debug.Assert( + SelectExportUserDataRegister(nggRegisters) == GsUserDataRegister); + + var queue = new SubmittedDcbState(); + queue.PendingSubmissions.Enqueue(new(0x1000, 8, 11, false)); + queue.PendingSubmissions.Enqueue(new(0x2000, 16, 12, true)); + System.Diagnostics.Debug.Assert( + queue.PendingSubmissions.Dequeue().SubmissionId == 11); + System.Diagnostics.Debug.Assert( + queue.PendingSubmissions.Dequeue().SubmissionId == 12); + + var control = (1u << 16) | (2u << 29); + var decoded = DecodeStandardReleaseMemControl(control); + System.Diagnostics.Debug.Assert(decoded.Destination == 1u); + System.Diagnostics.Debug.Assert(decoded.DataSelection == 2u); + System.Diagnostics.Debug.Assert( + PatchUInt32Bits(0xABCD_1234u, 0x00FF_0000u, 3u << 16) == + 0xAB03_1234u); + } + + private static void ValidateAcquireMemAndQueueResetDecoders() + { + var range = DecodeSubmittedAcquireMem( + 0x8000_7FC0u, + 0x0000_0123u, + 0x45u, + 0x89AB_CDEFu, + 0x0012_3456u, + 0x1_000Au, + 0x0001_0388u); + System.Diagnostics.Debug.Assert(range.Engine == 1u); + System.Diagnostics.Debug.Assert(range.CbDbControl == 0x7FC0u); + System.Diagnostics.Debug.Assert(range.SizeBytes == 0x0000_4500_0001_2300UL); + System.Diagnostics.Debug.Assert(range.BaseAddress == 0x1234_5689_ABCD_EF00UL); + System.Diagnostics.Debug.Assert(range.PollInterval == 0xAu); + System.Diagnostics.Debug.Assert(range.InvalidatesGuestResources); + System.Diagnostics.Debug.Assert(!range.CoversAllGuestMemory); + + var all = DecodeSubmittedAcquireMem(0, 0, 0, 0, 0, 0, 0x280u); + System.Diagnostics.Debug.Assert(all.CoversAllGuestMemory); + System.Diagnostics.Debug.Assert(all.InvalidatesGuestResources); + var explicitAll = DecodeSubmittedAcquireMem(0, 1, 0, 0, 0, 0, 0x103C0u); + System.Diagnostics.Debug.Assert(explicitAll.CoversAllGuestMemory); + + var queue = new SubmittedDcbState + { + QueueName = "validator", + ActiveSubmissionId = 7, + HasActiveSubmission = true, + IsSuspended = true, + IndexBufferAddress = 0x1000, + IndexBufferCount = 12, + IndexSize = 1, + InstanceCount = 4, + DrawIndexOffset = 2, + IndirectArgsAddress = 0x2000, + SawIndexedDraw = true, + GuestDrawKind = GuestDrawKind.FullscreenBarycentric, + }; + queue.CxRegisters.Add(1, 2); + queue.ShRegisters.Add(3, 4); + queue.UcRegisters.Add(5, 6); + queue.PendingSubmissions.Enqueue(new(0x3000, 2, 8, false)); + ResetSubmittedParserState(queue); + System.Diagnostics.Debug.Assert(queue.CxRegisters.Count == 0); + System.Diagnostics.Debug.Assert(queue.ShRegisters.Count == 0); + System.Diagnostics.Debug.Assert(queue.UcRegisters.Count == 0); + System.Diagnostics.Debug.Assert(queue.IndexBufferAddress == 0); + System.Diagnostics.Debug.Assert(queue.IndexBufferCount == 0); + System.Diagnostics.Debug.Assert(queue.IndexSize == 0); + System.Diagnostics.Debug.Assert(queue.InstanceCount == 1); + System.Diagnostics.Debug.Assert(queue.DrawIndexOffset == 0); + System.Diagnostics.Debug.Assert(queue.IndirectArgsAddress == 0); + System.Diagnostics.Debug.Assert(!queue.SawIndexedDraw); + System.Diagnostics.Debug.Assert(queue.GuestDrawKind == GuestDrawKind.None); + System.Diagnostics.Debug.Assert(queue.QueueName == "validator"); + System.Diagnostics.Debug.Assert(queue.ActiveSubmissionId == 7); + System.Diagnostics.Debug.Assert(queue.HasActiveSubmission); + System.Diagnostics.Debug.Assert(queue.IsSuspended); + System.Diagnostics.Debug.Assert(queue.PendingSubmissions.Count == 1); + } + + private static void ValidateDepthTargetDecoder() + { + var registers = new Dictionary + { + [DbDepthControl] = 0x2u | 0x4u | (1u << 4), + [DbDepthSizeXy] = 1919u | (1079u << 16), + [DbDepthClear] = BitConverter.SingleToUInt32Bits(1f), + [DbZInfo] = 3u | (24u << 4), + [DbZReadBase] = 0x0123_4567u, + [DbZWriteBase] = 0x0123_4567u, + [DbZReadBaseHi] = 2u, + [DbZWriteBaseHi] = 2u, + }; + var depth = DecodeDepthTarget(registers); + System.Diagnostics.Debug.Assert(depth is not null); + System.Diagnostics.Debug.Assert(depth.Width == 1920 && depth.Height == 1080); + System.Diagnostics.Debug.Assert(depth.GuestFormat == 3u); + System.Diagnostics.Debug.Assert(depth.SwizzleMode == 24u); + System.Diagnostics.Debug.Assert(depth.Address == 0x0000_0201_2345_6700UL); + System.Diagnostics.Debug.Assert(depth.ClearDepth == 1f); + } +#endif + + // SHARPEMU_GPU_WAIT_MODE=force reverts to the legacy behaviour of faking a + // satisfying value at parse time. Default (suspend) properly suspends the + // DCB on an unmet WAIT_REG_MEM and resumes it once the awaited completion + // label is genuinely written by a later submit — preserving cross-submit + // ordering so the work after a wait (e.g. the final composite) does not run + // ahead of the compute it samples. + private static readonly bool _gpuWaitSuspendEnabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_GPU_WAIT_MODE"), + "force", + StringComparison.OrdinalIgnoreCase); + + // Optional age for one-shot missing-producer diagnostics. Stale waits are + // never removed or force-satisfied in the default suspend mode: doing so + // advances a queue without its real cross-queue producer and can publish + // incomplete CPU/GPU state. Only SHARPEMU_GPU_WAIT_MODE=force retains the + // explicit legacy mutation path above. Default 0 disables age diagnostics. + private static readonly long _gpuWaitStaleTicks = + (long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_GPU_WAIT_FALLBACK_MS"), + out var fallbackMs) && fallbackMs >= 0 + ? fallbackMs + : 0L) * System.Diagnostics.Stopwatch.Frequency / 1000L; + + // Reads the WAIT_REG_MEM watched address, reference, mask, and 3-bit compare + // function for both the AGC NOP-encapsulated (RWaitMem32/64) and the standard + // ItWaitRegMem packet layouts. + private static bool TryParseSubmittedWait( + CpuContext ctx, + ulong packetAddress, + bool is64Bit, + bool isStandard, + out ulong waitAddress, + out ulong reference, + out ulong mask, + out uint compareFunction, + out uint controlValue) + { + waitAddress = 0; + reference = 0; + mask = 0; + compareFunction = 0; + controlValue = 0; + if (isStandard) + { + if (!TryReadUInt32(ctx, packetAddress + 4, out var stdControl) || + !TryReadUInt64(ctx, packetAddress + 8, out waitAddress) || + !TryReadUInt32(ctx, packetAddress + 16, out var stdRef) || + !TryReadUInt32(ctx, packetAddress + 20, out var stdMask)) + { + return false; + } + + compareFunction = stdControl & 0x7u; + controlValue = stdControl; + reference = stdRef; + mask = stdMask; + return true; + } + + if (!TryReadUInt64(ctx, packetAddress + 4, out waitAddress) || + !TryReadUInt32(ctx, packetAddress + (is64Bit ? 28u : 16u), out var control)) + { + return false; + } + + compareFunction = control & 0x7u; + controlValue = control; + if (is64Bit) + { + return TryReadUInt64(ctx, packetAddress + 12, out mask) && + TryReadUInt64(ctx, packetAddress + 20, out reference); + } + + if (!TryReadUInt32(ctx, packetAddress + 12, out var mask32) || + !TryReadUInt32(ctx, packetAddress + 20, out var reference32)) + { + return false; + } + + mask = mask32; + reference = reference32; + return true; + } + + // Returns true when the DCB should suspend parsing at this wait (its + // continuation was registered into GpuWaitRegistry); false to keep parsing + // (already satisfied, unreadable, or legacy force-satisfy mode). + private static bool HandleSubmittedWaitRegMem( + CpuContext ctx, + SubmittedDcbState state, + ulong commandAddress, + ulong packetAddress, + uint offset, + uint length, + uint dwordCount, + bool is64Bit, + bool isStandard, + bool tracePacket) + { + if (!TryParseSubmittedWait( + ctx, packetAddress, is64Bit, isStandard, + out var waitAddress, out var reference, out var mask, out var compareFunction, + out var controlValue)) + { + return false; + } + + // COMPARE_FUNC=0 is the hardware "always" condition. Reserved 7 is + // also fail-open; neither condition may register a waiter. Validate + // the watched memory before any read so null/malformed packets cannot + // become permanent entries keyed by address zero. + if (compareFunction is 0 or 7) + { + TraceSubmittedWait( + waitAddress, + 0, + mask, + reference, + compareFunction, + is64Bit ? 64 : 32, + tracePacket); + return false; + } + + var requiredAlignment = is64Bit ? sizeof(ulong) : sizeof(uint); + if (waitAddress == 0 || + mask == 0 || + waitAddress % (ulong)requiredAlignment != 0) + { + TraceAgc( + $"agc.dcb.wait_reject addr=0x{waitAddress:X16} " + + $"mask=0x{mask:X16} compare={compareFunction} bits=" + + $"{(is64Bit ? 64 : 32)} standard={isStandard} " + + $"packet=0x{packetAddress:X16} reason=invalid-address-or-mask"); + return false; + } + + ulong currentValue = 0; + bool hasCurrent; + if (is64Bit) + { + hasCurrent = TryReadUInt64(ctx, waitAddress, out currentValue); + } + else if (TryReadUInt32(ctx, waitAddress, out var current32)) + { + currentValue = current32; + hasCurrent = true; + } + else + { + hasCurrent = false; + } + + TraceSubmittedWait( + waitAddress, currentValue, mask, reference, compareFunction, + is64Bit ? 64 : 32, tracePacket); + + var waiter = new GpuWaitRegistry.WaitingDcb + { + CommandBufferAddress = commandAddress, + ResumeAddress = packetAddress + ((ulong)length * sizeof(uint)), + TotalDwords = dwordCount, + ResumeOffset = offset + length, + ReferenceValue = reference, + Mask = mask, + CompareFunction = compareFunction, + ControlValue = controlValue, + Is64Bit = is64Bit, + IsStandard = isStandard, + WaitAddress = waitAddress, + Memory = ctx.Memory, + QueueName = state.QueueName, + SubmissionId = state.ActiveSubmissionId, + RegisteredTicks = System.Diagnostics.Stopwatch.GetTimestamp(), + State = state, + }; + + if (hasCurrent && GpuWaitRegistry.Compare(waiter, currentValue)) + { + return false; // already satisfied — keep parsing + } + + if (!_gpuWaitSuspendEnabled) + { + if (hasCurrent) + { + ForceSatisfyGpuWait(ctx, waiter, currentValue); + } + + return false; + } + + if (!hasCurrent) + { + return false; // cannot evaluate the label — do not stall the DCB + } + + GpuWaitRegistry.Register(waitAddress, waiter); + var gpuState = _submittedGpuStates.GetValue( + ctx.Memory, + static _ => new SubmittedGpuState()); + EnsureGpuWaitMonitor(ctx, gpuState); + TraceWaitProducerState( + ctx.Memory, + waiter, + commandAddress, + packetAddress, + stale: false, + currentValue); if (tracePacket) { TraceAgc( - $"agc.dcb.write_data dst={destination} addr=0x{destinationAddress:X16} " + - $"count={dwordCount} increment={increment} wrote={wroteData}"); + $"agc.dcb.suspended addr=0x{waitAddress:X16} ref=0x{reference:X16} " + + $"mask=0x{mask:X16} cur=0x{currentValue:X16} cmp={compareFunction}"); + } + + return true; + } + + /// + /// Direct guest CPU stores can satisfy a GPU wait without crossing another + /// AGC import. Keep one low-frequency monitor per guest memory while waits + /// exist so those real stores wake their queues. The monitor never changes + /// a label: it uses the same masked comparison as submission-time parsing + /// and resumes only after the guest value genuinely satisfies the packet. + /// + private static void EnsureGpuWaitMonitor( + CpuContext submitContext, + SubmittedGpuState gpuState) + { + if (gpuState.WaitMonitorRunning) + { + return; + } + + gpuState.WaitMonitorRunning = true; + var monitorContext = new CpuContext( + submitContext.Memory, + submitContext.TargetGeneration); + ThreadPool.UnsafeQueueUserWorkItem( + static state => MonitorGpuWaits(state.Context, state.GpuState), + (Context: monitorContext, GpuState: gpuState), + preferLocal: false); + } + + private static void MonitorGpuWaits( + CpuContext ctx, + SubmittedGpuState gpuState) + { + var delayMilliseconds = 1; + while (true) + { + var madeProgress = false; + lock (gpuState.Gate) + { + var before = GpuWaitRegistry.CountForMemory(ctx.Memory); + if (before == 0) + { + gpuState.WaitMonitorRunning = false; + return; + } + + DrainResumableDcbs(ctx, gpuState, tracePackets: _traceAgc); + var after = GpuWaitRegistry.CountForMemory(ctx.Memory); + madeProgress = after < before; + if (madeProgress) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.wait_monitor_resumed count={before - after} " + + $"remaining={after}"); + } + if (after == 0) + { + gpuState.WaitMonitorRunning = false; + return; + } + } + + delayMilliseconds = madeProgress + ? 1 + : Math.Min(delayMilliseconds * 2, 16); + Thread.Sleep(delayMilliseconds); } } - private static void ApplySubmittedReleaseMem( + /// + /// Writes a value that satisfies the waiter's comparison. This deliberately + /// exists only behind SHARPEMU_GPU_WAIT_MODE=force for legacy A/B testing; + /// normal and stale waits must never mutate their watched label. + /// + private static void ForceSatisfyGpuWait( CpuContext ctx, + in GpuWaitRegistry.WaitingDcb waiter, + ulong value) + { + var address = waiter.WaitAddress; + var mask = waiter.Mask; + if (address == 0 || mask == 0) + { + return; + } + + var maskedRef = waiter.ReferenceValue & mask; + ulong? satisfyMasked = waiter.CompareFunction switch + { + 1 => maskedRef == 0 ? null : (maskedRef - 1) & mask, // < + 2 => maskedRef, // <= + 3 => maskedRef, // == + 4 => (~maskedRef) & mask, // != + 5 => maskedRef, // >= + 6 => maskedRef == mask ? null : (maskedRef + 1) & mask, // > + _ => null, + }; + + if (satisfyMasked is not { } satisfy) + { + return; + } + + var newValue = (value & ~mask) | (satisfy & mask); + if (waiter.Is64Bit) + { + ctx.TryWriteUInt64(address, newValue); + } + else + { + TryWriteUInt32(ctx, address, unchecked((uint)newValue)); + } + } + + // WAIT_REG_MEM packets whose condition is not met suspend their DCB into + // GpuWaitRegistry. Each submit re-checks every suspended DCB against current + // guest memory (labels are advanced by ReleaseMem/WriteData/DmaData packets + // or direct CPU writes) and resumes the ones now satisfied. A resumed DCB + // can itself write labels that unblock others, so loop to a fixed point. + private static void DrainResumableDcbs( + CpuContext ctx, + SubmittedGpuState gpuState, + bool tracePackets) + { + if (!_gpuWaitSuspendEnabled) + { + return; + } + + for (var pass = 0; pass < 256; pass++) + { + var woken = GpuWaitRegistry.CollectSatisfied(ctx.Memory, (address, is64Bit) => + is64Bit + ? TryReadUInt64(ctx, address, out var value64) ? value64 : (ulong?)null + : TryReadUInt32(ctx, address, out var value32) ? value32 : (ulong?)null); + + if (woken is null) + { + if (_gpuWaitStaleTicks > 0 && + GpuWaitRegistry.CollectUnreportedStale( + ctx.Memory, + System.Diagnostics.Stopwatch.GetTimestamp(), + _gpuWaitStaleTicks) is { } stale) + { + foreach (var waiter in stale) + { + ulong? currentValue = waiter.Is64Bit + ? TryReadUInt64(ctx, waiter.WaitAddress, out var value64) + ? value64 + : null + : TryReadUInt32(ctx, waiter.WaitAddress, out var value32) + ? value32 + : null; + TraceWaitProducerState( + ctx.Memory, + waiter, + waiter.CommandBufferAddress, + waiter.ResumeAddress, + stale: true, + currentValue); + } + } + + return; + } + + foreach (var waiter in woken) + { + ResumeSuspendedDcb(ctx, gpuState, waiter, tracePackets); + } + } + } + + private static void ResumeSuspendedDcb( + CpuContext ctx, + SubmittedGpuState gpuState, + in GpuWaitRegistry.WaitingDcb waiter, + bool tracePackets) + { + var state = waiter.State as SubmittedDcbState ?? gpuState.Graphics; + var remainingDwords = waiter.TotalDwords - waiter.ResumeOffset; + var waitedMilliseconds = waiter.RegisteredTicks == 0 + ? 0.0 + : (System.Diagnostics.Stopwatch.GetTimestamp() - waiter.RegisteredTicks) * + 1000.0 / System.Diagnostics.Stopwatch.Frequency; + TraceAgcShader( + $"agc.queue_resumed queue={waiter.QueueName} " + + $"submission={waiter.SubmissionId} label=0x{waiter.WaitAddress:X16} " + + $"resume=0x{waiter.ResumeAddress:X16} remaining_dwords={remainingDwords} " + + $"waited_ms={waitedMilliseconds:F3}"); + if (remainingDwords == 0) + { + state.IsSuspended = false; + state.HasActiveSubmission = false; + NotifySubmittedDcbCompleted(gpuState, state, waiter.SubmissionId); + PumpSubmittedQueue(ctx, gpuState, state); + return; + } + + if (tracePackets) + { + TraceAgc( + $"agc.dcb.resumed addr=0x{waiter.WaitAddress:X16} " + + $"resume=0x{waiter.ResumeAddress:X16} dwords={remainingDwords} forced=False"); + } + + System.Diagnostics.Debug.Assert(state.HasActiveSubmission); + System.Diagnostics.Debug.Assert(state.IsSuspended); + state.QueueName = waiter.QueueName ?? state.QueueName; + state.ActiveSubmissionId = waiter.SubmissionId; + state.IsSuspended = false; + if (ParseSubmittedDcb( + ctx, + gpuState, + state, + waiter.ResumeAddress, + remainingDwords, + tracePackets)) + { + state.IsSuspended = true; + return; + } + + state.HasActiveSubmission = false; + NotifySubmittedDcbCompleted(gpuState, state, waiter.SubmissionId); + PumpSubmittedQueue(ctx, gpuState, state); + } + + private static void TraceSubmittedWait( + ulong address, + ulong value, + ulong mask, + ulong reference, + uint compareFunction, + int bits, + bool tracePacket) + { + var maskedValue = value & mask; + var satisfied = compareFunction switch + { + 0 => true, + 1 => maskedValue < reference, + 2 => maskedValue <= reference, + 3 => maskedValue == reference, + 4 => maskedValue != reference, + 5 => maskedValue >= reference, + 6 => maskedValue > reference, + _ => true, + }; + if (!tracePacket && (satisfied || !ShouldTraceHotPath(ref _unsatisfiedWaitTraceCount))) + { + return; + } + + TraceAgc( + $"agc.dcb.wait_reg_mem bits={bits} addr=0x{address:X16} " + + $"value=0x{value:X16} mask=0x{mask:X16} ref=0x{reference:X16} " + + $"compare={compareFunction} satisfied={satisfied}"); + } + + private static void ApplySubmittedStandardReleaseMem( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, ulong packetAddress, bool tracePacket) { - if (!ctx.TryReadUInt32(packetAddress + 8, out var control) || - !ctx.TryReadUInt32(packetAddress + 12, out var destinationLo) || - !ctx.TryReadUInt32(packetAddress + 16, out var destinationHi) || - !ctx.TryReadUInt32(packetAddress + 20, out var dataLo) || - !ctx.TryReadUInt32(packetAddress + 24, out var dataHi)) + if (!TryReadUInt32(ctx, packetAddress + 8, out var control) || + !TryReadUInt32(ctx, packetAddress + 12, out var destinationLo) || + !TryReadUInt32(ctx, packetAddress + 16, out var destinationHi) || + !TryReadUInt32(ctx, packetAddress + 20, out var dataLo) || + !TryReadUInt32(ctx, packetAddress + 24, out var dataHi)) + { + return; + } + + var (destination, dataSelection) = DecodeStandardReleaseMemControl(control); + var destinationAddress = ((ulong)destinationHi << 32) | destinationLo; + var data = ((ulong)dataHi << 32) | dataLo; + var writeLength = dataSelection switch + { + 1 => (ulong)sizeof(uint), + 2 or 3 or 4 => (ulong)sizeof(ulong), + _ => 0UL, + }; + var writesGuestMemory = destination is 0 or 1 && + destinationAddress != 0 && + writeLength != 0; + + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + if (writesGuestMemory) + { + InvalidateDcbWindowIfOverlaps(destinationAddress, writeLength); + } + + var wroteData = writesGuestMemory && (dataSelection switch + { + 1 => TryWriteUInt32(ctx, destinationAddress, dataLo), + 2 => ctx.TryWriteUInt64(destinationAddress, data), + // Hardware counter writes are timing values sampled at the + // release point, not the immediate payload in ordinal 6/7. + 3 or 4 => ctx.TryWriteUInt64( + destinationAddress, + unchecked((ulong)System.Diagnostics.Stopwatch.GetTimestamp())), + _ => false, + }); + + if (tracePacket) + { + TraceAgc( + $"agc.dcb.release_mem_standard dst_sel={destination} " + + $"dst=0x{destinationAddress:X16} data_sel={dataSelection} " + + $"data=0x{data:X16} wrote={wroteData}"); + } + }, + $"release_mem_standard dst=0x{destinationAddress:X16} data=0x{data:X16}", + packetAddress, + writesGuestMemory ? destinationAddress : 0, + writesGuestMemory ? writeLength : 0); + } + + private static (uint Destination, uint DataSelection) + DecodeStandardReleaseMemControl(uint control) => + ( + Destination: (control >> 16) & 0x3u, + DataSelection: (control >> 29) & 0x7u); + + private static void ApplySubmittedReleaseMem( + CpuContext ctx, + SubmittedGpuState gpuState, + SubmittedDcbState state, + ulong packetAddress, + bool tracePacket) + { + if (!TryReadUInt32(ctx, packetAddress + 8, out var control) || + !TryReadUInt32(ctx, packetAddress + 12, out var destinationLo) || + !TryReadUInt32(ctx, packetAddress + 16, out var destinationHi) || + !TryReadUInt32(ctx, packetAddress + 20, out var dataLo) || + !TryReadUInt32(ctx, packetAddress + 24, out var dataHi)) { return; } @@ -3232,20 +5057,43 @@ public static class AgcExports var dataSelection = (control >> 16) & 0xFFu; var destinationAddress = ((ulong)destinationHi << 32) | destinationLo; var data = ((ulong)dataHi << 32) | dataLo; - - var wroteData = dataSelection switch + var writeLength = dataSelection switch { - 1 => ctx.TryWriteUInt32(destinationAddress, dataLo), - 2 or 3 => ctx.TryWriteUInt64(destinationAddress, data), - _ => false, + 1 => (ulong)sizeof(uint), + 2 or 3 => (ulong)sizeof(ulong), + _ => 0UL, }; + SubmitOrderedGpuSideEffect( + ctx, + gpuState, + state, + () => + { + InvalidateDcbWindowIfOverlaps(destinationAddress, writeLength); + var wroteData = dataSelection switch + { + 1 => TryWriteUInt32(ctx, destinationAddress, dataLo), + 2 => ctx.TryWriteUInt64(destinationAddress, data), + // Data selection 3 samples the GPU clock at the release + // point. The packet payload is ignored by hardware; Unity + // uses the nonzero timestamp as submit-completion state. + 3 => ctx.TryWriteUInt64( + destinationAddress, + unchecked((ulong)System.Diagnostics.Stopwatch.GetTimestamp())), + _ => false, + }; - if (tracePacket) - { - TraceAgc( - $"agc.dcb.release_mem dst=0x{destinationAddress:X16} data_sel={dataSelection} " + - $"data=0x{data:X16} wrote={wroteData}"); - } + if (tracePacket) + { + TraceAgc( + $"agc.dcb.release_mem dst=0x{destinationAddress:X16} " + + $"data_sel={dataSelection} data=0x{data:X16} wrote={wroteData}"); + } + }, + $"release_mem dst=0x{destinationAddress:X16} data=0x{data:X16}", + packetAddress, + dataSelection is 1 or 2 or 3 ? destinationAddress : 0, + writeLength); } private static void ApplySubmittedRegisters( @@ -3259,7 +5107,7 @@ public static class AgcExports if (op is ItSetShReg or ItSetContextReg or ItSetUconfigReg) { if (packetLength < 3 || - !ctx.TryReadUInt32(packetAddress + sizeof(uint), out var startRegister)) + !TryReadUInt32(ctx, packetAddress + sizeof(uint), out var startRegister)) { return; } @@ -3272,7 +5120,8 @@ public static class AgcExports }; for (uint index = 0; index < packetLength - 2; index++) { - if (!ctx.TryReadUInt32( + if (!TryReadUInt32( + ctx, packetAddress + 8 + ((ulong)index * sizeof(uint)), out var value)) { @@ -3288,8 +5137,8 @@ public static class AgcExports if (op != ItNop || register is not (RCxRegsIndirect or RShRegsIndirect or RUcRegsIndirect) || packetLength < 4 || - !ctx.TryReadUInt32(packetAddress + sizeof(uint), out var registerCount) || - !ctx.TryReadUInt64(packetAddress + 8, out var registersAddress)) + !TryReadUInt32(ctx, packetAddress + sizeof(uint), out var registerCount) || + !TryReadUInt64(ctx, packetAddress + 8, out var registersAddress)) { return; } @@ -3303,16 +5152,17 @@ public static class AgcExports for (uint index = 0; index < registerCount; index++) { var entryAddress = registersAddress + ((ulong)index * 8); - if (!ctx.TryReadUInt32(entryAddress, out var registerOffset) || - !ctx.TryReadUInt32(entryAddress + sizeof(uint), out var value)) + if (!TryReadUInt32(ctx, entryAddress, out var registerOffset) || + !TryReadUInt32(ctx, entryAddress + sizeof(uint), out var value)) { return; } - if (registerOffset != 0) - { - destination[registerOffset] = value; - } + // The indirect table has an explicit count; offset zero is a real + // context-register index (DB_RENDER_CONTROL), not a terminator. + // Dropping it leaves stale depth/render-control state active in + // later passes. + destination[registerOffset] = value; } } @@ -3328,25 +5178,20 @@ public static class AgcExports switch (op) { case ItDrawIndexAuto when packetLength >= 3: - return ctx.TryReadUInt32(packetAddress + 4, out drawCount); + return TryReadUInt32(ctx, packetAddress + 4, out drawCount); case ItDrawIndex2 when packetLength >= 6: state.DrawIndexOffset = 0; - return ctx.TryReadUInt32(packetAddress + 16, out drawCount); - // 5-dword form emitted by DcbDrawIndex (count at +4, ItIndexBase/ - // ItIndexBufferSize carried by separate preceding packets). - case ItDrawIndex2 when packetLength >= 5: - state.DrawIndexOffset = 0; - return ctx.TryReadUInt32(packetAddress + 4, out drawCount); + return TryReadUInt32(ctx, packetAddress + 16, out drawCount); case ItDrawIndexOffset2 when packetLength >= 5: - if (!ctx.TryReadUInt32(packetAddress + 8, out var indexOffset)) + if (!TryReadUInt32(ctx, packetAddress + 8, out var indexOffset)) { return false; } state.DrawIndexOffset = indexOffset; - return ctx.TryReadUInt32(packetAddress + 12, out drawCount); + return TryReadUInt32(ctx, packetAddress + 12, out drawCount); case ItDrawIndexMultiAuto when packetLength >= 4: - if (!ctx.TryReadUInt32(packetAddress + 12, out var control)) + if (!TryReadUInt32(ctx, packetAddress + 12, out var control)) { return false; } @@ -3355,12 +5200,13 @@ public static class AgcExports return true; case ItDrawIndirect or ItDrawIndexIndirect when packetLength >= 5 && state.IndirectArgsAddress != 0: - if (!ctx.TryReadUInt32(packetAddress + 4, out var dataOffset)) + if (!TryReadUInt32(ctx, packetAddress + 4, out var dataOffset)) { return false; } - return ctx.TryReadUInt32( + return TryReadUInt32( + ctx, state.IndirectArgsAddress + dataOffset, out drawCount); default: @@ -3390,24 +5236,48 @@ public static class AgcExports state.UcRegisters.TryGetValue(VgtPrimitiveType, out var primitiveType); var renderTargets = GetRenderTargets(state.CxRegisters); var drawSequence = ++gpuState.WorkSequence; + if (state.PendingTargetlessDraw is { } stalePendingDraw) + { + ReturnPooledDrawArrays( + stalePendingDraw, + globals: true, + vertex: true, + index: true); + state.PendingTargetlessDraw = null; + } state.TranslatedDraw = null; state.GuestDrawKind = GuestDrawKind.None; foreach (var target in renderTargets) { - state.RenderTargetWriters[target.Address] = new RenderTargetWriter( - drawSequence, - hasExportShader ? exportShaderAddress : 0, - hasPixelShader ? pixelShaderAddress : 0, - vertexCount, - primitiveType); + state.KnownRenderTargets[target.Address] = target; + // Colour exports originate in the pixel stage. A depth-only draw + // can leave old CB registers bound, but it must not become the + // advertised writer of those surfaces merely because they remain + // in state. + if (hasPixelShader) + { + state.RenderTargetWriters[target.Address] = new RenderTargetWriter( + drawSequence, + hasExportShader ? exportShaderAddress : 0, + pixelShaderAddress, + vertexCount, + primitiveType); + } - TraceAgcShader( - $"agc.rt_writer seq={drawSequence} target=0x{target.Address:X16} " + - $"fmt={target.Format} tile={target.TileMode} " + - $"size={target.Width}x{target.Height} vertices={vertexCount} " + - $"prim=0x{primitiveType:X} indexed={indexed} " + - $"es=0x{(hasExportShader ? exportShaderAddress : 0):X16} " + - $"ps=0x{(hasPixelShader ? pixelShaderAddress : 0):X16}"); + if (_traceAgcShader || + _tracePixelShaderAddress == pixelShaderAddress || + _traceRenderTargetAddress == target.Address) + { + Console.Error.WriteLine( + "[LOADER][TRACE] " + + $"agc.rt_writer seq={drawSequence} target=0x{target.Address:X16} " + + $"fmt={target.Format} tile={target.TileMode} " + + $"size={target.Width}x{target.Height} vertices={vertexCount} " + + $"prim=0x{primitiveType:X} indexed={indexed} " + + $"es=0x{(hasExportShader ? exportShaderAddress : 0):X16} " + + $"ps=0x{(hasPixelShader ? pixelShaderAddress : 0):X16} " + + $"color_write={(hasPixelShader ? 1 : 0)}"); + } } if (vertexCount == 0 || vertexCount > 1_048_576) @@ -3416,6 +5286,78 @@ public static class AgcExports } var translationError = string.Empty; + var depthState = DecodeDepthState(state.CxRegisters); + var depthTarget = DecodeDepthTarget(state.CxRegisters); + var hasDepthOnlyCandidate = hasExportShader && + !hasPixelShader && + depthTarget is not null && + (depthState.TestEnable || depthState.WriteEnable); + if (hasDepthOnlyCandidate && + TryCreateTranslatedDepthOnlyGuestDraw( + ctx, + state, + exportShaderAddress, + vertexCount, + indexed, + depthTarget!, + out var depthOnlyDraw, + out translationError)) + { + state.TranslatedDraw = depthOnlyDraw; + var activeDepthTarget = depthOnlyDraw.DepthTarget!; + var textures = CreateGuestDrawTextures( + ctx, + depthOnlyDraw.Textures, + out _); + var globalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(depthOnlyDraw); + var vertexBuffers = + CreateGuestVertexBuffers(depthOnlyDraw.VertexInputs); + var renderState = depthOnlyDraw.RenderState; + if (activeDepthTarget.ReadOnly && renderState.Depth.WriteEnable) + { + renderState = renderState with + { + Depth = renderState.Depth with { WriteEnable = false }, + }; + } + + TraceDrawCompact( + drawSequence, + depthOnlyDraw, + textures, + vertexBuffers); + GuestGpu.Current.SubmitDepthOnlyTranslatedDraw( + depthOnlyDraw.PixelShader, + textures, + globalMemoryBuffers, + depthOnlyDraw.AttributeCount, + activeDepthTarget, + depthOnlyDraw.VertexShader, + depthOnlyDraw.VertexCount, + depthOnlyDraw.InstanceCount, + depthOnlyDraw.PrimitiveType, + depthOnlyDraw.IndexBuffer, + vertexBuffers, + renderState, + depthOnlyDraw.PixelShaderAddress); + + if (_traceAgcShader) + { + TraceAgcShader( + $"agc.depth_only_draw seq={drawSequence} " + + $"es=0x{exportShaderAddress:X16} " + + $"depth=0x{activeDepthTarget.Address:X16}:" + + $"{activeDepthTarget.Width}x{activeDepthTarget.Height}:" + + $"fmt{activeDepthTarget.GuestFormat}/sw{activeDepthTarget.SwizzleMode} " + + $"test={(renderState.Depth.TestEnable ? 1 : 0)} " + + $"write={(renderState.Depth.WriteEnable ? 1 : 0)} " + + $"func={renderState.Depth.CompareOp} ro={(activeDepthTarget.ReadOnly ? 1 : 0)}"); + } + + return; + } + if (hasExportShader && hasPixelShader && hasPsInputEna && @@ -3425,58 +5367,219 @@ public static class AgcExports state, exportShaderAddress, pixelShaderAddress, + psInputEna, + psInputAddr, vertexCount, indexed, out var translatedDraw, out translationError)) { state.TranslatedDraw = translatedDraw; + if (TryGetHardwareColorResolveTargets( + state.CxRegisters, + out var resolveSource, + out var resolveDestination)) + { + state.KnownRenderTargets[resolveSource.Address] = resolveSource; + state.KnownRenderTargets[resolveDestination.Address] = resolveDestination; + ProvideRenderTargetInitialData(ctx, resolveSource); + if (VulkanVideoPresenter.TrySubmitGuestImageBlit( + resolveSource.Address, + resolveSource.Width, + resolveSource.Height, + resolveSource.Format, + resolveSource.NumberType, + resolveDestination.Address, + resolveDestination.Width, + resolveDestination.Height, + resolveDestination.Format, + resolveDestination.NumberType)) + { + state.RenderTargetWriters[resolveDestination.Address] = + new RenderTargetWriter( + drawSequence, + exportShaderAddress, + pixelShaderAddress, + vertexCount, + primitiveType); + TraceAgcShader( + $"agc.hardware_color_resolve seq={drawSequence} " + + $"src=0x{resolveSource.Address:X16}:" + + $"{resolveSource.Width}x{resolveSource.Height}:" + + $"fmt{resolveSource.Format}/num{resolveSource.NumberType} " + + $"dst=0x{resolveDestination.Address:X16}:" + + $"{resolveDestination.Width}x{resolveDestination.Height}:" + + $"fmt{resolveDestination.Format}/num{resolveDestination.NumberType}"); + ReturnPooledDrawArrays( + translatedDraw, + globals: true, + vertex: true, + index: true); + state.TranslatedDraw = null; + return; + } + + TraceAgcShader( + $"agc.hardware_color_resolve_unavailable seq={drawSequence} " + + $"src=0x{resolveSource.Address:X16} " + + $"dst=0x{resolveDestination.Address:X16}"); + } + var firstTarget = translatedDraw.RenderTargets.FirstOrDefault(); if (firstTarget.Address != 0) { - var textures = CreateGuestDrawTextures( + // Render every bound color target. A deferred G-buffer draw + // writes several targets in one guest pass; we render one bound + // target per Vulkan pass, each with the pixel variant that + // routes that target's MRT export slot to the fragment output. + // Every pass is enqueued in order on the same guest render + // queue. Share the immutable snapshots between those passes + // and let only the final pass return pooled arrays after its + // host upload. Copying the full vertex/global payload for each + // secondary target made deferred G-buffer draws allocate + // hundreds of MiB per second on the managed large-object heap. + var drawRenderTargets = translatedDraw.RenderTargets; + var lastTargetIndex = 0; + for (var targetIndex = 1; targetIndex < drawRenderTargets.Count; targetIndex++) + { + if (drawRenderTargets[targetIndex].Address != 0) + { + lastTargetIndex = targetIndex; + } + } + + var sharedTextures = CreateGuestDrawTextures( ctx, translatedDraw.Textures, out _); - var globalMemoryBuffers = - CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); - var vertexBuffers = + var sharedGlobalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(translatedDraw); + var sharedVertexBuffers = CreateGuestVertexBuffers(translatedDraw.VertexInputs); - TraceRectListVertices(translatedDraw, vertexBuffers); - TraceGrassDrawVertices(translatedDraw, textures, vertexBuffers); + TraceRectListVertices(translatedDraw, sharedVertexBuffers); + TraceGrassDrawVertices(translatedDraw, sharedTextures, sharedVertexBuffers); + TraceDrawCompact( + drawSequence, + translatedDraw, + sharedTextures, + sharedVertexBuffers); + foreach (var renderTarget in drawRenderTargets) + { + if (renderTarget.Address != 0) + { + ProvideRenderTargetInitialData(ctx, renderTarget); + } + } + GuestGpu.Current.SubmitOffscreenTranslatedDraw( translatedDraw.PixelShader, - textures, - globalMemoryBuffers, + sharedTextures, + sharedGlobalMemoryBuffers, translatedDraw.AttributeCount, translatedDraw.GuestTargets, - translatedDraw.VertexShader, - translatedDraw.VertexCount, - translatedDraw.InstanceCount, - translatedDraw.PrimitiveType, - translatedDraw.IndexBuffer, - vertexBuffers, - translatedDraw.RenderState); + translatedDraw.VertexShader, + translatedDraw.VertexCount, + translatedDraw.InstanceCount, + translatedDraw.PrimitiveType, + translatedDraw.IndexBuffer, + sharedVertexBuffers, + translatedDraw.RenderState, + translatedDraw.DepthTarget, + translatedDraw.PixelShaderAddress); } else { - var storageTarget = translatedDraw.Textures - .FirstOrDefault(binding => binding.IsStorage); - if (storageTarget is not null) + if (translatedDraw.DepthTarget is { } translatedDepthTarget) { var textures = CreateGuestDrawTextures( ctx, translatedDraw.Textures, out _); var globalMemoryBuffers = - CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); - GuestGpu.Current.SubmitStorageTranslatedDraw( + CreateTranslatedDrawGlobalBuffers(translatedDraw); + var vertexBuffers = + CreateGuestVertexBuffers(translatedDraw.VertexInputs); + var renderState = translatedDraw.RenderState; + if (translatedDepthTarget.ReadOnly && renderState.Depth.WriteEnable) + { + renderState = renderState with + { + Depth = renderState.Depth with { WriteEnable = false }, + }; + } + + TraceDrawCompact( + drawSequence, + translatedDraw, + textures, + vertexBuffers); + GuestGpu.Current.SubmitDepthOnlyTranslatedDraw( translatedDraw.PixelShader, textures, globalMemoryBuffers, translatedDraw.AttributeCount, - storageTarget.Descriptor.Width, - storageTarget.Descriptor.Height); + translatedDepthTarget, + translatedDraw.VertexShader, + translatedDraw.VertexCount, + translatedDraw.InstanceCount, + translatedDraw.PrimitiveType, + translatedDraw.IndexBuffer, + vertexBuffers, + renderState, + translatedDraw.PixelShaderAddress); + } + else + { + var storageTarget = translatedDraw.Textures + .FirstOrDefault(binding => binding.IsStorage); + if (storageTarget is not null) + { + var textures = CreateGuestDrawTextures( + ctx, + translatedDraw.Textures, + out _); + var globalMemoryBuffers = + CreateTranslatedDrawGlobalBuffers(translatedDraw); + TraceDrawCompact(drawSequence, translatedDraw, textures, []); + GuestGpu.Current.SubmitStorageTranslatedDraw( + translatedDraw.PixelShader, + textures, + globalMemoryBuffers, + translatedDraw.AttributeCount, + storageTarget.Descriptor.Width, + storageTarget.Descriptor.Height, + translatedDraw.PixelShaderAddress); + // The storage submit consumes the global buffers (the + // presenter returns them) but never the vertex/index + // arrays; return those here so they don't leak the pool. + ReturnPooledDrawArrays( + translatedDraw, + globals: false, + vertex: true, + index: true); + } + else + { + if (translatedDraw.Textures.Count != 0) + { + // Unity's PS5 final blit can omit CB registers and + // rely on the following AGC flip to name the scanout + // target. Retain that sampled draw until RFlip, then + // enqueue it against the known display surface before + // the ordered capture. + state.PendingTargetlessDraw = translatedDraw; + } + else + { + // No render target, storage sink or sampled source: + // nothing can consume this draw. + ReturnPooledDrawArrays( + translatedDraw, + globals: true, + vertex: true, + index: true); + } + } } } @@ -3489,25 +5592,40 @@ public static class AgcExports $"textures={translatedDraw.Textures.Count}"); } - lock (_submitTraceGate) + // Trace-only: gated on the flag so the dedup set and the dump — + // which reads pooled buffer data the presenter may already have + // recycled (harmless for diagnostics, garbage bytes at worst) — + // cost nothing in normal runs. + if (_traceAgcShader) { - var firstTextureAddress = translatedDraw.Textures.FirstOrDefault()?.Descriptor.Address ?? 0; - if (_tracedShaderDraws.Add( - (exportShaderAddress, pixelShaderAddress, firstTarget.Address, firstTextureAddress, vertexCount))) + lock (_submitTraceGate) { - TraceTranslatedGuestDraw( - ctx, - gpuState, - state, - translatedDraw, - psInputEna, - psInputAddr); + var firstTextureAddress = translatedDraw.Textures.FirstOrDefault()?.Descriptor.Address ?? 0; + if (_tracedShaderDraws.Add( + (exportShaderAddress, pixelShaderAddress, firstTarget.Address, firstTextureAddress, vertexCount))) + { + TraceTranslatedGuestDraw( + ctx, + gpuState, + state, + translatedDraw, + psInputEna, + psInputAddr); + } } } return; } + TraceDrawCompactMiss( + drawSequence, + vertexCount, + hasExportShader && hasPixelShader + ? translationError + : hasDepthOnlyCandidate && !string.IsNullOrEmpty(translationError) + ? $"depth-only: {translationError}" + : $"missing-shaders es={hasExportShader} ps={hasPixelShader} ena={hasPsInputEna} addr={hasPsInputAddr}"); TraceShaderTranslationMiss( ctx, state, @@ -3520,27 +5638,27 @@ public static class AgcExports psInputEna, hasPsInputAddr, psInputAddr, - hasExportShader && hasPixelShader ? translationError : null); + hasExportShader && hasPixelShader || hasDepthOnlyCandidate + ? translationError + : null); } - private static bool TryCreateTranslatedGuestDraw( + private static bool TryCreateTranslatedDepthOnlyGuestDraw( CpuContext ctx, SubmittedDcbState state, ulong exportShaderAddress, - ulong pixelShaderAddress, uint vertexCount, bool indexed, + GuestDepthTarget depthTarget, out TranslatedGuestDraw draw, out string error) { draw = default!; error = string.Empty; ulong exportShaderHeader; - ulong pixelShaderHeader; lock (_submitTraceGate) { _shaderHeadersByCode.TryGetValue(exportShaderAddress, out exportShaderHeader); - _shaderHeadersByCode.TryGetValue(pixelShaderAddress, out pixelShaderHeader); } if (!Gen5ShaderTranslator.TryCreateState( @@ -3558,28 +5676,285 @@ public static class AgcExports out var exportEvaluation, out error, resolveVertexInputs: true, - vertexRecordLimit: indexed ? null : vertexCount) || - !Gen5ShaderTranslator.TryCreateState( + requiredVertexRecordCount: TryGetRequiredVertexRecordCount( + ctx, + state, + vertexCount, + indexed, + out var depthVertexRecords) + ? depthVertexRecords + : null)) + { + return false; + } + + var exportFingerprint = _bakeScalars + ? ComputeShaderStateFingerprint(exportEvaluation) + : ComputeShaderStructuralFingerprint(exportEvaluation); + var cacheKey = ( + exportShaderAddress, + exportFingerprint, + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment); + _depthOnlyVertexShaderCache.TryGetValue(cacheKey, out var vertexShader); + + if (vertexShader is null) + { + var guestGlobalBufferCount = exportEvaluation.GlobalMemoryBindings.Count; + // CreateTranslatedDrawGlobalBuffers appends both stage scalar + // blocks. The pixel block is unused by the fixed fragment stage; + // the vertex block remains at guestCount+1, matching this layout. + var totalGlobalBufferCount = _bakeScalars + ? guestGlobalBufferCount + : guestGlobalBufferCount + 2; + if (!GuestGpu.Current.TryCompileVertexShader( + exportState, + exportEvaluation, + out vertexShader, + out error, + globalBufferBase: 0, + totalGlobalBufferCount: totalGlobalBufferCount, + imageBindingBase: 0, + scalarRegisterBufferIndex: _bakeScalars + ? -1 + : guestGlobalBufferCount + 1, + requiredVertexOutputCount: 0, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) + { + ReturnPooledEvaluationArrays(exportEvaluation); + return false; + } + + DumpCompiledShader( + "depth-vs", + exportShaderAddress, + exportFingerprint, + vertexShader!, + exportState.Program); + VulkanVideoPresenter.CountSpirvCompilation(); + _depthOnlyVertexShaderCache.TryAdd(cacheKey, vertexShader!); + } + + var textures = new List( + exportEvaluation.ImageBindings.Count); + foreach (var binding in exportEvaluation.ImageBindings) + { + if (!TryDecodeTextureDescriptor(binding.ResourceDescriptor, out var texture)) + { + if (_strictShaderDescriptors) + { + error = $"invalid export texture descriptor at pc=0x{binding.Pc:X}"; + ReturnPooledEvaluationArrays(exportEvaluation); + return false; + } + + texture = new TextureDescriptor( + 0, + 1, + 1, + Gen5TextureFormatR8G8B8A8Unorm, + 0, + 0, + 0, + 0, + 0, + 1, + 0xFAC); + } + + textures.Add(new TranslatedImageBinding( + texture, + Gen5ShaderTranslator.IsStorageImageOperation(binding.Opcode), + binding.MipLevel ?? 0, + binding.SamplerDescriptor)); + } + + IReadOnlyList vertexInputs = + exportEvaluation.VertexInputs ?? []; + state.UcRegisters.TryGetValue(VgtPrimitiveType, out var primitiveType); + var syntheticTarget = new RenderTargetDescriptor( + Slot: 0, + Address: 0, + depthTarget.Width, + depthTarget.Height, + Format: 0, + NumberType: 0, + TileMode: 0); + var renderState = CreateRenderState(state.CxRegisters, syntheticTarget) with + { + // A guest pass without a pixel shader has no colour exports. The + // presenter uses a private compatibility attachment, so disable + // all writes to it and expose only the persistent DB result. + Blends = [GuestBlendState.Default with { WriteMask = 0 }], + }; + if (depthTarget.Width == 1 && + depthTarget.Height == 1 && + renderState.Viewport is { } depthViewport) + { + var inferredWidth = (uint)Math.Clamp( + MathF.Ceiling(MathF.Abs(depthViewport.Width)), + 1f, + 16384f); + var inferredHeight = (uint)Math.Clamp( + MathF.Ceiling(MathF.Abs(depthViewport.Height)), + 1f, + 16384f); + if (inferredWidth > 1 || inferredHeight > 1) + { + depthTarget = depthTarget with + { + Width = inferredWidth, + Height = inferredHeight, + }; + syntheticTarget = syntheticTarget with + { + Width = inferredWidth, + Height = inferredHeight, + }; + renderState = CreateRenderState(state.CxRegisters, syntheticTarget) with + { + Blends = [GuestBlendState.Default with { WriteMask = 0 }], + }; + } + } + draw = new TranslatedGuestDraw( + exportShaderAddress, + PixelShaderAddress: 0, + primitiveType, + vertexShader!, + GuestGpu.Current.GetDepthOnlyFragmentShader(), + AttributeCount: 0, + vertexCount, + state.InstanceCount, + indexed ? CreateGuestIndexBuffer(ctx, state, vertexCount) : null, + textures, + exportEvaluation.GlobalMemoryBindings, + vertexInputs, + RenderTargets: [], + depthTarget, + GuestTargets: [], + renderState, + PixelUserData: [], + RawBlendControl: 0, + RawColorInfo: 0, + PixelInitialScalars: [], + exportEvaluation.InitialScalarRegisters); + return true; + } + + private static bool TryCreateTranslatedGuestDraw( + CpuContext ctx, + SubmittedDcbState state, + ulong exportShaderAddress, + ulong pixelShaderAddress, + uint psInputEna, + uint psInputAddr, + uint vertexCount, + bool indexed, + out TranslatedGuestDraw draw, + out string error) + { + draw = default!; + error = string.Empty; + ulong exportShaderHeader; + ulong pixelShaderHeader; + lock (_submitTraceGate) + { + _shaderHeadersByCode.TryGetValue(exportShaderAddress, out exportShaderHeader); + _shaderHeadersByCode.TryGetValue(pixelShaderAddress, out pixelShaderHeader); + } + + // Sequential (not short-circuited into one condition) so a failure + // after an evaluation succeeded can return that evaluation's pooled + // buffer arrays to the pool instead of leaking them. + if (!Gen5ShaderTranslator.TryCreateState( + ctx, + exportShaderAddress, + exportShaderHeader, + state.ShRegisters, + SelectExportUserDataRegister(state.ShRegisters), + out var exportState, + out error, + userDataScalarRegisterBase: NggUserDataScalarRegisterBase)) + { + return false; + } + + if (!Gen5ShaderScalarEvaluator.TryEvaluate( + ctx, + exportState, + out var exportEvaluation, + out error, + resolveVertexInputs: true, + requiredVertexRecordCount: TryGetRequiredVertexRecordCount( + ctx, + state, + vertexCount, + indexed, + out var vertexRecords) + ? vertexRecords + : null)) + { + return false; + } + + if (!Gen5ShaderTranslator.TryCreateState( ctx, pixelShaderAddress, pixelShaderHeader, state.ShRegisters, PsTextureUserDataRegister, out var pixelState, - out error) || - !Gen5ShaderScalarEvaluator.TryEvaluate( + out error)) + { + ReturnPooledEvaluationArrays(exportEvaluation); + return false; + } + + if (!Gen5ShaderScalarEvaluator.TryEvaluate( ctx, pixelState, out var pixelEvaluation, out error)) { + ReturnPooledEvaluationArrays(exportEvaluation); return false; } - var renderTargets = GetRenderTargets(state.CxRegisters) + if (pixelShaderAddress == 0x0000000500781200 && + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_GLOBALS") == "1") + { + TraceAstroTitlePixelGlobals(pixelEvaluation); + } + + if (pixelShaderAddress == 0x0000000500781200 && + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_GLOBALS_LIVE") == "1") + { + TraceAstroTitlePixelGlobalProbe(pixelEvaluation); + } + + // Every bound color target the shader exports to. Deferred renderers + // draw a multi-render-target G-buffer (up to eight slots) in one pass. + // Fall back to slot 0 if we cannot match any export to a bound target. + var allBoundTargets = GetRenderTargets(state.CxRegisters); + var renderTargets = allBoundTargets .Where(target => HasPixelColorExport(pixelState, target.Slot)) .OrderBy(target => target.Slot) .ToArray(); + if (renderTargets.Length == 0) + { + renderTargets = allBoundTargets + .Where(target => target.Slot == 0) + .ToArray(); + } + if (_traceAgcShader && allBoundTargets.Count > 1) + { + TraceAgcShader( + $"agc.mrt_filter ps=0x{pixelShaderAddress:X16} " + + $"bound=[{string.Join(",", allBoundTargets.Select(t => $"s{t.Slot}:0x{t.Address:X}:exp{(HasPixelColorExport(pixelState, t.Slot) ? 1 : 0)}"))}] " + + $"kept={renderTargets.Length}"); + } + var renderTargetOutputKinds = new Gen5PixelOutputKind[renderTargets.Length]; for (var index = 0; index < renderTargets.Length; index++) { @@ -3591,6 +5966,8 @@ public static class AgcExports { error = $"unsupported color target format={target.Format} number_type={target.NumberType}"; + ReturnPooledEvaluationArrays(exportEvaluation); + ReturnPooledEvaluationArrays(pixelEvaluation); return false; } } @@ -3607,8 +5984,12 @@ public static class AgcExports } var attributeCount = GetInterpolatedAttributeCount(pixelState); - var exportStateFingerprint = ComputeShaderStructureFingerprint(exportEvaluation); - var pixelStateFingerprint = ComputeShaderStructureFingerprint(pixelEvaluation); + var exportStateFingerprint = _bakeScalars + ? ComputeShaderStateFingerprint(exportEvaluation) + : ComputeShaderStructuralFingerprint(exportEvaluation); + var pixelStateFingerprint = _bakeScalars + ? ComputeShaderStateFingerprint(pixelEvaluation) + : ComputeShaderStructuralFingerprint(pixelEvaluation); var shaderKey = ( exportShaderAddress, exportStateFingerprint, @@ -3616,10 +5997,19 @@ public static class AgcExports pixelStateFingerprint, outputLayout, (uint)renderTargets.Length, - attributeCount); - var totalGlobalBuffers = + attributeCount, + psInputEna, + psInputAddr, + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment); + + var guestGlobalBuffers = pixelEvaluation.GlobalMemoryBindings.Count + exportEvaluation.GlobalMemoryBindings.Count; + // Two per-draw initial-scalar buffers ride after the guest buffers: + // [pixel guest][vertex guest][pixel sgprs][vertex sgprs]. + var totalGlobalBuffers = _bakeScalars + ? guestGlobalBuffers + : guestGlobalBuffers + 2; _graphicsShaderCache.TryGetValue(shaderKey, out var compiled); if (compiled.Vertex is null || compiled.Pixel is null) @@ -3640,19 +6030,28 @@ public static class AgcExports out var pixelShader, out error, globalBufferBase: 0, - totalGlobalBufferCount: totalGlobalBuffers + 2, + totalGlobalBufferCount: totalGlobalBuffers, imageBindingBase: 0, - scalarRegisterBufferIndex: totalGlobalBuffers) || + scalarRegisterBufferIndex: _bakeScalars ? -1 : guestGlobalBuffers, + pixelInputEnable: psInputEna, + pixelInputAddress: psInputAddr, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment) || !GuestGpu.Current.TryCompileVertexShader( exportState, exportEvaluation, out var vertexShader, out error, globalBufferBase: pixelEvaluation.GlobalMemoryBindings.Count, - totalGlobalBufferCount: totalGlobalBuffers + 2, + totalGlobalBufferCount: totalGlobalBuffers, imageBindingBase: pixelEvaluation.ImageBindings.Count, - scalarRegisterBufferIndex: totalGlobalBuffers + 1)) + scalarRegisterBufferIndex: _bakeScalars ? -1 : guestGlobalBuffers + 1, + requiredVertexOutputCount: (int)GetInterpolatedAttributeCount(pixelState), + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) { + ReturnPooledEvaluationArrays(exportEvaluation); + ReturnPooledEvaluationArrays(pixelEvaluation); return false; } @@ -3669,6 +6068,7 @@ public static class AgcExports pixelStateFingerprint, compiled.Pixel, pixelState.Program); + VulkanVideoPresenter.CountSpirvCompilation(); _graphicsShaderCache.TryAdd(shaderKey, compiled); } @@ -3681,15 +6081,34 @@ public static class AgcExports { if (!TryDecodeTextureDescriptor(binding.ResourceDescriptor, out var texture)) { - error = $"invalid texture descriptor at pc=0x{binding.Pc:X}"; - return false; + // A garbage/zeroed texture descriptor (from a per-draw descriptor + // setup race — the same root as scalar-load-failed) would drop + // the whole draw, so Demon's Souls' deferred-lighting/composite + // passes that produce the composite's feeder targets never run + // and the frame stays black. Substitute a 1x1 fallback binding so + // the pass still renders (that one sampled texture reads black) + // rather than dropping it entirely. STRICT reverts. + if (_strictShaderDescriptors) + { + error = $"invalid texture descriptor at pc=0x{binding.Pc:X}"; + ReturnPooledEvaluationArrays(exportEvaluation); + ReturnPooledEvaluationArrays(pixelEvaluation); + return false; + } + + texture = new TextureDescriptor( + 0, 1, 1, Gen5TextureFormatR8G8B8A8Unorm, 0, 0, 0, 0, 0, 1, 0xFAC); } - TraceAgcShader( - $"agc.texture_binding ps=0x{pixelShaderAddress:X16} es=0x{exportShaderAddress:X16} " + - $"pc=0x{binding.Pc:X} op={binding.Opcode} storage={(Gen5ShaderTranslator.IsStorageImageOperation(binding.Opcode) ? 1 : 0)} " + - $"decoded={FormatTextureDescriptor(texture)} " + - $"raw={FormatShaderDwords(binding.ResourceDescriptor)} sampler={FormatShaderDwords(binding.SamplerDescriptor)}"); + if (_traceAgcShader || _tracePixelShaderAddress == pixelShaderAddress) + { + Console.Error.WriteLine( + "[LOADER][TRACE] " + + $"agc.texture_binding ps=0x{pixelShaderAddress:X16} es=0x{exportShaderAddress:X16} " + + $"pc=0x{binding.Pc:X} op={binding.Opcode} storage={(Gen5ShaderTranslator.IsStorageImageOperation(binding.Opcode) ? 1 : 0)} " + + $"decoded={FormatTextureDescriptor(texture)} " + + $"raw={FormatShaderDwords(binding.ResourceDescriptor)} sampler={FormatShaderDwords(binding.SamplerDescriptor)}"); + } textures.Add( new TranslatedImageBinding( texture, @@ -3700,8 +6119,6 @@ public static class AgcExports var globalMemoryBindings = pixelEvaluation.GlobalMemoryBindings .Concat(exportEvaluation.GlobalMemoryBindings) - .Append(CreateScalarRegisterBinding(pixelEvaluation)) - .Append(CreateScalarRegisterBinding(exportEvaluation)) .ToArray(); IReadOnlyList vertexInputs = exportEvaluation.VertexInputs ?? []; @@ -3723,7 +6140,7 @@ public static class AgcExports primitiveType, compiled.Vertex, compiled.Pixel, - attributeCount, + GetInterpolatedAttributeCount(pixelState), vertexCount, state.InstanceCount, indexed ? CreateGuestIndexBuffer(ctx, state, vertexCount) : null, @@ -3731,26 +6148,100 @@ public static class AgcExports globalMemoryBindings, vertexInputs, renderTargets, + DecodeDepthTarget(state.CxRegisters), guestTargets, ApplyTransparentPremultipliedFillClear( CreateRenderState(state.CxRegisters, renderTargets, pixelState), textures, vertexInputs, - pixelEvaluation.InitialScalarRegisters)); + pixelEvaluation.InitialScalarRegisters), + pixelEvaluation.InitialScalarRegisters.Take(8).ToArray(), + state.CxRegisters.TryGetValue(CbBlend0Control, out var rawBlend) ? rawBlend : 0, + state.CxRegisters.TryGetValue( + CbColor0Info + renderTargets.FirstOrDefault().Slot * CbColorRegisterStride, + out var rawInfo) + ? rawInfo + : 0, + pixelEvaluation.InitialScalarRegisters, + exportEvaluation.InitialScalarRegisters); return true; } - private static readonly bool _transparentFillClearEnabled = !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_TRANSPARENT_FILL_CLEAR"), + private static int _tracedAstroTitlePixelGlobals; + private static int _tracedAstroTitlePixelGlobalProbe; + + private static void TraceAstroTitlePixelGlobalProbe(Gen5ShaderEvaluation evaluation) + { + const int probeOffset = 17216; + var draw = Interlocked.Increment(ref _tracedAstroTitlePixelGlobalProbe); + foreach (var (binding, index) in evaluation.GlobalMemoryBindings.Select((value, index) => (value, index))) + { + if (probeOffset + 16 > binding.DataLength) + { + continue; + } + + Console.Error.WriteLine( + $"[TITLE-GLOBALS-LIVE] draw={draw} binding={index} " + + $"base=0x{binding.BaseAddress:X16} offset=0x{probeOffset:X} " + + $"bytes={Convert.ToHexString(binding.Data.AsSpan(probeOffset, 16))}"); + } + } + + private static void TraceAstroTitlePixelGlobals(Gen5ShaderEvaluation evaluation) + { + if (Interlocked.Exchange(ref _tracedAstroTitlePixelGlobals, 1) != 0) + { + return; + } + + Console.Error.WriteLine( + $"[TITLE-GLOBALS] initial_s0_31=" + + string.Join(',', evaluation.InitialScalarRegisters + .Take(32) + .Select((value, index) => $"s{index}={value:X8}"))); + + var probeOffsets = new[] + { + 0, 16, 24, 32, 48, + 192, 256, 400, 432, + 17100, 17104, 17136, 17168, 17184, 17200, 17216, + }; + foreach (var binding in evaluation.GlobalMemoryBindings) + { + Console.Error.WriteLine( + $"[TITLE-GLOBALS] binding s{binding.ScalarAddress} " + + $"base=0x{binding.BaseAddress:X16} bytes={binding.DataLength} " + + $"pcs={string.Join(',', binding.InstructionPcs.Select(pc => $"0x{pc:X}"))}"); + foreach (var offset in probeOffsets) + { + if (offset < 0 || offset + 16 > binding.DataLength) + { + continue; + } + + Console.Error.WriteLine( + $"[TITLE-GLOBALS] s{binding.ScalarAddress}+0x{offset:X}=" + + Convert.ToHexString(binding.Data.AsSpan(offset, 16))); + } + } + } + + private static readonly bool _fillClearHack = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DISABLE_FILL_CLEAR"), "1", StringComparison.Ordinal); /// - /// Chowdren resets its effect layers with an untextured transparent-black - /// fill using premultiplied blending. With One/OneMinusSrcAlpha that draw - /// is otherwise a no-op, causing fog and vignette layers to accumulate. - /// Treat precisely that draw shape as an overwrite only when every MRT - /// attachment uses the same premultiplied blend pattern. + /// Treat an untextured fill that outputs pure transparent black through + /// premultiplied blending as an overwrite. Chowdren issues exactly this + /// draw once per frame to reset its effect layers (fog smoke, vignette + /// masks); under the blend factors it sets (One, OneMinusSrcAlpha) a + /// (0,0,0,0) source is a mathematical no-op, so without this the layers + /// accumulate until they saturate and the fog composites as a flat veil + /// over the whole scene. The workaround applies only when every MRT + /// attachment uses the same blend pattern. Disable with + /// SHARPEMU_DISABLE_FILL_CLEAR=1. /// private static GuestRenderState ApplyTransparentPremultipliedFillClear( GuestRenderState renderState, @@ -3758,7 +6249,7 @@ public static class AgcExports IReadOnlyList vertexInputs, IReadOnlyList pixelUserData) { - if (!_transparentFillClearEnabled || + if (!_fillClearHack || textures.Count != 0 || vertexInputs.Count != 0 || pixelUserData.Count < 4 || @@ -3769,6 +6260,7 @@ public static class AgcExports for (var index = 0; index < 4; index++) { + // Positive or negative zero. if ((pixelUserData[index] & 0x7FFF_FFFFu) != 0) { return renderState; @@ -3806,12 +6298,101 @@ public static class AgcExports var bytesPerIndex = is32Bit ? sizeof(uint) : sizeof(ushort); var byteOffset = checked((ulong)state.DrawIndexOffset * (uint)bytesPerIndex); var byteCount = checked((int)(indexCount * (uint)bytesPerIndex)); - var data = new byte[byteCount]; + var data = VulkanVideoPresenter.GuestDataPool.Rent(byteCount); + var span = data.AsSpan(0, byteCount); var address = state.IndexBufferAddress + byteOffset; - return (ctx.Memory.TryRead(address, data) || - KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, data)) - ? new GuestIndexBuffer(data, is32Bit) - : null; + if (ctx.Memory.TryRead(address, span) || + KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, span)) + { + return new GuestIndexBuffer(data, byteCount, is32Bit, Pooled: true); + } + + VulkanVideoPresenter.GuestDataPool.Return(data); + return null; + } + + private static bool TryGetRequiredVertexRecordCount( + CpuContext ctx, + SubmittedDcbState state, + uint drawCount, + bool indexed, + out uint recordCount) + { + recordCount = Math.Max(drawCount, Math.Max(state.InstanceCount, 1u)); + if (!indexed) + { + return true; + } + + if (state.IndexBufferAddress == 0 || drawCount == 0) + { + return false; + } + + var is32Bit = state.IndexSize != 0; + var bytesPerIndex = is32Bit ? sizeof(uint) : sizeof(ushort); + var byteOffset = checked((ulong)state.DrawIndexOffset * (uint)bytesPerIndex); + var address = state.IndexBufferAddress + byteOffset; + const int chunkBytes = 64 * 1024; + var scratch = VulkanVideoPresenter.GuestDataPool.Rent(chunkBytes); + var remaining = drawCount; + var maxIndex = 0u; + var sawIndex = false; + try + { + while (remaining != 0) + { + var chunkIndices = (int)Math.Min( + remaining, + (uint)(chunkBytes / bytesPerIndex)); + var bytes = chunkIndices * bytesPerIndex; + var span = scratch.AsSpan(0, bytes); + if (!ctx.Memory.TryRead(address, span) && + !KernelMemoryCompatExports.TryReadTrackedLibcHeap(address, span)) + { + return false; + } + + for (var index = 0; index < chunkIndices; index++) + { + var value = is32Bit + ? BinaryPrimitives.ReadUInt32LittleEndian( + span.Slice(index * sizeof(uint), sizeof(uint))) + : BinaryPrimitives.ReadUInt16LittleEndian( + span.Slice(index * sizeof(ushort), sizeof(ushort))); + if (value == (is32Bit ? uint.MaxValue : ushort.MaxValue)) + { + // Primitive-restart markers do not address vertex data. + continue; + } + + maxIndex = Math.Max(maxIndex, value); + sawIndex = true; + } + + address += (uint)bytes; + remaining -= (uint)chunkIndices; + } + } + finally + { + VulkanVideoPresenter.GuestDataPool.Return(scratch); + } + + var indexedRecords = sawIndex && maxIndex != uint.MaxValue + ? maxIndex + 1 + : 1u; + recordCount = Math.Max(indexedRecords, Math.Max(state.InstanceCount, 1u)); + if (_traceVertexRanges && + Interlocked.Increment(ref _tracedVertexRangeCount) <= 512) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.vertex_range indexed=1 draw_count={drawCount} " + + $"max_index={(sawIndex ? maxIndex : 0)} records={recordCount} " + + $"instances={state.InstanceCount} index_size={(is32Bit ? 32 : 16)} " + + $"index_addr=0x{state.IndexBufferAddress:X16} offset={state.DrawIndexOffset}"); + } + return true; } private static bool HasPixelColorExport(Gen5ShaderState state, uint target) => @@ -3838,35 +6419,40 @@ public static class AgcExports return (uint)(maxAttribute + 1); } - private const int ShaderScalarRegisterCount = 256; + private static readonly bool _bakeScalars = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_BAKE_SGPRS"), + "1", + StringComparison.Ordinal); - private static Gen5GlobalMemoryBinding CreateScalarRegisterBinding( - Gen5ShaderEvaluation evaluation) + /// + /// Fingerprint of everything that shapes the translated SPIR-V besides + /// scalar register values (those arrive in a per-draw buffer): the + /// resolved binding set with its format-shaping descriptor words, vertex + /// input layouts, and compute system registers. Value churn in user data + /// no longer forces a new translation and pipeline. + /// + private static ulong ComputeShaderStructuralFingerprint(Gen5ShaderEvaluation evaluation) { - var data = new byte[ShaderScalarRegisterCount * sizeof(uint)]; - var registers = evaluation.InitialScalarRegisters; - var count = Math.Min(registers.Count, ShaderScalarRegisterCount); - for (var index = 0; index < count; index++) - { - var value = registers[index]; - var offset = index * sizeof(uint); - data[offset] = (byte)value; - data[offset + 1] = (byte)(value >> 8); - data[offset + 2] = (byte)(value >> 16); - data[offset + 3] = (byte)(value >> 24); - } - - return new Gen5GlobalMemoryBinding(0, 0, [], data); - } - - private static ulong ComputeShaderStructureFingerprint(Gen5ShaderEvaluation evaluation) - { - const ulong offsetBasis = 14695981039346656037UL; const ulong prime = 1099511628211UL; - var hash = offsetBasis; + var hash = 14695981039346656037UL; void Mix(ulong value) => hash = (hash ^ value) * prime; - Mix((ulong)evaluation.GlobalMemoryBindings.Count); + foreach (var binding in evaluation.ImageBindings) + { + Mix(binding.Pc); + Mix((ulong)(uint)binding.Opcode.GetHashCode()); + if (binding.ResourceDescriptor.Count > 1) + { + // The generated image type depends only on unified format. + // Bounds are queried from the bound view in SPIR-V; guest image + // addresses, dimensions, swizzles and sampler state are all + // runtime descriptor data and must not create pipeline variants. + Mix(binding.ResourceDescriptor[1] & 0x1FF0_0000u); + } + + Mix(binding.MipLevel ?? 0xFFFF_FFFFUL); + } + foreach (var binding in evaluation.GlobalMemoryBindings) { Mix(binding.ScalarAddress); @@ -3877,27 +6463,8 @@ public static class AgcExports } } - Mix((ulong)evaluation.ImageBindings.Count); - foreach (var image in evaluation.ImageBindings) - { - Mix(image.Pc); - Mix((ulong)(uint)image.Opcode.GetHashCode()); - foreach (var word in image.ResourceDescriptor) - { - Mix(word); - } - - foreach (var word in image.SamplerDescriptor) - { - Mix(word); - } - - Mix(image.MipLevel ?? uint.MaxValue); - } - if (evaluation.VertexInputs is { } vertexInputs) { - Mix((ulong)vertexInputs.Count); foreach (var input in vertexInputs) { Mix(input.Pc); @@ -3906,6 +6473,7 @@ public static class AgcExports Mix(input.DataFormat); Mix(input.NumberFormat); Mix(input.Stride); + Mix(input.OffsetBytes); } } @@ -3930,6 +6498,16 @@ public static class AgcExports hash = (hash ^ value) * prime; } + // Baked-scalar mode has no runtime state block from which the shader + // can load descriptor-alignment biases, so the low guest address bits + // remain part of the generated module and must participate in its key. + foreach (var binding in evaluation.GlobalMemoryBindings) + { + hash = (hash ^ ( + binding.BaseAddress & + (VulkanVideoPresenter.GuestStorageBufferOffsetAlignment - 1))) * prime; + } + if (evaluation.ComputeSystemRegisters is { } computeSystemRegisters) { hash = (hash ^ (computeSystemRegisters.WorkGroupXRegister ?? uint.MaxValue)) * prime; @@ -3941,8 +6519,36 @@ public static class AgcExports return hash; } + private static bool TryGetHardwareColorResolveTargets( + IReadOnlyDictionary registers, + out RenderTargetDescriptor source, + out RenderTargetDescriptor destination) + { + source = default; + destination = default; + if (!registers.TryGetValue(CbColorControl, out var colorControl) || + ((colorControl >> 4) & 0x7u) != 3u) + { + return false; + } + + // CB_COLOR_CONTROL.MODE=RESOLVE uses color slot 0 as the multisampled + // source and slot 1 as the single-sample destination. CB_TARGET_MASK + // still enables only slot 0, so treating this like a normal MRT draw + // rewrites the source and leaves the following composite's input blank. + var boundTargets = GetRenderTargets(registers, includeMaskedTargets: true); + source = boundTargets.FirstOrDefault(target => target.Slot == 0); + destination = boundTargets.FirstOrDefault(target => target.Slot == 1); + return source.Address != 0 && + destination.Address != 0 && + source.Width == destination.Width && + source.Height == destination.Height && + source.Format == destination.Format; + } + private static IReadOnlyList GetRenderTargets( - IReadOnlyDictionary registers) + IReadOnlyDictionary registers, + bool includeMaskedTargets = false) { var hasTargetMask = registers.TryGetValue(CbTargetMask, out var targetMask); var targets = new List(ColorTargetCount); @@ -3960,7 +6566,8 @@ public static class AgcExports var address = ((ulong)(baseHigh & 0xFFu) << 40) | ((ulong)baseLow << 8); var writeMask = (targetMask >> ((int)slot * 4)) & 0xFu; - if (address == 0 || (hasTargetMask && writeMask == 0)) + if (address == 0 || + (!includeMaskedTargets && hasTargetMask && writeMask == 0)) { continue; } @@ -3978,6 +6585,19 @@ public static class AgcExports return targets; } + private static GuestRenderState CreateRenderState( + IReadOnlyDictionary registers, + RenderTargetDescriptor target) + { + var scissor = DecodeScissor(registers, target.Width, target.Height); + return new GuestRenderState( + [DecodeBlendState(registers, target.Slot)], + scissor, + DecodeViewport(registers, target.Width, target.Height, scissor), + DecodeRasterState(registers), + DecodeDepthState(registers)); + } + private static GuestRenderState CreateRenderState( IReadOnlyDictionary registers, IReadOnlyList targets, @@ -4000,7 +6620,108 @@ public static class AgcExports }; }).ToArray(), scissor, - DecodeViewport(registers, target.Width, target.Height, scissor)); + DecodeViewport(registers, target.Width, target.Height, scissor), + DecodeRasterState(registers), + DecodeDepthState(registers)); + } + + // DB_DEPTH_CONTROL (context register 0x200): Z_ENABLE bit1, Z_WRITE_ENABLE + // bit2, ZFUNC bits[6:4] (GCN compare, matches Vulkan CompareOp ordering). + private const uint DbDepthControl = 0x200; + + private static GuestDepthState DecodeDepthState( + IReadOnlyDictionary registers) + { + if (!registers.TryGetValue(DbDepthControl, out var control)) + { + return GuestDepthState.Default; + } + + var testEnable = (control & 0x2u) != 0; + var writeEnable = (control & 0x4u) != 0; + var compareOp = (control >> 4) & 0x7u; + return new GuestDepthState(testEnable, writeEnable, compareOp); + } + + private static GuestDepthTarget? DecodeDepthTarget( + IReadOnlyDictionary registers) + { + var depthState = DecodeDepthState(registers); + if (!depthState.TestEnable && !depthState.WriteEnable) + { + return null; + } + + if (!registers.TryGetValue(DbZInfo, out var zInfo) || + !registers.TryGetValue(DbDepthSizeXy, out var sizeXy)) + { + return null; + } + + var guestFormat = zInfo & 0x3u; + if (guestFormat == 0) + { + return null; + } + + registers.TryGetValue(DbZReadBase, out var readBase); + registers.TryGetValue(DbZWriteBase, out var writeBase); + registers.TryGetValue(DbZReadBaseHi, out var readBaseHi); + registers.TryGetValue(DbZWriteBaseHi, out var writeBaseHi); + var readAddress = ((ulong)(readBaseHi & 0xFFu) << 40) | ((ulong)readBase << 8); + var writeAddress = ((ulong)(writeBaseHi & 0xFFu) << 40) | ((ulong)writeBase << 8); + if (readAddress == 0 && writeAddress == 0) + { + return null; + } + + var width = (sizeXy & 0x3FFFu) + 1; + var height = ((sizeXy >> 16) & 0x3FFFu) + 1; + if (width == 0 || height == 0 || width > 16384 || height > 16384) + { + return null; + } + + registers.TryGetValue(DbDepthView, out var depthView); + var clearDepth = registers.TryGetValue(DbDepthClear, out var clearBits) + ? BitConverter.UInt32BitsToSingle(clearBits) + : 1f; + if (!float.IsFinite(clearDepth) || clearDepth < 0f || clearDepth > 1f) + { + clearDepth = 1f; + } + + return new GuestDepthTarget( + readAddress, + writeAddress, + width, + height, + guestFormat, + (zInfo >> 4) & 0x1Fu, + clearDepth, + ReadOnly: (depthView & (1u << 24)) != 0 || writeAddress == 0); + } + + // PA_SU_SC_MODE_CNTL (context register 0x205) carries face culling, the + // front-face winding and polygon (wireframe) mode. + private const uint PaSuScModeCntl = 0x205; + + private static GuestRasterState DecodeRasterState( + IReadOnlyDictionary registers) + { + if (!registers.TryGetValue(PaSuScModeCntl, out var mode)) + { + return GuestRasterState.Default; + } + + var cullFront = (mode & 0x1u) != 0; + var cullBack = (mode & 0x2u) != 0; + var frontFaceClockwise = (mode & 0x4u) != 0; + var polyMode = (mode >> 3) & 0x3u; + var frontPtype = (mode >> 5) & 0x7u; + // POLY_MODE != 0 with a line front primitive type renders wireframe. + var wireframe = polyMode != 0 && frontPtype == 1; + return new GuestRasterState(cullFront, cullBack, frontFaceClockwise, wireframe); } private static GuestBlendState DecodeBlendState( @@ -4056,7 +6777,19 @@ public static class AgcExports windowOffsetY = (short)(windowOffset >> 16); } - IntersectScissorPair(registers, PaScScreenScissorTl, PaScScreenScissorBr, ref left, ref top, ref right, ref bottom); + // AGC reset-state blocks can carry an all-zero screen-scissor pair as + // an unpatched placeholder while the generic/viewport scissors hold + // the active bounds. Treat only that exact reset value as absent. A + // nonzero empty rectangle remains meaningful and still clips the draw. + IntersectScissorPair( + registers, + PaScScreenScissorTl, + PaScScreenScissorBr, + ref left, + ref top, + ref right, + ref bottom, + ignoreAllZeroPair: true); IntersectScissorPair( registers, PaScWindowScissorTl, @@ -4188,9 +6921,19 @@ public static class AgcExports ref int right, ref int bottom, int offsetX = 0, - int offsetY = 0) + int offsetY = 0, + bool ignoreAllZeroPair = false) { - if (!TryDecodeScissorPair(registers, tlRegister, brRegister, out var pairLeft, out var pairTop, out var pairRight, out var pairBottom)) + if (!TryDecodeScissorPair( + registers, + tlRegister, + brRegister, + out var pairLeft, + out var pairTop, + out var pairRight, + out var pairBottom, + out var allZero) || + (ignoreAllZeroPair && allZero)) { return; } @@ -4213,18 +6956,21 @@ public static class AgcExports out int left, out int top, out int right, - out int bottom) + out int bottom, + out bool allZero) { left = 0; top = 0; right = 0; bottom = 0; + allZero = false; if (!registers.TryGetValue(tlRegister, out var tl) || !registers.TryGetValue(brRegister, out var br)) { return false; } + allZero = tl == 0 && br == 0; left = (int)(tl & 0x7FFFu); top = (int)((tl >> 16) & 0x7FFFu); right = (int)(br & 0x7FFFu); @@ -4247,6 +6993,12 @@ public static class AgcExports draw.RenderTargets.Select(target => $"{target.Slot}:0x{target.Address:X16}:{target.Width}x{target.Height}:" + $"fmt{target.Format}/num{target.NumberType}/tile{target.TileMode}")); + var depthTarget = draw.DepthTarget is { } depth + ? $"0x{depth.Address:X16}:{depth.Width}x{depth.Height}:" + + $"fmt{depth.GuestFormat}/sw{depth.SwizzleMode}:" + + $"read=0x{depth.ReadAddress:X16}/write=0x{depth.WriteAddress:X16}:" + + $"clear={depth.ClearDepth:0.######}/ro={(depth.ReadOnly ? 1 : 0)}" + : "none"; var probes = new Dictionary(); var textures = string.Join( ',', @@ -4284,11 +7036,11 @@ public static class AgcExports var buffers = string.Join( ',', draw.GlobalMemoryBindings.Select((binding, index) => - $"{index}:0x{binding.BaseAddress:X16}:{binding.Data.Length}:" + - Convert.ToHexString(binding.Data.AsSpan(0, Math.Min(binding.Data.Length, 256))))); + $"{index}:0x{binding.BaseAddress:X16}:{binding.DataLength}:" + + Convert.ToHexString(binding.Data.AsSpan(0, Math.Min(binding.DataLength, 256))))); var indices = draw.IndexBuffer is { } indexBuffer ? $"{(indexBuffer.Is32Bit ? 32 : 16)}:" + - Convert.ToHexString(indexBuffer.Data.AsSpan(0, Math.Min(indexBuffer.Data.Length, 32))) + Convert.ToHexString(indexBuffer.Data.AsSpan(0, Math.Min(indexBuffer.Length, 32))) : "none"; var vertexInputs = draw.VertexInputs.Count == 0 ? "none" @@ -4338,7 +7090,7 @@ public static class AgcExports $"write_mask=0x{blend.WriteMask:X} scissor={scissor} viewport={viewport} " + $"raster=[{raster}] " + $"ps_ena=0x{psInputEna:X8} ps_addr=0x{psInputAddr:X8} " + - $"targets=[{targets}] textures=[{textures}] " + + $"targets=[{targets}] depth=[{depthTarget}] textures=[{textures}] " + $"buffers=[{buffers}] vertex=[{vertexInputs}] indices=[{indices}]"); } @@ -4370,6 +7122,243 @@ public static class AgcExports return textures; } + /// + /// Guest storage buffers for a translated draw, followed by the per-draw + /// initial scalar registers of each stage (pixel then vertex), matching + /// the binding layout the shaders were compiled against. + /// + private static IReadOnlyList CreateTranslatedDrawGlobalBuffers( + TranslatedGuestDraw translatedDraw) + { + var buffers = CreateGuestMemoryBuffers(translatedDraw.GlobalMemoryBindings); + if (_bakeScalars) + { + return buffers; + } + + var combined = new List(buffers.Count + 2); + combined.AddRange(buffers); + var runtimeStateLength = GetRuntimeScalarBufferLength( + translatedDraw.GlobalMemoryBindings.Count); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarState( + translatedDraw.PixelInitialScalars, + translatedDraw.GlobalMemoryBindings), + runtimeStateLength, + Pooled: true)); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarState( + translatedDraw.VertexInitialScalars, + translatedDraw.GlobalMemoryBindings), + runtimeStateLength, + Pooled: true)); + return combined; + } + + private static IReadOnlyList + CreateGlobalBufferOwnershipView( + IReadOnlyList buffers, + bool ownsPooledData) + { + var view = new GuestMemoryBuffer[buffers.Count]; + for (var index = 0; index < buffers.Count; index++) + { + var buffer = buffers[index]; + view[index] = buffer with + { + Pooled = ownsPooledData && buffer.Pooled, + }; + } + + return view; + } + + /// + /// Present-time variant: the flip path can reuse the same translated + /// draw across several flips and swapchain retries, so it must not wrap + /// the (pooled, single-consumption) binding arrays. Buffer contents are + /// re-read from guest memory instead, which also presents current data. + /// + private static IReadOnlyList CreateTranslatedDrawGlobalBuffersForPresent( + CpuContext ctx, + TranslatedGuestDraw translatedDraw) + { + var bindings = translatedDraw.GlobalMemoryBindings; + var combined = new List(bindings.Count + 2); + foreach (var binding in bindings) + { + var data = new byte[Math.Max(binding.DataLength, sizeof(uint))]; + var guestMemoryBacked = binding.BaseAddress != 0 && + (ctx.Memory.TryRead(binding.BaseAddress, data) || + KernelMemoryCompatExports.TryReadTrackedLibcHeap(binding.BaseAddress, data)); + if (!guestMemoryBacked) + { + // Keep the zero-filled buffer; layout must match the shader. + } + + combined.Add(new GuestMemoryBuffer( + binding.BaseAddress, + data, + data.Length, + Pooled: false, + Writable: binding.Writable, + WriteBackToGuest: binding.WriteBackToGuest && guestMemoryBacked)); + } + + if (!_bakeScalars) + { + var runtimeStateLength = GetRuntimeScalarBufferLength(bindings.Count); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarStateUnpooled( + translatedDraw.PixelInitialScalars, + bindings), + runtimeStateLength, + Pooled: false)); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarStateUnpooled( + translatedDraw.VertexInitialScalars, + bindings), + runtimeStateLength, + Pooled: false)); + } + + return combined; + } + + private static int GetRuntimeScalarBufferLength(int bindingCount) => + checked((256 + bindingCount) * sizeof(uint)); + + private static byte[] PackRuntimeScalarState( + IReadOnlyList registers, + IReadOnlyList bindings) + { + var bytes = VulkanVideoPresenter.GuestDataPool.Rent( + GetRuntimeScalarBufferLength(bindings.Count)); + PackRuntimeScalarStateInto(bytes, registers, bindings); + return bytes; + } + + private static byte[] PackRuntimeScalarStateUnpooled( + IReadOnlyList registers, + IReadOnlyList bindings) + { + var bytes = new byte[GetRuntimeScalarBufferLength(bindings.Count)]; + PackRuntimeScalarStateInto(bytes, registers, bindings); + return bytes; + } + + private static void PackRuntimeScalarStateInto( + byte[] bytes, + IReadOnlyList registers, + IReadOnlyList bindings) + { + PackScalarRegistersInto(bytes, registers); + var biasOffset = 256 * sizeof(uint); + for (var index = 0; index < bindings.Count; index++) + { + var byteBias = checked((uint)( + bindings[index].BaseAddress & + (VulkanVideoPresenter.GuestStorageBufferOffsetAlignment - 1))); + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(biasOffset + index * sizeof(uint), sizeof(uint)), + byteBias); + } + } + + private static void PackScalarRegistersInto(byte[] bytes, IReadOnlyList registers) + { + if (registers is uint[] { Length: >= 256 } array) + { + // Guest scalar registers are little-endian dwords and the host + // is x86-64, so a bulk copy replaces 256 per-element writes. + System.Runtime.InteropServices.MemoryMarshal + .AsBytes(array.AsSpan(0, 256)) + .CopyTo(bytes); + return; + } + + // Rented arrays carry stale bytes; clear the packed window first. + Array.Clear(bytes, 0, 256 * sizeof(uint)); + var count = Math.Min(registers.Count, 256); + for (var index = 0; index < count; index++) + { + BinaryPrimitives.WriteUInt32LittleEndian( + bytes.AsSpan(index * sizeof(uint)), + registers[index]); + } + } + + /// + /// Returns the pooled buffer arrays an evaluation produced. Called only + /// on translation-failure paths, where no + /// is built to take ownership; on success the draw's consumers return them. + /// + private static void ReturnPooledEvaluationArrays(Gen5ShaderEvaluation evaluation) + { + foreach (var binding in evaluation.GlobalMemoryBindings) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + + if (evaluation.VertexInputs is { } vertexInputs) + { + foreach (var binding in vertexInputs) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + } + } + + /// + /// Returns pooled data arrays a translated draw owns but did not hand to + /// a presenter consumer. The offscreen path hands globals, vertex and + /// index buffers to the presenter (which returns them), so it passes all + /// three false; other draw sinks pass true for whatever they dropped. + /// + private static void ReturnPooledDrawArrays( + TranslatedGuestDraw draw, + bool globals, + bool vertex, + bool index) + { + if (globals) + { + foreach (var binding in draw.GlobalMemoryBindings) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + } + + if (vertex) + { + foreach (var binding in draw.VertexInputs) + { + if (binding.DataPooled) + { + VulkanVideoPresenter.GuestDataPool.Return(binding.Data); + } + } + } + + if (index && draw.IndexBuffer is { Pooled: true } indexBuffer) + { + VulkanVideoPresenter.GuestDataPool.Return(indexBuffer.Data); + } + } + private static IReadOnlyList CreateGuestMemoryBuffers( IReadOnlyList bindings) { @@ -4378,12 +7367,42 @@ public static class AgcExports { buffers[index] = new GuestMemoryBuffer( bindings[index].BaseAddress, - bindings[index].Data); + bindings[index].Data, + bindings[index].DataLength, + bindings[index].DataPooled, + bindings[index].Writable, + bindings[index].WriteBackToGuest); } return buffers; } + /// + /// Guest storage buffers for a compute dispatch followed by its initial + /// scalar registers. Dispatch-specific SGPR values remain runtime data so + /// one translated pipeline serves every matching shader/resource shape. + /// + private static IReadOnlyList CreateTranslatedComputeGlobalBuffers( + Gen5ShaderEvaluation evaluation) + { + var buffers = CreateGuestMemoryBuffers(evaluation.GlobalMemoryBindings); + if (_bakeScalars) + { + return buffers; + } + + var combined = new List(buffers.Count + 1); + combined.AddRange(buffers); + combined.Add(new GuestMemoryBuffer( + 0, + PackRuntimeScalarState( + evaluation.InitialScalarRegisters, + evaluation.GlobalMemoryBindings), + GetRuntimeScalarBufferLength(evaluation.GlobalMemoryBindings.Count), + Pooled: true)); + return combined; + } + private static IReadOnlyList CreateGuestVertexBuffers( IReadOnlyList bindings) { @@ -4399,171 +7418,113 @@ public static class AgcExports binding.BaseAddress, binding.Stride, binding.OffsetBytes, - binding.Data); + binding.Data, + binding.DataLength, + binding.DataPooled); } return buffers; } - private static bool TryCreateGuestDrawTexture( - CpuContext ctx, - TextureDescriptor descriptor, - bool isStorage, - uint mipLevel, - IReadOnlyList samplerDescriptor, - out GuestDrawTexture texture) + private static IReadOnlyList + CreateVertexBufferOwnershipView( + IReadOnlyList buffers, + bool ownsPooledData) { - texture = default!; - if (descriptor.Type != Gen5TextureType2D || - descriptor.Width == 0 || - descriptor.Height == 0 || - descriptor.Width > 8192 || - descriptor.Height > 8192) + var view = new GuestVertexBuffer[buffers.Count]; + for (var index = 0; index < buffers.Count; index++) { - TraceTextureFallback(descriptor, "invalid-descriptor"); - texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); - return true; - } - - var sourceWidth = descriptor.TileMode == 0 - ? GetLinearTexturePitch( - Math.Max(descriptor.Width, descriptor.Pitch), - descriptor.Format) - : descriptor.Width; - var sourceByteCount = GetTextureByteCount( - descriptor.Format, - sourceWidth, - descriptor.Height); - if (sourceByteCount == 0 || - sourceByteCount > MaxPresentedTextureBytes || - sourceByteCount > int.MaxValue) - { - TraceTextureFallback( - descriptor, - $"invalid-byte-count:{sourceByteCount}"); - texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); - return true; - } - - if (!isStorage && - descriptor.Address != 0 && - GuestGpu.Current.IsGpuGuestImageAvailable( - descriptor.Address, - descriptor.Format, - descriptor.NumberType)) - { - texture = new GuestDrawTexture( - descriptor.Address, - descriptor.Width, - descriptor.Height, - descriptor.Format, - descriptor.NumberType, - [], - IsFallback: false, - IsStorage: false, - MipLevels: descriptor.MipLevels, - MipLevel: mipLevel, - Pitch: sourceWidth, - TileMode: descriptor.TileMode, - DstSelect: descriptor.DstSelect, - Sampler: ToGuestSampler(samplerDescriptor)); - return true; - } - - if (isStorage) - { - var initialPixels = Array.Empty(); - if (descriptor.Address != 0) + var buffer = buffers[index]; + view[index] = buffer with { - var storageSource = new byte[(int)sourceByteCount]; - if ((ctx.Memory.TryRead(descriptor.Address, storageSource) || - KernelMemoryCompatExports.TryReadTrackedLibcHeapGpuAlias( - descriptor.Address, - storageSource)) && - storageSource.AsSpan().IndexOfAnyExcept((byte)0) >= 0) - { - initialPixels = storageSource; - } + Pooled = ownsPooledData && buffer.Pooled, + }; + } + + return view; + } + + private static GuestIndexBuffer? CreateIndexBufferOwnershipView( + GuestIndexBuffer? buffer, + bool ownsPooledData) => + buffer is null + ? null + : buffer with { Pooled = ownsPooledData && buffer.Pooled }; + + // BCn block-compressed guest formats and the bytes per 4x4 block. + private static int GetBlockCompressedBlockBytes(uint format) => format switch + { + 169 or 170 or 175 or 176 => 8, + 171 or 172 or 173 or 174 or 177 or 178 or 179 or 180 or 181 or 182 => 16, + _ => 0, + }; + + /// + /// Deswizzles a tiled texture source into linear layout when tiling is + /// enabled and the format is understood; returns null to keep the raw + /// bytes (linear surfaces, unknown modes, or non-power-of-two elements). + /// + private static bool TryGetTextureElementLayout( + TextureDescriptor descriptor, + uint sourceWidth, + out int elementsWide, + out int elementsHigh, + out int bytesPerElement) + { + var blockBytes = GetBlockCompressedBlockBytes(descriptor.Format); + if (blockBytes != 0) + { + bytesPerElement = blockBytes; + elementsWide = (int)((sourceWidth + 3) / 4); + elementsHigh = (int)((descriptor.Height + 3) / 4); + } + else + { + bytesPerElement = (int)GetTextureBytesPerTexel(descriptor.Format); + if (bytesPerElement == 0) + { + elementsWide = 0; + elementsHigh = 0; + return false; } - texture = new GuestDrawTexture( - descriptor.Address, - descriptor.Width, - descriptor.Height, - descriptor.Format, - descriptor.NumberType, - initialPixels, - IsFallback: descriptor.Address == 0, - IsStorage: true, - MipLevels: descriptor.MipLevels, - MipLevel: mipLevel, - Pitch: sourceWidth, - TileMode: descriptor.TileMode, - DstSelect: descriptor.DstSelect, - Sampler: ToGuestSampler(samplerDescriptor)); - return true; + elementsWide = (int)sourceWidth; + elementsHigh = (int)descriptor.Height; } - var source = new byte[(int)sourceByteCount]; - if (!ctx.Memory.TryRead(descriptor.Address, source) && - !KernelMemoryCompatExports.TryReadTrackedLibcHeapGpuAlias( - descriptor.Address, - source)) - { - TraceTextureFallback( - descriptor, - $"guest-read-failed:{sourceByteCount}"); - texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); - return true; - } - - TraceTextureHash(descriptor, source); - - var nonZero = 0; - for (var i = 0; i < source.Length; i++) - { - if (source[i] != 0) - { - nonZero++; - if (nonZero >= 64) - { - break; - } - } - } - - TraceAgcShader( - $"agc.texture_source addr=0x{descriptor.Address:X16} " + - $"fmt={descriptor.Format} num={descriptor.NumberType} tile={descriptor.TileMode} " + - $"size={descriptor.Width}x{descriptor.Height} pitch={descriptor.Pitch} " + - $"dst=0x{descriptor.DstSelect:X3} " + - $"bytes={source.Length} nonzero64={nonZero}"); - DumpTextureSourceIfRequested(descriptor, sourceWidth, source); - - var rgba = source; - texture = new GuestDrawTexture( - descriptor.Address, - descriptor.Width, - descriptor.Height, - descriptor.Format, - descriptor.NumberType, - rgba, - IsFallback: false, - IsStorage: isStorage, - MipLevels: descriptor.MipLevels, - MipLevel: mipLevel, - Pitch: sourceWidth, - TileMode: descriptor.TileMode, - DstSelect: descriptor.DstSelect, - Sampler: ToGuestSampler(samplerDescriptor)); return true; } - private static int _textureFallbackTraceCount; - - private static void TraceTextureFallback( + private static byte[]? TryDetileTextureSource( TextureDescriptor descriptor, - string reason) + uint sourceWidth, + int logicalByteCount, + byte[] source) + { + if (!GnmTiling.NeedsDetile(descriptor.TileMode) || + !TryGetTextureElementLayout( + descriptor, + sourceWidth, + out var elementsWide, + out var elementsHigh, + out var bytesPerElement)) + { + return null; + } + + var linear = new byte[logicalByteCount]; + return GnmTiling.TryDetile( + source, + linear, + descriptor.TileMode, + elementsWide, + elementsHigh, + bytesPerElement) + ? linear + : null; + } + + private static void TraceTextureFallback(TextureDescriptor descriptor, string reason) { var mode = Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); if ((!string.Equals(mode, "1", StringComparison.Ordinal) && @@ -4582,7 +7543,383 @@ public static class AgcExports $"dst=0x{descriptor.DstSelect:X3}"); } + private static bool TryCreateGuestDrawTexture( + CpuContext ctx, + TextureDescriptor descriptor, + bool isStorage, + uint mipLevel, + IReadOnlyList samplerDescriptor, + out GuestDrawTexture texture) + { + texture = default!; + if ((descriptor.Type != Gen5TextureType1D && + descriptor.Type != Gen5TextureType2D) || + descriptor.Width == 0 || + descriptor.Height == 0 || + descriptor.Width > 8192 || + descriptor.Height > 8192) + { + TraceTextureFallback(descriptor, "invalid-descriptor"); + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + var sourceWidth = descriptor.TileMode == 0 + ? GetLinearTexturePitch( + Math.Max(descriptor.Width, descriptor.Pitch), + descriptor.Height, + descriptor.Format) + : descriptor.Width; + var sourceByteCount = GetTextureByteCount( + descriptor.Format, + sourceWidth, + descriptor.Height); + if (sourceByteCount == 0 || + sourceByteCount > MaxPresentedTextureBytes || + sourceByteCount > int.MaxValue) + { + TraceTextureFallback( + descriptor, + $"invalid-byte-count:{sourceByteCount}"); + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + + var physicalSourceByteCount = sourceByteCount; + if (GnmTiling.NeedsDetile(descriptor.TileMode) && + TryGetTextureElementLayout( + descriptor, + sourceWidth, + out var elementsWide, + out var elementsHigh, + out var bytesPerElement) && + GnmTiling.TryGetTiledByteCount( + descriptor.TileMode, + elementsWide, + elementsHigh, + bytesPerElement, + out var tiledByteCount)) + { + physicalSourceByteCount = tiledByteCount; + } + + if (physicalSourceByteCount > MaxPresentedTextureBytes || + physicalSourceByteCount > int.MaxValue) + { + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + + if (!isStorage && + descriptor.Address != 0 && + VulkanVideoPresenter.IsGuestImageAvailable( + descriptor.Address, + descriptor.Format, + descriptor.NumberType)) + { + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + [], + IsFallback: false, + IsStorage: false, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: ToGuestSampler(samplerDescriptor)); + return true; + } + + if (isStorage) + { + var initialPixels = Array.Empty(); + var uploadKnown = descriptor.Address != 0 && + VulkanVideoPresenter.IsGuestImageUploadKnown( + descriptor.Address, + descriptor.Format, + descriptor.NumberType); + var readSucceeded = false; + var linearNonzero = false; + if (descriptor.Address != 0 && !uploadKnown) + { + // Storage images can be pre-populated in tiled guest memory + // just like sampled images. Reading only the logical linear + // byte count both truncates 64 KiB swizzle blocks and uploads + // tiled bytes as scanlines. Read the full physical footprint + // and run the same AddrLib-derived detile path used below for + // sampled textures before seeding the Vulkan image. + var storageSource = new byte[(int)physicalSourceByteCount]; + if (ctx.Memory.TryRead(descriptor.Address, storageSource)) + { + readSucceeded = true; + var linearStorage = TryDetileTextureSource( + descriptor, + sourceWidth, + checked((int)sourceByteCount), + storageSource) ?? storageSource + .AsSpan(0, checked((int)sourceByteCount)) + .ToArray(); + if (linearStorage.AsSpan().IndexOfAnyExcept((byte)0) >= 0) + { + linearNonzero = true; + initialPixels = linearStorage; + } + } + } + + if (ParseOptionalHexAddress( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_STORAGE_IMAGE_INIT_ADDRESS")) == + descriptor.Address) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.storage_initial_data " + + $"addr=0x{descriptor.Address:X16} op_storage={isStorage} " + + $"upload_known={uploadKnown} read={readSucceeded} " + + $"nonzero={linearNonzero} initial_bytes={initialPixels.Length} " + + $"logical_bytes={sourceByteCount} physical_bytes={physicalSourceByteCount} " + + $"size={descriptor.Width}x{descriptor.Height} pitch={sourceWidth} " + + $"fmt={descriptor.Format} num={descriptor.NumberType} " + + $"tile={descriptor.TileMode} mip={mipLevel}"); + } + + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + initialPixels, + IsFallback: descriptor.Address == 0, + IsStorage: true, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: ToGuestSampler(samplerDescriptor)); + return true; + } + + // When the presenter already holds this exact texture identity in + // its cache, the texel copy below would be discarded on arrival; for + // scenes that sample large textures every draw this copy dominated + // CPU time. The dirty peek closes the race with eviction: a texture + // the guest rewrote must ship fresh texels with this draw, because + // the render thread evicts the stale cache entry before executing it + // (skipping would leave the draw with no pixels and a fallback + // texture for the frame — visible flicker on animated textures). + var sampler = ToGuestSampler(samplerDescriptor); + if (!_textureCopySkipDisabled && + descriptor.Address != 0 && + !SharpEmu.HLE.GuestImageWriteTracker.PeekDirty(descriptor.Address) && + VulkanVideoPresenter.IsTextureContentCached( + new VulkanVideoPresenter.TextureContentIdentity( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + descriptor.DstSelect, + descriptor.TileMode, + sourceWidth, + sampler))) + { + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + [], + IsFallback: false, + IsStorage: false, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: sampler); + return true; + } + + var source = new byte[(int)physicalSourceByteCount]; + if (!ctx.Memory.TryRead(descriptor.Address, source)) + { + TraceTextureFallback( + descriptor, + $"guest-read-failed:{sourceByteCount}"); + texture = CreateFallbackGuestDrawTexture(isStorage, descriptor.Format, descriptor.NumberType); + return true; + } + + if (_traceAgcShader) + { + var nonZero = 0; + for (var i = 0; i < source.Length; i++) + { + if (source[i] != 0) + { + nonZero++; + if (nonZero >= 64) + { + break; + } + } + } + + TraceAgcShader( + $"agc.texture_source addr=0x{descriptor.Address:X16} " + + $"fmt={descriptor.Format} num={descriptor.NumberType} tile={descriptor.TileMode} " + + $"size={descriptor.Width}x{descriptor.Height} pitch={descriptor.Pitch} " + + $"dst=0x{descriptor.DstSelect:X3} " + + $"bytes={source.Length} logical_bytes={sourceByteCount} nonzero64={nonZero}"); + } + DumpTextureSourceIfRequested(descriptor, sourceWidth, source); + + var rgba = TryDetileTextureSource( + descriptor, + sourceWidth, + checked((int)sourceByteCount), + source) ?? source.AsSpan(0, checked((int)sourceByteCount)).ToArray(); + DumpLinearTextureIfRequested(descriptor, sourceWidth, rgba); + texture = new GuestDrawTexture( + descriptor.Address, + descriptor.Width, + descriptor.Height, + descriptor.Format, + descriptor.NumberType, + rgba, + IsFallback: false, + IsStorage: isStorage, + MipLevels: descriptor.MipLevels, + MipLevel: mipLevel, + BaseMipLevel: descriptor.ViewBaseLevel, + ResourceMipLevels: descriptor.ResourceMipLevels, + Pitch: sourceWidth, + TileMode: descriptor.TileMode, + DstSelect: descriptor.DstSelect, + Sampler: ToGuestSampler(samplerDescriptor)); + return true; + } + + + + /// + /// On PS5 render targets alias guest memory, so pixels the game wrote with + /// the CPU are visible before the first GPU draw (Chowdren pre-fills its + /// fog/overlay layers that way). Seed newly created Vulkan guest images + /// with the current guest memory contents to preserve that base layer. + /// + private static void ProvideRenderTargetInitialData( + CpuContext ctx, + RenderTargetDescriptor target) + { + if (!VulkanVideoPresenter.GuestImageWantsInitialData(target.Address)) + { + return; + } + + var byteCount = (ulong)target.Width * target.Height * 4; + if (byteCount == 0 || byteCount > MaxPresentedTextureBytes) + { + return; + } + + var initialData = new byte[byteCount]; + var readOk = ctx.Memory.TryRead(target.Address, initialData); + var nonZero = readOk && initialData.AsSpan().IndexOfAnyExcept((byte)0) >= 0; + if (_traceDraws && _rtSeedTraced.Add(target.Address)) + { + Console.Error.WriteLine( + $"[RTSEED] addr=0x{target.Address:X} {target.Width}x{target.Height} " + + $"read={readOk} nonZero={nonZero}"); + } + + if (nonZero) + { + VulkanVideoPresenter.ProvideGuestImageInitialData(target.Address, initialData); + } + } + + private static readonly HashSet _rtSeedTraced = new(); + + private static void TraceDrawCompact( + ulong sequence, + TranslatedGuestDraw draw, + IReadOnlyList textures, + IReadOnlyList vertexBuffers) + { + if (!_traceDraws) + { + return; + } + + var target = draw.RenderTargets.FirstOrDefault(); + var blend = draw.RenderState.Blend; + var viewport = draw.RenderState.Viewport is { } vp + ? $"{vp.X:0.#},{vp.Y:0.#},{vp.Width:0.#}x{vp.Height:0.#}" + : "none"; + var textureList = string.Join( + '|', + textures.Select(texture => + $"0x{texture.Address:X}:{texture.Width}x{texture.Height}" + + $":f{texture.Format}/n{texture.NumberType}/d{texture.DstSelect:X3}" + + (texture.IsFallback ? ":FALLBACK" : string.Empty))); + var positions = string.Empty; + var positionBuffer = vertexBuffers.FirstOrDefault(buffer => buffer.Location == 0); + if (positionBuffer is { Length: >= 8 }) + { + var stride = Math.Max(positionBuffer.Stride, 4u); + var vertexTotal = (int)((positionBuffer.Length - positionBuffer.OffsetBytes) / stride); + var sampled = new List(); + foreach (var vertex in new[] { 0, 1, vertexTotal - 1 }) + { + var baseOffset = (int)(positionBuffer.OffsetBytes + vertex * stride); + if (vertex < 0 || baseOffset + 8 > positionBuffer.Length) + { + continue; + } + + sampled.Add( + $"{BitConverter.ToSingle(positionBuffer.Data, baseOffset):0.##}," + + $"{BitConverter.ToSingle(positionBuffer.Data, baseOffset + 4):0.##}"); + } + + positions = string.Join(';', sampled); + } + + Console.Error.WriteLine( + $"[DRAW] seq={sequence} es=0x{draw.ExportShaderAddress:X} ps=0x{draw.PixelShaderAddress:X} " + + $"target=0x{target.Address:X}:{target.Width}x{target.Height}:f{target.Format}/n{target.NumberType} " + + $"prim=0x{draw.PrimitiveType:X} verts={draw.VertexCount} indexed={draw.IndexBuffer is not null} " + + $"blend={(blend.Enable ? 1 : 0)}:{blend.ColorSrcFactor}/{blend.ColorDstFactor}/{blend.ColorFunc}" + + $":a{blend.AlphaSrcFactor}/{blend.AlphaDstFactor}/{blend.AlphaFunc}/s{(blend.SeparateAlphaBlend ? 1 : 0)} " + + $"mask=0x{blend.WriteMask:X} viewport={viewport} textures={textureList} pos={positions} " + + $"ps_s0..3={string.Join(',', draw.PixelUserData.Take(4).Select(value => BitConverter.UInt32BitsToSingle(value).ToString("0.###")))} " + + $"rawblend=0x{draw.RawBlendControl:X8} info=0x{draw.RawColorInfo:X8}"); + } + + private static void TraceDrawCompactMiss(ulong sequence, uint vertexCount, string error) + { + if (!_traceDraws) + { + return; + } + + Console.Error.WriteLine($"[DRAW] seq={sequence} MISS verts={vertexCount} error={error}"); + } private static int _grassTraceCount; @@ -4606,13 +7943,13 @@ public static class AgcExports { text.Append( $"\n loc={buffer.Location} fmt={buffer.DataFormat}/{buffer.NumberFormat}x{buffer.ComponentCount} " + - $"stride={buffer.Stride} offset={buffer.OffsetBytes} bytes={buffer.Data.Length}"); + $"stride={buffer.Stride} offset={buffer.OffsetBytes} bytes={buffer.Length}"); var stride = Math.Max(buffer.Stride, 4u); - var maxVerts = Math.Min(6, (int)((buffer.Data.Length - buffer.OffsetBytes) / stride)); + var maxVerts = Math.Min(6, (int)((buffer.Length - buffer.OffsetBytes) / stride)); for (var vertex = 0; vertex < maxVerts; vertex++) { var baseOffset = (int)(buffer.OffsetBytes + vertex * stride); - var components = Math.Min(4, (int)((buffer.Data.Length - baseOffset) / 4)); + var components = Math.Min(4, (int)((buffer.Length - baseOffset) / 4)); text.Append($"\n v{vertex}:"); for (var c = 0; c < components; c++) { @@ -4645,7 +7982,7 @@ public static class AgcExports for (var vertex = 0; vertex < 3; vertex++) { var baseOffset = (int)(buffer.OffsetBytes + vertex * stride); - if (baseOffset + 16 > buffer.Data.Length) + if (baseOffset + 16 > buffer.Length) { break; } @@ -4702,12 +8039,47 @@ public static class AgcExports $"-p{sourcePitch}-f{descriptor.Format}-t{descriptor.TileMode}.bin"); File.WriteAllBytes(path, source); } - catch (Exception exception) + catch (IOException) + { + } + } + + /// + /// Writes the bytes after detiling when SHARPEMU_TEXTURE_LINEAR_DUMP_DIR is + /// set. Keeping this separate from the raw-source dump makes AddrLib + /// equation changes directly inspectable with ordinary image tools. + /// + private static void DumpLinearTextureIfRequested( + in TextureDescriptor descriptor, + uint sourcePitch, + byte[] source) + { + var directory = Environment.GetEnvironmentVariable("SHARPEMU_TEXTURE_LINEAR_DUMP_DIR"); + if (string.IsNullOrWhiteSpace(directory)) + { + return; + } + + var key = $"linear-0x{descriptor.Address:X}-{descriptor.Width}x{descriptor.Height}"; + var occurrence = _textureDumpKeys.AddOrUpdate(key, 1, static (_, count) => count + 1); + if ((occurrence > 3 && occurrence % 500 >= 3) || + Interlocked.Increment(ref _textureDumpCount) > 200) + { + return; + } + + var index = _textureDumpCount; + try + { + Directory.CreateDirectory(directory); + var path = Path.Combine( + directory, + $"{index:D3}-0x{descriptor.Address:X}-{descriptor.Width}x{descriptor.Height}" + + $"-p{sourcePitch}-f{descriptor.Format}-t{descriptor.TileMode}.linear.bin"); + File.WriteAllBytes(path, source); + } + catch (IOException) { - // A bad SHARPEMU_TEXTURE_DUMP_DIR (permissions, invalid path) - // must not take the emulator down; the dump is a debug aid. - Console.Error.WriteLine( - $"[LOADER][WARN] Texture dump failed: {exception.Message}"); } } @@ -4731,34 +8103,6 @@ public static class AgcExports MipLevel: 0); } - private static void TraceTextureHash(TextureDescriptor descriptor, ReadOnlySpan source) - { - if (!_traceTextureHashes || - descriptor.Address == 0 || - descriptor.Width > 256 || - descriptor.Height > 256) - { - return; - } - - var hash = ComputeFingerprint(source); - var key = (descriptor.Address, descriptor.Width, descriptor.Height); - lock (_textureHashTraceGate) - { - if (_tracedTextureHashes.TryGetValue(key, out var previousHash) && - previousHash == hash) - { - return; - } - - _tracedTextureHashes[key] = hash; - } - - Console.Error.WriteLine( - $"[LOADER][TRACE] agc.texture_hash addr=0x{descriptor.Address:X16} " + - $"size={descriptor.Width}x{descriptor.Height} bytes={source.Length} hash=0x{hash:X16}"); - } - private static GuestSampler ToGuestSampler(IReadOnlyList descriptor) => descriptor.Count >= 4 ? new GuestSampler( @@ -4807,52 +8151,246 @@ public static class AgcExports dispatch = default; ulong dimensionsAddress; uint initiator; + string dispatchSource; if (opcode == ItDispatchDirect) { if (packetLength < 5 || - !ctx.TryReadUInt32(packetAddress + 16, out initiator)) + !TryReadUInt32(ctx, packetAddress + 16, out initiator)) { return false; } dimensionsAddress = packetAddress + 4; + dispatchSource = "direct"; } else if (packetLength >= 4) { - if (!ctx.TryReadUInt64(packetAddress + 4, out dimensionsAddress) || - !ctx.TryReadUInt32(packetAddress + 12, out initiator)) + if (!TryReadUInt64(ctx, packetAddress + 4, out dimensionsAddress) || + !TryReadUInt32(ctx, packetAddress + 12, out initiator)) { return false; } + + dispatchSource = "absolute-indirect"; } else { if (packetLength < 3 || state.IndirectArgsAddress == 0 || - !ctx.TryReadUInt32(packetAddress + 4, out var dataOffset) || - !ctx.TryReadUInt32(packetAddress + 8, out initiator)) + !TryReadUInt32(ctx, packetAddress + 4, out var dataOffset) || + !TryReadUInt32(ctx, packetAddress + 8, out initiator)) { return false; } dimensionsAddress = state.IndirectArgsAddress + dataOffset; + dispatchSource = "base-indirect"; } if ((initiator & 1) == 0 || - !ctx.TryReadUInt32(dimensionsAddress, out var groupCountX) || - !ctx.TryReadUInt32(dimensionsAddress + 4, out var groupCountY) || - !ctx.TryReadUInt32(dimensionsAddress + 8, out var groupCountZ) || - groupCountX == 0 || - groupCountY == 0 || - groupCountZ == 0) + !TryReadUInt32(ctx, dimensionsAddress, out var dispatchEndX) || + !TryReadUInt32(ctx, dimensionsAddress + 4, out var dispatchEndY) || + !TryReadUInt32(ctx, dimensionsAddress + 8, out var dispatchEndZ)) { return false; } - dispatch = new ComputeDispatch(groupCountX, groupCountY, groupCountZ); + if (dispatchEndX == 0 || dispatchEndY == 0 || dispatchEndZ == 0) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + "zero-dimension"); + } + + // When FORCE_START_AT_000 is clear, RDNA2 interprets the three packet + // values as end coordinates, not group counts. Vulkan expresses the + // same operation as vkCmdDispatchBase(base, end - base). Ignoring the + // COMPUTE_START registers turned small high-base clears into apparent + // multi-million/billion-group dispatches and forced an unsafe cap. + const uint forceStartAtZero = 1u << 2; + const uint partialThreadGroupEnabled = 1u << 1; + const uint useThreadDimensions = 1u << 5; + uint baseGroupX = 0; + uint baseGroupY = 0; + uint baseGroupZ = 0; + if ((initiator & forceStartAtZero) == 0) + { + state.ShRegisters.TryGetValue(ComputeStartX, out baseGroupX); + state.ShRegisters.TryGetValue(ComputeStartY, out baseGroupY); + state.ShRegisters.TryGetValue(ComputeStartZ, out baseGroupZ); + } + + var localSizeX = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadX); + var localSizeY = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadY); + var localSizeZ = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadZ); + uint groupCountX; + uint groupCountY; + uint groupCountZ; + var threadCountX = uint.MaxValue; + var threadCountY = uint.MaxValue; + var threadCountZ = uint.MaxValue; + if ((initiator & useThreadDimensions) != 0) + { + // In thread-dimension mode the packet contains thread counts, not + // group end coordinates. Vulkan still dispatches whole workgroups, + // so round up and pass the exact exclusive thread bounds to the + // translated shader. Its entry guard disables invocations in the + // partially populated final group before any guest instruction. + var startThreadX = (ulong)baseGroupX * localSizeX; + var startThreadY = (ulong)baseGroupY * localSizeY; + var startThreadZ = (ulong)baseGroupZ * localSizeZ; + if ((ulong)dispatchEndX <= startThreadX || + (ulong)dispatchEndY <= startThreadY || + (ulong)dispatchEndZ <= startThreadZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"thread-end-not-after-base(" + + $"{startThreadX}x{startThreadY}x{startThreadZ})"); + } + + groupCountX = CeilDivide((ulong)dispatchEndX - startThreadX, localSizeX); + groupCountY = CeilDivide((ulong)dispatchEndY - startThreadY, localSizeY); + groupCountZ = CeilDivide((ulong)dispatchEndZ - startThreadZ, localSizeZ); + threadCountX = dispatchEndX; + threadCountY = dispatchEndY; + threadCountZ = dispatchEndZ; + } + else + { + if (dispatchEndX <= baseGroupX || + dispatchEndY <= baseGroupY || + dispatchEndZ <= baseGroupZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"end-not-after-base({baseGroupX}x{baseGroupY}x{baseGroupZ})"); + } + + groupCountX = dispatchEndX - baseGroupX; + groupCountY = dispatchEndY - baseGroupY; + groupCountZ = dispatchEndZ - baseGroupZ; + } + + if ((initiator & partialThreadGroupEnabled) != 0) + { + var partialSizeX = GetComputePartialSize(state.ShRegisters, ComputeNumThreadX); + var partialSizeY = GetComputePartialSize(state.ShRegisters, ComputeNumThreadY); + var partialSizeZ = GetComputePartialSize(state.ShRegisters, ComputeNumThreadZ); + if (partialSizeX == 0 || partialSizeX > localSizeX || + partialSizeY == 0 || partialSizeY > localSizeY || + partialSizeZ == 0 || partialSizeZ > localSizeZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"invalid-partial-size({partialSizeX}x{partialSizeY}x{partialSizeZ}/" + + $"{localSizeX}x{localSizeY}x{localSizeZ})"); + } + + if (partialSizeX != localSizeX || + partialSizeY != localSizeY || + partialSizeZ != localSizeZ) + { + return RejectComputeDispatch( + dimensionsAddress, + initiator, + dispatchSource, + dispatchEndX, + dispatchEndY, + dispatchEndZ, + $"unrepresentable-partial-group({partialSizeX}x{partialSizeY}x{partialSizeZ}/" + + $"{localSizeX}x{localSizeY}x{localSizeZ})"); + } + } + + var waveLaneCount = (initiator & (1u << 15)) != 0 ? 32u : 64u; + + if (_traceAgcShader && + ((ulong)groupCountX * groupCountY * groupCountZ >= 1_000_000UL || + groupCountX >= 1_000_000u)) + { + lock (_submitTraceGate) + { + if (_tracedDispatchArguments.Add( + (dimensionsAddress, groupCountX, groupCountY, groupCountZ))) + { + TraceAgcShader( + $"agc.dispatch_args source={dispatchSource} op=0x{opcode:X2} " + + $"queue={state.QueueName} submission={state.ActiveSubmissionId} " + + $"packet=0x{packetAddress:X16} len={packetLength} " + + $"dims=0x{dimensionsAddress:X16} " + + $"raw={dispatchEndX:X8}/{dispatchEndY:X8}/{dispatchEndZ:X8} " + + $"base={baseGroupX:X8}/{baseGroupY:X8}/{baseGroupZ:X8} " + + $"count={groupCountX:X8}/{groupCountY:X8}/{groupCountZ:X8} " + + $"wave={waveLaneCount} " + + $"initiator=0x{initiator:X8} " + + $"indirect_base=0x{state.IndirectArgsAddress:X16}"); + } + } + } + + dispatch = new ComputeDispatch( + groupCountX, + groupCountY, + groupCountZ, + baseGroupX, + baseGroupY, + baseGroupZ, + waveLaneCount, + IsIndirect: opcode == ItDispatchIndirect, + threadCountX, + threadCountY, + threadCountZ); return true; } + private static uint CeilDivide(ulong value, uint divisor) => + checked((uint)((value + divisor - 1) / divisor)); + + private static bool RejectComputeDispatch( + ulong dimensionsAddress, + uint initiator, + string source, + uint rawX, + uint rawY, + uint rawZ, + string reason) + { + lock (_submitTraceGate) + { + if (_rejectedDispatchArguments.Count < 256 && + _rejectedDispatchArguments.Add((dimensionsAddress, initiator, reason))) + { + Console.Error.WriteLine( + $"[LOADER][WARN] agc.dispatch_reject source={source} " + + $"dims=0x{dimensionsAddress:X16} raw={rawX:X8}/{rawY:X8}/{rawZ:X8} " + + $"initiator=0x{initiator:X8} reason={reason}"); + } + } + + return false; + } + private static void ObserveComputeDispatch( CpuContext ctx, SubmittedGpuState gpuState, @@ -4948,17 +8486,73 @@ public static class AgcExports var localSizeX = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadX); var localSizeY = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadY); var localSizeZ = GetComputeLocalSize(state.ShRegisters, ComputeNumThreadZ); + if (_traceComputeShaderAddress == shaderAddress) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] agc.compute_dispatch_trace seq={sequence} " + + $"cs=0x{shaderAddress:X16} " + + $"groups={dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ} " + + $"base={dispatch.BaseGroupX}x{dispatch.BaseGroupY}x{dispatch.BaseGroupZ} " + + $"local={localSizeX}x{localSizeY}x{localSizeZ} " + + $"bindings=[{string.Join(',', descriptions)}]"); + } + + var writesGlobalMemory = evaluation.GlobalMemoryBindings.Any(static binding => + binding.Writable); var gpuDispatch = false; + var evaluationHandledByCpu = false; var computeError = string.Empty; - if ((hasStorageBinding || evaluation.GlobalMemoryBindings.Count != 0) && + if (!hasStorageBinding && + writesGlobalMemory && + TrySubmitMaskedDwordCopyKernel( + ctx, + shaderState.Program, + evaluation, + dispatch, + localSizeX, + localSizeY, + localSizeZ, + out var semanticCopySequence, + out var copyDescription)) + { + gpuDispatch = true; + evaluationHandledByCpu = true; + TraceAgcShader( + $"agc.compute_semantic_fast_path cs=0x{shaderAddress:X16} " + + $"queue={state.QueueName} submission={state.ActiveSubmissionId} " + + copyDescription); + // The scalar evaluator snapshots guest buffers while parsing the + // command stream. Do not let another submission (or the CPU) + // observe that snapshot until the semantic replacement has + // reached the same CPU-visible completion point as a translated + // writable-buffer dispatch below. Returning early here allowed + // the guest to reuse a transient heap while its delayed clear was + // still queued, so the clear could erase newly constructed CPU + // objects. Waiting on the work sequence also retires preceding + // Vulkan writes before the next evaluator snapshot is captured. + if (!VulkanVideoPresenter.WaitForGuestWork(semanticCopySequence)) + { + computeError = + $"semantic-global-write-sync-timeout sequence={semanticCopySequence}"; + } + } + else if ((hasStorageBinding || writesGlobalMemory) && (ulong)localSizeX * localSizeY * localSizeZ <= 1024) { var shaderKey = ( shaderAddress, - ComputeShaderStateFingerprint(evaluation), + _bakeScalars + ? ComputeShaderStateFingerprint(evaluation) + : ComputeShaderStructuralFingerprint(evaluation), localSizeX, localSizeY, - localSizeZ); + localSizeZ, + dispatch.WaveLaneCount, + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment); + var guestGlobalBufferCount = evaluation.GlobalMemoryBindings.Count; + var totalGlobalBufferCount = _bakeScalars + ? guestGlobalBufferCount + : guestGlobalBufferCount + 1; _computeShaderCache.TryGetValue(shaderKey, out var computeShader); if (computeShader is null && @@ -4969,7 +8563,14 @@ public static class AgcExports localSizeY, localSizeZ, out computeShader, - out computeError)) + out computeError, + totalGlobalBufferCount, + initialScalarBufferIndex: _bakeScalars + ? -1 + : guestGlobalBufferCount, + waveLaneCount: dispatch.WaveLaneCount, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) { DumpCompiledShader( "cs", @@ -4988,16 +8589,32 @@ public static class AgcExports translatedBindings, out _); var globalMemoryBuffers = - CreateGuestMemoryBuffers(evaluation.GlobalMemoryBindings); - GuestGpu.Current.SubmitComputeDispatch( + CreateTranslatedComputeGlobalBuffers(evaluation); + var workSequence = GuestGpu.Current.SubmitComputeDispatch( shaderAddress, computeShader, textures, globalMemoryBuffers, dispatch.GroupCountX, dispatch.GroupCountY, - dispatch.GroupCountZ); + dispatch.GroupCountZ, + dispatch.BaseGroupX, + dispatch.BaseGroupY, + dispatch.BaseGroupZ, + localSizeX, + localSizeY, + localSizeZ, + dispatch.IsIndirect, + writesGlobalMemory, + dispatch.ThreadCountX, + dispatch.ThreadCountY, + dispatch.ThreadCountZ); gpuDispatch = true; + if (writesGlobalMemory && + !VulkanVideoPresenter.WaitForGuestWork(workSequence)) + { + computeError = $"global-write-sync-timeout sequence={workSequence}"; + } } } @@ -5007,17 +8624,324 @@ public static class AgcExports { if (_tracedComputeShaders.Add(shaderAddress)) { + var globalBuffers = evaluation.GlobalMemoryBindings.Count == 0 + ? string.Empty + : $" global_buffers=[{string.Join(',', evaluation.GlobalMemoryBindings.Select( + binding => $"0x{binding.BaseAddress:X16}:{binding.DataLength}"))}]"; + var scalarProbe = string.Join( + ',', + evaluation.InitialScalarRegisters + .Take(16) + .Select((value, index) => $"s{index}={value:X8}")); + var globalProbes = evaluation.GlobalMemoryBindings.Count == 0 + ? string.Empty + : $" global_heads=[{string.Join(',', evaluation.GlobalMemoryBindings.Select( + binding => + $"0x{binding.BaseAddress:X16}:" + + Convert.ToHexString(binding.Data.AsSpan( + 0, + Math.Min(binding.DataLength, 16)))))}]"; + var globalDescriptors = evaluation.GlobalMemoryBindings.Count == 0 + ? string.Empty + : $" global_descriptors=[{string.Join(',', evaluation.GlobalMemoryBindings.Select( + binding => + $"s{binding.ScalarAddress}=" + + string.Join(':', evaluation.ScalarRegisters + .Skip(checked((int)binding.ScalarAddress)) + .Take(4) + .Select(value => $"{value:X8}"))))}]"; + var opcodes = string.Join( + ',', + shaderState.Program.Instructions + .Select(instruction => instruction.Opcode) + .Distinct() + .Take(48)); TraceAgcShader( $"agc.compute_shader cs=0x{shaderAddress:X16} " + $"groups={dispatch.GroupCountX}x{dispatch.GroupCountY}x{dispatch.GroupCountZ} " + + $"base={dispatch.BaseGroupX}x{dispatch.BaseGroupY}x{dispatch.BaseGroupZ} " + + $"wave={dispatch.WaveLaneCount} " + $"local={localSizeX}x{localSizeY}x{localSizeZ} " + $"sys={DescribeComputeSystemRegisters(computeSystemRegisters)} " + - $"gpu={gpuDispatch} blits={blitCount} " + - $"globals={evaluation.GlobalMemoryBindings.Count}" + + $"gpu={gpuDispatch} blits={blitCount} globals={evaluation.GlobalMemoryBindings.Count} " + + $"global_writes={writesGlobalMemory}" + (computeError.Length == 0 ? string.Empty : $" error={computeError}") + + $" sgprs=[{scalarProbe}]" + + globalBuffers + + globalProbes + + globalDescriptors + + $" opcodes=[{opcodes}]" + $" bindings=[{string.Join(',', descriptions)}]"); } } + + if (evaluationHandledByCpu) + { + ReturnPooledEvaluationArrays(evaluation); + } + } + + /// + /// Recognizes the SDK's masked-dword resource initialization kernel and + /// executes its exact semantics over the guest-memory window that the + /// emulator can map. The guest dispatches this kernel over multi-gigabyte + /// virtual heaps (up to ~67 million 64-lane workgroups); translating every + /// out-of-window invocation to Vulkan dominated startup despite those + /// stores being bounds-discarded. This is a semantic kernel replacement, + /// not a generic dispatch cap: the complete instruction shape and SGPR + /// bindings must match before the ordered CPU action is used. + /// + private static bool TrySubmitMaskedDwordCopyKernel( + CpuContext ctx, + Gen5ShaderProgram program, + Gen5ShaderEvaluation evaluation, + ComputeDispatch dispatch, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + out long workSequence, + out string description) + { + workSequence = 0; + description = string.Empty; + var instructions = program.Instructions; + string[] expectedOpcodes = + [ + "SMovB32", + "STtraceData", + "SInstPrefetch", + "VLshlAddU32", + "SBufferLoadDword", + "SWaitcnt", + "VCmpxGtU32", + "SCbranchExecz", + "SBufferLoadDword", + "SWaitcnt", + "VAndB32", + "BufferLoadFormatX", + "SWaitcnt", + "BufferStoreFormatX", + "SEndpgm", + ]; + if (instructions.Count != expectedOpcodes.Length || + !instructions.Select(static instruction => instruction.Opcode) + .SequenceEqual(expectedOpcodes) || + !IsExactMaskedDwordCopyInstructionShape(instructions) || + dispatch.BaseGroupX != 0 || + dispatch.BaseGroupY != 0 || + dispatch.BaseGroupZ != 0 || + dispatch.GroupCountY != 1 || + dispatch.GroupCountZ != 1 || + localSizeX != 64 || + localSizeY != 1 || + localSizeZ != 1 || + evaluation.ComputeSystemRegisters?.WorkGroupXRegister != 12) + { + return false; + } + + var control = evaluation.GlobalMemoryBindings.SingleOrDefault( + static binding => binding.ScalarAddress == 8 && !binding.Writable); + var source = evaluation.GlobalMemoryBindings.SingleOrDefault( + static binding => binding.ScalarAddress == 0 && !binding.Writable); + var destination = evaluation.GlobalMemoryBindings.SingleOrDefault( + static binding => binding.ScalarAddress == 4 && + binding.Writable && + binding.WriteBackToGuest); + if (control is null || source is null || destination is null || + control.DataLength < 2 * sizeof(uint) || + source.DataLength < sizeof(uint) || + destination.BaseAddress == 0 || + destination.DataLength < sizeof(uint) || + !IsExactMaskedDwordCopyDescriptor( + evaluation.InitialScalarRegisters, + source.ScalarAddress, + source.BaseAddress) || + !IsExactMaskedDwordCopyDescriptor( + evaluation.InitialScalarRegisters, + destination.ScalarAddress, + destination.BaseAddress)) + { + return false; + } + + var elementCount = BinaryPrimitives.ReadUInt32LittleEndian( + control.Data.AsSpan(0, sizeof(uint))); + var sourceMask = BinaryPrimitives.ReadUInt32LittleEndian( + control.Data.AsSpan(sizeof(uint), sizeof(uint))); + var dispatchedThreads = dispatch.ThreadCountX != uint.MaxValue + ? dispatch.ThreadCountX + : Math.Min( + (ulong)uint.MaxValue, + (ulong)dispatch.GroupCountX * localSizeX); + var writableDwords = (uint)(destination.DataLength / sizeof(uint)); + var outputDwords = (uint)Math.Min( + Math.Min((ulong)elementCount, dispatchedThreads), + writableDwords); + if (outputDwords == 0) + { + return false; + } + + var output = new byte[checked((int)outputDwords * sizeof(uint))]; + var outputWords = System.Runtime.InteropServices.MemoryMarshal.Cast( + output.AsSpan()); + if (sourceMask == 0) + { + outputWords.Fill(BinaryPrimitives.ReadUInt32LittleEndian( + source.Data.AsSpan(0, sizeof(uint)))); + } + else + { + var sourceWords = System.Runtime.InteropServices.MemoryMarshal.Cast( + source.Data.AsSpan(0, source.DataLength - (source.DataLength % sizeof(uint)))); + for (uint index = 0; index < outputDwords; index++) + { + var sourceIndex = index & sourceMask; + outputWords[(int)index] = sourceIndex < (uint)sourceWords.Length + ? sourceWords[(int)sourceIndex] + : 0; + } + } + + var destinationAddress = destination.BaseAddress; + workSequence = VulkanVideoPresenter.SubmitOrderedGuestAction( + () => + { + if (!ctx.Memory.TryWrite(destinationAddress, output)) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] AGC masked-copy fast path failed " + + $"dst=0x{destinationAddress:X16} bytes={output.Length}"); + return; + } + + GuestImageWriteTracker.Track( + destinationAddress, + (ulong)output.Length, + VulkanVideoPresenter.CurrentGuestWorkSequenceForDiagnostics, + "agc.masked-dword-copy"); + }, + $"masked_dword_copy dst=0x{destinationAddress:X16} bytes={output.Length}"); + description = + $"dst=0x{destinationAddress:X16} bytes={output.Length} " + + $"elements={elementCount} mask=0x{sourceMask:X8} " + + $"dispatch={dispatch.GroupCountX}x{localSizeX}"; + return workSequence > 0; + } + + private static bool IsExactMaskedDwordCopyInstructionShape( + IReadOnlyList instructions) + { + static bool IsOperand( + Gen5Operand operand, + Gen5OperandKind kind, + uint value) => + operand.Kind == kind && operand.Value == value; + + static bool IsBufferControl( + Gen5ShaderInstruction instruction, + uint vectorAddress, + uint vectorData, + uint scalarResource) => + instruction.Control is Gen5BufferMemoryControl + { + DwordCount: 1, + OffsetBytes: 0, + IndexEnabled: true, + OffsetEnabled: false, + } control && + control.VectorAddress == vectorAddress && + control.VectorData == vectorData && + control.ScalarResource == scalarResource; + + static bool IsScalarLoad( + Gen5ShaderInstruction instruction, + int offsetBytes) => + instruction.Control is Gen5ScalarMemoryControl + { + DestinationCount: 1, + DynamicOffsetRegister: null, + } control && + control.ImmediateOffsetBytes == offsetBytes && + instruction.Destinations.Count == 1 && + IsOperand( + instruction.Destinations[0], + Gen5OperandKind.ScalarRegister, + 106) && + instruction.Sources.Count >= 1 && + IsOperand( + instruction.Sources[0], + Gen5OperandKind.ScalarRegister, + 8); + + // This replacement depends on the operands as much as the opcode + // sequence. Reversing V_CMPX_GT or enabling offen on either MUBUF + // operation changes the set or address of written lanes. + var globalId = instructions[3]; + var compare = instructions[6]; + var sourceIndex = instructions[10]; + var load = instructions[11]; + var store = instructions[13]; + return + globalId.Destinations.Count == 1 && + IsOperand(globalId.Destinations[0], Gen5OperandKind.VectorRegister, 0) && + globalId.Sources.Count == 3 && + IsOperand(globalId.Sources[0], Gen5OperandKind.ScalarRegister, 12) && + IsOperand(globalId.Sources[1], Gen5OperandKind.EncodedConstant, 134) && + IsOperand(globalId.Sources[2], Gen5OperandKind.VectorRegister, 0) && + IsScalarLoad(instructions[4], offsetBytes: 0) && + compare.Sources.Count == 2 && + IsOperand(compare.Sources[0], Gen5OperandKind.ScalarRegister, 106) && + IsOperand(compare.Sources[1], Gen5OperandKind.VectorRegister, 0) && + instructions[7].Words.Count == 1 && + (instructions[7].Words[0] & 0xFFFFu) == 9 && + IsScalarLoad(instructions[8], offsetBytes: sizeof(uint)) && + sourceIndex.Destinations.Count == 1 && + IsOperand(sourceIndex.Destinations[0], Gen5OperandKind.VectorRegister, 1) && + sourceIndex.Sources.Count == 2 && + IsOperand(sourceIndex.Sources[0], Gen5OperandKind.ScalarRegister, 106) && + IsOperand(sourceIndex.Sources[1], Gen5OperandKind.VectorRegister, 0) && + IsBufferControl(load, vectorAddress: 1, vectorData: 1, scalarResource: 0) && + IsBufferControl(store, vectorAddress: 0, vectorData: 1, scalarResource: 4); + } + + private static bool IsExactMaskedDwordCopyDescriptor( + IReadOnlyList scalarRegisters, + uint scalarBase, + ulong expectedBaseAddress) + { + if (scalarBase + 3 >= scalarRegisters.Count) + { + return false; + } + + var word0 = scalarRegisters[(int)scalarBase]; + var word1 = scalarRegisters[(int)scalarBase + 1]; + var word3 = scalarRegisters[(int)scalarBase + 3]; + var baseAddress = word0 | ((ulong)(word1 & 0xFFFFu) << 32); + var stride = (word1 >> 16) & 0x3FFFu; + var cacheSwizzle = (word1 & (1u << 30)) != 0; + var swizzleEnabled = (word1 & (1u << 31)) != 0; + var unifiedFormat = (word3 >> 12) & 0x7Fu; + var addTidEnabled = (word3 & (1u << 23)) != 0; + var outOfBoundsSelect = (word3 >> 28) & 0x3u; + var type = word3 >> 30; + var dstSelectX = word3 & 0x7u; + + // RDNA2 tables 35 and 37: OOB_SELECT=0 is structured indexing, so + // NUM_RECORDS counts stride-sized records. FORMAT=20 is 32_UINT and + // dst_sel_x=4 selects its R component. ADD_TID and either swizzle bit + // alter addressing and are therefore outside this replacement. + return baseAddress == expectedBaseAddress && + stride == sizeof(uint) && + !cacheSwizzle && + !swizzleEnabled && + unifiedFormat == 20 && + !addTidEnabled && + outOfBoundsSelect == 0 && + type == 0 && + dstSelectX == 4; } private static Gen5ComputeSystemRegisters DecodeComputeSystemRegisters( @@ -5069,6 +8993,29 @@ public static class AgcExports private static uint SelectExportUserDataRegister( IReadOnlyDictionary registers) { + // RSRC2 is the authoritative stage selector: its USER_SGPR field + // describes the hardware SGPR window even when the shader has zero + // user-data dwords and therefore no USER_DATA register was written. + // GFX10 NGG export shaders use the GS user-data bank (RSRC2 at 0x8B), + // while their program address is carried in the ES/NGG registers. + // Looking only for a populated USER_DATA range made those shaders + // fall through to ES (0xCC) and reject every graphics draw because + // the unrelated ES RSRC2 register at 0xCB was legitimately absent. + if (HasShaderResource2(registers, GsUserDataRegister)) + { + return GsUserDataRegister; + } + + if (HasShaderResource2(registers, EsUserDataRegister)) + { + return EsUserDataRegister; + } + + if (HasShaderResource2(registers, VsUserDataRegister)) + { + return VsUserDataRegister; + } + if (HasUserDataRange(registers, GsUserDataRegister)) { return GsUserDataRegister; @@ -5091,6 +9038,11 @@ public static class AgcExports : EsUserDataRegister; } + private static bool HasShaderResource2( + IReadOnlyDictionary registers, + uint userDataBaseRegister) => + registers.ContainsKey(userDataBaseRegister - 1); + private static bool HasUserDataRange( IReadOnlyDictionary registers, uint startRegister) @@ -5127,10 +9079,17 @@ public static class AgcExports uint register) { return registers.TryGetValue(register, out var value) - ? Math.Max(value & 0x3FFu, 1u) + ? Math.Max(value & 0xFFFFu, 1u) : 1u; } + private static uint GetComputePartialSize( + IReadOnlyDictionary registers, + uint register) => + registers.TryGetValue(register, out var value) + ? value >> 16 + : 0u; + private static int TryApplySoftwareComputeBlits( CpuContext ctx, ulong shaderAddress, @@ -5167,10 +9126,12 @@ public static class AgcExports cachedSourceTexture.Width, cachedSourceTexture.Height, cachedSourceTexture.Format, + cachedSourceTexture.NumberType, texture.Address, texture.Width, texture.Height, - texture.Format)) + texture.Format, + texture.NumberType)) { blits++; TraceAgcShader( @@ -5362,16 +9323,6 @@ public static class AgcExports 12 => 8UL, 13 => 12UL, 14 => 16UL, - 20 => 4UL, - 22 => 8UL, - 29 => 4UL, - 36 => 1UL, - 49 => 1UL, - Gen5TextureFormatR8G8B8A8Unorm => 4UL, - 62 => 4UL, - 64 => 4UL, - Gen5TextureFormatR16G16B16A16Float => 8UL, - 75 => 8UL, _ => 0UL, }; @@ -5383,29 +9334,27 @@ public static class AgcExports return checked((ulong)width * height * bytesPerTexel); } - var blockBytes = format switch - { - 169 or 170 => 8UL, - 171 or 172 or 173 or 174 or 175 or 176 or - 177 or 178 or 179 or 180 or 181 or 182 => 16UL, - _ => 0UL, - }; + var blockBytes = (ulong)GetBlockCompressedBlockBytes(format); return blockBytes == 0 ? 0 : checked(((ulong)width + 3) / 4 * (((ulong)height + 3) / 4) * blockBytes); } - private static uint GetLinearTexturePitch(uint pitch, uint format) + private static uint GetLinearTexturePitch(uint pitch, uint height, uint format) { var bytesPerTexel = GetTextureBytesPerTexel(format); - if (bytesPerTexel == 0) + if (bytesPerTexel == 0 || height == 0) { return pitch; } - // GFX10 ADDR_SW_LINEAR aligns each row to 256 bytes. - var pitchAlignment = Math.Max(1UL, 256UL / bytesPerTexel); - return checked((uint)AlignUp(pitch, pitchAlignment)); + // GNM linear surfaces align the row pitch to 256 bytes, so a 32px + // RGBA8 texture is stored with a 64px (256-byte) pitch and a 288px + // one with 320px. Reading at the unpadded width made every padded + // tail land on the next row, which showed as transparent gaps every + // other row on small tiles and diagonal dashes on wider surfaces. + var pitchBytes = AlignUp((ulong)pitch * bytesPerTexel, 256UL); + return checked((uint)(pitchBytes / bytesPerTexel)); } private static ulong AlignUp(ulong value, ulong alignment) => @@ -5441,6 +9390,17 @@ public static class AgcExports return; } + // Translation failures are compatibility issues, not merely verbose + // shader diagnostics. Report each distinct failure once even when AGC + // tracing is disabled so normal runs preserve the missing opcode or + // unsupported translation reason needed to fix the game. + if (firstFailure) + { + Console.Error.WriteLine( + $"[COMPAT][SHADER] ps=0x{pixelShaderAddress:X16} " + + $"es=0x{exportShaderAddress:X16} error={translationError}"); + } + if ((!hasPixelShader || !hasPsInputEna || !hasPsInputAddr) && TryMarkMissingPixelShaderBindingsTrace()) { @@ -5449,6 +9409,29 @@ public static class AgcExports DescribeShaderRegisterCandidates(ctx, state.ShRegisters)); } + if (!hasPixelShader) + { + state.CxRegisters.TryGetValue(DbDepthControl, out var rawDepthControl); + state.CxRegisters.TryGetValue(DbZInfo, out var rawZInfo); + state.CxRegisters.TryGetValue(DbDepthSizeXy, out var rawDepthSize); + state.CxRegisters.TryGetValue(DbDepthView, out var rawDepthView); + var depthState = DecodeDepthState(state.CxRegisters); + var depthTarget = DecodeDepthTarget(state.CxRegisters); + TraceAgcShader( + $"agc.shader_depth_state control=0x{rawDepthControl:X8} " + + $"zinfo=0x{rawZInfo:X8} size=0x{rawDepthSize:X8} " + + $"view=0x{rawDepthView:X8} " + + $"test={(depthState.TestEnable ? 1 : 0)} " + + $"write={(depthState.WriteEnable ? 1 : 0)} " + + $"func={depthState.CompareOp} " + + (depthTarget is null + ? "target=none" + : $"target=0x{depthTarget.Address:X16}:" + + $"{depthTarget.Width}x{depthTarget.Height}:" + + $"fmt{depthTarget.GuestFormat}/sw{depthTarget.SwizzleMode}:" + + $"ro={(depthTarget.ReadOnly ? 1 : 0)}")); + } + var shaderDecode = string.Empty; if (hasExportShader && hasPixelShader) { @@ -5512,7 +9495,7 @@ public static class AgcExports TraceAgcShader( $"agc.shader_global_binding ps=0x{pixelShaderAddress:X16} " + $"saddr=s{binding.ScalarAddress} " + - $"base=0x{binding.BaseAddress:X16} bytes={binding.Data.Length} " + + $"base=0x{binding.BaseAddress:X16} bytes={binding.DataLength} " + $"pcs={string.Join(',', binding.InstructionPcs.Select(pc => $"0x{pc:X}"))}"); } @@ -5521,7 +9504,11 @@ public static class AgcExports evaluation, [new(0, 0, Gen5PixelOutputKind.Float)], out var compiledPixel, - out var compileError)) + out var compileError, + pixelInputEnable: psInputEna, + pixelInputAddress: psInputAddr, + storageBufferOffsetAlignment: + VulkanVideoPresenter.GuestStorageBufferOffsetAlignment)) { TraceAgcShader( $"agc.shader_spirv ps=0x{pixelShaderAddress:X16} " + @@ -5604,7 +9591,8 @@ public static class AgcExports .Take(16) .Select(candidate => { - var type = ctx.TryReadByte( + var type = TryReadByte( + ctx, candidate.Header + ShaderTypeOffset, out var shaderType) ? shaderType.ToString() @@ -5640,7 +9628,7 @@ public static class AgcExports { descriptor = default; if (packetLength < 10 || - !ctx.TryReadUInt32(packetAddress + 4, out var startRegister)) + !TryReadUInt32(ctx, packetAddress + 4, out var startRegister)) { return false; } @@ -5656,10 +9644,10 @@ public static class AgcExports packetAddress + 8 + ((ulong)(PsTextureUserDataRegister - startRegister) * sizeof(uint)); - Span fields = stackalloc uint[4]; + Span fields = stackalloc uint[8]; for (var i = 0; i < fields.Length; i++) { - if (!ctx.TryReadUInt32(descriptorAddress + ((ulong)i * sizeof(uint)), out fields[i])) + if (!TryReadUInt32(ctx, descriptorAddress + ((ulong)i * sizeof(uint)), out fields[i])) { return false; } @@ -5678,32 +9666,51 @@ public static class AgcExports return false; } - // GFX10/RDNA2 T# layout: WIDTH is split across word1[31:30] (lo 2 bits) - // and word2[11:0] (hi 12 bits); FORMAT is the combined 9-bit field at - // word1[28:20]. Verified against Kyty's decode of the same game - // descriptors (fmt=56=8_8_8_8_UNORM, extent 1280x720, sw_mode 27). - // GNM T# exposes a 38-bit baseaddr256 field, but RPCSX and the - // Demon's Souls descriptors both show that only the low 32 bits are - // part of the guest GPU VA. The upper baseaddr bits carry resource - // metadata and produce bogus addresses such as 0x00003804... if used. - var address = ((ulong)(uint)((((ulong)fields[1] << 32) | fields[0]) & 0x3F_FFFF_FFFFUL)) << 8; - var width = (((fields[1] >> 30) & 0x3u) | ((fields[2] & 0xFFFu) << 2)) + 1; - var height = ((fields[2] >> 14) & 0x3FFFu) + 1; - var format = (fields[1] >> 20) & 0x1FFu; - var numberType = (fields[1] >> 26) & 0xFu; + // RDNA2 ISA table 45: BASE_ADDRESS is addr[47:8], WIDTH is the full + // 16-bit field split across word1/word2, and HEIGHT is word2[29:14]. + // Keeping the high base byte is required for legal guest VAs above + // 1 TiB; it is not descriptor metadata. + var address = (((ulong)(fields[1] & 0xFFu) << 32) | fields[0]) << 8; + var width = (((fields[1] >> 30) & 0x3u) | ((fields[2] & 0x3FFFu) << 2)) + 1; + var height = ((fields[2] >> 14) & 0xFFFFu) + 1; + var unifiedFormat = (fields[1] >> 20) & 0x1FFu; + if (unifiedFormat == 0 || + !Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out var format, + out var numberType)) + { + return false; + } var tileMode = (fields[3] >> 20) & 0x1Fu; var type = (fields[3] >> 28) & 0xFu; var baseLevel = (fields[3] >> 12) & 0xFu; var lastLevel = (fields[3] >> 16) & 0xFu; - // The 128-bit RDNA2 2D resource derives pitch[12:0] from width; - // the optional extension word only supplies pitch[13]. - var pitch = width; - if (fields.Count >= 5) - { - pitch = ((((width - 1) & 0x1FFFu) | (((fields[4] >> 13) & 1u) << 13)) + 1); - } + var bcSwizzle = (fields[3] >> 25) & 0x7u; + var hasExtendedDescriptor = fields.Count >= 8; + var word4 = fields.Count >= 5 ? fields[4] : 0u; + var depthOrLastSlice = (word4 & 0x1FFFu) + 1; + var baseArray = (word4 >> 16) & 0x1FFFu; + // In a 256-bit 1D/2D/2D-MSAA descriptor word4[13:0] is + // (pitch-1). A zeroed upper half denotes the common 128-bit resource, + // where pitch is implicit; use width rather than inventing pitch=1. + var pitch = type is 8u or 9u or 14u && word4 != 0 + ? (word4 & 0x3FFFu) + 1 + : width; + var depth = type is 10u or 11u or 12u or 13u or 15u + ? depthOrLastSlice + : 1u; + var word5 = fields.Count >= 6 ? fields[5] : 0u; + var arrayPitch = word5 & 0xFu; + var maxMip = (word5 >> 4) & 0xFu; + var minLod = (fields[1] >> 8) & 0xFFFu; + var minLodWarn = (word5 >> 8) & 0xFFFu; + var word6 = fields.Count >= 7 ? fields[6] : 0u; + var word7 = fields.Count >= 8 ? fields[7] : 0u; + var metadataAddress = ((((ulong)word7 << 8) | (word6 >> 24)) << 8); + var descriptorFlags = word6 & 0x00FF_FFFFu; var dstSelect = fields[3] & 0xFFFu; - if (address == 0 || width == 0 || height == 0) + if (address == 0 || width == 0 || height == 0 || type is >= 1 and <= 7) { return false; } @@ -5719,7 +9726,17 @@ public static class AgcExports baseLevel, lastLevel, pitch, - dstSelect); + dstSelect, + depth, + baseArray, + arrayPitch, + maxMip, + minLod, + minLodWarn, + bcSwizzle, + metadataAddress, + descriptorFlags, + hasExtendedDescriptor); return true; } @@ -5730,8 +9747,15 @@ public static class AgcExports var tileMode = 0u; if (fields.Count >= 4) { - format = (fields[1] >> 20) & 0x1FFu; - numberType = (fields[1] >> 26) & 0xFu; + var unifiedFormat = (fields[1] >> 20) & 0x1FFu; + if (!Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out format, + out numberType)) + { + format = Gen5TextureFormatR8G8B8A8Unorm; + numberType = 0; + } tileMode = (fields[3] >> 20) & 0x1Fu; if (format == 0) { @@ -5774,8 +9798,14 @@ public static class AgcExports destination.PixelFormat is not ( VideoOutPixelFormatA8R8G8B8Srgb or VideoOutPixelFormatA8B8G8R8Srgb or - VideoOutPixelFormatB8G8R8A8Unorm or - VideoOutPixelFormatR8G8B8A8Unorm)) + VideoOutPixelFormat2R8G8B8A8Srgb or + VideoOutPixelFormat2B8G8R8A8Srgb or + VideoOutPixelFormat2R10G10B10A2 or + VideoOutPixelFormat2B10G10R10A2 or + VideoOutPixelFormat2R10G10B10A2Srgb or + VideoOutPixelFormat2B10G10R10A2Srgb or + VideoOutPixelFormat2R10G10B10A2Bt2100Pq or + VideoOutPixelFormat2B10G10R10A2Bt2100Pq)) { return false; } @@ -5814,7 +9844,9 @@ public static class AgcExports var destinationRow = new byte[checked((int)destinationPitch * 4)]; var rgbaDestination = destination.PixelFormat is VideoOutPixelFormatA8B8G8R8Srgb or - VideoOutPixelFormatR8G8B8A8Unorm; + VideoOutPixelFormat2R8G8B8A8Srgb; + var packed10Destination = + VideoOutExports.IsPacked10BitPixelFormat(destination.PixelFormat); for (uint y = 0; y < destination.Height; y++) { var sourceY = (uint)(((ulong)y * source.Height) / destination.Height); @@ -5823,7 +9855,24 @@ public static class AgcExports var sourceX = (uint)(((ulong)x * source.Width) / destination.Width); var sourceOffset = checked((int)(((ulong)sourceY * source.Width + sourceX) * 4)); var destinationOffset = checked((int)x * 4); - if (rgbaDestination) + if (packed10Destination) + { + if (!VideoOutExports.TryPackRgba8Pixel( + destination.PixelFormat, + sourceBytes[sourceOffset + 0], + sourceBytes[sourceOffset + 1], + sourceBytes[sourceOffset + 2], + sourceBytes[sourceOffset + 3], + out var packed)) + { + return false; + } + + BinaryPrimitives.WriteUInt32LittleEndian( + destinationRow.AsSpan(destinationOffset, sizeof(uint)), + packed); + } + else if (rgbaDestination) { destinationRow[destinationOffset + 0] = sourceBytes[sourceOffset + 0]; destinationRow[destinationOffset + 1] = sourceBytes[sourceOffset + 1]; @@ -5836,7 +9885,10 @@ public static class AgcExports destinationRow[destinationOffset + 2] = sourceBytes[sourceOffset + 0]; } - destinationRow[destinationOffset + 3] = sourceBytes[sourceOffset + 3]; + if (!packed10Destination) + { + destinationRow[destinationOffset + 3] = sourceBytes[sourceOffset + 3]; + } } var destinationAddress = destination.Address + ((ulong)y * destinationPitch * 4); @@ -5886,7 +9938,7 @@ public static class AgcExports var payloadCount = Math.Min(length - 1, 32u); for (uint i = 0; i < payloadCount; i++) { - if (!ctx.TryReadUInt32(packetAddress + ((ulong)(i + 1) * sizeof(uint)), out var value)) + if (!TryReadUInt32(ctx, packetAddress + ((ulong)(i + 1) * sizeof(uint)), out var value)) { return; } @@ -5897,8 +9949,8 @@ public static class AgcExports if (op != ItNop || register is not (RCxRegsIndirect or RShRegsIndirect or RUcRegsIndirect) || length < 4 || - !ctx.TryReadUInt32(packetAddress + 4, out var registerCount) || - !ctx.TryReadUInt64(packetAddress + 8, out var registersAddress)) + !TryReadUInt32(ctx, packetAddress + 4, out var registerCount) || + !TryReadUInt64(ctx, packetAddress + 8, out var registersAddress)) { return; } @@ -5909,8 +9961,8 @@ public static class AgcExports for (uint i = 0; i < tracedCount; i++) { var entryAddress = registersAddress + ((ulong)i * 8); - if (!ctx.TryReadUInt32(entryAddress, out var registerOffset) || - !ctx.TryReadUInt32(entryAddress + 4, out var value)) + if (!TryReadUInt32(ctx, entryAddress, out var registerOffset) || + !TryReadUInt32(ctx, entryAddress + 4, out var value)) { TraceAgc($"agc.dcb.indirect_read_failed space={registerSpace} index={i} addr=0x{entryAddress:X16}"); return; @@ -5927,9 +9979,9 @@ public static class AgcExports private static bool PatchShaderProgramRegisters(CpuContext ctx, ulong headerAddress, ulong codeAddress) { - if (!ctx.TryReadUInt64(headerAddress + ShaderShRegistersOffset, out var shRegistersAddress) || - !ctx.TryReadByte(headerAddress + ShaderTypeOffset, out var shaderType) || - !ctx.TryReadByte(headerAddress + ShaderNumShRegistersOffset, out var registerCount)) + if (!TryReadUInt64(ctx, headerAddress + ShaderShRegistersOffset, out var shRegistersAddress) || + !TryReadByte(ctx, headerAddress + ShaderTypeOffset, out var shaderType) || + !TryReadByte(ctx, headerAddress + ShaderNumShRegistersOffset, out var registerCount)) { return false; } @@ -5939,8 +9991,8 @@ public static class AgcExports return false; } - if (!ctx.TryReadUInt32(shRegistersAddress, out var loRegister) || - !ctx.TryReadUInt32(shRegistersAddress + 8, out var hiRegister)) + if (!TryReadUInt32(ctx, shRegistersAddress, out var loRegister) || + !TryReadUInt32(ctx, shRegistersAddress + 8, out var hiRegister)) { return false; } @@ -5971,12 +10023,12 @@ public static class AgcExports var loValue = (uint)((codeAddress >> 8) & 0xFFFF_FFFFUL); var hiValue = (uint)((codeAddress >> 40) & 0xFFUL); - return ctx.TryWriteUInt32(shRegistersAddress + sizeof(uint), loValue) && - ctx.TryWriteUInt32(shRegistersAddress + 8 + sizeof(uint), hiValue); + return TryWriteUInt32(ctx, shRegistersAddress + sizeof(uint), loValue) && + TryWriteUInt32(ctx, shRegistersAddress + 8 + sizeof(uint), hiValue); } private static bool IsEsGeometryShaderType(byte shaderType) => - shaderType is 2 or 4 or 6; + shaderType is 2 or 6; private static int SetIndirectPatchAddress(CpuContext ctx, string registerSpace) { @@ -5984,13 +10036,13 @@ public static class AgcExports var registersAddress = ctx[CpuRegister.Rsi]; if (commandAddress == 0 || registersAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryWriteUInt32(commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(registersAddress >> 32))) + if (!TryWriteUInt32(ctx, commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(registersAddress >> 32))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceAgc($"agc.patch_{registerSpace}_addr cmd=0x{commandAddress:X16} regs=0x{registersAddress:X16}"); @@ -5998,19 +10050,75 @@ public static class AgcExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + private static int PatchWriteDataControlByte(CpuContext ctx, int byteIndex) + { + if (!TryResolveWriteDataPatchArguments( + ctx, + ctx[CpuRegister.Rdi], + ctx[CpuRegister.Rsi], + out var commandAddress, + out var value) || + !TryReadUInt32(ctx, commandAddress + 4, out var control)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var shift = byteIndex * 8; + var patchedControl = (control & ~(0xFFu << shift)) | (((uint)value & 0xFFu) << shift); + return TryWriteUInt32(ctx, commandAddress + 4, patchedControl) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + private static bool TryResolveWriteDataPatchArguments( + CpuContext ctx, + ulong first, + ulong second, + out ulong commandAddress, + out ulong value) + { + if (IsWriteDataPacket(ctx, first)) + { + commandAddress = first; + value = second; + return true; + } + + if (IsWriteDataPacket(ctx, second)) + { + commandAddress = second; + value = first; + return true; + } + + commandAddress = 0; + value = 0; + return false; + } + + private static bool IsWriteDataPacket(CpuContext ctx, ulong commandAddress) + { + if (!TryGetPacketIdentity(ctx, commandAddress, out var op, out var register)) + { + return false; + } + + return op == ItWriteData || (op == ItNop && register == RWriteData); + } + private static int AddIndirectPatchRegisters(CpuContext ctx, string registerSpace) { var commandAddress = ctx[CpuRegister.Rdi]; var registerCount = (uint)ctx[CpuRegister.Rsi]; if (commandAddress == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadUInt32(commandAddress + 4, out var currentCount) || - !ctx.TryWriteUInt32(commandAddress + 4, currentCount + registerCount)) + if (!TryReadUInt32(ctx, commandAddress + 4, out var currentCount) || + !TryWriteUInt32(ctx, commandAddress + 4, currentCount + registerCount)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceAgc($"agc.patch_{registerSpace}_add cmd=0x{commandAddress:X16} add={registerCount} total={currentCount + registerCount}"); @@ -6029,10 +10137,10 @@ public static class AgcExports } if (!TryAllocateCommandDwords(ctx, commandBufferAddress, 4, out var commandAddress) || - !ctx.TryWriteUInt32(commandAddress, Pm4(4, ItNop, packetRegister)) || - !ctx.TryWriteUInt32(commandAddress + 4, registerCount) || - !ctx.TryWriteUInt32(commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || - !ctx.TryWriteUInt32(commandAddress + 12, (uint)(registersAddress >> 32))) + !TryWriteUInt32(ctx, commandAddress, Pm4(4, ItNop, packetRegister)) || + !TryWriteUInt32(ctx, commandAddress + 4, registerCount) || + !TryWriteUInt32(ctx, commandAddress + 8, (uint)(registersAddress & 0xFFFF_FFFFUL)) || + !TryWriteUInt32(ctx, commandAddress + 12, (uint)(registersAddress >> 32))) { return ReturnPointer(ctx, 0); } @@ -6045,22 +10153,55 @@ public static class AgcExports { commandAddress = 0; if (sizeDwords == 0 || - !ctx.TryReadUInt64(commandBufferAddress + CommandBufferCursorUpOffset, out var cursorUp) || - !ctx.TryReadUInt64(commandBufferAddress + CommandBufferCursorDownOffset, out var cursorDown) || - !ctx.TryReadUInt64(commandBufferAddress + CommandBufferCallbackOffset, out var callback) || - !ctx.TryReadUInt32(commandBufferAddress + CommandBufferReservedDwOffset, out var reservedDwords)) + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out var cursorUp) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out var cursorDown) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCallbackOffset, out var callback) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferUserDataOffset, out var userData) || + !TryReadUInt32(ctx, commandBufferAddress + CommandBufferReservedDwOffset, out var reservedDwords)) { return false; } - var availableDwords = cursorDown >= cursorUp - ? Math.Min((cursorDown - cursorUp) / sizeof(uint), uint.MaxValue) - : 0; - var remainingDwords = (uint)Math.Max(availableDwords, reservedDwords) - reservedDwords; + var remainingDwords = GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords); if (sizeDwords > remainingDwords) { TraceAgc($"agc.cmd_alloc_full buf=0x{commandBufferAddress:X16} need={sizeDwords} remaining={remainingDwords} callback=0x{callback:X16}"); - return false; + var scheduler = GuestThreadExecution.Scheduler; + ulong callbackResult = 0; + string? callbackError = null; + if (callback == 0 || + scheduler is null || + !scheduler.TryCallGuestFunction( + ctx, + callback, + commandBufferAddress, + (ulong)sizeDwords + reservedDwords, + userData, + 0, + 0, + "agc_command_buffer_full", + out callbackResult, + out callbackError)) + { + TraceAgc( + $"agc.cmd_alloc_callback_failed buf=0x{commandBufferAddress:X16} " + + $"callback=0x{callback:X16} result=0x{callbackResult:X16} " + + $"error={callbackError ?? "none"}"); + return false; + } + + TraceAgc( + $"agc.cmd_alloc_callback_complete buf=0x{commandBufferAddress:X16} " + + $"callback=0x{callback:X16} result=0x{callbackResult:X16}"); + + if (!TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorUpOffset, out cursorUp) || + !TryReadUInt64(ctx, commandBufferAddress + CommandBufferCursorDownOffset, out cursorDown) || + !TryReadUInt32(ctx, commandBufferAddress + CommandBufferReservedDwOffset, out reservedDwords) || + sizeDwords > GetRemainingCommandDwords(cursorUp, cursorDown, reservedDwords)) + { + TraceAgc($"agc.cmd_alloc_callback_no_space buf=0x{commandBufferAddress:X16} need={sizeDwords}"); + return false; + } } var nextCursor = cursorUp + ((ulong)sizeDwords * sizeof(uint)); @@ -6073,21 +10214,34 @@ public static class AgcExports return true; } + private static uint GetRemainingCommandDwords( + ulong cursorUp, + ulong cursorDown, + uint reservedDwords) + { + var availableDwords = cursorDown >= cursorUp + ? Math.Min((cursorDown - cursorUp) / sizeof(uint), uint.MaxValue) + : 0; + return availableDwords > reservedDwords + ? (uint)availableDwords - reservedDwords + : 0; + } + private static bool CopyShaderRegister(CpuContext ctx, ulong sourceAddress, ulong destinationAddress) { - if (!ctx.TryReadUInt32(sourceAddress, out var offset) || - !ctx.TryReadUInt32(sourceAddress + sizeof(uint), out var value)) + if (!TryReadUInt32(ctx, sourceAddress, out var offset) || + !TryReadUInt32(ctx, sourceAddress + sizeof(uint), out var value)) { return false; } - return ctx.TryWriteUInt32(destinationAddress, offset) && - ctx.TryWriteUInt32(destinationAddress + sizeof(uint), value); + return TryWriteUInt32(ctx, destinationAddress, offset) && + TryWriteUInt32(ctx, destinationAddress + sizeof(uint), value); } private static bool RelocatePointerField(CpuContext ctx, ulong fieldAddress) { - if (!ctx.TryReadUInt64(fieldAddress, out var relativeAddress)) + if (!TryReadUInt64(ctx, fieldAddress, out var relativeAddress)) { return false; } @@ -6254,6 +10408,12 @@ public static class AgcExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)(int)result); + return (int)result; + } + private static uint Pm4(uint lengthDwords, uint op, uint register) => 0xC0000000u | ((((ushort)lengthDwords - 2u) & 0x3FFFu) << 16) | @@ -6263,6 +10423,102 @@ public static class AgcExports private static uint Pm4Length(uint header) => ((header >> 16) & 0x3FFFu) + 2u; + private static bool TryReadByte(CpuContext ctx, ulong address, out byte value) + { + Span buffer = stackalloc byte[1]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = buffer[0]; + return true; + } + + // A submitted command buffer is bulk-copied once per submit and served + // from this thread-local window: the previous per-dword reads each took + // the guest-memory reader lock and ran a region binary search, which + // dominated submit parsing (thousands of locked 4-byte reads per DCB). + [ThreadStatic] + private static byte[]? _dcbWindowBuffer; + [ThreadStatic] + private static ulong _dcbWindowStart; + [ThreadStatic] + private static int _dcbWindowByteLength; + + /// + /// Drops the bulk-read window when a self-patching command buffer writes + /// into its own bytes during parse, so subsequent reads see live guest + /// memory instead of the pre-write snapshot. Self-patching is rare, so + /// paying live-read cost for the rest of that one submit is acceptable. + /// + private static void InvalidateDcbWindowIfOverlaps(ulong address, ulong length) + { + if (_dcbWindowBuffer is null || length == 0) + { + return; + } + + var windowEnd = _dcbWindowStart + (ulong)_dcbWindowByteLength; + if (address < windowEnd && address + length > _dcbWindowStart) + { + _dcbWindowBuffer = null; + _dcbWindowByteLength = 0; + } + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + if (_dcbWindowBuffer is { } window && + address >= _dcbWindowStart && + address - _dcbWindowStart + sizeof(uint) <= (ulong)_dcbWindowByteLength) + { + value = BinaryPrimitives.ReadUInt32LittleEndian( + window.AsSpan((int)(address - _dcbWindowStart))); + return true; + } + + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + if (_dcbWindowBuffer is { } window && + address >= _dcbWindowStart && + address - _dcbWindowStart + sizeof(ulong) <= (ulong)_dcbWindowByteLength) + { + value = BinaryPrimitives.ReadUInt64LittleEndian( + window.AsSpan((int)(address - _dcbWindowStart))); + return true; + } + + Span buffer = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + private static bool TryReadGuestCString( CpuContext ctx, ulong address, @@ -6278,7 +10534,7 @@ public static class AgcExports var values = new List(Math.Min(maximumLength, 128)); for (var index = 0; index < maximumLength; index++) { - if (!ctx.TryReadByte(address + (ulong)index, out var value)) + if (!TryReadByte(ctx, address + (ulong)index, out var value)) { bytes = []; return false; @@ -6305,7 +10561,7 @@ public static class AgcExports { op = 0; register = 0; - if (commandAddress == 0 || !ctx.TryReadUInt32(commandAddress, out var header)) + if (commandAddress == 0 || !TryReadUInt32(ctx, commandAddress, out var header)) { return false; } @@ -6384,6 +10640,62 @@ public static class AgcExports return count <= 8 || count % 100_000 == 0; } + // Interpolated-string handlers gated on the trace flags: when tracing is + // off (the normal case) the compiler skips every AppendFormatted call, so + // the interpolation never runs. These functions are on the hottest guest + // paths — e.g. AddIndirectPatchRegisters fires tens of thousands of times + // per second — and previously formatted a discarded string every call. + [System.Runtime.CompilerServices.InterpolatedStringHandler] + private ref struct AgcTraceHandler + { + private System.Runtime.CompilerServices.DefaultInterpolatedStringHandler _inner; + private readonly bool _enabled; + + public AgcTraceHandler(int literalLength, int formattedCount, out bool shouldAppend) + { + _enabled = _traceAgc; + shouldAppend = _enabled; + _inner = _enabled + ? new System.Runtime.CompilerServices.DefaultInterpolatedStringHandler(literalLength, formattedCount) + : default; + } + + public void AppendLiteral(string value) => _inner.AppendLiteral(value); + public void AppendFormatted(T value) => _inner.AppendFormatted(value); + public void AppendFormatted(T value, string? format) => _inner.AppendFormatted(value, format); + public string ToStringAndClear() => _enabled ? _inner.ToStringAndClear() : string.Empty; + } + + [System.Runtime.CompilerServices.InterpolatedStringHandler] + private ref struct AgcShaderTraceHandler + { + private System.Runtime.CompilerServices.DefaultInterpolatedStringHandler _inner; + private readonly bool _enabled; + + public AgcShaderTraceHandler(int literalLength, int formattedCount, out bool shouldAppend) + { + _enabled = _traceAgcShader; + shouldAppend = _enabled; + _inner = _enabled + ? new System.Runtime.CompilerServices.DefaultInterpolatedStringHandler(literalLength, formattedCount) + : default; + } + + public void AppendLiteral(string value) => _inner.AppendLiteral(value); + public void AppendFormatted(T value) => _inner.AppendFormatted(value); + public void AppendFormatted(T value, string? format) => _inner.AppendFormatted(value, format); + public string ToStringAndClear() => _enabled ? _inner.ToStringAndClear() : string.Empty; + } + + private static void TraceAgc( + [System.Runtime.CompilerServices.InterpolatedStringHandlerArgument] ref AgcTraceHandler message) + { + if (_traceAgc) + { + Console.Error.WriteLine($"[LOADER][TRACE] {message.ToStringAndClear()}"); + } + } + private static void TraceAgc(string message) { if (!_traceAgc) @@ -6394,6 +10706,15 @@ public static class AgcExports Console.Error.WriteLine($"[LOADER][TRACE] {message}"); } + private static void TraceAgcShader( + [System.Runtime.CompilerServices.InterpolatedStringHandlerArgument] ref AgcShaderTraceHandler message) + { + if (_traceAgcShader) + { + Console.Error.WriteLine($"[LOADER][TRACE] {message.ToStringAndClear()}"); + } + } + private static void TraceAgcShader(string message) { if (!_traceAgcShader) @@ -6412,8 +10733,34 @@ public static class AgcExports private static string FormatTextureDescriptor(TextureDescriptor descriptor) => $"addr=0x{descriptor.Address:X16} {descriptor.Width}x{descriptor.Height} " + $"fmt={descriptor.Format} num={descriptor.NumberType} tile={descriptor.TileMode} " + - $"type={descriptor.Type} levels={descriptor.BaseLevel}-{descriptor.LastLevel} " + - $"pitch={descriptor.Pitch} dst=0x{descriptor.DstSelect:X3}"; + $"type={descriptor.Type} depth={descriptor.Depth} base_array={descriptor.BaseArray} " + + $"levels={descriptor.BaseLevel}-{descriptor.LastLevel}/max{descriptor.MaxMip} " + + $"pitch={descriptor.Pitch} array_pitch={descriptor.ArrayPitch} " + + $"lod={descriptor.MinLod:X3}/{descriptor.MinLodWarn:X3} " + + $"bc={descriptor.BcSwizzle} meta=0x{descriptor.MetadataAddress:X16} " + + $"flags=0x{descriptor.DescriptorFlags:X6} dst=0x{descriptor.DstSelect:X3}"; + + private static ulong? ParseOptionalHexAddress(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var span = value.AsSpan().Trim(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) + ? address + : null; + } private static void DumpCompiledShader( string stage, @@ -6431,6 +10778,27 @@ public static class AgcExports return; } + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_DUMP_SPIRV_ADDRESS"); + if (!string.IsNullOrWhiteSpace(addressFilter)) + { + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + if (!ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var filteredAddress) || + shaderAddress != filteredAddress) + { + return; + } + } + var directory = Path.Combine(AppContext.BaseDirectory, "shader-dumps"); Directory.CreateDirectory(directory); var name = $"{shaderAddress:X16}-{stateFingerprint:X16}.{stage}"; @@ -6471,85 +10839,297 @@ public static class AgcExports $"[LOADER][TRACE] agc.create_shader dst=0x{destinationAddress:X16} header=0x{headerAddress:X16} code=0x{codeAddress:X16} {detail}"); } - // Packet layouts mirror what DcbWaitRegMem emits. - // 32-bit (ItNop/RWaitMem32): +4 addrLo, +8 addrHi, +12 mask, +16 cmp|op<<8, +20 ref. - // 64-bit (ItNop/RWaitMem64): +4 addrLo, +8 addrHi, +12 maskLo, +16 maskHi, - // +20 refLo, +24 refHi, +28 cmp|op<<8, +32 poll. - private static bool TryParseWaitRegMem( - CpuContext ctx, - ulong addr, - bool is64, - out ulong waitAddr, - out ulong refVal, - out ulong mask, - out uint cmpFunc) + [SysAbiExport( + Nid = "xSAR0LTcRKM", + ExportName = "sceAgcDcbJump", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbJump(CpuContext ctx) { - waitAddr = refVal = mask = 0; - cmpFunc = 0; - - if (!ctx.TryReadUInt32(addr + 4, out var lo) || - !ctx.TryReadUInt32(addr + 8, out var hi)) + var dcb = ctx[CpuRegister.Rdi]; + var target = ctx[CpuRegister.Rsi]; + var sizeDwords = (uint)ctx[CpuRegister.Rdx]; + if (dcb == 0) { - return false; + return ReturnPointer(ctx, 0); } - waitAddr = ((ulong)hi << 32) | lo; - if (!is64) + if (!TryAllocateCommandDwords(ctx, dcb, 4, out var cmd) || + !ctx.TryWriteUInt32(cmd, Pm4(4, ItIndirectBuffer, RZero)) || + !ctx.TryWriteUInt32(cmd + 4, (uint)(target & 0xFFFF_FFFFUL)) || + !ctx.TryWriteUInt32(cmd + 8, (uint)((target >> 32) & 0xFFFFUL)) || + !ctx.TryWriteUInt32(cmd + 12, sizeDwords & 0xFFFFF)) { - if (!ctx.TryReadUInt32(addr + 12, out var mask32) || - !ctx.TryReadUInt32(addr + 16, out var cmpRaw) || - !ctx.TryReadUInt32(addr + 20, out var refVal32)) + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, cmd); + } + + [SysAbiExport( + Nid = "bbFueFP+J4k", + ExportName = "sceAgcDcbSetPredication", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DcbSetPredication(CpuContext ctx) + { + var dcb = ctx[CpuRegister.Rdi]; + var address = ctx[CpuRegister.Rsi]; + if (dcb == 0) + { + return ReturnPointer(ctx, 0); + } + + if (!TryAllocateCommandDwords(ctx, dcb, 3, out var cmd) || + !ctx.TryWriteUInt32(cmd, Pm4(3, ItNop, RZero)) || + !ctx.TryWriteUInt32(cmd + 4, (uint)(address & 0xFFFF_FFFFUL)) || + !ctx.TryWriteUInt32(cmd + 8, (uint)(address >> 32))) + { + return ReturnPointer(ctx, 0); + } + + return ReturnPointer(ctx, cmd); + } + + [SysAbiExport( + Nid = "w6Dj1VJt5qY", + ExportName = "sceAgcSetPacketPredication", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int SetPacketPredication(CpuContext ctx) + { + // Global predication toggle on a packet; a no-op is safe for rendering. + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ABI (reversed from Quake): rdi = array of DCB base addresses (u64 each), + // rsi = array of DCB sizes in dwords (u32 each), rdx = buffer count. + [SysAbiExport( + Nid = "6UzEidRZwkg", + ExportName = "sceAgcDriverSubmitMultiDcbs", + Target = Generation.Gen5, + LibraryName = "libSceAgcDriver")] + public static int DriverSubmitMultiDcbs(CpuContext ctx) + { + var addressArray = ctx[CpuRegister.Rdi]; + var sizeArray = ctx[CpuRegister.Rsi]; + var bufferCount = (uint)ctx[CpuRegister.Rdx]; + if (addressArray == 0 || sizeArray == 0 || bufferCount == 0 || bufferCount > 4096) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var tracePackets = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal); + + var gpuState = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (gpuState.Gate) + { + Gen5ShaderScalarEvaluator.BeginGlobalMemoryReadScope(); + try { - return false; + for (uint i = 0; i < bufferCount; i++) + { + if (!ctx.TryReadUInt64(addressArray + i * 8, out var commandAddress) || + commandAddress == 0 || + !ctx.TryReadUInt32(sizeArray + i * 4, out var dwordCount) || + dwordCount == 0) + { + continue; + } + + if (tracePackets) + { + TraceAgc( + $"agc.driver_submit_multi_dcbs index={i}/{bufferCount} " + + $"addr=0x{commandAddress:X16} dwords={dwordCount}"); + } + + ParseSubmittedDcb(ctx, gpuState, gpuState.Graphics, commandAddress, dwordCount, tracePackets); + } + + DrainResumableDcbs(ctx, gpuState, tracePackets); + } + finally + { + Gen5ShaderScalarEvaluator.EndGlobalMemoryReadScope(); + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "AOLcoIkQDgM", + ExportName = "sceAgcDriverQueryResourceRegistrationUserMemoryRequirements", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverQueryResourceRegistrationUserMemoryRequirements(CpuContext ctx) + { + var sizeAddress = ctx[CpuRegister.Rdi]; + var resourceCount = ctx[CpuRegister.Rsi]; + var ownerCount = ctx[CpuRegister.Rdx]; + if (sizeAddress == 0 || resourceCount == 0 || ownerCount == 0) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ulong requiredSize; + try + { + requiredSize = checked( + resourceCount * ResourceRegistrationBytesPerResource + + ownerCount * ResourceRegistrationBytesPerOwner); + } + catch (OverflowException) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + if (!ctx.TryWriteUInt64(sizeAddress, requiredSize)) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceAgc( + $"agc.driver_query_resource_registration_memory resources={resourceCount} " + + $"owners={ownerCount} bytes=0x{requiredSize:X}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + } + + [SysAbiExport( + Nid = "F0Y42t-3e18", + ExportName = "sceAgcDriverInitResourceRegistration", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverInitResourceRegistration(CpuContext ctx) + { + var memoryAddress = ctx[CpuRegister.Rdi]; + var memorySize = ctx[CpuRegister.Rsi]; + var ownerCount = ctx[CpuRegister.Rdx]; + if (memoryAddress == 0 || memorySize == 0 || ownerCount == 0 || ownerCount > uint.MaxValue) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (state.Gate) + { + state.ResourceRegistrationInitialized = true; + state.ResourceRegistrationMemory = memoryAddress; + state.ResourceRegistrationMemorySize = memorySize; + state.ResourceRegistrationMaxOwners = (uint)ownerCount; + state.ResourceOwners.Clear(); + state.RegisteredResources.Clear(); + state.DefaultOwner = DefaultAgcOwner; + state.NextOwner = 1; + state.NextResource = 1; + } + + TraceAgc( + $"agc.driver_init_resource_registration memory=0x{memoryAddress:X16} " + + $"bytes=0x{memorySize:X} owners={ownerCount}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + } + + [SysAbiExport( + Nid = "U9ueyEhSkF4", + ExportName = "sceAgcDriverRegisterDefaultOwner", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverRegisterDefaultOwner(CpuContext ctx) + { + var owner = (uint)ctx[CpuRegister.Rdi]; + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (state.Gate) + { + state.DefaultOwner = owner; + } + + TraceAgc($"agc.driver_register_default_owner owner={owner}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + } + + [SysAbiExport( + Nid = "X-Nm5KLREeg", + ExportName = "sceAgcDriverRegisterOwner", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverRegisterOwner(CpuContext ctx) + { + var ownerAddress = ctx[CpuRegister.Rdi]; + var nameAddress = ctx[CpuRegister.Rsi]; + if (ownerAddress == 0 || nameAddress == 0 || + !TryReadGuestCString( + ctx, + nameAddress, + ResourceRegistrationMaxNameLength, + out var nameBytes)) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + uint owner; + lock (state.Gate) + { + if (!state.ResourceRegistrationInitialized || + state.ResourceRegistrationMaxOwners != 0 && + state.ResourceOwners.Count >= state.ResourceRegistrationMaxOwners) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - mask = mask32; - refVal = refVal32; - cmpFunc = cmpRaw & 0x7; - return true; + owner = state.NextOwner; + while (owner == state.DefaultOwner || state.ResourceOwners.ContainsKey(owner)) + { + owner++; + if (owner == 0) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + } + + state.NextOwner = owner + 1; + state.ResourceOwners.Add(owner, System.Text.Encoding.UTF8.GetString(nameBytes)); } - if (!ctx.TryReadUInt32(addr + 12, out var maskLo) || - !ctx.TryReadUInt32(addr + 16, out var maskHi) || - !ctx.TryReadUInt32(addr + 20, out var refLo) || - !ctx.TryReadUInt32(addr + 24, out var refHi) || - !ctx.TryReadUInt32(addr + 28, out var cmpRaw64)) + if (!ctx.TryWriteUInt32(ownerAddress, owner)) { - return false; + lock (state.Gate) + { + state.ResourceOwners.Remove(owner); + } + + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - mask = ((ulong)maskHi << 32) | maskLo; - refVal = ((ulong)refHi << 32) | refLo; - cmpFunc = cmpRaw64 & 0x7; - return true; + TraceAgc( + $"agc.driver_register_owner out=0x{ownerAddress:X16} owner={owner} " + + $"name={System.Text.Encoding.UTF8.GetString(nameBytes)}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } - // Standard ItWaitRegMem (7 dwords, 32-bit compare): - // +4 cmp|(op&1)<<8, +8 addrLo, +12 addrHi, +16 ref, +20 mask, +24 poll. - private static bool TryParseStandardWaitRegMem( - CpuContext ctx, - ulong addr, - out ulong waitAddr, - out ulong refVal, - out ulong mask, - out uint cmpFunc) + [SysAbiExport( + Nid = "pWLG7WOpVcw", + ExportName = "sceAgcDriverUnregisterResource", + Target = Generation.Gen5, + LibraryName = "libSceAgc")] + public static int DriverUnregisterResource(CpuContext ctx) { - waitAddr = refVal = mask = 0; - cmpFunc = 0; - - if (!ctx.TryReadUInt32(addr + 4, out var cmpRaw) || - !ctx.TryReadUInt32(addr + 8, out var lo) || - !ctx.TryReadUInt32(addr + 12, out var hi) || - !ctx.TryReadUInt32(addr + 16, out var reference) || - !ctx.TryReadUInt32(addr + 20, out var mask32)) + var resourceHandle = (uint)ctx[CpuRegister.Rdi]; + var state = _submittedGpuStates.GetValue(ctx.Memory, static _ => new SubmittedGpuState()); + lock (state.Gate) { - return false; + if (!state.RegisteredResources.Remove(resourceHandle)) + { + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } } - cmpFunc = cmpRaw & 0x7; - waitAddr = ((ulong)hi << 32) | lo; - refVal = reference; - mask = mask32; - return true; + TraceAgc($"agc.driver_unregister_resource handle={resourceHandle}"); + return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } } diff --git a/src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs b/src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs new file mode 100644 index 0000000..1b089fc --- /dev/null +++ b/src/SharpEmu.Libs/Agc/AgcPrimaryRegisterDefaults.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Agc; + +public static partial class AgcExports +{ + // Exact Gen5 primary register defaults originally reverse-engineered by Kyty (MIT). + // Hashes, offsets, and values match the tables consumed by the PS5 AGC SDK. + private static RegisterDefaultGroup[] CreatePrimaryRegisterDefaults() => + [ + // context register groups + new(0u, 0u, 0xE24F806Du, [new(0x202u, 0x00CC0010u)]), // CB_COLOR_CONTROL + new(0u, 1u, 0xF6C28182u, [new(0x109u, 0x00000000u)]), // CB_DCC_CONTROL + new(0u, 2u, 0x6F6E55A5u, [new(0x104u, 0x00000000u)]), // CB_RMI_GL2_CACHE_CONTROL + new(0u, 3u, 0x0BC65DA4u, [new(0x08Fu, 0x00000000u)]), // CB_SHADER_MASK + new(0u, 4u, 0x9E5AD592u, [new(0x08Eu, 0x0000000Fu)]), // CB_TARGET_MASK + new(0u, 5u, 0xBB513B98u, [new(0x2DCu, 0x0000AA00u)]), // DB_ALPHA_TO_MASK + new(0u, 6u, 0xAB64B23Bu, [new(0x001u, 0x00000000u)]), // DB_COUNT_CONTROL + new(0u, 7u, 0x53C39964u, [new(0x200u, 0x00000000u)]), // DB_DEPTH_CONTROL + new(0u, 8u, 0x01396B11u, [new(0x201u, 0x00000000u)]), // DB_EQAA + new(0u, 9u, 0x7D42019Au, [new(0x000u, 0x00000000u)]), // DB_RENDER_CONTROL + new(0u, 10u, 0x3548F523u, [new(0x006u, 0x00000000u)]), // PS_SHADER_SAMPLE_EXCLUSION_MASK + new(0u, 11u, 0xF43AD28Au, [new(0x01Fu, 0x00000000u)]), // DB_RMI_L2_CACHE_CONTROL + new(0u, 12u, 0x6DE4C312u, [new(0x203u, 0x00000000u)]), // DB_SHADER_CONTROL + new(0u, 13u, 0x00A77AE0u, [new(0x2B0u, 0x00000000u)]), // DB_SRESULTS_COMPARE_STATE0 + new(0u, 14u, 0x00A779B7u, [new(0x2B1u, 0x00000000u)]), // DB_SRESULTS_COMPARE_STATE1 + new(0u, 15u, 0x5100100Cu, [new(0x10Cu, 0x00000000u)]), // DB_STENCILREFMASK + new(0u, 16u, 0x59958BBAu, [new(0x10Du, 0x00000000u)]), // DB_STENCILREFMASK_BF + new(0u, 17u, 0x0C06F17Cu, [new(0x10Bu, 0x00000000u)]), // DB_STENCIL_CONTROL + new(0u, 18u, 0x6F104B72u, [new(0x1FFu, 0x00000000u)]), // GE_MAX_OUTPUT_PER_SUBGROUP + new(0u, 19u, 0x25C70D9Cu, [new(0x204u, 0x00000000u)]), // PA_CL_CLIP_CNTL + new(0u, 20u, 0x3881201Eu, [new(0x20Du, 0x00000000u)]), // PA_CL_OBJPRIM_ID_CNTL + new(0u, 21u, 0x09AFDDAFu, [new(0x206u, 0x0000043Fu)]), // PA_CL_VTE_CNTL + new(0u, 22u, 0x367D63CFu, [new(0x2F8u, 0x00000000u)]), // PA_SC_AA_CONFIG + new(0u, 23u, 0x43707DB8u, [new(0x083u, 0x0000FFFFu)]), // PA_SC_CLIPRECT_RULE + new(0u, 24u, 0xF6AE26BAu, [new(0x313u, 0x00000000u)]), // PA_SC_CONSERVATIVE_RASTERIZATION_CNTL + new(0u, 25u, 0x1B917652u, [new(0x800003FEu, 0x00000000u)]), // PA_SC_FSR_ENABLE + new(0u, 26u, 0x94B1E4F7u, [new(0x0EAu, 0x00000000u)]), // PA_SC_HORIZ_GRID + new(0u, 27u, 0xE3661B6Cu, [new(0x0E9u, 0x00000000u)]), // PA_SC_LEFT_VERT_GRID + new(0u, 28u, 0x1EB8D73Au, [new(0x292u, 0x00000002u)]), // PA_SC_MODE_CNTL_0 + new(0u, 29u, 0x15051FA3u, [new(0x293u, 0x00000000u)]), // PA_SC_MODE_CNTL_1 + new(0u, 30u, 0x9C51A7F1u, [new(0x0E8u, 0x00000000u)]), // PA_SC_RIGHT_VERT_GRID + new(0u, 31u, 0xA20EFC70u, [new(0x080u, 0x00000000u)]), // PA_SC_WINDOW_OFFSET + new(0u, 32u, 0x0EC09F6Eu, [new(0x211u, 0x00000000u)]), // PA_STATE_STEREO_X + new(0u, 33u, 0x34A7D6D3u, [new(0x210u, 0x00000000u)]), // PA_STEREO_CNTL + new(0u, 34u, 0xCE831B94u, [new(0x08Du, 0x00000000u)]), // PA_SU_HARDWARE_SCREEN_OFFSET + new(0u, 35u, 0x5CC72A74u, [new(0x282u, 0x00000008u)]), // PA_SU_LINE_CNTL + new(0u, 36u, 0x3B77713Cu, [new(0x281u, 0xFFFF0000u)]), // PA_SU_POINT_MINMAX + new(0u, 37u, 0x40F64410u, [new(0x280u, 0x00080008u)]), // PA_SU_POINT_SIZE + new(0u, 38u, 0x69441268u, [new(0x2DFu, 0x00000000u)]), // PA_SU_POLY_OFFSET_CLAMP + new(0u, 39u, 0x2E418B83u, [new(0x2DEu, 0x000001E9u)]), // PA_SU_POLY_OFFSET_DB_FMT_CNTL + new(0u, 40u, 0xA00D0C8Du, [new(0x205u, 0x00000240u)]), // PA_SU_SC_MODE_CNTL + new(0u, 41u, 0xB1289FB3u, [new(0x20Cu, 0x00000001u)]), // PA_SU_SMALL_PRIM_FILTER_CNTL + new(0u, 42u, 0x144832FBu, [new(0x2F9u, 0x0000002Du)]), // PA_SU_VTX_CNTL + new(0u, 43u, 0x9890D9FAu, [new(0x1BAu, 0x00000000u)]), // SPI_TMPRING_SIZE + new(0u, 44u, 0x9016FAF1u, [new(0x2A6u, 0x00000000u)]), // VGT_DRAW_PAYLOAD_CNTL + new(0u, 45u, 0x4B73CE27u, [new(0x2CEu, 0x00000400u)]), // VGT_GS_MAX_VERT_OUT + new(0u, 46u, 0x5F5A3E7Bu, [new(0x29Bu, 0x00000002u)]), // VGT_GS_OUT_PRIM_TYPE + new(0u, 47u, 0xD4AF3A51u, [new(0x2D6u, 0x00000000u)]), // VGT_LS_HS_CONFIG + new(0u, 48u, 0x6CF4F543u, [new(0x2A3u, 0xFFFFFFFFu)]), // VGT_PRIMITIVEID_RESET + new(0u, 49u, 0x5FB86CCBu, [new(0x2A1u, 0x00000000u)]), // VGT_PRIMITIVEID_EN + new(0u, 50u, 0xEDEFA188u, [new(0x2ADu, 0x00000000u)]), // VGT_REUSE_OFF + new(0u, 51u, 0xD0DE9EE6u, [new(0x2D5u, 0x00000000u)]), // VGT_SHADER_STAGES_EN + new(0u, 52u, 0xC5831803u, [new(0x2D4u, 0x88101000u)]), // VGT_TESS_DISTRIBUTION + new(0u, 53u, 0x8E6DE84Bu, [new(0x2DBu, 0x00000000u)]), // VGT_TF_PARAM + new(0u, 54u, 0xD0771662u, [new(0x2F5u, 0x00000000u), new(0x2F6u, 0x00000000u)]), // PA_SC_CENTROID_PRIORITY_0, PA_SC_CENTROID_PRIORITY_1 + new(0u, 55u, 0x569F7444u, [new(0x2FEu, 0x00000000u)]), // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 + new(0u, 56u, 0x5C6637CDu, [new(0x30Eu, 0xFFFFFFFFu), new(0x30Fu, 0xFFFFFFFFu)]), // PA_SC_AA_MASK_X0Y0_X1Y0, PA_SC_AA_MASK_X0Y1_X1Y1 + new(0u, 57u, 0xCAE3E690u, [new(0x311u, 0x00000002u), new(0x312u, 0x03FF0080u)]), // PA_SC_BINNER_CNTL_0, PA_SC_BINNER_CNTL_1 + new(0u, 58u, 0x43FBD769u, [new(0x105u, 0x00000000u), new(0x107u, 0x00000000u), new(0x106u, 0x00000000u), new(0x108u, 0x00000000u)]), // CB_BLEND_RED, CB_BLEND_BLUE, CB_BLEND_GREEN, CB_BLEND_ALPHA + new(0u, 59u, 0xEF550356u, [new(0x1E0u, 0x20010001u)]), // CB_BLEND0_CONTROL + new(0u, 60u, 0x8F52E279u, [new(0x020u, 0x00000000u), new(0x021u, 0x00000000u)]), // TA_BC_BASE_ADDR, TA_BC_BASE_ADDR_HI + new(0u, 61u, 0x1F2D8149u, [new(0x084u, 0x00000000u), new(0x085u, 0x20002000u)]), // PA_SC_CLIPRECT_0_TL, PA_SC_CLIPRECT_0_BR + new(0u, 62u, 0x853D0614u, [new(0x800003FFu, 0x00000000u)]), // CX_NOP + new(0u, 63u, 0x4413C6F9u, [new(0x008u, 0x00000000u), new(0x009u, 0x00000000u)]), // DB_DEPTH_BOUNDS_MIN, DB_DEPTH_BOUNDS_MAX + new(0u, 64u, 0x67096014u, [new(0x010u, 0x80000000u), new(0x011u, 0x20000000u), new(0x012u, 0x00000000u), new(0x013u, 0x00000000u), new(0x014u, 0x00000000u), new(0x015u, 0x00000000u), new(0x01Au, 0x00000000u), new(0x01Bu, 0x00000000u), new(0x01Cu, 0x00000000u), new(0x01Du, 0x00000000u), new(0x01Eu, 0x00000000u), new(0x002u, 0x00000000u), new(0x005u, 0x00000000u), new(0x007u, 0x00000000u), new(0x00Bu, 0x00000000u), new(0x00Au, 0x00000000u)]), // DB_Z_INFO, DB_STENCIL_INFO, DB_Z_READ_BASE, DB_STENCIL_READ_BASE, DB_Z_WRITE_BASE, DB_STENCIL_WRITE_BASE, DB_Z_READ_BASE_HI, DB_STENCIL_READ_BASE_HI, DB_Z_WRITE_BASE_HI, DB_STENCIL_WRITE_BASE_HI, DB_HTILE_DATA_BASE_HI, DB_DEPTH_VIEW, DB_HTILE_DATA_BASE, DB_DEPTH_SIZE_XY, DB_DEPTH_CLEAR, DB_STENCIL_CLEAR + new(0u, 65u, 0x88F5E915u, [new(0x0EBu, 0xFF00FF00u), new(0x0ECu, 0x00000000u)]), // PA_SC_FOV_WINDOW_LR, PA_SC_FOV_WINDOW_TB + new(0u, 66u, 0x033F1EFFu, [new(0x800003FCu, 0x00000000u), new(0x800003FDu, 0x00000000u)]), // FSR_RECURSIONS0, FSR_RECURSIONS1 + new(0u, 67u, 0x918106BBu, [new(0x090u, 0x80000000u), new(0x091u, 0x40004000u)]), // PA_SC_GENERIC_SCISSOR_TL, PA_SC_GENERIC_SCISSOR_BR + new(0u, 68u, 0x95F0E7ACu, [new(0x2FAu, 0x4E7E0000u), new(0x2FBu, 0x4E7E0000u), new(0x2FCu, 0x4E7E0000u), new(0x2FDu, 0x4E7E0000u)]), // PA_CL_GB_VERT_CLIP_ADJ, PA_CL_GB_VERT_DISC_ADJ, PA_CL_GB_HORZ_CLIP_ADJ, PA_CL_GB_HORZ_DISC_ADJ + new(0u, 69u, 0xB48CBAB2u, [new(0x2E2u, 0x00000000u), new(0x2E3u, 0x00000000u)]), // PA_SU_POLY_OFFSET_BACK_SCALE, PA_SU_POLY_OFFSET_BACK_OFFSET + new(0u, 70u, 0x05BB3BC6u, [new(0x2E0u, 0x00000000u), new(0x2E1u, 0x00000000u)]), // PA_SU_POLY_OFFSET_FRONT_SCALE, PA_SU_POLY_OFFSET_FRONT_OFFSET + new(0u, 71u, 0x94FABA07u, [new(0x003u, 0x00000000u), new(0x004u, 0x00000000u)]), // DB_RENDER_OVERRIDE, DB_RENDER_OVERRIDE2 + new(0u, 72u, 0x38E92C91u, [new(0x318u, 0x00000000u), new(0x31Bu, 0x00000000u), new(0x31Cu, 0x00000000u), new(0x31Du, 0x00000000u), new(0x31Eu, 0x00000048u), new(0x31Fu, 0x00000000u), new(0x321u, 0x00000000u), new(0x323u, 0x00000000u), new(0x324u, 0x00000000u), new(0x325u, 0x00000000u), new(0x390u, 0x00000000u), new(0x398u, 0x00000000u), new(0x3A0u, 0x00000000u), new(0x3A8u, 0x00000000u), new(0x3B0u, 0x00000000u), new(0x3B8u, 0x0006C000u)]), // CB_COLOR0_BASE, CB_COLOR0_VIEW, CB_COLOR0_INFO, CB_COLOR0_ATTRIB, CB_COLOR0_DCC_CONTROL, CB_COLOR0_CMASK, CB_COLOR0_FMASK, CB_COLOR0_CLEAR_WORD0, CB_COLOR0_CLEAR_WORD1, CB_COLOR0_DCC_BASE, CB_COLOR0_BASE_EXT, CB_COLOR0_CMASK_BASE_EXT, CB_COLOR0_FMASK_BASE_EXT, CB_COLOR0_DCC_BASE_EXT, CB_COLOR0_ATTRIB2, CB_COLOR0_ATTRIB3 + new(0u, 73u, 0x0B177B43u, [new(0x00Cu, 0x00000000u), new(0x00Du, 0x40004000u)]), // PA_SC_SCREEN_SCISSOR_TL, PA_SC_SCREEN_SCISSOR_BR + new(0u, 74u, 0x48531062u, [new(0x191u, 0x00000000u)]), // SPI_PS_INPUT_CNTL_0 + new(0u, 75u, 0xAAA964B9u, [new(0x16Fu, 0x00000000u), new(0x170u, 0x00000000u), new(0x171u, 0x00000000u), new(0x172u, 0x00000000u)]), // PA_CL_UCP_0_X, PA_CL_UCP_0_Y, PA_CL_UCP_0_Z, PA_CL_UCP_0_W + new(0u, 76u, 0x7690AF6Fu, [new(0x10Fu, 0x4E7E0000u), new(0x111u, 0x4E7E0000u), new(0x113u, 0x4E7E0000u), new(0x110u, 0x00000000u), new(0x112u, 0x00000000u), new(0x114u, 0x00000000u), new(0x094u, 0x80000000u), new(0x095u, 0x40004000u), new(0x0B4u, 0x00000000u), new(0x0B5u, 0x00000000u)]), // PA_CL_VPORT_XSCALE, PA_CL_VPORT_YSCALE, PA_CL_VPORT_ZSCALE, PA_CL_VPORT_XOFFSET, PA_CL_VPORT_YOFFSET, PA_CL_VPORT_ZOFFSET, PA_SC_VPORT_SCISSOR_0_TL, PA_SC_VPORT_SCISSOR_0_BR, PA_SC_VPORT_ZMIN_0, PA_SC_VPORT_ZMAX_0 + new(0u, 77u, 0x078D7060u, [new(0x081u, 0x80000000u), new(0x082u, 0x40004000u)]), // PA_SC_WINDOW_SCISSOR_TL, PA_SC_WINDOW_SCISSOR_BR + // shader register groups + new(1u, 0u, 0x5D6E3EC7u, [new(0x212u, 0x00000000u)]), // COMPUTE_PGM_RSRC1 + new(1u, 1u, 0x57E7079Au, [new(0x213u, 0x00000000u)]), // COMPUTE_PGM_RSRC2 + new(1u, 2u, 0x7467FAFDu, [new(0x228u, 0x00000000u)]), // COMPUTE_PGM_RSRC3 + new(1u, 3u, 0x9E826B50u, [new(0x215u, 0x00000000u)]), // COMPUTE_RESOURCE_LIMITS + new(1u, 4u, 0xDC484F18u, [new(0x218u, 0x00000000u)]), // COMPUTE_TMPRING_SIZE + new(1u, 5u, 0x5DA8BCA3u, [new(0x08Au, 0x00000000u)]), // SPI_SHADER_PGM_RSRC1_GS + new(1u, 6u, 0x5CA726D8u, [new(0x10Au, 0x00000000u)]), // SPI_SHADER_PGM_RSRC1_HS + new(1u, 7u, 0x5DD28360u, [new(0x00Au, 0x00000000u)]), // SPI_SHADER_PGM_RSRC1_PS + new(1u, 8u, 0x57EFA0BEu, [new(0x08Bu, 0x00000000u)]), // SPI_SHADER_PGM_RSRC2_GS + new(1u, 9u, 0x502363D5u, [new(0x10Bu, 0x00000000u)]), // SPI_SHADER_PGM_RSRC2_HS + new(1u, 10u, 0x506D14BDu, [new(0x00Bu, 0x00000000u)]), // SPI_SHADER_PGM_RSRC2_PS + new(1u, 11u, 0xB2609506u, [new(0x224u, 0x00000000u)]), // COMPUTE_USER_ACCUM_0 + new(1u, 12u, 0x9E5CFB8Au, [new(0x107u, 0x00000000u), new(0x087u, 0x00000000u), new(0x007u, 0x00000000u)]), // SPI_SHADER_PGM_RSRC3_HS, SPI_SHADER_PGM_RSRC3_GS, SPI_SHADER_PGM_RSRC3_PS + new(1u, 13u, 0xC918DF3Eu, [new(0x20Cu, 0x00000000u), new(0x20Du, 0x00000000u)]), // COMPUTE_PGM_LO, COMPUTE_PGM_HI + new(1u, 14u, 0xC9751C9Cu, [new(0x0C8u, 0x00000000u), new(0x0C9u, 0x00000000u)]), // SPI_SHADER_PGM_LO_ES, SPI_SHADER_PGM_HI_ES + new(1u, 15u, 0xC97EF77Au, [new(0x088u, 0x00000000u), new(0x089u, 0x00000000u)]), // SPI_SHADER_PGM_LO_GS, SPI_SHADER_PGM_HI_GS + new(1u, 16u, 0xC927C6B9u, [new(0x108u, 0x00000000u), new(0x109u, 0x00000000u)]), // SPI_SHADER_PGM_LO_HS, SPI_SHADER_PGM_HI_HS + new(1u, 17u, 0xC92A1EC5u, [new(0x148u, 0x00000000u), new(0x149u, 0x00000000u)]), // SPI_SHADER_PGM_LO_LS, SPI_SHADER_PGM_HI_LS + new(1u, 18u, 0xC9E01B31u, [new(0x008u, 0x00000000u), new(0x009u, 0x00000000u)]), // SPI_SHADER_PGM_LO_PS, SPI_SHADER_PGM_HI_PS + new(1u, 19u, 0x50685F29u, [new(0x800002FFu, 0x00000000u)]), // SH_NOP + new(1u, 20u, 0xB26219CAu, [new(0x0B2u, 0x00000000u)]), // SPI_SHADER_USER_ACCUM_ESGS_0 + new(1u, 21u, 0xB25B6CF9u, [new(0x132u, 0x00000000u)]), // SPI_SHADER_USER_ACCUM_LSHS_0 + new(1u, 22u, 0xB2F86101u, [new(0x032u, 0x00000000u)]), // SPI_SHADER_USER_ACCUM_PS_0 + new(1u, 23u, 0x07E3B155u, [new(0x082u, 0x00000000u), new(0x083u, 0x00000000u)]), // SPI_SHADER_USER_DATA_ADDR_LO_GS, SPI_SHADER_USER_DATA_ADDR_HI_GS + new(1u, 24u, 0x07E383C6u, [new(0x102u, 0x00000000u), new(0x103u, 0x00000000u)]), // SPI_SHADER_USER_DATA_ADDR_LO_HS, SPI_SHADER_USER_DATA_ADDR_HI_HS + new(1u, 25u, 0xBDA98653u, [new(0x240u, 0x00000000u)]), // COMPUTE_USER_DATA_0 + new(1u, 26u, 0xBDBD1D0Fu, [new(0x08Cu, 0x00000000u)]), // SPI_SHADER_USER_DATA_GS_0 + new(1u, 27u, 0xBD946FD4u, [new(0x10Cu, 0x00000000u)]), // SPI_SHADER_USER_DATA_HS_0 + new(1u, 28u, 0xBDF02A4Cu, [new(0x00Cu, 0x00000000u)]), // SPI_SHADER_USER_DATA_PS_0 + // uconfig register groups + new(2u, 0u, 0x19E93E85u, [new(0x41Fu, 0x00000000u)]), // GDS_OA_ADDRESS + new(2u, 1u, 0x3B5C2AF3u, [new(0x41Du, 0x00000000u)]), // GDS_OA_CNTL + new(2u, 2u, 0x47974A35u, [new(0x41Eu, 0x00000000u)]), // GDS_OA_COUNTER + new(2u, 3u, 0x105971C2u, [new(0x25Bu, 0x00000000u)]), // GE_CNTL + new(2u, 4u, 0x7D137765u, [new(0x24Au, 0x00000000u)]), // GE_INDX_OFFSET + new(2u, 5u, 0xD187FEBCu, [new(0x24Bu, 0x00000000u)]), // GE_MULTI_PRIM_IB_RESET_EN + new(2u, 6u, 0x12F854ACu, [new(0x25Fu, 0x00000000u)]), // GE_STEREO_CNTL + new(2u, 7u, 0x40D49AD1u, [new(0x262u, 0x00000000u)]), // GE_USER_VGPR_EN + new(2u, 8u, 0x8C0923DAu, [new(0x80003FF4u, 0x00000000u)]), // FSR_EXTEND_SUBPIXEL_ROUNDING + new(2u, 9u, 0xBB8DF494u, [new(0x80003FFDu, 0x00000000u)]), // TEXTURE_GRADIENT_CONTROL + new(2u, 10u, 0xF6D8A76Eu, [new(0x382u, 0x40000040u)]), // TEXTURE_GRADIENT_FACTORS + new(2u, 11u, 0x7620F1E9u, [new(0x248u, 0x00000000u)]), // VGT_OBJECT_ID + new(2u, 12u, 0x9EBFAB10u, [new(0x242u, 0x00000000u)]), // VGT_PRIMITIVE_TYPE + new(2u, 13u, 0x98A09D0Eu, [new(0x380u, 0x00000000u), new(0x381u, 0x00000000u)]), // TA_CS_BC_BASE_ADDR, TA_CS_BC_BASE_ADDR_HI + new(2u, 14u, 0x195D37D2u, [new(0x80003FF5u, 0x00000000u), new(0x80003FF6u, 0x00000000u)]), // FSR_ALPHA_VALUE0, FSR_ALPHA_VALUE1 + new(2u, 15u, 0xF9EC4F85u, [new(0x80003FF7u, 0x00000000u), new(0x80003FF8u, 0x00000000u), new(0x80003FF9u, 0x00000000u), new(0x80003FFAu, 0x00000000u)]), // FSR_CONTROL_POINT0, FSR_CONTROL_POINT1, FSR_CONTROL_POINT2, FSR_CONTROL_POINT3 + new(2u, 16u, 0x4626B750u, [new(0x80003FFBu, 0x00000000u), new(0x80003FFCu, 0x00000000u)]), // FSR_WINDOW0, FSR_WINDOW1 + new(2u, 17u, 0x4CC673A0u, [new(0x80003FFEu, 0x00000000u)]), // MEMORY_MAPPING_MASK + new(2u, 18u, 0xDE5B3431u, [new(0x80003FFFu, 0x00000000u)]), // UC_NOP + new(2u, 19u, 0x036AC8A6u, [new(0x25Cu, 0x00000000u)]), // GE_USER_VGPR1 + ]; +} diff --git a/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs b/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs index aa824cc..d6dedbc 100644 --- a/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs +++ b/src/SharpEmu.Libs/Agc/AgcShaderCompilerHooks.cs @@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using SharpEmu.Libs.Kernel; +using SharpEmu.Libs.VideoOut; using SharpEmu.ShaderCompiler; namespace SharpEmu.Libs.Agc; @@ -26,5 +27,7 @@ internal static class AgcShaderCompilerHooks { Gen5ShaderScalarEvaluator.FallbackMemoryReader = KernelMemoryCompatExports.TryReadTrackedLibcHeap; + Gen5ShaderScalarEvaluator.GlobalMemoryPool = + VulkanVideoPresenter.GuestDataPool; } } diff --git a/src/SharpEmu.Libs/Agc/GnmTiling.cs b/src/SharpEmu.Libs/Agc/GnmTiling.cs new file mode 100644 index 0000000..2fe0c00 --- /dev/null +++ b/src/SharpEmu.Libs/Agc/GnmTiling.cs @@ -0,0 +1,486 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.Libs.Agc; + +/// +/// Deswizzles RDNA2 (GFX10) tiled texture surfaces into linear layout so they +/// can be uploaded to Vulkan. PS5 stores most textures in a swizzled layout +/// selected by the 5-bit SWIZZLE_MODE in the image descriptor; uploading those +/// bytes verbatim samples as garbage. +/// +/// The GFX10 addressing uses power-of-two swizzle blocks (256 B / 4 KiB / +/// 64 KiB) whose internal element order follows the standard (S), display (D), +/// render (R), or z-order/depth (Z) equations. This implements the exact base +/// S and Z 2D single-sample modes plus the PS5's RB+ 64 KiB Z_X/R_X equations; +/// other D/R and pipe/bank-XOR modes stay opt-in while their complete AddrLib +/// equations are being ported. +/// +internal static class GnmTiling +{ + // Oberon uses the 16-pipe / 8-pixel-packer RB+ topology. These are the + // single-sample 64 KiB equations generated by AMD AddrLib for that exact + // topology. Each entry describes one address bit as an XOR of X/Y bits. + private readonly record struct AddressBit(uint XMask, uint YMask); + + private static readonly AddressBit[][] RbPlus64KRenderX = + [ + // 1 byte/element: nibble01=0, nibble2=307, nibble3=379. + [X(0), X(1), X(2), X(3), Y(0), Y(1), Y(2), Y(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(6), Y(6), XY(7, 8), XY(8, 7)], + // 2 bytes/element: nibble01=1, nibble2=307, nibble3=389. + [Zero, X(0), X(1), X(2), Y(0), Y(1), Y(2), X(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(3), X(6), XY(7, 7), XY(8, 6)], + // 4 bytes/element: nibble01=39, nibble2=307, nibble3=381. + [Zero, Zero, X(0), X(1), Y(0), Y(1), X(2), Y(2), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(3), Y(3), XY(6, 7), XY(7, 6)], + // 8 bytes/element: nibble01=6, nibble2=307, nibble3=382. + [Zero, Zero, Zero, X(0), Y(0), X(1), X(2), Y(1), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(2), X(3), XY(7, 3), XY(6, 6)], + // 16 bytes/element: nibble01=7, nibble2=307, nibble3=390. + [Zero, Zero, Zero, Zero, X(0), Y(0), X(1), Y(1), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(2), Y(2), XY(6, 3), XY(3, 6)], + ]; + + private static readonly AddressBit[][] RbPlus64KDepthX = + [ + // 1 byte/element: nibble01=8, nibble2=306, nibble3=379. + [X(0), Y(0), X(1), Y(1), X(2), Y(2), X(3), Y(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(6), Y(6), XY(7, 8), XY(8, 7)], + // 2 bytes/element: nibble01=9, nibble2=306, nibble3=389. + [Zero, X(0), Y(0), X(1), Y(1), X(2), Y(2), X(3), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(3), X(6), XY(7, 7), XY(8, 6)], + // 4 bytes/element: nibble01=10, nibble2=306, nibble3=381. + [Zero, Zero, X(0), Y(0), X(1), Y(1), X(2), Y(2), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(3), Y(3), XY(6, 7), XY(7, 6)], + // 8 bytes/element: nibble01=11, nibble2=307, nibble3=382. + [Zero, Zero, Zero, X(0), Y(0), X(1), Y(1), X(2), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + Y(2), X(3), XY(7, 3), XY(6, 6)], + // 16 bytes/element is identical to R_X for a 2D single-sample image. + [Zero, Zero, Zero, Zero, X(0), Y(0), X(1), Y(1), + XY(7, 4, 7), XY(4, 4), XY(6, 5), XY(5, 6), + X(2), Y(2), XY(6, 3), XY(3, 6)], + ]; + + private static readonly AddressBit[][] RbPlus64KStandard = + [ + // GFX10_SW_64K_S_RBPLUS_PATINFO, 1 byte/element. + [X(0), X(1), X(2), X(3), Y(0), Y(1), Y(2), Y(3), + Y(4), X(4), Y(5), X(5), Y(6), X(6), Y(7), X(7)], + // 2 bytes/element. + [Zero, X(0), X(1), X(2), Y(0), Y(1), Y(2), X(3), + Y(3), X(4), Y(4), X(5), Y(5), X(6), Y(6), X(7)], + // 4 bytes/element. + [Zero, Zero, X(0), X(1), Y(0), Y(1), Y(2), X(2), + Y(3), X(3), Y(4), X(4), Y(5), X(5), Y(6), X(6)], + // 8 bytes/element (also BC1/BC4 compressed blocks). + [Zero, Zero, Zero, X(0), Y(0), Y(1), X(1), X(2), + Y(2), X(3), Y(3), X(4), Y(4), X(5), Y(5), X(6)], + // 16 bytes/element (also 16-byte BC compressed blocks). + [Zero, Zero, Zero, Zero, Y(0), Y(1), X(0), X(1), + Y(2), X(2), Y(3), X(3), Y(4), X(4), Y(5), X(5)], + ]; + + // GFX10 4K_S has a separate 12-bit micro-tile equation. It is not the + // generic x/y interleave used by the 64K standard block; using that larger + // equation leaves a regular grid in linearized atlases. + private static readonly AddressBit[][] Standard4K = + [ + [X(0), X(1), X(2), X(3), Y(0), Y(1), Y(2), Y(3), + Y(4), X(4), Y(5), X(5)], + [Zero, X(0), X(1), X(2), Y(0), Y(1), Y(2), X(3), + Y(3), X(4), Y(4), X(5)], + [Zero, Zero, X(0), X(1), Y(0), Y(1), Y(2), X(2), + Y(3), X(3), Y(4), X(4)], + [Zero, Zero, Zero, X(0), Y(0), Y(1), X(1), X(2), + Y(2), X(3), Y(3), X(4)], + [Zero, Zero, Zero, Zero, Y(0), Y(1), X(0), X(1), + Y(2), X(2), Y(3), X(3)], + ]; + + private static readonly bool _enabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DETILE"), + "1", + StringComparison.Ordinal); + + private static readonly bool _disabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DETILE"), + "0", + StringComparison.Ordinal); + + private static readonly HashSet _reportedModes = new(); + + public static bool Enabled => _enabled || !_disabled; + + /// + /// Base S/Z modes and the Oberon RB+ 64 KiB Z_X/R_X modes for which this + /// implementation carries the exact AddrLib address equations. Other + /// bank/pipe-XOR variants remain opt-in. + /// + private static bool IsTrustedByDefault(uint swizzleMode) => + // Exact base S/Z modes. D/R use different GFX10 swizzle equations and + // the T/X modes additionally apply pipe/bank XOR between blocks. + swizzleMode is 1 or 4 or 5 or 8 or 9 or 24 or 27; + + // Detile a surface when it is verified-correct by default (trusted base mode), + // or when the user opts the approximate modes in with SHARPEMU_DETILE=1. + // SHARPEMU_DETILE=0 forces the old raw-upload behavior for everything. + private static bool ShouldDetile(uint swizzleMode) => + swizzleMode != 0 && !_disabled && (_enabled || IsTrustedByDefault(swizzleMode)); + + /// + /// True when a surface with the given swizzle mode needs deswizzling. + /// Mode 0 is linear and never needs it. + /// + public static bool NeedsDetile(uint swizzleMode) => ShouldDetile(swizzleMode); + + /// + /// Gets the physical byte span occupied by a tiled mip. GNM allocates whole + /// swizzle blocks even when the logical image is much smaller than a block; + /// reading only the logical texel count truncates most source offsets during + /// detiling (a 64x64 BC1 mode-9 image is 2 KiB logically but occupies one + /// 64 KiB swizzle block). + /// + public static bool TryGetTiledByteCount( + uint swizzleMode, + int elementsWide, + int elementsHigh, + int bytesPerElement, + out ulong byteCount) + { + byteCount = 0; + if (!ShouldDetile(swizzleMode) || + elementsWide <= 0 || + elementsHigh <= 0 || + bytesPerElement <= 0 || + !TryGetSwizzleKind(swizzleMode, out _, out var blockBytes)) + { + return false; + } + + var bppLog2 = BitLog2((uint)bytesPerElement); + if (bppLog2 < 0) + { + return false; + } + + var blockElements = blockBytes >> bppLog2; + var (blockWidth, blockHeight) = SquareBlockDimensions(blockElements); + if (blockWidth == 0 || blockHeight == 0) + { + return false; + } + + var blocksWide = ((ulong)elementsWide + (ulong)blockWidth - 1) / (ulong)blockWidth; + var blocksHigh = ((ulong)elementsHigh + (ulong)blockHeight - 1) / (ulong)blockHeight; + try + { + byteCount = checked(blocksWide * blocksHigh * (ulong)blockBytes); + return true; + } + catch (OverflowException) + { + byteCount = 0; + return false; + } + } + + /// + /// Deswizzles into linear row-major order. + /// Elements are pixels for uncompressed formats and 4x4 blocks for + /// block-compressed formats, so callers pass the element grid dimensions + /// and the bytes per element. Returns false (leaving output untouched) for + /// unsupported swizzle modes so the caller can fall back to the raw bytes. + /// + public static bool TryDetile( + ReadOnlySpan tiled, + Span linear, + uint swizzleMode, + int elementsWide, + int elementsHigh, + int bytesPerElement) + { + if (!ShouldDetile(swizzleMode) || elementsWide <= 0 || elementsHigh <= 0 || bytesPerElement <= 0) + { + return false; + } + + if (!TryGetSwizzleKind(swizzleMode, out var kind, out var blockBytes)) + { + ReportUnsupported(swizzleMode); + return false; + } + + var bppLog2 = BitLog2((uint)bytesPerElement); + if (bppLog2 < 0) + { + // Non-power-of-two element size (e.g. 24-bit) is not swizzled in a + // way this equation models. + ReportUnsupported(swizzleMode); + return false; + } + + // Block dimensions in elements: a swizzle block holds blockBytes bytes, + // laid out as a square-ish power-of-two element grid scaled by bpp. + var blockElements = blockBytes >> bppLog2; + var (blockWidth, blockHeight) = SquareBlockDimensions(blockElements); + if (blockWidth == 0 || blockHeight == 0) + { + ReportUnsupported(swizzleMode); + return false; + } + + var blocksPerRow = (elementsWide + blockWidth - 1) / blockWidth; + var requiredLinear = (long)elementsWide * elementsHigh * bytesPerElement; + if (linear.Length < requiredLinear) + { + return false; + } + + // Precompute the within-block element offset for each (x, y) inside a + // single block. The swizzle equation only depends on the in-block + // coordinates, so this table is reused for every block — turning the + // per-pixel bit-interleave (a loop + calls) into a single array lookup. + // Detiling a 2048x2048 texture is millions of elements; without this the + // per-pixel math makes DETILE unusably slow during asset streaming. + var hasExactXorPattern = TryGetExactXorPattern(swizzleMode, bppLog2, out var xorPattern); + var blockTable = hasExactXorPattern ? [] : new int[blockWidth * blockHeight]; + for (var by = 0; !hasExactXorPattern && by < blockHeight; by++) + { + for (var bx = 0; bx < blockWidth; bx++) + { + blockTable[by * blockWidth + bx] = (int)(kind == SwizzleKind.ZOrder + ? MortonInterleave((uint)bx, (uint)by, blockWidth, blockHeight) + : StandardSwizzleOffset((uint)bx, (uint)by, blockWidth, blockHeight)); + } + } + + for (var y = 0; y < elementsHigh; y++) + { + var blockY = y / blockHeight; + var inBlockY = y % blockHeight; + var rowBlockBase = (long)blockY * blocksPerRow; + var tableRowBase = inBlockY * blockWidth; + var destRowBase = (long)y * elementsWide * bytesPerElement; + for (var x = 0; x < elementsWide; x++) + { + var blockX = x / blockWidth; + var inBlockX = x % blockWidth; + + var blockIndex = rowBlockBase + blockX; + var sourceByte = hasExactXorPattern + ? blockIndex * blockBytes + ComputePatternOffset((uint)x, (uint)y, xorPattern) + : (blockIndex * blockElements + blockTable[tableRowBase + inBlockX]) * + (long)bytesPerElement; + var destByte = destRowBase + (long)x * bytesPerElement; + if (sourceByte + bytesPerElement > tiled.Length || + destByte + bytesPerElement > linear.Length) + { + continue; + } + + tiled.Slice((int)sourceByte, bytesPerElement) + .CopyTo(linear.Slice((int)destByte, bytesPerElement)); + } + } + + return true; + } + + private enum SwizzleKind + { + Standard, + ZOrder, + } + + private static readonly AddressBit Zero = new(0, 0); + + private static AddressBit X(int bit) => new(1u << bit, 0); + + private static AddressBit Y(int bit) => new(0, 1u << bit); + + private static AddressBit XY(int xBit, int yBit) => new(1u << xBit, 1u << yBit); + + private static AddressBit XY(int xBit, int yBit1, int yBit2) => + new(1u << xBit, (1u << yBit1) | (1u << yBit2)); + + private static bool TryGetExactXorPattern( + uint swizzleMode, + int bytesPerElementLog2, + out AddressBit[] pattern) + { + pattern = []; + if ((uint)bytesPerElementLog2 >= RbPlus64KRenderX.Length) + { + return false; + } + + pattern = swizzleMode switch + { + 5 => Standard4K[bytesPerElementLog2], + 9 => RbPlus64KStandard[bytesPerElementLog2], + 24 => RbPlus64KDepthX[bytesPerElementLog2], + 27 => RbPlus64KRenderX[bytesPerElementLog2], + _ => [], + }; + return pattern.Length != 0; + } + + private static long ComputePatternOffset(uint x, uint y, AddressBit[] pattern) + { + uint offset = 0; + for (var bit = 0; bit < pattern.Length; bit++) + { + var equation = pattern[bit]; + var parity = (System.Numerics.BitOperations.PopCount(x & equation.XMask) + + System.Numerics.BitOperations.PopCount(y & equation.YMask)) & 1; + offset |= (uint)parity << bit; + } + + return offset; + } + + private static bool TryGetSwizzleKind(uint swizzleMode, out SwizzleKind kind, out int blockBytes) + { + // GFX10 AddrLib SWIZZLE_MODE enumeration: + // 1-3 = 256 B S/D/R + // 4-7 = 4 KiB Z/S/D/R + // 8-11 = 64 KiB Z/S/D/R + // 16-19 = 64 KiB Z/S/D/R _T + // 20-23 = 4 KiB Z/S/D/R _X + // 24-27 = 64 KiB Z/S/D/R _X + // The pipe/bank XOR (_T/_X) affects which block a given tile lands in, + // but the *within-block* element order matches the base S/Z equation, + // which is what dominates visible correctness. We model the block + // interior and treat blocks as linear-ordered. + kind = SwizzleKind.Standard; + blockBytes = 0; + switch (swizzleMode) + { + case 4: case 8: case 16: case 20: case 24: + kind = SwizzleKind.ZOrder; + break; + case 1: case 2: case 3: + case 5: case 6: case 7: + case 9: case 10: case 11: + case 17: case 18: case 19: + case 21: case 22: case 23: + case 25: case 26: case 27: + kind = SwizzleKind.Standard; + break; + default: + return false; + } + + blockBytes = swizzleMode switch + { + >= 1 and <= 3 => 256, + >= 4 and <= 7 => 4096, + >= 20 and <= 23 => 4096, + _ => 65536, + }; + return true; + } + + // Standard (S) 2D swizzle: within a block, the element order interleaves x + // and y bits with x taking the low bit — the AMD "standard" microtile. + private static long StandardSwizzleOffset(uint x, uint y, int blockWidth, int blockHeight) + { + var widthBits = BitLog2((uint)blockWidth); + var heightBits = BitLog2((uint)blockHeight); + long offset = 0; + var outBit = 0; + var xi = 0; + var yi = 0; + while (xi < widthBits || yi < heightBits) + { + if (xi < widthBits) + { + offset |= (long)((x >> xi) & 1u) << outBit++; + xi++; + } + + if (yi < heightBits) + { + offset |= (long)((y >> yi) & 1u) << outBit++; + yi++; + } + } + + return offset; + } + + // Z-order (Morton) swizzle: pure bit interleave, y taking the low bit. + private static long MortonInterleave(uint x, uint y, int blockWidth, int blockHeight) + { + var widthBits = BitLog2((uint)blockWidth); + var heightBits = BitLog2((uint)blockHeight); + long offset = 0; + var outBit = 0; + var xi = 0; + var yi = 0; + while (xi < widthBits || yi < heightBits) + { + if (yi < heightBits) + { + offset |= (long)((y >> yi) & 1u) << outBit++; + yi++; + } + + if (xi < widthBits) + { + offset |= (long)((x >> xi) & 1u) << outBit++; + xi++; + } + } + + return offset; + } + + private static (int Width, int Height) SquareBlockDimensions(int blockElements) + { + if (blockElements <= 0 || (blockElements & (blockElements - 1)) != 0) + { + return (0, 0); + } + + var totalBits = BitLog2((uint)blockElements); + // Split as evenly as possible with width >= height (x gets the extra bit). + var widthBits = (totalBits + 1) / 2; + var heightBits = totalBits - widthBits; + return (1 << widthBits, 1 << heightBits); + } + + private static int BitLog2(uint value) + { + if (value == 0 || (value & (value - 1)) != 0) + { + return -1; + } + + return System.Numerics.BitOperations.TrailingZeroCount(value); + } + + private static void ReportUnsupported(uint swizzleMode) + { + lock (_reportedModes) + { + if (!_reportedModes.Add(swizzleMode)) + { + return; + } + } + + Console.Error.WriteLine( + $"[LOADER][WARN] GNM detile: unsupported swizzle mode {swizzleMode}; texture uploaded linear."); + } +} diff --git a/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs b/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs index 80c4822..dd8b3fe 100644 --- a/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs +++ b/src/SharpEmu.Libs/Agc/GpuWaitRegistry.cs @@ -3,10 +3,17 @@ namespace SharpEmu.Libs.Agc; -// Holds DCBs whose parsing was suspended on an unsatisfied WAIT_REG_MEM condition. -// AgcExports re-checks every waiter against guest memory on each submit and resumes -// the ones whose condition became true (labels are advanced by ReleaseMem/WriteData/ -// DmaData packets or by direct CPU writes). +/// +/// Holds DCBs whose parsing was suspended on an unsatisfied WAIT_REG_MEM +/// condition. AgcExports re-checks every waiter against guest memory on each +/// submit and resumes the ones whose condition became true (labels are advanced +/// by ReleaseMem / WriteData / DmaData packets, or by direct CPU writes). +/// +/// This preserves cross-submit ordering: the work that follows a wait inside a +/// DCB is only queued once the awaited completion label is genuinely written, +/// instead of being force-satisfied at parse time and running ahead of the +/// compute/graphics work it depends on (which produced a black composite). +/// internal static class GpuWaitRegistry { public struct WaitingDcb @@ -19,13 +26,56 @@ internal static class GpuWaitRegistry public ulong ReferenceValue; public ulong Mask; public uint CompareFunction; + public uint ControlValue; public bool Is64Bit; + public bool IsStandard; + public object? Memory; + public string? QueueName; + public ulong SubmissionId; + // Stopwatch timestamp captured at registration. Stale waiters remain + // registered; this only controls one-shot diagnostics. + public long RegisteredTicks; + public bool StaleReported; public object? State; } private static readonly object _gate = new(); private static readonly Dictionary> _waiters = new(); + public static int Count + { + get + { + lock (_gate) + { + var total = 0; + foreach (var (_, list) in _waiters) + { + total += list.Count; + } + + return total; + } + } + } + + public static int CountForMemory(object memory) + { + lock (_gate) + { + var total = 0; + foreach (var (_, list) in _waiters) + { + foreach (var waiter in list) + { + total += ReferenceEquals(waiter.Memory, memory) ? 1 : 0; + } + } + + return total; + } + } + public static void Register(ulong address, WaitingDcb waiter) { waiter.WaitAddress = address; @@ -41,9 +91,15 @@ internal static class GpuWaitRegistry } } - // Re-evaluates every registered waiter. readValue receives (address, is64Bit) and - // returns null when the memory is unreadable; such waiters are kept registered. - public static List? CollectSatisfied(Func readValue) + /// + /// Re-evaluates every registered waiter. + /// receives (address, is64Bit) and returns null when the memory is + /// unreadable; such waiters are kept registered. Returns the waiters whose + /// condition is now satisfied (removed from the registry), or null. + /// + public static List? CollectSatisfied( + object memory, + Func readValue) { List? woken = null; lock (_gate) @@ -53,6 +109,11 @@ internal static class GpuWaitRegistry { for (var i = list.Count - 1; i >= 0; i--) { + if (!ReferenceEquals(list[i].Memory, memory)) + { + continue; + } + var value = readValue(address, list[i].Is64Bit); if (value is null || !Compare(list[i], value.Value)) { @@ -71,7 +132,7 @@ internal static class GpuWaitRegistry } } - if (emptied != null) + if (emptied is not null) { foreach (var address in emptied) { @@ -83,20 +144,122 @@ internal static class GpuWaitRegistry return woken; } + /// + /// Returns waiters that have remained unsatisfied longer than + /// exactly once, without removing them or + /// changing their labels. Missing GPU work must fail closed: advancing a + /// command buffer without its real producer corrupts cross-queue ordering. + /// + public static List? CollectUnreportedStale( + object memory, + long nowTicks, + long maxAgeTicks) + { + List? stale = null; + lock (_gate) + { + foreach (var (_, list) in _waiters) + { + for (var i = list.Count - 1; i >= 0; i--) + { + var waiter = list[i]; + if (!ReferenceEquals(waiter.Memory, memory) || + waiter.StaleReported || + nowTicks - waiter.RegisteredTicks < maxAgeTicks) + { + continue; + } + + stale ??= new List(); + waiter.StaleReported = true; + list[i] = waiter; + stale.Add(waiter); + } + } + } + + return stale; + } + + /// + /// Returns watched labels overlapped by a newly discovered producer. Used + /// only for diagnostics; producer completion still wakes through the + /// normal CollectSatisfied path after the ordered memory write executes. + /// + public static List<(ulong Address, int Count)> SnapshotInRange( + object memory, + ulong start, + ulong length) + { + var matches = new List<(ulong Address, int Count)>(); + if (length == 0) + { + return matches; + } + + var end = start > ulong.MaxValue - length ? ulong.MaxValue : start + length; + lock (_gate) + { + foreach (var (address, list) in _waiters) + { + var matchingCount = 0; + var any64Bit = false; + foreach (var waiter in list) + { + if (!ReferenceEquals(waiter.Memory, memory)) + { + continue; + } + + matchingCount++; + any64Bit |= waiter.Is64Bit; + } + + if (matchingCount == 0) + { + continue; + } + + var width = any64Bit + ? sizeof(ulong) + : sizeof(uint); + var waitEnd = address > ulong.MaxValue - (ulong)width + ? ulong.MaxValue + : address + (ulong)width; + if (start < waitEnd && address < end) + { + matches.Add((address, matchingCount)); + } + } + } + + return matches; + } + public static bool Compare(in WaitingDcb waiter, ulong value) { var masked = value & waiter.Mask; + var reference = waiter.ReferenceValue & waiter.Mask; return waiter.CompareFunction switch { - 1 => masked < waiter.ReferenceValue, - 2 => masked <= waiter.ReferenceValue, - 3 => masked == waiter.ReferenceValue, - 4 => masked != waiter.ReferenceValue, - 5 => masked >= waiter.ReferenceValue, - 6 => masked > waiter.ReferenceValue, - // 0 is "always" in the PM4 encoding and 7 is reserved; treating both as - // satisfied keeps a malformed packet from suspending its DCB forever. + 0 => true, + 1 => masked < reference, + 2 => masked <= reference, + 3 => masked == reference, + 4 => masked != reference, + 5 => masked >= reference, + 6 => masked > reference, + // 7 is reserved; treating it as satisfied keeps a malformed packet + // from suspending forever. _ => true, }; } + + public static void Clear() + { + lock (_gate) + { + _waiters.Clear(); + } + } } diff --git a/src/SharpEmu.Libs/Ajm/AjmExports.cs b/src/SharpEmu.Libs/Ajm/AjmExports.cs index c9b8741..7c1dfd5 100644 --- a/src/SharpEmu.Libs/Ajm/AjmExports.cs +++ b/src/SharpEmu.Libs/Ajm/AjmExports.cs @@ -7,31 +7,11 @@ namespace SharpEmu.Libs.Ajm; public static class AjmExports { - private const int OrbisAjmErrorInvalidParameter = unchecked((int)0x80930005); - private static int _nextContextId; - [SysAbiExport( Nid = "dl+4eHSzUu4", ExportName = "sceAjmInitialize", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAjm")] - public static int Initialize(CpuContext ctx) - { - var reserved = ctx[CpuRegister.Rdi]; - var outContextAddress = ctx[CpuRegister.Rsi]; - - // AJM requires a zero reserved argument. - if (reserved != 0 || outContextAddress == 0) - { - return ctx.SetReturn(OrbisAjmErrorInvalidParameter); - } - - var contextId = unchecked((uint)Interlocked.Increment(ref _nextContextId)); - if (!ctx.TryWriteUInt32(outContextAddress, contextId)) - { - return ctx.SetReturn(OrbisAjmErrorInvalidParameter); - } - - return ctx.SetReturn(0); - } + public static int Initialize(CpuContext ctx) => + SharpEmu.Libs.Audio.AjmExports.AjmInitialize(ctx); } diff --git a/src/SharpEmu.Libs/Ampr/AmprExports.cs b/src/SharpEmu.Libs/Ampr/AmprExports.cs index 0c41b05..0dc0cfe 100644 --- a/src/SharpEmu.Libs/Ampr/AmprExports.cs +++ b/src/SharpEmu.Libs/Ampr/AmprExports.cs @@ -36,6 +36,7 @@ public static class AmprExports public ulong Buffer; public ulong Size; public ulong WriteOffset; + public ulong CommandCount; } private sealed class CachedHostFile @@ -267,53 +268,19 @@ public static class AmprExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - ulong bytesRead = 0; - - // Unregistered/missing files are zero-filled instead of failing: games queue - // speculative reads and only consume the bytes on success paths. - if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath) || !File.Exists(hostPath)) + if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath)) { - if (destination != 0 && size > 0) - { - int chunkSize = (int)Math.Min(size, 4096); - Span zeros = stackalloc byte[chunkSize]; - zeros.Clear(); - while (bytesRead < size) - { - int currentChunk = (int)Math.Min((ulong)chunkSize, size - bytesRead); - if (!ctx.Memory.TryWrite(destination + bytesRead, zeros[..currentChunk])) - { - break; - } - - bytesRead += (ulong)currentChunk; - } - } - - TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead, "(missing)", (int)OrbisGen2Result.ORBIS_GEN2_OK); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead: 0, hostPath, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - // Offset -1 means "continue after the previous read of this file id". - if (fileOffset == unchecked((ulong)(long)-1)) - { - fileOffset = PakDirectoryTracker.ResolveSequentialOffset(fileId, size); - } - else if (fileOffset > long.MaxValue) - { - fileOffset = 0; - } - - var result = TryReadFileToGuestMemory(ctx, hostPath, fileOffset, destination, size, out bytesRead); + var result = TryReadFileToGuestMemory(ctx, hostPath, fileOffset, destination, size, out var bytesRead); if (result != (int)OrbisGen2Result.ORBIS_GEN2_OK) { TraceAmprRead(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead, hostPath, result); return result; } - PakDirectoryTracker.OnReadCompleted(ctx, fileId, destination, fileOffset, bytesRead); - if (!AppendReadFileRecord(ctx, commandBuffer, fileId, destination, size, fileOffset, bytesRead)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; @@ -418,6 +385,35 @@ public static class AmprExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + [SysAbiExport( + Nid = "gzndltBEzWc", + ExportName = "sceAmprCommandBufferGetNumCommands", + Target = Generation.Gen5, + LibraryName = "libSceAmpr")] + public static int CommandBufferGetNumCommands(CpuContext ctx) + { + var commandBuffer = ctx[CpuRegister.Rdi]; + if (commandBuffer == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryGetCommandBufferState(ctx, commandBuffer, out _, out _, out var state) || state is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ulong commandCount; + lock (state) + { + commandCount = state.CommandCount; + } + + TraceAmpr(ctx, "get_num_commands", commandBuffer, commandCount, 0); + ctx[CpuRegister.Rax] = commandCount; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "H896Pt-yB4I", ExportName = "sceAmprCommandBufferWriteKernelEventQueue_04_00", @@ -533,7 +529,7 @@ public static class AmprExports var offset = 0UL; while (offset < writeOffset) { - if (!ctx.TryReadUInt32(buffer + offset, out var recordType)) + if (!TryReadUInt32(ctx, buffer + offset, out var recordType)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -649,6 +645,7 @@ public static class AmprExports state.Buffer = buffer; state.Size = size; state.WriteOffset = writeOffset; + state.CommandCount = 0; } } @@ -681,6 +678,7 @@ public static class AmprExports state.Buffer = buffer; state.Size = size; state.WriteOffset = 0; + state.CommandCount = 0; } return true; @@ -911,6 +909,7 @@ public static class AmprExports } state.WriteOffset += recordSize; + state.CommandCount++; } return true; @@ -963,6 +962,19 @@ public static class AmprExports return true; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + private static void TraceAmpr(CpuContext ctx, string operation, ulong commandBuffer, ulong arg0, ulong arg1) { if (!_traceAmpr) diff --git a/src/SharpEmu.Libs/Audio/AjmExports.cs b/src/SharpEmu.Libs/Audio/AjmExports.cs new file mode 100644 index 0000000..7ac0a1a --- /dev/null +++ b/src/SharpEmu.Libs/Audio/AjmExports.cs @@ -0,0 +1,104 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Threading; + +namespace SharpEmu.Libs.Audio; + +public static class AjmExports +{ + private static readonly ConcurrentDictionary Contexts = new(); + private static int _nextContextId; + public static int AjmInitialize(CpuContext ctx) + { + var reserved = ctx[CpuRegister.Rdi]; + var outputAddress = ctx[CpuRegister.Rsi]; + if (reserved != 0 || outputAddress == 0) + { + return unchecked((int)0x806A0001); + } + + var contextId = unchecked((uint)Interlocked.Increment(ref _nextContextId)); + Span value = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(value, contextId); + if (!ctx.Memory.TryWrite(outputAddress, value)) + { + return unchecked((int)0x806A0001); + } + + Contexts[contextId] = 0; + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] ajm.initialize reserved={reserved} out=0x{outputAddress:X16} context={contextId}"); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "MHur6qCsUus", + ExportName = "sceAjmFinalize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmFinalize(CpuContext ctx) + { + Contexts.TryRemove(unchecked((uint)ctx[CpuRegister.Rdi]), out _); + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "Q3dyFuwGn64", + ExportName = "sceAjmModuleRegister", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmModuleRegister(CpuContext ctx) + { + var contextId = unchecked((uint)ctx[CpuRegister.Rdi]); + var codecType = unchecked((uint)ctx[CpuRegister.Rsi]); + var reserved = ctx[CpuRegister.Rdx]; + if (reserved != 0 || !Contexts.ContainsKey(contextId)) + { + return unchecked((int)0x806A0001); + } + + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AJM"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] ajm.module_register context={contextId} codec={codecType} reserved={reserved}"); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "Wi7DtlLV+KI", + ExportName = "sceAjmModuleUnregister", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmModuleUnregister(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "MmpF1XsQiHw", + ExportName = "sceAjmBatchInitialize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAjm")] + public static int AjmBatchInitialize(CpuContext ctx) + { + // The caller owns and initializes the batch storage. This API resets + // its submission cursor on hardware; FMOD does not consume a return + // value or an additional output object here. + ctx[CpuRegister.Rax] = 0; + return 0; + } +} diff --git a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs index 730e3e9..18b3751 100644 --- a/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs +++ b/src/SharpEmu.Libs/Audio/AudioOut2Exports.cs @@ -3,22 +3,70 @@ using SharpEmu.HLE; using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Threading; namespace SharpEmu.Libs.Audio; public static class AudioOut2Exports { - // Sized from guest evidence, not SDK headers: Quake keeps its - // SceAudioOut2ContextParam on the stack with the frame canary at param+0x60, - // and an earlier 0x80-byte ResetParam write zeroed that canary - // (__stack_chk_fail right after sceAudioOut2UserCreate, which silently killed - // the whole audio init). Stay well below 0x60 and only write the prefix we - // populate. - private const int AudioOut2ContextParamSize = 0x30; + // FMOD's PS5 backend allocates this ABI structure as four 16-byte lanes. + // Clearing 0x80 bytes here overwrote the caller's stack canary immediately + // following the 0x40-byte parameter block. + private const int AudioOut2ContextParamSize = 0x40; private const int AudioOut2ContextMemorySize = 0x10000; + private const int AudioOut2ContextMemoryAlignment = 0x10000; private static long _nextContextHandle = 1; private static long _nextUserHandle = 1; private static int _nextPortId; + private static long _pushTraceCount; + + // Per-context audio parameters captured at ContextCreate so ContextAdvance + // can pace to the real playback cadence (grain samples at the sample rate). + private static readonly ConcurrentDictionary Contexts = new(); + + private sealed class ContextState + { + private readonly object _paceGate = new(); + private long _nextAdvanceTimestamp; + + public ContextState(uint frequency, uint channels, uint grainSamples) + { + Frequency = frequency == 0 ? 48000 : frequency; + Channels = channels == 0 ? 2 : channels; + GrainSamples = grainSamples == 0 ? 256 : grainSamples; + } + + public uint Frequency { get; } + public uint Channels { get; } + public uint GrainSamples { get; } + + // Blocks the advancing thread until one grain worth of wall-clock time + // has elapsed since the previous advance, matching hardware timing so + // audio-gated titles neither spin nor drift ahead. + public void PaceAdvance() + { + long delay; + lock (_paceGate) + { + var now = Stopwatch.GetTimestamp(); + if (_nextAdvanceTimestamp < now) + { + _nextAdvanceTimestamp = now; + } + + delay = _nextAdvanceTimestamp - now; + _nextAdvanceTimestamp += checked( + (long)Math.Ceiling(Stopwatch.Frequency * (double)GrainSamples / Frequency)); + } + + if (delay > 0) + { + Thread.Sleep(TimeSpan.FromSeconds((double)delay / Stopwatch.Frequency)); + } + } + } [SysAbiExport( Nid = "g2tViFIohHE", @@ -41,7 +89,7 @@ public static class AudioOut2Exports var paramAddress = ctx[CpuRegister.Rdi]; if (paramAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } Span param = stackalloc byte[AudioOut2ContextParamSize]; @@ -52,8 +100,8 @@ public static class AudioOut2Exports BinaryPrimitives.WriteUInt32LittleEndian(param[0x0C..], 0x400); return ctx.Memory.TryWrite(paramAddress, param) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -64,15 +112,22 @@ public static class AudioOut2Exports public static int AudioOut2ContextQueryMemory(CpuContext ctx) { var paramAddress = ctx[CpuRegister.Rdi]; - var outMemorySizeAddress = ctx[CpuRegister.Rsi]; - if (paramAddress == 0 || outMemorySizeAddress == 0) + var memoryInfoAddress = ctx[CpuRegister.Rsi]; + if (paramAddress == 0 || memoryInfoAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - - return ctx.TryWriteUInt64(outMemorySizeAddress, AudioOut2ContextMemorySize) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + + Span memoryInfo = stackalloc byte[0x20]; + memoryInfo.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x00..], AudioOut2ContextMemorySize); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x08..], AudioOut2ContextMemoryAlignment); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x10..], AudioOut2ContextMemorySize); + BinaryPrimitives.WriteUInt64LittleEndian(memoryInfo[0x18..], AudioOut2ContextMemoryAlignment); + + return ctx.Memory.TryWrite(memoryInfoAddress, memoryInfo) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -88,13 +143,34 @@ public static class AudioOut2Exports var outContextAddress = ctx[CpuRegister.Rcx]; if (paramAddress == 0 || memoryAddress == 0 || memorySize == 0 || outContextAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + // Read channels/frequency/grain from the reset-param blob so the + // context can pace advances to the real audio cadence. + uint channels = 2; + uint frequency = 48000; + uint grain = 256; + Span param = stackalloc byte[AudioOut2ContextParamSize]; + if (ctx.Memory.TryRead(paramAddress, param)) + { + var pc = BinaryPrimitives.ReadUInt32LittleEndian(param[0x04..]); + var pf = BinaryPrimitives.ReadUInt32LittleEndian(param[0x08..]); + var pg = BinaryPrimitives.ReadUInt32LittleEndian(param[0x0C..]); + if (pc is > 0 and <= 8) channels = pc; + if (pf is >= 8000 and <= 192000) frequency = pf; + // Values below one cache line are flags/counts in observed PS5 + // callers, not audio grains. Keep the hardware-sized default. + if (pg is >= 64 and <= 0x4000) grain = pg; + TraceAudioOut2($"context-param address=0x{paramAddress:X} bytes={Convert.ToHexString(param)}"); } var handle = (ulong)Interlocked.Increment(ref _nextContextHandle); - return ctx.TryWriteUInt64(outContextAddress, handle) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + Contexts[handle] = new ContextState(frequency, channels, grain); + TraceAudioOut2($"context-create handle=0x{handle:X} frequency={frequency} channels={channels} grain={grain} memory=0x{memoryAddress:X} size=0x{memorySize:X}"); + return TryWriteUInt64(ctx, outContextAddress, handle) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -102,7 +178,77 @@ public static class AudioOut2Exports ExportName = "sceAudioOut2ContextDestroy", Target = Generation.Gen5, LibraryName = "libSceAudioOut2")] - public static int AudioOut2ContextDestroy(CpuContext ctx) => ctx.SetReturn(0); + public static int AudioOut2ContextDestroy(CpuContext ctx) + { + Contexts.TryRemove(ctx[CpuRegister.Rdi], out _); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "DxGyV8dtOR8", + ExportName = "sceAudioOut2ContextBedWrite", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextBedWrite(CpuContext ctx) => SetReturn(ctx, 0); + + [SysAbiExport( + Nid = "aII9h5nli9U", + ExportName = "sceAudioOut2ContextPush", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextPush(CpuContext ctx) + { + var handle = ctx[CpuRegister.Rdi]; + var traceCount = Interlocked.Increment(ref _pushTraceCount); + if (traceCount <= 16) + { + TraceAudioOut2($"context-push count={traceCount} rdi=0x{handle:X} rsi=0x{ctx[CpuRegister.Rsi]:X} rdx=0x{ctx[CpuRegister.Rdx]:X} rcx=0x{ctx[CpuRegister.Rcx]:X}"); + } + + if (Contexts.TryGetValue(handle, out var context)) + { + // FMOD's PS5 output path uses ContextPush as the submission clock + // and does not call ContextAdvance. Pace pushes to one hardware + // grain so the feeder cannot outrun playback and starve the game. + context.PaceAdvance(); + } + + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "PE2zHMqLSHs", + ExportName = "sceAudioOut2ContextAdvance", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextAdvance(CpuContext ctx) + { + // Advancing renders one grain of audio on hardware; pace it to the same + // wall-clock cadence so the guest audio thread runs at the right speed. + if (Contexts.TryGetValue(ctx[CpuRegister.Rdi], out var context)) + { + context.PaceAdvance(); + } + + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "R7d0F1g2qsU", + ExportName = "sceAudioOut2ContextGetQueueLevel", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2ContextGetQueueLevel(CpuContext ctx) + { + // The advance path paces synchronously, so the queue is always drained. + var levelAddress = ctx[CpuRegister.Rsi]; + if (levelAddress != 0) + { + _ = TryWriteUInt64(ctx, levelAddress, 0); + } + + return SetReturn(ctx, 0); + } [SysAbiExport( Nid = "JK2wamZPzwM", @@ -117,16 +263,23 @@ public static class AudioOut2Exports var contextAddress = ctx[CpuRegister.Rcx]; if (type < 0 || type > 255 || paramAddress == 0 || outPortAddress == 0 || contextAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var portId = unchecked((uint)Interlocked.Increment(ref _nextPortId)) & 0xFF; var handle = 0x2000_0000UL | ((ulong)(uint)type << 16) | portId; - return ctx.TryWriteUInt64(outPortAddress, handle) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return TryWriteUInt64(ctx, outPortAddress, handle) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "8XTArSPyWHk", + ExportName = "sceAudioOut2PortSetAttributes", + Target = Generation.Gen5, + LibraryName = "libSceAudioOut2")] + public static int AudioOut2PortSetAttributes(CpuContext ctx) => SetReturn(ctx, 0); + [SysAbiExport( Nid = "gatEUKG+Ea4", ExportName = "sceAudioOut2PortGetState", @@ -138,7 +291,7 @@ public static class AudioOut2Exports var stateAddress = ctx[CpuRegister.Rsi]; if (handle == 0 || stateAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var type = (int)((handle >> 16) & 0xFF); @@ -151,8 +304,8 @@ public static class AudioOut2Exports BinaryPrimitives.WriteInt16LittleEndian(state[0x04..], -1); return ctx.Memory.TryWrite(stateAddress, state) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -165,7 +318,7 @@ public static class AudioOut2Exports var infoAddress = ctx[CpuRegister.Rdi]; if (infoAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } Span info = stackalloc byte[0x40]; @@ -175,8 +328,8 @@ public static class AudioOut2Exports BinaryPrimitives.WriteUInt32LittleEndian(info[0x08..], 48000); return ctx.Memory.TryWrite(infoAddress, info) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -184,14 +337,14 @@ public static class AudioOut2Exports ExportName = "sceAudioOut2PortDestroy", Target = Generation.Gen5, LibraryName = "libSceAudioOut2")] - public static int AudioOut2PortDestroy(CpuContext ctx) => ctx.SetReturn(0); + public static int AudioOut2PortDestroy(CpuContext ctx) => SetReturn(ctx, 0); [SysAbiExport( Nid = "IaZXJ9M79uo", ExportName = "sceAudioOut2UserDestroy", Target = Generation.Gen5, LibraryName = "libSceAudioOut2")] - public static int AudioOut2UserDestroy(CpuContext ctx) => ctx.SetReturn(0); + public static int AudioOut2UserDestroy(CpuContext ctx) => SetReturn(ctx, 0); [SysAbiExport( Nid = "xywYcRB7nbQ", @@ -202,14 +355,36 @@ public static class AudioOut2Exports { var userId = unchecked((int)ctx[CpuRegister.Rdi]); var outUserAddress = ctx[CpuRegister.Rsi]; - if ((userId != 0 && userId != 1 && userId != 1000 && userId != 255) || outUserAddress == 0) + if ((userId != 0 && userId != 1 && userId != 1000 && userId != 0x10000000 && userId != 255) || + outUserAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var handle = (ulong)Interlocked.Increment(ref _nextUserHandle); - return ctx.TryWriteUInt64(outUserAddress, handle) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return TryWriteUInt64(ctx, outUserAddress, handle) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + private static bool TryWriteUInt64(CpuContext ctx, ulong address, ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + + private static void TraceAudioOut2(string message) + { + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AUDIO_OUT2"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine($"[LOADER][TRACE] audio_out2.{message}"); + } } } diff --git a/src/SharpEmu.Libs/Audio/AudioOutExports.cs b/src/SharpEmu.Libs/Audio/AudioOutExports.cs index a044e58..a736b8c 100644 --- a/src/SharpEmu.Libs/Audio/AudioOutExports.cs +++ b/src/SharpEmu.Libs/Audio/AudioOutExports.cs @@ -50,6 +50,7 @@ public static class AudioOutExports public int BytesPerSample { get; } public bool IsFloat { get; } public IHostAudioStream? Backend { get; } + public volatile float Volume = 1.0f; public int BufferByteLength => checked((int)BufferLength * Channels * BytesPerSample); @@ -198,7 +199,8 @@ public static class AudioOutExports checked((int)port.BufferLength), port.Channels, port.BytesPerSample, - port.IsFloat); + port.IsFloat, + port.Volume); if (!port.Backend.Submit(output.AsSpan(0, outputLength))) { port.PaceSilence(); @@ -225,10 +227,43 @@ public static class AudioOutExports public static int AudioOutSetVolume(CpuContext ctx) { var handle = unchecked((int)ctx[CpuRegister.Rdi]); - return ctx.SetReturn( - Ports.ContainsKey(handle) - ? 0 - : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + var channelFlags = unchecked((uint)ctx[CpuRegister.Rsi]); + var volumeArrayAddress = ctx[CpuRegister.Rdx]; + if (!Ports.TryGetValue(handle, out var port)) + { + return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + const int unityVolume = 32768; + var maxVolume = 0; + var found = false; + if (volumeArrayAddress != 0) + { + Span raw = stackalloc byte[sizeof(int)]; + for (var channel = 0; channel < 8; channel++) + { + if ((channelFlags & (1u << channel)) == 0) + { + continue; + } + + if (!ctx.Memory.TryRead(volumeArrayAddress + (ulong)(channel * sizeof(int)), raw)) + { + return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + var value = System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(raw); + maxVolume = Math.Max(maxVolume, value); + found = true; + } + } + + if (found) + { + port.Volume = Math.Clamp(maxVolume / (float)unityVolume, 0f, 1f); + } + + return ctx.SetReturn(0); } public static void ShutdownAllPorts() diff --git a/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs b/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs index a15ce19..6e65b4c 100644 --- a/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs +++ b/src/SharpEmu.Libs/Audio/AudioPcmConversion.cs @@ -21,7 +21,8 @@ internal static class AudioPcmConversion int frames, int channels, int bytesPerSample, - bool isFloat) + bool isFloat, + float volume) { var sourceFrameSize = checked(channels * bytesPerSample); for (var frame = 0; frame < frames; frame++) @@ -31,6 +32,8 @@ internal static class AudioPcmConversion var right = channels == 1 ? left : ReadSample(sourceFrame, 1, bytesPerSample, isFloat); + left = ApplyVolume(left, volume); + right = ApplyVolume(right, volume); BinaryPrimitives.WriteInt16LittleEndian(destination[(frame * OutputFrameSize)..], left); BinaryPrimitives.WriteInt16LittleEndian(destination[((frame * OutputFrameSize) + 2)..], right); } @@ -52,4 +55,10 @@ internal static class AudioPcmConversion var value = Math.Clamp(BitConverter.Int32BitsToSingle(bits), -1.0f, 1.0f); return checked((short)MathF.Round(value * short.MaxValue)); } + + private static short ApplyVolume(short sample, float volume) + { + var scaled = MathF.Round(sample * Math.Clamp(volume, 0.0f, 1.0f)); + return (short)Math.Clamp(scaled, short.MinValue, short.MaxValue); + } } diff --git a/src/SharpEmu.Libs/Audio/FmodCompatExports.cs b/src/SharpEmu.Libs/Audio/FmodCompatExports.cs new file mode 100644 index 0000000..35e65fc --- /dev/null +++ b/src/SharpEmu.Libs/Audio/FmodCompatExports.cs @@ -0,0 +1,47 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Buffers.Binary; +using System.Collections.Concurrent; + +namespace SharpEmu.Libs.Audio; + +public static class FmodCompatExports +{ + private static readonly ConcurrentDictionary SetOutputCalls = new(); + + [SysAbiExport( + Nid = "uPLTdl3psGk", + Target = Generation.Gen5, + LibraryName = "libfmod")] + public static int FmodSystemSetOutput(CpuContext ctx) + { + var system = ctx[CpuRegister.Rdi]; + var output = unchecked((int)ctx[CpuRegister.Rsi]); + var callCount = SetOutputCalls.AddOrUpdate(system, 1, static (_, count) => count + 1); + var resetPrematureInit = callCount <= 2; + if (resetPrematureInit && system != 0) + { + Span zeroByte = stackalloc byte[1]; + Span outputBytes = stackalloc byte[sizeof(int)]; + Span zeroInt = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(outputBytes, output); + _ = ctx.Memory.TryWrite(system + 0x08, zeroByte); + _ = ctx.Memory.TryWrite(system + 0x116D0, outputBytes); + _ = ctx.Memory.TryWrite(system + 0x116D4, zeroInt); + } + + if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_FMOD"), "1", StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] fmod.system_set_output system=0x{system:X16} output={output} call={callCount} reset_premature_init={resetPrematureInit} -> 0"); + } + + // The PS5 FMOD object arrives with its initialized byte set before Unity has + // applied the startup output configuration. Preserve those settings while + // clearing the premature marker so Studio can execute the real core init. + ctx[CpuRegister.Rax] = 0; + return 0; + } +} diff --git a/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs b/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs index 2153bc7..f34c0f3 100644 --- a/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs +++ b/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs @@ -3,14 +3,114 @@ using SharpEmu.HLE; using SharpEmu.Libs.Kernel; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Globalization; +using System.Text; namespace SharpEmu.Libs.AvPlayer; public static class AvPlayerExports { private const int InvalidParameters = unchecked((int)0x806A0001); + private const int OperationFailed = unchecked((int)0x806A0002); + private const int FrameBufferCount = 3; + private const int FrameInfoSize = 40; + private const int FrameInfoExSize = 104; + private const int StreamInfoSize = 40; + private const int MaxGuestPathLength = 4096; private static readonly object StateGate = new(); - private static readonly HashSet Players = new(); + private static readonly Dictionary Players = new(); + private static int _traceCount; + + private sealed class PlayerState : IDisposable + { + public required ulong Handle { get; init; } + public bool AutoStart { get; init; } + public ulong AllocatorObject { get; init; } + public ulong AllocateTextureCallback { get; init; } + public ulong EventObject { get; init; } + public ulong EventCallback { get; init; } + public string? SourcePath { get; set; } + public int Width { get; set; } + public int Height { get; set; } + public double FramesPerSecond { get; set; } = 30.0; + public ulong DurationMilliseconds { get; set; } + public bool Started { get; set; } + public bool Paused { get; set; } + public bool Looping { get; set; } + public bool EndOfStream { get; set; } + public Process? Decoder { get; set; } + public Stream? DecoderOutput { get; set; } + public Process? AudioDecoder { get; set; } + public Stream? AudioDecoderOutput { get; set; } + public Stopwatch PlaybackClock { get; } = new(); + public byte[]? RawFrame { get; set; } + public byte[]? RawAudioFrame { get; set; } + public byte[]? PaddedFrame { get; set; } + public ulong[] GuestBuffers { get; } = new ulong[FrameBufferCount]; + public bool TextureAllocatorFailed { get; set; } + public int GuestBufferStride { get; set; } + public int NextGuestBuffer { get; set; } + public ulong LastGuestBuffer { get; set; } + public long NextFrameIndex { get; set; } + public ulong AudioBufferBase { get; set; } + public int NextAudioBuffer { get; set; } + public long NextAudioFrameIndex { get; set; } + + public void Dispose() + { + DecoderOutput?.Dispose(); + DecoderOutput = null; + AudioDecoderOutput?.Dispose(); + AudioDecoderOutput = null; + if (Decoder is not null) + { + try + { + if (!Decoder.HasExited) + { + Decoder.Kill(entireProcessTree: true); + } + } + catch (InvalidOperationException) + { + } + finally + { + Decoder.Dispose(); + Decoder = null; + } + } + if (AudioDecoder is not null) + { + try + { + if (!AudioDecoder.HasExited) + { + AudioDecoder.Kill(entireProcessTree: true); + } + } + catch (InvalidOperationException) + { + } + finally + { + AudioDecoder.Dispose(); + AudioDecoder = null; + } + } + } + + public void ResetPlayback() + { + Dispose(); + PlaybackClock.Reset(); + NextFrameIndex = 0; + NextAudioFrameIndex = 0; + EndOfStream = false; + } + } [SysAbiExport( Nid = "aS66RI0gGgo", @@ -19,7 +119,8 @@ public static class AvPlayerExports LibraryName = "libSceAvPlayer")] public static int AvPlayerInit(CpuContext ctx) { - if (ctx[CpuRegister.Rdi] == 0 || + var initDataAddress = ctx[CpuRegister.Rdi]; + if (initDataAddress == 0 || !KernelMemoryCompatExports.TryAllocateHleData(ctx, 0x40, 16, out var handle)) { ctx[CpuRegister.Rax] = 0; @@ -28,60 +129,22 @@ public static class AvPlayerExports lock (StateGate) { - Players.Add(handle); + Players.Add(handle, new PlayerState + { + Handle = handle, + AutoStart = TryReadByte(ctx, initDataAddress + 108, out var autoStart) && autoStart != 0, + AllocatorObject = TryReadUInt64(ctx, initDataAddress, out var allocatorObject) ? allocatorObject : 0, + AllocateTextureCallback = TryReadUInt64(ctx, initDataAddress + 24, out var allocateTexture) ? allocateTexture : 0, + EventObject = TryReadUInt64(ctx, initDataAddress + 80, out var eventObject) ? eventObject : 0, + EventCallback = TryReadUInt64(ctx, initDataAddress + 88, out var eventCallback) ? eventCallback : 0, + }); } + Trace($"init handle=0x{handle:X16} alloc_texture=0x{Players[handle].AllocateTextureCallback:X16}"); ctx[CpuRegister.Rax] = handle; return unchecked((int)handle); } - [SysAbiExport( - Nid = "KMcEa+rHsIo", - ExportName = "sceAvPlayerAddSource", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerAddSource(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "JdksQu8pNdQ", - ExportName = "sceAvPlayerGetVideoDataEx", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerGetVideoDataEx(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - // "UbQoYawOsfY" is sceAvPlayerIsActive, not sceAvPlayerGetVideoDataEx (the previous NID here - // was wrong - verified by hashing both names against scripts/ps5_names.txt). No player is - // ever actually active in this HLE implementation, so report false/inactive. - [SysAbiExport( - Nid = "UbQoYawOsfY", - ExportName = "sceAvPlayerIsActive", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerIsActive(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return 0; - } - - [SysAbiExport( - Nid = "Wnp1OVcrZgk", - ExportName = "sceAvPlayerGetAudioData", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceAvPlayer")] - public static int AvPlayerGetAudioData(CpuContext ctx) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "HD1YKVU26-M", ExportName = "sceAvPlayerPostInit", @@ -93,13 +156,55 @@ public static class AvPlayerExports var dataAddress = ctx[CpuRegister.Rsi]; lock (StateGate) { - return ctx.SetReturn( - handle != 0 && dataAddress != 0 && Players.Contains(handle) + return SetReturn( + ctx, + handle != 0 && dataAddress != 0 && Players.ContainsKey(handle) ? 0 : InvalidParameters); } } + [SysAbiExport( + Nid = "o9eWRkSL+M4", + ExportName = "sceAvPlayerInitEx", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerInitEx(CpuContext ctx) + { + var initDataAddress = ctx[CpuRegister.Rdi]; + var playerOutAddress = ctx[CpuRegister.Rsi]; + if (initDataAddress == 0 || + playerOutAddress == 0 || + !KernelMemoryCompatExports.TryAllocateHleData(ctx, 0x40, 16, out var handle) || + !ctx.TryWriteUInt64(playerOutAddress, handle)) + { + return SetReturn(ctx, InvalidParameters); + } + + lock (StateGate) + { + Players.Add(handle, new PlayerState + { + Handle = handle, + AutoStart = TryReadByte(ctx, initDataAddress + 164, out var autoStart) && autoStart != 0, + AllocatorObject = TryReadUInt64(ctx, initDataAddress + 8, out var allocatorObject) ? allocatorObject : 0, + AllocateTextureCallback = TryReadUInt64(ctx, initDataAddress + 32, out var allocateTexture) ? allocateTexture : 0, + EventObject = TryReadUInt64(ctx, initDataAddress + 88, out var eventObject) ? eventObject : 0, + EventCallback = TryReadUInt64(ctx, initDataAddress + 96, out var eventCallback) ? eventCallback : 0, + }); + } + + Trace($"init_ex handle=0x{handle:X16} alloc_texture=0x{Players[handle].AllocateTextureCallback:X16}"); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "eBTreZ84JFY", + ExportName = "sceAvPlayerSetLogCallback", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetLogCallback(CpuContext ctx) => SetReturn(ctx, 0); + [SysAbiExport( Nid = "NkJwDzKmIlw", ExportName = "sceAvPlayerClose", @@ -107,9 +212,1063 @@ public static class AvPlayerExports LibraryName = "libSceAvPlayer")] public static int AvPlayerClose(CpuContext ctx) { + PlayerState? player; lock (StateGate) { - return ctx.SetReturn(Players.Remove(ctx[CpuRegister.Rdi]) ? 0 : InvalidParameters); + if (!Players.Remove(ctx[CpuRegister.Rdi], out player)) + { + return SetReturn(ctx, InvalidParameters); + } + } + + player.Dispose(); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "KMcEa+rHsIo", + ExportName = "sceAvPlayerAddSource", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerAddSource(CpuContext ctx) + { + if (!TryReadNullTerminatedUtf8(ctx, ctx[CpuRegister.Rsi], MaxGuestPathLength, out var path)) + { + return SetReturn(ctx, InvalidParameters); + } + + return AddSource(ctx, path); + } + + [SysAbiExport( + Nid = "x8uvuFOPZhU", + ExportName = "sceAvPlayerAddSourceEx", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerAddSourceEx(CpuContext ctx) + { + var uriType = unchecked((uint)ctx[CpuRegister.Rsi]); + var detailsAddress = ctx[CpuRegister.Rdx]; + if (uriType != 0 || detailsAddress == 0 || + !ctx.TryReadUInt64(detailsAddress, out var pathAddress) || + !TryReadUInt32(ctx, detailsAddress + sizeof(ulong), out var pathLength) || + pathLength == 0 || pathLength > MaxGuestPathLength || + !TryReadUtf8(ctx, pathAddress, checked((int)pathLength), out var path)) + { + return SetReturn(ctx, InvalidParameters); + } + + return AddSource(ctx, path.TrimEnd('\0')); + } + + [SysAbiExport( + Nid = "ET4Gr-Uu07s", + ExportName = "sceAvPlayerStart", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerStart(CpuContext ctx) + { + PlayerState player; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer) || foundPlayer.SourcePath is null) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + player.Started = true; + player.Paused = false; + player.EndOfStream = false; + Trace($"start handle=0x{player.Handle:X16}"); + } + + // Event callbacks are guest code and can immediately query the player. + // Never hold StateGate while waiting for one or the callback deadlocks + // when it re-enters an AvPlayer export on another guest worker. + NotifyEvent(ctx, player, 3); // StatePlay + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "ZC17w3vB5Lo", + ExportName = "sceAvPlayerStop", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerStop(CpuContext ctx) + { + PlayerState player; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer)) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + player.ResetPlayback(); + player.Started = false; + } + + NotifyEvent(ctx, player, 1); // StateStop + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "9y5v+fGN4Wk", + ExportName = "sceAvPlayerPause", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerPause(CpuContext ctx) + { + PlayerState player; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer)) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + player.Paused = true; + player.PlaybackClock.Stop(); + } + + + NotifyEvent(ctx, player, 4); // StatePause + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "w5moABNwnRY", + ExportName = "sceAvPlayerResume", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerResume(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + player.Paused = false; + if (player.Decoder is not null) + { + player.PlaybackClock.Start(); + } + return SetReturn(ctx, 0); + } + } + + [SysAbiExport( + Nid = "OVths0xGfho", + ExportName = "sceAvPlayerSetLooping", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetLooping(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + player.Looping = ctx[CpuRegister.Rsi] != 0; + return SetReturn(ctx, 0); + } + } + + [SysAbiExport( + Nid = "ODJK2sn9w4A", + ExportName = "sceAvPlayerEnableStream", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerEnableStream(CpuContext ctx) => ValidatePlayer(ctx); + + [SysAbiExport( + Nid = "k-q+xOxdc3E", + ExportName = "sceAvPlayerSetAvSyncMode", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetAvSyncMode(CpuContext ctx) + { + Trace($"set_av_sync_mode handle=0x{ctx[CpuRegister.Rdi]:X16} mode={ctx[CpuRegister.Rsi]}"); + return ValidatePlayer(ctx); + } + + [SysAbiExport( + Nid = "ctTAcF5DiKQ", + ExportName = "sceAvPlayerGetStreamInfoEx", + Target = Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerSetDecoderMode(CpuContext ctx) => ValidatePlayer(ctx); + + [SysAbiExport( + Nid = "XC9wM+xULz8", + ExportName = "sceAvPlayerJumpToTime", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerJumpToTime(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + player.ResetPlayback(); + player.Started = true; + return SetReturn(ctx, 0); + } + } + + [SysAbiExport( + Nid = "yN7Jhuv8g24", + ExportName = "sceAvPlayerVprintf", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerVprintf(CpuContext ctx) => SetReturn(ctx, 0); + + [SysAbiExport( + Nid = "UbQoYawOsfY", + ExportName = "sceAvPlayerIsActive", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerIsActive(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn( + ctx, + Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) && + player.Started && !player.EndOfStream ? 1 : 0); + } + } + + [SysAbiExport( + Nid = "o3+RWnHViSg", + ExportName = "sceAvPlayerGetVideoData", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetVideoData(CpuContext ctx) => GetVideoData(ctx, extended: false); + + [SysAbiExport( + Nid = "JdksQu8pNdQ", + ExportName = "sceAvPlayerGetVideoDataEx", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetVideoDataEx(CpuContext ctx) => GetVideoData(ctx, extended: true); + + [SysAbiExport( + Nid = "Wnp1OVcrZgk", + ExportName = "sceAvPlayerGetAudioData", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetAudioData(CpuContext ctx) + { + var infoAddress = ctx[CpuRegister.Rsi]; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) || + infoAddress == 0 || !player.Started || player.Paused || player.EndOfStream || + player.SourcePath is null || !EnsureAudioDecoder(player)) + { + return SetReturn(ctx, 0); + } + + const int samplesPerFrame = 1024; + const int channelCount = 2; + const int sampleRate = 48_000; + const int audioFrameSize = samplesPerFrame * channelCount * sizeof(short); + if (player.RawAudioFrame is null || + !ReadExactly(player.AudioDecoderOutput, player.RawAudioFrame)) + { + return SetReturn(ctx, 0); + } + if (player.AudioBufferBase == 0) + { + if (!KernelMemoryCompatExports.TryAllocateHleData( + ctx, + audioFrameSize * 8UL, + 0x100, + out var audioBufferBase)) + { + return SetReturn(ctx, 0); + } + player.AudioBufferBase = audioBufferBase; + } + + var bufferAddress = player.AudioBufferBase + + checked((ulong)(player.NextAudioBuffer * audioFrameSize)); + player.NextAudioBuffer = (player.NextAudioBuffer + 1) % 8; + if (!ctx.Memory.TryWrite(bufferAddress, player.RawAudioFrame)) + { + return SetReturn(ctx, 0); + } + + var timestamp = checked((ulong)(player.NextAudioFrameIndex * samplesPerFrame * 1000L / sampleRate)); + player.NextAudioFrameIndex++; + Span info = stackalloc byte[FrameInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(info[0..], bufferAddress); + BinaryPrimitives.WriteUInt64LittleEndian(info[16..], timestamp); + BinaryPrimitives.WriteUInt16LittleEndian(info[24..], channelCount); + BinaryPrimitives.WriteUInt32LittleEndian(info[28..], sampleRate); + BinaryPrimitives.WriteUInt32LittleEndian(info[32..], audioFrameSize); + if (!ctx.Memory.TryWrite(infoAddress, info)) + { + return SetReturn(ctx, 0); + } + Trace($"audio_frame handle=0x{player.Handle:X16} ts={timestamp} data=0x{bufferAddress:X16}"); + return SetReturn(ctx, 1); + } + } + + [SysAbiExport( + Nid = "wwM99gjFf1Y", + ExportName = "sceAvPlayerCurrentTime", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerCurrentTime(CpuContext ctx) + { + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player)) + { + return SetReturn(ctx, InvalidParameters); + } + + var milliseconds = (ulong)player.PlaybackClock.ElapsedMilliseconds; + ctx[CpuRegister.Rax] = milliseconds; + return unchecked((int)milliseconds); + } + } + + [SysAbiExport( + Nid = "hdTyRzCXQeQ", + ExportName = "sceAvPlayerStreamCount", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerStreamCount(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn(ctx, Players.ContainsKey(ctx[CpuRegister.Rdi]) ? 2 : InvalidParameters); + } + } + + [SysAbiExport( + Nid = "d8FcbzfAdQw", + ExportName = "sceAvPlayerGetStreamInfo", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceAvPlayer")] + public static int AvPlayerGetStreamInfo(CpuContext ctx) + { + var streamIndex = unchecked((uint)ctx[CpuRegister.Rsi]); + var infoAddress = ctx[CpuRegister.Rdx]; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) || + streamIndex > 1 || infoAddress == 0 || player.Width <= 0 || player.Height <= 0) + { + return SetReturn(ctx, InvalidParameters); + } + + Span info = stackalloc byte[StreamInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt32LittleEndian(info[0..], streamIndex); // 0=video, 1=audio + if (streamIndex == 0) + { + BinaryPrimitives.WriteUInt32LittleEndian(info[8..], checked((uint)player.Width)); + BinaryPrimitives.WriteUInt32LittleEndian(info[12..], checked((uint)player.Height)); + BinaryPrimitives.WriteSingleLittleEndian(info[16..], (float)player.Width / player.Height); + } + else + { + BinaryPrimitives.WriteUInt16LittleEndian(info[8..], 2); + BinaryPrimitives.WriteUInt32LittleEndian(info[12..], 48_000); + } + BinaryPrimitives.WriteUInt64LittleEndian(info[24..], player.DurationMilliseconds); + if (!ctx.Memory.TryWrite(infoAddress, info)) + { + return SetReturn(ctx, InvalidParameters); + } + + return SetReturn(ctx, 0); + } + } + + private static int AddSource(CpuContext ctx, string guestPath) + { + PlayerState player; + bool autoStart; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var foundPlayer)) + { + return SetReturn(ctx, InvalidParameters); + } + player = foundPlayer; + + var hostPath = ResolveGuestPath(guestPath); + if (hostPath is null || !ProbeVideo(hostPath, out var width, out var height, out var fps, out var duration)) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Could not open guest video '{guestPath}' (resolved '{hostPath ?? ""}')."); + return SetReturn(ctx, OperationFailed); + } + + player.ResetPlayback(); + player.SourcePath = hostPath; + player.Width = width; + player.Height = height; + player.FramesPerSecond = fps; + player.DurationMilliseconds = duration; + player.Started = player.AutoStart; + autoStart = player.AutoStart; + Trace($"source guest='{guestPath}' host='{hostPath}' {width}x{height} fps={fps:F3} duration_ms={duration} auto_start={player.AutoStart}"); + } + + + NotifyEvent(ctx, player, 2); // StateReady + if (autoStart) + { + NotifyEvent(ctx, player, 3); // StatePlay + } + return SetReturn(ctx, 0); + } + + private static int GetVideoData(CpuContext ctx, bool extended) + { + var infoAddress = ctx[CpuRegister.Rsi]; + lock (StateGate) + { + if (!Players.TryGetValue(ctx[CpuRegister.Rdi], out var player) || + infoAddress == 0 || !player.Started || player.Paused || player.EndOfStream || + player.SourcePath is null) + { + return SetReturn(ctx, 0); + } + + if (!EnsureDecoder(player)) + { + player.EndOfStream = true; + return SetReturn(ctx, 0); + } + + var fps = Math.Max(1.0, player.FramesPerSecond); + var expectedFrame = (long)Math.Floor(player.PlaybackClock.Elapsed.TotalSeconds * fps); + while (player.NextFrameIndex < expectedFrame) + { + if (!ReadFrame(player)) + { + return FinishStream(ctx, player); + } + player.NextFrameIndex++; + } + + if (!ReadFrame(player)) + { + return FinishStream(ctx, player); + } + + var timestamp = checked((ulong)Math.Round(player.NextFrameIndex * 1000.0 / fps)); + player.NextFrameIndex++; + if (!WriteVideoFrame(ctx, player, infoAddress, timestamp, extended)) + { + return SetReturn(ctx, 0); + } + + Trace($"video_frame handle=0x{player.Handle:X16} ex={extended} ts={timestamp} data=0x{player.LastGuestBuffer:X16}"); + return SetReturn(ctx, 1); + } + } + + private static int FinishStream(CpuContext ctx, PlayerState player) + { + if (player.Looping) + { + player.ResetPlayback(); + player.Started = true; + } + else + { + player.EndOfStream = true; + player.PlaybackClock.Stop(); + } + return SetReturn(ctx, 0); + } + + private static bool EnsureDecoder(PlayerState player) + { + if (player.DecoderOutput is not null) + { + return true; + } + + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null || player.SourcePath is null) + { + Console.Error.WriteLine("[AVPLAYER][ERROR] FFmpeg was not found. Set SHARPEMU_FFMPEG_PATH."); + return false; + } + + var startInfo = new ProcessStartInfo(ffmpeg) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-nostdin"); + startInfo.ArgumentList.Add("-hide_banner"); + startInfo.ArgumentList.Add("-loglevel"); + startInfo.ArgumentList.Add("error"); + startInfo.ArgumentList.Add("-i"); + startInfo.ArgumentList.Add(player.SourcePath); + startInfo.ArgumentList.Add("-map"); + startInfo.ArgumentList.Add("0:v:0"); + startInfo.ArgumentList.Add("-an"); + startInfo.ArgumentList.Add("-pix_fmt"); + startInfo.ArgumentList.Add("nv12"); + startInfo.ArgumentList.Add("-f"); + startInfo.ArgumentList.Add("rawvideo"); + startInfo.ArgumentList.Add("pipe:1"); + + try + { + player.Decoder = Process.Start(startInfo); + if (player.Decoder is null) + { + return false; + } + player.Decoder.ErrorDataReceived += (_, eventArgs) => + { + if (!string.IsNullOrWhiteSpace(eventArgs.Data)) + { + Console.Error.WriteLine($"[AVPLAYER][FFMPEG] {eventArgs.Data}"); + } + }; + player.Decoder.BeginErrorReadLine(); + player.DecoderOutput = player.Decoder.StandardOutput.BaseStream; + player.RawFrame = new byte[checked(player.Width * player.Height * 3 / 2)]; + player.PlaybackClock.Start(); + Trace($"decoder_started pid={player.Decoder.Id} source='{player.SourcePath}'"); + return true; + } + catch (Exception exception) when (exception is IOException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Failed to launch FFmpeg: {exception.Message}"); + player.Dispose(); + return false; + } + } + + private static bool EnsureAudioDecoder(PlayerState player) + { + if (player.AudioDecoderOutput is not null) + { + return true; + } + + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null || player.SourcePath is null) + { + return false; + } + + var startInfo = new ProcessStartInfo(ffmpeg) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-nostdin"); + startInfo.ArgumentList.Add("-hide_banner"); + startInfo.ArgumentList.Add("-loglevel"); + startInfo.ArgumentList.Add("error"); + startInfo.ArgumentList.Add("-i"); + startInfo.ArgumentList.Add(player.SourcePath); + startInfo.ArgumentList.Add("-map"); + startInfo.ArgumentList.Add("0:a:0"); + startInfo.ArgumentList.Add("-vn"); + startInfo.ArgumentList.Add("-ac"); + startInfo.ArgumentList.Add("2"); + startInfo.ArgumentList.Add("-ar"); + startInfo.ArgumentList.Add("48000"); + startInfo.ArgumentList.Add("-f"); + startInfo.ArgumentList.Add("s16le"); + startInfo.ArgumentList.Add("pipe:1"); + + try + { + player.AudioDecoder = Process.Start(startInfo); + if (player.AudioDecoder is null) + { + return false; + } + player.AudioDecoder.ErrorDataReceived += (_, eventArgs) => + { + if (!string.IsNullOrWhiteSpace(eventArgs.Data)) + { + Console.Error.WriteLine($"[AVPLAYER][FFMPEG-AUDIO] {eventArgs.Data}"); + } + }; + player.AudioDecoder.BeginErrorReadLine(); + player.AudioDecoderOutput = player.AudioDecoder.StandardOutput.BaseStream; + player.RawAudioFrame = new byte[1024 * 2 * sizeof(short)]; + Trace($"audio_decoder_started pid={player.AudioDecoder.Id} source='{player.SourcePath}'"); + return true; + } + catch (Exception exception) when (exception is IOException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Failed to launch FFmpeg audio decoder: {exception.Message}"); + player.AudioDecoderOutput?.Dispose(); + player.AudioDecoderOutput = null; + player.AudioDecoder?.Dispose(); + player.AudioDecoder = null; + return false; + } + } + + private static bool ReadFrame(PlayerState player) + { + if (player.DecoderOutput is null || player.RawFrame is null) + { + return false; + } + + try + { + return ReadExactly(player.DecoderOutput, player.RawFrame); + } + catch (IOException exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] FFmpeg stream read failed: {exception.Message}"); + return false; + } + } + + private static bool ReadExactly(Stream? stream, byte[] buffer) + { + if (stream is null) + { + return false; + } + var offset = 0; + while (offset < buffer.Length) + { + var read = stream.Read(buffer, offset, buffer.Length - offset); + if (read == 0) + { + return false; + } + offset += read; + } + return true; + } + + private static bool WriteVideoFrame( + CpuContext ctx, + PlayerState player, + ulong infoAddress, + ulong timestamp, + bool extended) + { + if (player.RawFrame is null) + { + return false; + } + + var alignedWidth = AlignUp(player.Width, 16); + var alignedHeight = AlignUp(player.Height, 16); + var bufferStride = checked(alignedWidth * alignedHeight * 3 / 2); + if (player.GuestBuffers[0] == 0) + { + if (!AllocateGuestVideoBuffers(ctx, player, bufferStride)) + { + return false; + } + player.GuestBufferStride = bufferStride; + } + + var frameData = player.RawFrame; + if (!extended && (alignedWidth != player.Width || alignedHeight != player.Height)) + { + player.PaddedFrame ??= new byte[bufferStride]; + player.PaddedFrame.AsSpan().Clear(); + for (var row = 0; row < player.Height; row++) + { + player.RawFrame.AsSpan(row * player.Width, player.Width) + .CopyTo(player.PaddedFrame.AsSpan(row * alignedWidth, player.Width)); + } + var rawChromaOffset = player.Width * player.Height; + var paddedChromaOffset = alignedWidth * alignedHeight; + for (var row = 0; row < player.Height / 2; row++) + { + player.RawFrame.AsSpan(rawChromaOffset + (row * player.Width), player.Width) + .CopyTo(player.PaddedFrame.AsSpan(paddedChromaOffset + (row * alignedWidth), player.Width)); + } + frameData = player.PaddedFrame; + } + + var bufferAddress = player.GuestBuffers[player.NextGuestBuffer]; + player.NextGuestBuffer = (player.NextGuestBuffer + 1) % FrameBufferCount; + player.LastGuestBuffer = bufferAddress; + if (!ctx.Memory.TryWrite(bufferAddress, frameData)) + { + return false; + } + + Span info = extended + ? stackalloc byte[FrameInfoExSize] + : stackalloc byte[FrameInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(info[0..], bufferAddress); + BinaryPrimitives.WriteUInt64LittleEndian(info[16..], timestamp); + BinaryPrimitives.WriteUInt32LittleEndian(info[24..], checked((uint)(extended ? player.Width : alignedWidth))); + BinaryPrimitives.WriteUInt32LittleEndian(info[28..], checked((uint)(extended ? player.Height : alignedHeight))); + BinaryPrimitives.WriteSingleLittleEndian(info[32..], 1.0f); + if (extended) + { + BinaryPrimitives.WriteUInt32LittleEndian(info[60..], checked((uint)player.Width)); + info[64] = 8; + info[65] = 8; + } + return ctx.Memory.TryWrite(infoAddress, info); + } + + private static bool AllocateGuestVideoBuffers(CpuContext ctx, PlayerState player, int bufferSize) + { + var scheduler = GuestThreadExecution.Scheduler; + if (!player.TextureAllocatorFailed && player.AllocateTextureCallback != 0 && scheduler is not null) + { + for (var index = 0; index < player.GuestBuffers.Length; index++) + { + if (!scheduler.TryCallGuestFunction( + ctx, + player.AllocateTextureCallback, + player.AllocatorObject, + 0x100, + checked((ulong)bufferSize), + 0, + 0, + "avplayer_allocate_texture", + out var buffer, + out var error) || buffer == 0) + { + Console.Error.WriteLine( + $"[AVPLAYER][ERROR] Guest texture allocation failed index={index} " + + $"callback=0x{player.AllocateTextureCallback:X16}: {error ?? "returned null"}"); + player.TextureAllocatorFailed = true; + Array.Clear(player.GuestBuffers); + break; + } + player.GuestBuffers[index] = buffer; + Trace($"texture_buffer index={index} data=0x{buffer:X16} size={bufferSize}"); + } + if (!player.TextureAllocatorFailed) + { + return true; + } + } + + if (!KernelMemoryCompatExports.TryAllocateHleData( + ctx, + checked((ulong)bufferSize * FrameBufferCount), + 0x1000, + out var bufferBase)) + { + return false; + } + for (var index = 0; index < player.GuestBuffers.Length; index++) + { + player.GuestBuffers[index] = bufferBase + checked((ulong)(index * bufferSize)); + } + Console.Error.WriteLine("[AVPLAYER][WARN] Guest texture allocator unavailable; using generic HLE memory."); + return true; + } + + private static bool ProbeVideo( + string path, + out int width, + out int height, + out double framesPerSecond, + out ulong durationMilliseconds) + { + width = 0; + height = 0; + framesPerSecond = 30.0; + durationMilliseconds = 0; + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null) + { + return false; + } + var ffprobe = Path.Combine(Path.GetDirectoryName(ffmpeg) ?? string.Empty, "ffprobe"); + if (!File.Exists(ffprobe)) + { + return false; + } + + var startInfo = new ProcessStartInfo(ffprobe) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add("error"); + startInfo.ArgumentList.Add("-select_streams"); + startInfo.ArgumentList.Add("v:0"); + startInfo.ArgumentList.Add("-show_entries"); + startInfo.ArgumentList.Add("stream=width,height,avg_frame_rate,duration"); + startInfo.ArgumentList.Add("-of"); + startInfo.ArgumentList.Add("default=noprint_wrappers=1"); + startInfo.ArgumentList.Add(path); + + try + { + using var process = Process.Start(startInfo); + if (process is null) + { + return false; + } + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (process.ExitCode != 0) + { + Console.Error.WriteLine($"[AVPLAYER][FFPROBE] {error.Trim()}"); + return false; + } + + foreach (var line in output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var separator = line.IndexOf('='); + if (separator < 1) + { + continue; + } + var key = line[..separator]; + var value = line[(separator + 1)..]; + switch (key) + { + case "width": + _ = int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out width); + break; + case "height": + _ = int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out height); + break; + case "avg_frame_rate": + var parts = value.Split('/'); + if (parts.Length == 2 && + double.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var numerator) && + double.TryParse(parts[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var denominator) && + denominator != 0) + { + framesPerSecond = numerator / denominator; + } + break; + case "duration": + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var duration)) + { + durationMilliseconds = checked((ulong)Math.Max(0, Math.Round(duration * 1000.0))); + } + break; + } + } + return width > 0 && height > 0 && framesPerSecond > 0; + } + catch (Exception exception) when (exception is IOException or InvalidOperationException or System.ComponentModel.Win32Exception) + { + Console.Error.WriteLine($"[AVPLAYER][ERROR] Failed to probe video: {exception.Message}"); + return false; + } + } + + private static string? FindFfmpeg() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_FFMPEG_PATH"); + if (!string.IsNullOrWhiteSpace(configured) && File.Exists(configured)) + { + return configured; + } + foreach (var candidate in new[] { "/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg" }) + { + if (File.Exists(candidate)) + { + return candidate; + } + } + return null; + } + + private static string? ResolveGuestPath(string guestPath) + { + if (string.IsNullOrWhiteSpace(guestPath)) + { + return null; + } + + var normalized = guestPath.Replace('\\', '/'); + if (Uri.TryCreate(normalized, UriKind.Absolute, out var uri) && uri.IsFile) + { + normalized = uri.LocalPath; + } + if (File.Exists(normalized)) + { + return Path.GetFullPath(normalized); + } + + var app0 = Environment.GetEnvironmentVariable("SHARPEMU_APP0_DIR"); + if (string.IsNullOrWhiteSpace(app0)) + { + return null; + } + foreach (var prefix in new[] { "app0:/", "/app0/", "app0:", "/app0" }) + { + if (normalized.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + normalized = normalized[prefix.Length..]; + break; + } + } + var candidate = Path.GetFullPath(Path.Combine(app0, normalized.TrimStart('/'))); + var root = Path.GetFullPath(app0).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar; + return candidate.StartsWith(root, StringComparison.OrdinalIgnoreCase) && File.Exists(candidate) + ? candidate + : null; + } + + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int maxLength, out string value) + { + value = string.Empty; + if (address == 0 || maxLength <= 0) + { + return false; + } + var bytes = new List(Math.Min(maxLength, 256)); + Span single = stackalloc byte[1]; + for (var index = 0; index < maxLength; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, single)) + { + return false; + } + if (single[0] == 0) + { + value = Encoding.UTF8.GetString(bytes.ToArray()); + return true; + } + bytes.Add(single[0]); + } + return false; + } + + private static bool TryReadUtf8(CpuContext ctx, ulong address, int length, out string value) + { + value = string.Empty; + if (address == 0 || length <= 0) + { + return false; + } + var bytes = new byte[length]; + if (!ctx.Memory.TryRead(address, bytes)) + { + return false; + } + value = Encoding.UTF8.GetString(bytes); + return true; + } + + private static bool TryReadByte(CpuContext ctx, ulong address, out byte value) + { + Span buffer = stackalloc byte[1]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + value = buffer[0]; + return true; + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) => + ctx.TryReadUInt64(address, out value); + + private static void NotifyEvent(CpuContext ctx, PlayerState player, ulong eventId) + { + if (player.EventCallback == 0) + { + Trace($"event skipped handle=0x{player.Handle:X16} id={eventId} callback=0"); + return; + } + + var scheduler = GuestThreadExecution.Scheduler; + string? error = null; + if (scheduler is null || + !scheduler.TryCallGuestFunction( + ctx, + player.EventCallback, + player.EventObject, + eventId, + 0, + 0, + 0, + $"avplayer_event_{eventId}", + out _, + out error)) + { + Console.Error.WriteLine( + $"[AVPLAYER][WARN] Event callback failed handle=0x{player.Handle:X16} " + + $"event={eventId} callback=0x{player.EventCallback:X16}: {error ?? "scheduler unavailable"}"); + return; + } + + Trace($"event handle=0x{player.Handle:X16} id={eventId} callback=0x{player.EventCallback:X16}"); + } + + private static int AlignUp(int value, int alignment) => + checked((value + alignment - 1) & -alignment); + + private static int ValidatePlayer(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn(ctx, Players.ContainsKey(ctx[CpuRegister.Rdi]) ? 0 : InvalidParameters); + } + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + + private static void Trace(string message) + { + var count = Interlocked.Increment(ref _traceCount); + if (count <= 32 || count % 300 == 0) + { + Console.Error.WriteLine($"[AVPLAYER][INFO] {message}"); } } } diff --git a/src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs b/src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs new file mode 100644 index 0000000..d3d5ead --- /dev/null +++ b/src/SharpEmu.Libs/Bink/Bink2MovieBridge.cs @@ -0,0 +1,403 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Runtime.InteropServices; +using System.Buffers.Binary; + +namespace SharpEmu.Libs.Bink; + +/// +/// Optional host-side Bink 2 bridge for games that ship a static Bink player. +/// +/// The game in that case never imports libSceVideodec, so an HLE video-decoder +/// export cannot see its movie frames. Kernel file opens identify the active +/// .bk2 file and the presenter requests BGRA frames from a tiny native adapter. +/// The adapter is deliberately a separate, user-supplied library: Bink 2 is a +/// proprietary SDK and SharpEmu must neither bundle it nor depend on its ABI. +/// +internal static class Bink2MovieBridge +{ + private const uint MaxDimension = 16384; + private static readonly object Gate = new(); + private static NativeAdapter? _adapter; + private static string? _activePath; + private static IntPtr _activeMovie; + private static Bink2MovieInfo _activeInfo; + private static byte[]? _frameBuffer; + private static bool _usingDummyMovie; + private static bool _loadAttempted; + private static bool _availabilityReported; + + /// + /// Returns true when the guest should receive a normal "file not found" + /// result for a Bink movie. This is the safe default without a decoder: + /// games that treat movies as optional fall through to their next state + /// rather than submitting an empty Bink GPU texture forever. + /// + internal static bool ShouldSkipGuestMovie(string hostPath) => + hostPath.EndsWith(".bk2", StringComparison.OrdinalIgnoreCase) && + ResolveMode() == MovieMode.Skip; + + internal static void ObserveGuestMovie(string hostPath) + { + if (!hostPath.EndsWith(".bk2", StringComparison.OrdinalIgnoreCase) || + !File.Exists(hostPath)) + { + return; + } + + lock (Gate) + { + if (string.Equals(_activePath, hostPath, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (ResolveMode() == MovieMode.Dummy) + { + AttachDummyMovieLocked(hostPath); + return; + } + + var adapter = GetAdapterLocked(); + if (adapter is null) + { + return; + } + + CloseActiveLocked(); + if (!adapter.TryOpen(hostPath, out var movie, out var info)) + { + Console.Error.WriteLine( + "[LOADER][WARN] Bink2 bridge could not open movie '" + + Path.GetFileName(hostPath) + "'."); + return; + } + + if (!IsValid(info)) + { + adapter.Close(movie); + Console.Error.WriteLine( + "[LOADER][WARN] Bink2 bridge rejected invalid movie dimensions for '" + + Path.GetFileName(hostPath) + "'."); + return; + } + + _activePath = hostPath; + _activeMovie = movie; + _activeInfo = info; + _frameBuffer = GC.AllocateUninitializedArray(GetFrameBufferLength(info)); + Console.Error.WriteLine( + "[LOADER][INFO] Bink2 bridge attached: " + Path.GetFileName(hostPath) + " " + + info.Width + "x" + info.Height + " @ " + + info.FramesPerSecondNumerator + "/" + info.FramesPerSecondDenominator + " fps."); + } + } + + internal static bool TryDecodeNextFrame( + out byte[] pixels, + out uint width, + out uint height) + { + lock (Gate) + { + pixels = []; + width = 0; + height = 0; + if (_adapter is null || _activeMovie == IntPtr.Zero || _frameBuffer is null) + { + if (_usingDummyMovie && _frameBuffer is not null) + { + pixels = _frameBuffer; + width = _activeInfo.Width; + height = _activeInfo.Height; + return true; + } + + return false; + } + + unsafe + { + fixed (byte* destination = _frameBuffer) + { + if (!_adapter.DecodeNextBgra( + _activeMovie, + (IntPtr)destination, + _activeInfo.Width * 4, + (uint)_frameBuffer.Length)) + { + return false; + } + } + } + + pixels = _frameBuffer; + width = _activeInfo.Width; + height = _activeInfo.Height; + return true; + } + } + + private static bool IsValid(Bink2MovieInfo info) => + info.Width > 0 && info.Height > 0 && + info.Width <= MaxDimension && info.Height <= MaxDimension && + (ulong)info.Width * info.Height * 4 <= int.MaxValue; + + private static int GetFrameBufferLength(Bink2MovieInfo info) => + checked((int)((ulong)info.Width * info.Height * 4)); + + private static MovieMode ResolveMode() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_BINK_MODE"); + if (string.Equals(configured, "dummy", StringComparison.OrdinalIgnoreCase)) + { + return MovieMode.Dummy; + } + + if (string.Equals(configured, "native", StringComparison.OrdinalIgnoreCase)) + { + return MovieMode.Native; + } + + if (string.Equals(configured, "skip", StringComparison.OrdinalIgnoreCase)) + { + return MovieMode.Skip; + } + + // With no SDK adapter present, returning "not found" makes optional + // cinematics advance. Supplying either an explicit path or the normal + // side-by-side adapter enables native playback automatically. + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("SHARPEMU_BINK2_BRIDGE")) || + EnumerateAdapterCandidates().Any(File.Exists)) + { + return MovieMode.Native; + } + + return MovieMode.Skip; + } + + private static void AttachDummyMovieLocked(string hostPath) + { + if (!TryReadBinkInfo(hostPath, out var info) || !IsValid(info)) + { + Console.Error.WriteLine( + "[LOADER][WARN] Bink dummy could not read movie header '" + + Path.GetFileName(hostPath) + "'."); + return; + } + + CloseActiveLocked(); + _activePath = hostPath; + _activeInfo = info; + _frameBuffer = GC.AllocateUninitializedArray(GetFrameBufferLength(info)); + FillDummyFrame(_frameBuffer, info.Width, info.Height); + _usingDummyMovie = true; + Console.Error.WriteLine( + "[LOADER][INFO] Bink dummy attached: " + Path.GetFileName(hostPath) + " " + + info.Width + "x" + info.Height + "."); + } + + private static bool TryReadBinkInfo(string path, out Bink2MovieInfo info) + { + info = default; + Span header = stackalloc byte[32]; + try + { + using var stream = File.OpenRead(path); + if (stream.Read(header) != header.Length || + !header[..4].SequenceEqual("KB2j"u8)) + { + return false; + } + + info = new Bink2MovieInfo( + BinaryPrimitives.ReadUInt32LittleEndian(header.Slice(0x14, 4)), + BinaryPrimitives.ReadUInt32LittleEndian(header.Slice(0x18, 4)), + BinaryPrimitives.ReadUInt32LittleEndian(header.Slice(0x1C, 4)), + 1); + return true; + } + catch (IOException) + { + return false; + } + } + + private static void FillDummyFrame(byte[] pixels, uint width, uint height) + { + for (var y = 0u; y < height; y++) + { + for (var x = 0u; x < width; x++) + { + var offset = checked((int)(((ulong)y * width + x) * 4)); + var band = ((x / 96) + (y / 96)) & 1; + pixels[offset] = band == 0 ? (byte)0x28 : (byte)0x18; + pixels[offset + 1] = band == 0 ? (byte)0x18 : (byte)0x28; + pixels[offset + 2] = 0x10; + pixels[offset + 3] = 0xFF; + } + } + } + + private static NativeAdapter? GetAdapterLocked() + { + if (_loadAttempted) + { + return _adapter; + } + + _loadAttempted = true; + foreach (var candidate in EnumerateAdapterCandidates()) + { + if (!NativeLibrary.TryLoad(candidate, out var library)) + { + continue; + } + + if (NativeAdapter.TryCreate(library, out var adapter)) + { + _adapter = adapter; + Console.Error.WriteLine("[LOADER][INFO] Bink2 bridge loaded: " + candidate); + return adapter; + } + + NativeLibrary.Free(library); + } + + if (!_availabilityReported) + { + _availabilityReported = true; + Console.Error.WriteLine( + "[LOADER][INFO] Bink2 bridge unavailable; install the licensed adapter and set SHARPEMU_BINK2_BRIDGE."); + } + + return null; + } + + private static IEnumerable EnumerateAdapterCandidates() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_BINK2_BRIDGE"); + if (!string.IsNullOrWhiteSpace(configured)) + { + yield return configured; + } + + var baseDirectory = AppContext.BaseDirectory; + if (OperatingSystem.IsMacOS()) + { + yield return Path.Combine(baseDirectory, "libsharpemu_bink2_bridge.dylib"); + } + else if (OperatingSystem.IsWindows()) + { + yield return Path.Combine(baseDirectory, "sharpemu_bink2_bridge.dll"); + } + else + { + yield return Path.Combine(baseDirectory, "libsharpemu_bink2_bridge.so"); + } + } + + private static void CloseActiveLocked() + { + if (_activeMovie != IntPtr.Zero) + { + _adapter?.Close(_activeMovie); + } + + _activePath = null; + _activeMovie = IntPtr.Zero; + _activeInfo = default; + _frameBuffer = null; + _usingDummyMovie = false; + } + + [StructLayout(LayoutKind.Sequential)] + private readonly struct Bink2MovieInfo + { + public readonly uint Width; + public readonly uint Height; + public readonly uint FramesPerSecondNumerator; + public readonly uint FramesPerSecondDenominator; + + internal Bink2MovieInfo( + uint width, + uint height, + uint framesPerSecondNumerator, + uint framesPerSecondDenominator) + { + Width = width; + Height = height; + FramesPerSecondNumerator = framesPerSecondNumerator; + FramesPerSecondDenominator = framesPerSecondDenominator; + } + } + + private enum MovieMode + { + Skip, + Dummy, + Native, + } + + private sealed class NativeAdapter + { + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int OpenUtf8Delegate(IntPtr pathUtf8, out IntPtr movie, out Bink2MovieInfo info); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int DecodeNextBgraDelegate(IntPtr movie, IntPtr destination, uint stride, uint destinationBytes); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate void CloseDelegate(IntPtr movie); + + private readonly OpenUtf8Delegate _openUtf8; + private readonly DecodeNextBgraDelegate _decodeNextBgra; + private readonly CloseDelegate _close; + + private NativeAdapter( + OpenUtf8Delegate openUtf8, + DecodeNextBgraDelegate decodeNextBgra, + CloseDelegate close) + { + _openUtf8 = openUtf8; + _decodeNextBgra = decodeNextBgra; + _close = close; + } + + internal static bool TryCreate(IntPtr library, out NativeAdapter? adapter) + { + adapter = null; + if (!NativeLibrary.TryGetExport(library, "sharpemu_bink2_open_utf8", out var open) || + !NativeLibrary.TryGetExport(library, "sharpemu_bink2_decode_next_bgra", out var decode) || + !NativeLibrary.TryGetExport(library, "sharpemu_bink2_close", out var close)) + { + return false; + } + + adapter = new NativeAdapter( + Marshal.GetDelegateForFunctionPointer(open), + Marshal.GetDelegateForFunctionPointer(decode), + Marshal.GetDelegateForFunctionPointer(close)); + return true; + } + + internal bool TryOpen(string path, out IntPtr movie, out Bink2MovieInfo info) + { + var utf8 = Marshal.StringToCoTaskMemUTF8(path); + try + { + return _openUtf8(utf8, out movie, out info) != 0 && movie != IntPtr.Zero; + } + finally + { + Marshal.FreeCoTaskMem(utf8); + } + } + + internal bool DecodeNextBgra(IntPtr movie, IntPtr destination, uint stride, uint destinationBytes) => + _decodeNextBgra(movie, destination, stride, destinationBytes) != 0; + + internal void Close(IntPtr movie) => _close(movie); + } +} diff --git a/src/SharpEmu.Libs/Codec/CodecExports.cs b/src/SharpEmu.Libs/Codec/CodecExports.cs new file mode 100644 index 0000000..fbf85ac --- /dev/null +++ b/src/SharpEmu.Libs/Codec/CodecExports.cs @@ -0,0 +1,104 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using SharpEmu.Libs.Kernel; +using System.Collections.Concurrent; +using System.Threading; + +namespace SharpEmu.Libs.Codec; + +/// +/// libSceVideodec / libSceAudiodec handle management. Actual H.264/HEVC and +/// AAC/AT9 decoding requires an external codec, which is out of scope; these +/// exports keep the decoder lifecycle resolvable (create/decode/flush/delete) +/// and report "no output produced" so guests advance instead of failing on +/// unresolved imports. +/// +public static class CodecExports +{ + private const int Ok = 0; + private const int VideodecErrorInvalidArg = unchecked((int)0x80620801); + private const int AudiodecErrorInvalidArg = unchecked((int)0x807F0002); + + private static readonly ConcurrentDictionary VideoDecoders = new(); + private static readonly ConcurrentDictionary AudioDecoders = new(); + private static long _nextHandle = 1; + + // ---- Video decoder ---- + + [SysAbiExport(Nid = "qkgRiwHyheU", ExportName = "sceVideodecCreateDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecCreateDecoder(CpuContext ctx) + { + var outHandleAddress = ctx[CpuRegister.Rdx]; + if (outHandleAddress == 0) + { + return SetReturn(ctx, VideodecErrorInvalidArg); + } + + var handle = (ulong)Interlocked.Increment(ref _nextHandle); + VideoDecoders[handle] = 1; + return TryWriteHandle(ctx, outHandleAddress, handle) ? SetReturn(ctx, Ok) : SetReturn(ctx, VideodecErrorInvalidArg); + } + + [SysAbiExport(Nid = "q0W5GJMovMs", ExportName = "sceVideodecDecode", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecDecode(CpuContext ctx) + { + // No decoder is present: report success with no picture produced. + return SetReturn(ctx, VideoDecoders.ContainsKey(ctx[CpuRegister.Rdi]) ? Ok : VideodecErrorInvalidArg); + } + + [SysAbiExport(Nid = "jeigLlKdp5I", ExportName = "sceVideodecFlush", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecFlush(CpuContext ctx) => + SetReturn(ctx, VideoDecoders.ContainsKey(ctx[CpuRegister.Rdi]) ? Ok : VideodecErrorInvalidArg); + + [SysAbiExport(Nid = "U0kpGF1cl90", ExportName = "sceVideodecDeleteDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideodec")] + public static int VideodecDeleteDecoder(CpuContext ctx) + { + VideoDecoders.TryRemove(ctx[CpuRegister.Rdi], out _); + return SetReturn(ctx, Ok); + } + + // ---- Audio decoder ---- + + [SysAbiExport(Nid = "O3f1sLMWRvs", ExportName = "sceAudiodecCreateDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAudiodec")] + public static int AudiodecCreateDecoder(CpuContext ctx) + { + var handle = (ulong)Interlocked.Increment(ref _nextHandle); + AudioDecoders[handle] = 1; + // sceAudiodec returns the handle directly (>= 0) or a negative error. + ctx[CpuRegister.Rax] = handle; + return unchecked((int)handle); + } + + [SysAbiExport(Nid = "KHXHMDLkILw", ExportName = "sceAudiodecDecode", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAudiodec")] + public static int AudiodecDecode(CpuContext ctx) + { + // No decoder present: report success with zero output samples so the + // caller treats the frame as silent rather than erroring. + return SetReturn(ctx, AudioDecoders.ContainsKey(ctx[CpuRegister.Rdi]) ? Ok : AudiodecErrorInvalidArg); + } + + [SysAbiExport(Nid = "Tp+ZEy69mLk", ExportName = "sceAudiodecDeleteDecoder", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceAudiodec")] + public static int AudiodecDeleteDecoder(CpuContext ctx) + { + AudioDecoders.TryRemove(ctx[CpuRegister.Rdi], out _); + return SetReturn(ctx, Ok); + } + + private static bool TryWriteHandle(CpuContext ctx, ulong address, ulong handle) => + ctx.TryWriteUInt64(address, handle); + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs b/src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs new file mode 100644 index 0000000..71a4624 --- /dev/null +++ b/src/SharpEmu.Libs/CommonDialog/ErrorDialogExports.cs @@ -0,0 +1,106 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Threading; + +namespace SharpEmu.Libs.CommonDialog; + +/// +/// libSceErrorDialog. There is no host error dialog, so an opened dialog +/// immediately reports finished — this keeps guests that block on the dialog +/// status (a common fatal-error path) from spinning forever. +/// +public static class ErrorDialogExports +{ + private const int AlreadyInitialized = unchecked((int)0x80ED0001); + private const int NotInitialized = unchecked((int)0x80ED0002); + private const int ArgNull = unchecked((int)0x80ED0005); + + private const int StatusNone = 0; + private const int StatusInitialized = 1; + private const int StatusFinished = 3; + + private static int _initialized; + private static int _status; + + [SysAbiExport( + Nid = "I88KChlynSs", + ExportName = "sceErrorDialogInitialize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogInitialize(CpuContext ctx) + { + var result = Interlocked.Exchange(ref _initialized, 1) == 0 ? 0 : AlreadyInitialized; + if (result == 0) + { + Volatile.Write(ref _status, StatusInitialized); + } + + return SetReturn(ctx, result); + } + + [SysAbiExport( + Nid = "M2ZF-ClLhgY", + ExportName = "sceErrorDialogOpen", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogOpen(CpuContext ctx) + { + if (ctx[CpuRegister.Rdi] == 0) + { + return SetReturn(ctx, ArgNull); + } + + if (Volatile.Read(ref _initialized) == 0) + { + return SetReturn(ctx, NotInitialized); + } + + Volatile.Write(ref _status, StatusFinished); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "t2FvHRXzgqk", + ExportName = "sceErrorDialogGetStatus", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogGetStatus(CpuContext ctx) => SetReturn(ctx, Volatile.Read(ref _status)); + + [SysAbiExport( + Nid = "WWiGuh9XfgQ", + ExportName = "sceErrorDialogUpdateStatus", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogUpdateStatus(CpuContext ctx) => SetReturn(ctx, Volatile.Read(ref _status)); + + [SysAbiExport( + Nid = "ekXHb1kDBl0", + ExportName = "sceErrorDialogClose", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogClose(CpuContext ctx) + { + Volatile.Write(ref _status, StatusFinished); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "9XAxK2PMwk8", + ExportName = "sceErrorDialogTerminate", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceErrorDialog")] + public static int ErrorDialogTerminate(CpuContext ctx) + { + Volatile.Write(ref _status, StatusNone); + Interlocked.Exchange(ref _initialized, 0); + return SetReturn(ctx, 0); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/Fiber/FiberExports.cs b/src/SharpEmu.Libs/Fiber/FiberExports.cs index 30a873d..ea2f99e 100644 --- a/src/SharpEmu.Libs/Fiber/FiberExports.cs +++ b/src/SharpEmu.Libs/Fiber/FiberExports.cs @@ -3,6 +3,7 @@ using System.Buffers.Binary; using System.Collections.Concurrent; +using System.Diagnostics; using System.Text; using SharpEmu.HLE; @@ -22,6 +23,10 @@ public static class FiberExports private const uint FiberStateIdle = 2; private const uint FiberStateTerminated = 3; private const uint FiberFlagContextSizeCheck = 0x10; + private const uint FiberFlagSetFpuRegs = 0x100; + private const uint Firmware350BuildVersion = 0x03500000; + private const ushort InitialFpuControlWord = 0x037F; + private const uint InitialMxcsr = 0x9FC0; private const int FiberErrorNull = unchecked((int)0x80590001); private const int FiberErrorAlignment = unchecked((int)0x80590002); @@ -45,13 +50,26 @@ public static class FiberExports private static int _contextSizeCheck; - [ThreadStatic] - private static ulong _currentFiberAddress; - private static readonly object _fiberGate = new(); private static readonly ConcurrentDictionary _continuations = new(); - private static readonly ConcurrentDictionary _returnTargets = new(); private static readonly ConcurrentDictionary _stackRanges = new(); + private static readonly ConcurrentDictionary _threadStates = new(); + + static FiberExports() + { + RunFiberSelfChecks(); + } + + public static void ResetRuntimeState() + { + lock (_fiberGate) + { + _continuations.Clear(); + _stackRanges.Clear(); + _threadStates.Clear(); + Volatile.Write(ref _contextSizeCheck, 0); + } + } [SysAbiExport( Nid = "hVYD7Ou2pCQ", @@ -61,6 +79,7 @@ public static class FiberExports public static int FiberInitialize(CpuContext ctx) { var optParam = ReadStackArg64(ctx, 0); + var buildVersion = unchecked((uint)ReadStackArg64(ctx, 1)); return FiberInitializeCore( ctx, ctx[CpuRegister.Rdi], @@ -70,7 +89,8 @@ public static class FiberExports ctx[CpuRegister.R8], ctx[CpuRegister.R9], optParam, - flags: 0); + flags: 0, + buildVersion); } [SysAbiExport( @@ -82,6 +102,7 @@ public static class FiberExports { var optParam = ReadStackArg64(ctx, 0); var flags = unchecked((uint)ReadStackArg64(ctx, 1)); + var buildVersion = unchecked((uint)ReadStackArg64(ctx, 2)); return FiberInitializeCore( ctx, ctx[CpuRegister.Rdi], @@ -91,7 +112,8 @@ public static class FiberExports ctx[CpuRegister.R8], ctx[CpuRegister.R9], optParam, - flags); + flags, + buildVersion); } [SysAbiExport( @@ -104,17 +126,17 @@ public static class FiberExports var optParam = ctx[CpuRegister.Rdi]; if (optParam == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } if ((optParam & 7) != 0) { - return ctx.SetReturn(FiberErrorAlignment); + return SetReturn(ctx, FiberErrorAlignment); } - return ctx.TryWriteUInt32(optParam, FiberOptSignature) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + return TryWriteUInt32(ctx, optParam, FiberOptSignature) + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } [SysAbiExport( @@ -127,24 +149,23 @@ public static class FiberExports var fiber = ctx[CpuRegister.Rdi]; if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } - if (!ctx.TryReadUInt32(fiber + FiberStateOffset, out var state)) + if (!TryReadUInt32(ctx, fiber + FiberStateOffset, out var state)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (state != FiberStateIdle) { - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } _continuations.TryRemove(fiber, out _); - _returnTargets.TryRemove(fiber, out _); _stackRanges.TryRemove(fiber, out _); - _ = ctx.TryWriteUInt32(fiber + FiberStateOffset, FiberStateTerminated); - return ctx.SetReturn(0); + _ = TryWriteUInt32(ctx, fiber + FiberStateOffset, FiberStateTerminated); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -229,74 +250,47 @@ public static class FiberExports var fiberAddress = ResolveCurrentFiberAddress(ctx); if (fiberAddress == 0) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } if (GuestThreadExecution.Scheduler is not { SupportsGuestContextTransfer: true } || !GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } var returnArgument = ctx[CpuRegister.Rdi]; var argOnRunAddress = ctx[CpuRegister.Rsi]; - if (argOnRunAddress != 0 && !ctx.TryWriteUInt64(argOnRunAddress, 0)) - { - return ctx.SetReturn(FiberErrorInvalid); - } - GuestCpuContinuation transferTarget; - ulong previousFiber; lock (_fiberGate) { _continuations[fiberAddress] = new FiberContinuation( CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), argOnRunAddress); - if (!_returnTargets.TryRemove(fiberAddress, out var returnTarget)) + var threadKey = GetThreadKey(ctx); + if (!_threadStates.TryGetValue(threadKey, out var threadState) || + !TryWriteResumeArgument(ctx, threadState.RootContinuation, returnArgument)) { _continuations.TryRemove(fiberAddress, out _); - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } - previousFiber = returnTarget.PreviousFiber; - if (previousFiber != 0) + if (!TryWriteUInt32(ctx, fiberAddress + FiberStateOffset, FiberStateIdle)) { - if (!_continuations.TryRemove(previousFiber, out var previousContinuation) || - !TryWriteResumeArgument(ctx, previousContinuation, returnArgument) || - !ctx.TryWriteUInt32(previousFiber + FiberStateOffset, FiberStateRun)) - { - _continuations.TryRemove(fiberAddress, out _); - return ctx.SetReturn(FiberErrorState); - } - - transferTarget = previousContinuation.Context with { Rax = 0 }; - } - else - { - if (!returnTarget.ThreadContinuation.HasValue || - !TryWriteResumeArgument(ctx, returnTarget.ThreadContinuation.Value, returnArgument)) - { - _continuations.TryRemove(fiberAddress, out _); - return ctx.SetReturn(FiberErrorState); - } - - transferTarget = returnTarget.ThreadContinuation.Value.Context with { Rax = 0 }; + return SetReturn(ctx, FiberErrorInvalid); } - if (!ctx.TryWriteUInt32(fiberAddress + FiberStateOffset, FiberStateIdle)) - { - return ctx.SetReturn(FiberErrorInvalid); - } + transferTarget = threadState.RootContinuation.Context with { Rax = 0 }; + _threadStates.TryRemove(threadKey, out _); } - _currentFiberAddress = previousFiber; - _ = GuestThreadExecution.EnterFiber(previousFiber); + _ = GuestThreadExecution.EnterFiber(0); GuestThreadExecution.RequestCurrentContextTransfer(transferTarget); TraceFiber( - $"return fiber=0x{fiberAddress:X16} to=0x{previousFiber:X16} " + + $"return-to-thread fiber=0x{fiberAddress:X16} " + $"resume=0x{transferTarget.Rip:X16} rsp=0x{transferTarget.Rsp:X16} arg=0x{returnArgument:X16}"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -309,18 +303,18 @@ public static class FiberExports var outAddress = ctx[CpuRegister.Rdi]; if (outAddress == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } var fiberAddress = ResolveCurrentFiberAddress(ctx); if (fiberAddress == 0) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } - return ctx.TryWriteUInt64(outAddress, fiberAddress) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + return TryWriteUInt64(ctx, outAddress, fiberAddress) + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } [SysAbiExport( @@ -334,35 +328,35 @@ public static class FiberExports var info = ctx[CpuRegister.Rsi]; if (info == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } - if (!ctx.TryReadUInt64(info, out var size) || size != FiberInfoSize) + if (!TryReadUInt64(ctx, info, out var size) || size != FiberInfoSize) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (!TryReadFiberFields(ctx, fiber, out var fields)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - if (!ctx.TryWriteUInt64(info + 8, fields.Entry) || - !ctx.TryWriteUInt64(info + 16, fields.ArgOnInitialize) || - !ctx.TryWriteUInt64(info + 24, fields.ContextAddress) || - !ctx.TryWriteUInt64(info + 32, fields.ContextSize) || + if (!TryWriteUInt64(ctx, info + 8, fields.Entry) || + !TryWriteUInt64(ctx, info + 16, fields.ArgOnInitialize) || + !TryWriteUInt64(ctx, info + 24, fields.ContextAddress) || + !TryWriteUInt64(ctx, info + 32, fields.ContextSize) || !TryWriteName(ctx, info + 40, fields.Name) || - !ctx.TryWriteUInt64(info + 72, ulong.MaxValue)) + !TryWriteUInt64(ctx, info + 72, ulong.MaxValue)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -376,22 +370,22 @@ public static class FiberExports var nameAddress = ctx[CpuRegister.Rsi]; if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } if (nameAddress == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxNameLength + 1, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxNameLength + 1, out var name)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } return TryWriteName(ctx, fiber + FiberNameOffset, name) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } [SysAbiExport( @@ -403,12 +397,12 @@ public static class FiberExports { if (ctx[CpuRegister.Rdi] != 0) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } return Interlocked.Exchange(ref _contextSizeCheck, 1) == 0 - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorState); + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorState); } [SysAbiExport( @@ -419,8 +413,8 @@ public static class FiberExports public static int FiberStopContextSizeCheck(CpuContext ctx) { return Interlocked.Exchange(ref _contextSizeCheck, 0) == 1 - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorState); + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorState); } [SysAbiExport( @@ -433,17 +427,18 @@ public static class FiberExports var outAddress = ctx[CpuRegister.Rdi]; if (outAddress == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } - if (ResolveCurrentFiberAddress(ctx) == 0) + if (ResolveCurrentFiberAddress(ctx) == 0 || + !_threadStates.TryGetValue(GetThreadKey(ctx), out var threadState)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } - return ctx.TryWriteUInt64(outAddress, ctx[CpuRegister.Rbp]) - ? ctx.SetReturn(0) - : ctx.SetReturn(FiberErrorInvalid); + return TryWriteUInt64(ctx, outAddress, threadState.RootContinuation.Context.Rbp) + ? SetReturn(ctx, 0) + : SetReturn(ctx, FiberErrorInvalid); } private static int FiberInitializeCore( @@ -455,69 +450,70 @@ public static class FiberExports ulong contextAddress, ulong contextSize, ulong optParam, - uint flags) + uint flags, + uint buildVersion) { if (fiber == 0 || nameAddress == 0 || entry == 0) { - return ctx.SetReturn(FiberErrorNull); + return SetReturn(ctx, FiberErrorNull); } if ((fiber & 7) != 0 || (contextAddress & 15) != 0 || (optParam & 7) != 0) { - return ctx.SetReturn(FiberErrorAlignment); + return SetReturn(ctx, FiberErrorAlignment); } if (contextSize != 0 && contextSize < FiberContextMinimumSize) { - return ctx.SetReturn(FiberErrorRange); + return SetReturn(ctx, FiberErrorRange); } if ((contextSize & 15) != 0 || (contextAddress == 0 && contextSize != 0) || (contextAddress != 0 && contextSize == 0)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (optParam != 0 && - (!ctx.TryReadUInt32(optParam, out var optMagic) || optMagic != FiberOptSignature)) + (!TryReadUInt32(ctx, optParam, out var optMagic) || optMagic != FiberOptSignature)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxNameLength + 1, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxNameLength + 1, out var name)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } - if (Volatile.Read(ref _contextSizeCheck) != 0) - { - flags |= FiberFlagContextSizeCheck; - } + flags = ApplyInitializationFlags( + flags, + buildVersion, + Volatile.Read(ref _contextSizeCheck) != 0); - if (!ctx.TryWriteUInt32(fiber + FiberMagicStartOffset, FiberSignature0) || - !ctx.TryWriteUInt32(fiber + FiberStateOffset, FiberStateIdle) || - !ctx.TryWriteUInt64(fiber + FiberEntryOffset, entry) || - !ctx.TryWriteUInt64(fiber + FiberArgOnInitializeOffset, argOnInitialize) || - !ctx.TryWriteUInt64(fiber + FiberContextAddressOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextSizeOffset, contextSize) || + if (!TryWriteUInt32(ctx, fiber + FiberMagicStartOffset, FiberSignature0) || + !TryWriteUInt32(ctx, fiber + FiberStateOffset, FiberStateIdle) || + !TryWriteUInt64(ctx, fiber + FiberEntryOffset, entry) || + !TryWriteUInt64(ctx, fiber + FiberArgOnInitializeOffset, argOnInitialize) || + !TryWriteUInt64(ctx, fiber + FiberContextAddressOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextSizeOffset, contextSize) || !TryWriteName(ctx, fiber + FiberNameOffset, name) || - !ctx.TryWriteUInt64(fiber + FiberContextPointerOffset, 0) || - !ctx.TryWriteUInt32(fiber + FiberFlagsOffset, flags) || - !ctx.TryWriteUInt64(fiber + FiberContextStartOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextEndOffset, contextAddress == 0 ? 0 : contextAddress + contextSize) || - !ctx.TryWriteUInt32(fiber + FiberMagicEndOffset, FiberSignature1)) + !TryWriteUInt64(ctx, fiber + FiberContextPointerOffset, 0) || + !TryWriteUInt32(ctx, fiber + FiberFlagsOffset, flags) || + !TryWriteUInt64(ctx, fiber + FiberContextStartOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextEndOffset, contextAddress == 0 ? 0 : contextAddress + contextSize) || + !TryWriteUInt32(ctx, fiber + FiberMagicEndOffset, FiberSignature1)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (contextAddress != 0) { - if (!ctx.TryWriteUInt64(contextAddress, FiberStackSignature)) + if (!TryWriteUInt64(ctx, contextAddress, FiberStackSignature)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if ((flags & FiberFlagContextSizeCheck) != 0) @@ -531,8 +527,10 @@ public static class FiberExports _stackRanges[fiber] = new FiberStackRange(contextAddress, contextSize); } - TraceFiber($"init fiber=0x{fiber:X16} entry=0x{entry:X16} ctx=0x{contextAddress:X16} size=0x{contextSize:X} name='{name}'"); - return ctx.SetReturn(0); + TraceFiber( + $"init fiber=0x{fiber:X16} entry=0x{entry:X16} ctx=0x{contextAddress:X16} " + + $"size=0x{contextSize:X} flags=0x{flags:X} build=0x{buildVersion:X8} name='{name}'"); + return SetReturn(ctx, 0); } private static int FiberRunCore( @@ -547,12 +545,12 @@ public static class FiberExports { if (!TryValidateFiber(ctx, fiber, out var error)) { - return ctx.SetReturn(error); + return SetReturn(ctx, error); } if (!TryReadFiberFields(ctx, fiber, out var fields)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (attachContextAddress != 0 || attachContextSize != 0) @@ -560,7 +558,7 @@ public static class FiberExports var attachResult = AttachContext(ctx, fiber, attachContextAddress, attachContextSize, ref fields); if (attachResult != 0) { - return ctx.SetReturn(attachResult); + return SetReturn(ctx, attachResult); } } @@ -568,78 +566,96 @@ public static class FiberExports if ((isSwitch && previousFiber == 0) || (!isSwitch && previousFiber != 0)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } if (previousFiber == fiber) { - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } if (GuestThreadExecution.Scheduler is not { SupportsGuestContextTransfer: true } || !GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame)) { - return ctx.SetReturn(FiberErrorPermission); + return SetReturn(ctx, FiberErrorPermission); } GuestCpuContinuation transferTarget; var resumed = false; lock (_fiberGate) { + var threadKey = GetThreadKey(ctx); if (!TryReadFiberFields(ctx, fiber, out fields)) { - return ctx.SetReturn(FiberErrorInvalid); + return SetReturn(ctx, FiberErrorInvalid); } if (fields.State != FiberStateIdle) { TraceFiber($"run-state-error reason={reason} fiber=0x{fiber:X16} state=0x{fields.State:X8}"); - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } - FiberContinuation targetContinuation; + FiberContinuation targetContinuation = default; if (_continuations.TryGetValue(fiber, out var savedContinuation)) { targetContinuation = savedContinuation; resumed = true; } - else if (!TryCreateInitialContinuation(ctx, fields, argOnRun, out targetContinuation)) - { - return ctx.SetReturn(FiberErrorInvalid); - } - - if (resumed && !TryWriteResumeArgument(ctx, targetContinuation, argOnRun)) - { - return ctx.SetReturn(FiberErrorInvalid); - } - var callerContinuation = new FiberContinuation( CaptureContinuation(ctx, frame.ReturnRip, frame.ResumeRsp, frame.ReturnSlotAddress), outArgumentAddress); + if (!resumed) + { + var rootStackTop = _threadStates.TryGetValue(threadKey, out var existingThreadState) + ? existingThreadState.RootContinuation.Context.Rsp + : callerContinuation.Context.Rsp; + if (!TryCreateInitialContinuation( + ctx, + fields, + argOnRun, + rootStackTop, + out targetContinuation)) + { + return SetReturn(ctx, FiberErrorInvalid); + } + } + else if (!TryWriteResumeArgument(ctx, targetContinuation, argOnRun)) + { + return SetReturn(ctx, FiberErrorInvalid); + } + if (previousFiber != 0) { - if (!ctx.TryReadUInt32(previousFiber + FiberStateOffset, out var previousState) || + if (!TryReadUInt32(ctx, previousFiber + FiberStateOffset, out var previousState) || previousState != FiberStateRun || - !ctx.TryWriteUInt32(previousFiber + FiberStateOffset, FiberStateIdle)) + !TryWriteUInt32(ctx, previousFiber + FiberStateOffset, FiberStateIdle)) { - return ctx.SetReturn(FiberErrorState); + return SetReturn(ctx, FiberErrorState); } _continuations[previousFiber] = callerContinuation; - _returnTargets[fiber] = new FiberReturnTarget(previousFiber, null); } else { - _returnTargets[fiber] = new FiberReturnTarget(0, callerContinuation); + if (_threadStates.ContainsKey(threadKey)) + { + return SetReturn(ctx, FiberErrorPermission); + } + + _threadStates[threadKey] = new FiberThreadState(callerContinuation, fiber, previousFiber); } - if (!ctx.TryWriteUInt32(fiber + FiberStateOffset, FiberStateRun)) + if (!TryWriteUInt32(ctx, fiber + FiberStateOffset, FiberStateRun)) { if (previousFiber != 0) { _continuations.TryRemove(previousFiber, out _); - _ = ctx.TryWriteUInt32(previousFiber + FiberStateOffset, FiberStateRun); + _ = TryWriteUInt32(ctx, previousFiber + FiberStateOffset, FiberStateRun); } - _returnTargets.TryRemove(fiber, out _); - return ctx.SetReturn(FiberErrorInvalid); + else + { + _threadStates.TryRemove(threadKey, out _); + } + return SetReturn(ctx, FiberErrorInvalid); } if (resumed) @@ -648,35 +664,66 @@ public static class FiberExports } transferTarget = targetContinuation.Context with { Rax = 0 }; + if (_threadStates.TryGetValue(threadKey, out var activeState)) + { + _threadStates[threadKey] = activeState with + { + CurrentFiber = fiber, + PreviousFiber = previousFiber, + }; + } } - _currentFiberAddress = fiber; _ = GuestThreadExecution.EnterFiber(fiber); GuestThreadExecution.RequestCurrentContextTransfer(transferTarget); TraceFiber( $"transfer reason={reason} from=0x{previousFiber:X16} to=0x{fiber:X16} resume={resumed} " + $"rip=0x{transferTarget.Rip:X16} rsp=0x{transferTarget.Rsp:X16} arg=0x{argOnRun:X16}"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } private static bool TryCreateInitialContinuation( CpuContext ctx, FiberFields fields, ulong argOnRun, + ulong rootStackTop, out FiberContinuation continuation) { continuation = default; - if (fields.ContextAddress == 0 || fields.ContextSize < FiberContextMinimumSize) + ulong stackEnd; + if (fields.ContextAddress == 0) + { + if (rootStackTop < 32) + { + return false; + } + + // Contextless fibers are specified to execute on the root + // sceFiberRun stack. Keep the suspended import return record + // above the entry stack and use a synthetic transfer slot below it. + stackEnd = rootStackTop & ~15UL; + } + else + { + if (fields.ContextSize < FiberContextMinimumSize) + { + return false; + } + + stackEnd = fields.ContextAddress + fields.ContextSize; + } + + if (!TryCalculateInitialStackLayout(stackEnd, out var entryRsp, out var transferSlot)) + { + return false; + } + if (!TryWriteUInt64(ctx, transferSlot, fields.Entry) || + !TryWriteUInt64(ctx, entryRsp, 0)) { return false; } - var stackEnd = fields.ContextAddress + fields.ContextSize; - var entryRsp = (stackEnd & ~15UL) - sizeof(ulong); - if (!ctx.TryWriteUInt64(entryRsp, 0)) - { - return false; - } + var setFpuRegisters = (fields.Flags & FiberFlagSetFpuRegs) != 0; continuation = new FiberContinuation( new GuestCpuContinuation( @@ -698,7 +745,12 @@ public static class FiberExports 0, 0, 0, - 0), + 0, + 0, + 0, + setFpuRegisters ? InitialFpuControlWord : ctx.FpuControlWord, + setFpuRegisters ? InitialMxcsr : ctx.Mxcsr, + RestoreFullFpuState: setFpuRegisters), 0); return true; } @@ -708,7 +760,7 @@ public static class FiberExports FiberContinuation continuation, ulong argument) => continuation.ArgOnRunAddress == 0 || - ctx.TryWriteUInt64(continuation.ArgOnRunAddress, argument); + TryWriteUInt64(ctx, continuation.ArgOnRunAddress, argument); private static GuestCpuContinuation CaptureContinuation( CpuContext ctx, @@ -731,26 +783,99 @@ public static class FiberExports ctx[CpuRegister.Rdi], ctx[CpuRegister.R8], ctx[CpuRegister.R9], + ctx[CpuRegister.R10], + ctx[CpuRegister.R11], ctx[CpuRegister.R12], ctx[CpuRegister.R13], ctx[CpuRegister.R14], - ctx[CpuRegister.R15]); + ctx[CpuRegister.R15], + ctx.FpuControlWord, + ctx.Mxcsr, + RestoreFullFpuState: false); private static ulong ResolveCurrentFiberAddress(CpuContext ctx) { - if (_currentFiberAddress != 0) - { - return _currentFiberAddress; - } - if (GuestThreadExecution.CurrentFiberAddress != 0) { return GuestThreadExecution.CurrentFiberAddress; } + if (_threadStates.TryGetValue(GetThreadKey(ctx), out var threadState) && + threadState.CurrentFiber != 0) + { + return threadState.CurrentFiber; + } + return TryFindFiberByStack(ctx, out var fiberAddress) ? fiberAddress : 0; } + private static ulong GetThreadKey(CpuContext ctx) + { + var handle = GuestThreadExecution.CurrentGuestThreadHandle; + if (handle != 0) + { + return handle; + } + + // The main guest thread does not always have a scheduler handle. Its + // ABI thread pointer is stable across native/managed transitions. + return ctx.FsBase != 0 ? ctx.FsBase : 1; + } + + private static uint ApplyInitializationFlags( + uint flags, + uint buildVersion, + bool contextSizeCheck) + { + if (buildVersion >= Firmware350BuildVersion) + { + flags |= FiberFlagSetFpuRegs; + } + if (contextSizeCheck) + { + flags |= FiberFlagContextSizeCheck; + } + + return flags; + } + + private static bool TryCalculateInitialStackLayout( + ulong stackEnd, + out ulong entryRsp, + out ulong transferSlot) + { + var alignedEnd = stackEnd & ~15UL; + if (alignedEnd < 2 * sizeof(ulong)) + { + entryRsp = 0; + transferSlot = 0; + return false; + } + + entryRsp = alignedEnd - sizeof(ulong); + transferSlot = entryRsp - sizeof(ulong); + return true; + } + + [Conditional("DEBUG")] + private static void RunFiberSelfChecks() + { + Debug.Assert( + ApplyInitializationFlags(0, Firmware350BuildVersion - 1, false) == 0, + "Pre-3.50 fibers unexpectedly enable SetFpuRegs."); + Debug.Assert( + ApplyInitializationFlags(0, Firmware350BuildVersion, false) == FiberFlagSetFpuRegs, + "3.50+ fibers must initialize x87/MXCSR control state."); + Debug.Assert( + ApplyInitializationFlags(1, Firmware350BuildVersion, true) == + (1u | FiberFlagSetFpuRegs | FiberFlagContextSizeCheck), + "Fiber initialization discarded caller/internal flags."); + Debug.Assert( + TryCalculateInitialStackLayout(0x1017, out var rsp, out var slot) && + rsp == 0x1008 && slot == 0x1000 && (rsp & 15) == 8, + "Initial fiber transfer slot does not produce SysV entry alignment."); + } + internal static ulong GetCurrentFiberAddressForDiagnostics(CpuContext ctx) => ResolveCurrentFiberAddress(ctx); @@ -779,11 +904,11 @@ public static class FiberExports return FiberErrorInvalid; } - if (!ctx.TryWriteUInt64(fiber + FiberContextAddressOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextSizeOffset, contextSize) || - !ctx.TryWriteUInt64(fiber + FiberContextStartOffset, contextAddress) || - !ctx.TryWriteUInt64(fiber + FiberContextEndOffset, contextAddress + contextSize) || - !ctx.TryWriteUInt64(contextAddress, FiberStackSignature)) + if (!TryWriteUInt64(ctx, fiber + FiberContextAddressOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextSizeOffset, contextSize) || + !TryWriteUInt64(ctx, fiber + FiberContextStartOffset, contextAddress) || + !TryWriteUInt64(ctx, fiber + FiberContextEndOffset, contextAddress + contextSize) || + !TryWriteUInt64(ctx, contextAddress, FiberStackSignature)) { return FiberErrorInvalid; } @@ -839,8 +964,8 @@ public static class FiberExports return false; } - if (!ctx.TryReadUInt32(fiber + FiberMagicStartOffset, out var magicStart) || - !ctx.TryReadUInt32(fiber + FiberMagicEndOffset, out var magicEnd) || + if (!TryReadUInt32(ctx, fiber + FiberMagicStartOffset, out var magicStart) || + !TryReadUInt32(ctx, fiber + FiberMagicEndOffset, out var magicEnd) || magicStart != FiberSignature0 || magicEnd != FiberSignature1) { @@ -855,12 +980,12 @@ public static class FiberExports private static bool TryReadFiberFields(CpuContext ctx, ulong fiber, out FiberFields fields) { fields = default; - if (!ctx.TryReadUInt32(fiber + FiberStateOffset, out var state) || - !ctx.TryReadUInt64(fiber + FiberEntryOffset, out var entry) || - !ctx.TryReadUInt64(fiber + FiberArgOnInitializeOffset, out var argOnInitialize) || - !ctx.TryReadUInt64(fiber + FiberContextAddressOffset, out var contextAddress) || - !ctx.TryReadUInt64(fiber + FiberContextSizeOffset, out var contextSize) || - !ctx.TryReadUInt32(fiber + FiberFlagsOffset, out var flags) || + if (!TryReadUInt32(ctx, fiber + FiberStateOffset, out var state) || + !TryReadUInt64(ctx, fiber + FiberEntryOffset, out var entry) || + !TryReadUInt64(ctx, fiber + FiberArgOnInitializeOffset, out var argOnInitialize) || + !TryReadUInt64(ctx, fiber + FiberContextAddressOffset, out var contextAddress) || + !TryReadUInt64(ctx, fiber + FiberContextSizeOffset, out var contextSize) || + !TryReadUInt32(ctx, fiber + FiberFlagsOffset, out var flags) || !TryReadInlineName(ctx, fiber + FiberNameOffset, out var name)) { return false; @@ -898,6 +1023,46 @@ public static class FiberExports return 0; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt64(CpuContext ctx, ulong address, ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + private static bool TryWriteName(CpuContext ctx, ulong address, string name) { Span buffer = stackalloc byte[MaxNameLength + 1]; @@ -925,6 +1090,37 @@ public static class FiberExports return true; } + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int capacity, out string value) + { + var bytes = new byte[capacity]; + Span current = stackalloc byte[1]; + for (var index = 0; index < bytes.Length; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, current)) + { + value = string.Empty; + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, index); + return true; + } + + bytes[index] = current[0]; + } + + value = Encoding.UTF8.GetString(bytes); + return true; + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + private static void TraceFiber(string message) { if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_FIBER"), "1", StringComparison.Ordinal)) @@ -946,9 +1142,10 @@ public static class FiberExports GuestCpuContinuation Context, ulong ArgOnRunAddress); - private readonly record struct FiberReturnTarget( - ulong PreviousFiber, - FiberContinuation? ThreadContinuation); + private sealed record FiberThreadState( + FiberContinuation RootContinuation, + ulong CurrentFiber, + ulong PreviousFiber); private readonly record struct FiberStackRange(ulong Start, ulong Size) { diff --git a/src/SharpEmu.Libs/Font/FontExports.cs b/src/SharpEmu.Libs/Font/FontExports.cs new file mode 100644 index 0000000..b02d14d --- /dev/null +++ b/src/SharpEmu.Libs/Font/FontExports.cs @@ -0,0 +1,293 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Font; + +public static class FontExports +{ + private static readonly object AllocationGate = new(); + private static ulong _librarySelectionAddress; + private static ulong _rendererSelectionAddress; + + [SysAbiExport( + Nid = "whrS4oksXc4", + ExportName = "sceFontMemoryInit", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int MemoryInit(CpuContext ctx) + { + var descriptorAddress = ctx[CpuRegister.Rdi]; + var regionAddress = ctx[CpuRegister.Rsi]; + var regionSize = (uint)ctx[CpuRegister.Rdx]; + var interfaceAddress = ctx[CpuRegister.Rcx]; + var mspaceAddress = ctx[CpuRegister.R8]; + var destroyCallback = ctx[CpuRegister.R9]; + if (descriptorAddress == 0 || + !TryWriteUInt32(ctx, descriptorAddress, 0x00000F00) || + !TryWriteUInt32(ctx, descriptorAddress + 0x04, regionSize) || + !ctx.TryWriteUInt64(descriptorAddress + 0x08, regionAddress) || + !ctx.TryWriteUInt64(descriptorAddress + 0x10, mspaceAddress) || + !ctx.TryWriteUInt64(descriptorAddress + 0x18, interfaceAddress) || + !ctx.TryWriteUInt64(descriptorAddress + 0x20, destroyCallback) || + !ctx.TryWriteUInt64(descriptorAddress + 0x28, 0) || + !ctx.TryWriteUInt64(descriptorAddress + 0x30, 0) || + !ctx.TryWriteUInt64(descriptorAddress + 0x38, mspaceAddress)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetSuccess(ctx); + } + + [SysAbiExport( + Nid = "oM+XCzVG3oM", + ExportName = "sceFontSelectLibraryFt", + Target = Generation.Gen5, + LibraryName = "libSceFontFt")] + public static int SelectLibraryFt(CpuContext ctx) => + ReturnSelection(ctx, ref _librarySelectionAddress, 0x38); + + [SysAbiExport( + Nid = "Xx974EW-QFY", + ExportName = "sceFontSelectRendererFt", + Target = Generation.Gen5, + LibraryName = "libSceFontFt")] + public static int SelectRendererFt(CpuContext ctx) => + ReturnSelection(ctx, ref _rendererSelectionAddress, 0x100); + + [SysAbiExport( + Nid = "n590hj5Oe-k", + ExportName = "sceFontCreateLibraryWithEdition", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int CreateLibraryWithEdition(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.Rcx], 0x100, magic: 0x0F01); + + [SysAbiExport( + Nid = "WaSFJoRWXaI", + ExportName = "sceFontCreateRendererWithEdition", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int CreateRendererWithEdition(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.Rcx], 0x100, magic: 0x0F07); + + [SysAbiExport( + Nid = "cKYtVmeSTcw", + ExportName = "sceFontOpenFontSet", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int OpenFontSet(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.R8], 0x100, magic: 0x0F02); + + [SysAbiExport( + Nid = "KXUpebrFk1U", + ExportName = "sceFontOpenFontMemory", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int OpenFontMemory(CpuContext ctx) => + CreateOpaqueHandle(ctx, ctx[CpuRegister.R8], 0x100, magic: 0x0F02); + + [SysAbiExport( + Nid = "JzCH3SCFnAU", + ExportName = "sceFontOpenFontInstance", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int OpenFontInstance(CpuContext ctx) + { + var sourceHandle = ctx[CpuRegister.Rdi]; + var setupHandle = ctx[CpuRegister.Rsi]; + var outputAddress = ctx[CpuRegister.Rdx]; + if (outputAddress == 0) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + if (setupHandle != 0) + { + return ctx.TryWriteUInt64(outputAddress, setupHandle) + ? SetSuccess(ctx) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (!TryAllocateOpaque(ctx, 0x100, out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (sourceHandle != 0) + { + Span source = stackalloc byte[0x100]; + if (ctx.Memory.TryRead(sourceHandle, source)) + { + _ = ctx.Memory.TryWrite(handle, source); + } + } + + _ = TryWriteUInt16(ctx, handle, 0x0F02); + return ctx.TryWriteUInt64(outputAddress, handle) + ? SetSuccess(ctx) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "SsRbbCiWoGw", + ExportName = "sceFontSupportSystemFonts", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int SupportSystemFonts(CpuContext ctx) => SetSuccess(ctx); + + [SysAbiExport( + Nid = "mz2iTY0MK4A", + ExportName = "sceFontSupportExternalFonts", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int SupportExternalFonts(CpuContext ctx) => SetSuccess(ctx); + + [SysAbiExport( + Nid = "CUKn5pX-NVY", + ExportName = "sceFontAttachDeviceCacheBuffer", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int AttachDeviceCacheBuffer(CpuContext ctx) => SetSuccess(ctx); + + [SysAbiExport( + Nid = "IQtleGLL5pQ", + ExportName = "sceFontGetRenderCharGlyphMetrics", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int GetRenderCharGlyphMetrics(CpuContext ctx) + { + var metricsAddress = ctx[CpuRegister.Rdx]; + if (metricsAddress == 0) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + var values = new[] { 8.0f, 16.0f, 0.0f, 12.0f, 8.0f, 0.0f, 0.0f, 16.0f }; + for (var index = 0; index < values.Length; index++) + { + if (!TryWriteUInt32( + ctx, + metricsAddress + (ulong)(index * sizeof(float)), + BitConverter.SingleToUInt32Bits(values[index]))) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + } + + return SetSuccess(ctx); + } + + [SysAbiExport( + Nid = "gdUCnU0gHdI", + ExportName = "sceFontRenderSurfaceInit", + Target = Generation.Gen5, + LibraryName = "libSceFont")] + public static int RenderSurfaceInit(CpuContext ctx) + { + var surfaceAddress = ctx[CpuRegister.Rdi]; + var bufferAddress = ctx[CpuRegister.Rsi]; + var widthBytes = (uint)ctx[CpuRegister.Rdx]; + var pixelBytes = (uint)ctx[CpuRegister.Rcx] & 0xFF; + var width = (uint)ctx[CpuRegister.R8]; + var height = (uint)ctx[CpuRegister.R9]; + if (surfaceAddress == 0 || + !ctx.TryWriteUInt64(surfaceAddress, bufferAddress) || + !TryWriteUInt32(ctx, surfaceAddress + 0x08, widthBytes) || + !TryWriteUInt32(ctx, surfaceAddress + 0x0C, pixelBytes) || + !TryWriteUInt32(ctx, surfaceAddress + 0x10, width) || + !TryWriteUInt32(ctx, surfaceAddress + 0x14, height) || + !TryWriteUInt32(ctx, surfaceAddress + 0x18, 0) || + !TryWriteUInt32(ctx, surfaceAddress + 0x1C, 0) || + !TryWriteUInt32(ctx, surfaceAddress + 0x20, width) || + !TryWriteUInt32(ctx, surfaceAddress + 0x24, height)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetSuccess(ctx); + } + + private static int ReturnSelection(CpuContext ctx, ref ulong selectionAddress, uint objectSize) + { + if (ctx[CpuRegister.Rdi] != 0) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + lock (AllocationGate) + { + if (selectionAddress == 0) + { + if (!TryAllocateOpaque(ctx, 0x20, out selectionAddress) || + !TryWriteUInt32(ctx, selectionAddress, 0) || + !TryWriteUInt32(ctx, selectionAddress + 4, objectSize)) + { + selectionAddress = 0; + } + } + } + + ctx[CpuRegister.Rax] = selectionAddress; + return 0; + } + + private static int CreateOpaqueHandle(CpuContext ctx, ulong outputAddress, int size, ushort magic) + { + if (outputAddress == 0 || !TryAllocateOpaque(ctx, size, out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (!TryWriteUInt16(ctx, handle, magic) || !ctx.TryWriteUInt64(outputAddress, handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetSuccess(ctx); + } + + private static bool TryAllocateOpaque(CpuContext ctx, int size, out ulong address) + { + address = 0; + if (ctx.Memory is not IGuestMemoryAllocator allocator || + !allocator.TryAllocateGuestMemory((ulong)size, 0x10, out address)) + { + return false; + } + + Span bytes = stackalloc byte[size]; + bytes.Clear(); + return ctx.Memory.TryWrite(address, bytes); + } + + private static bool TryWriteUInt16(CpuContext ctx, ulong address, ushort value) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + BinaryPrimitives.WriteUInt16LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static int SetSuccess(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)(int)result); + return (int)result; + } +} diff --git a/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs b/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs index 759b40d..99f90dc 100644 --- a/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs +++ b/src/SharpEmu.Libs/Gpu/GuestGpuTypes.cs @@ -22,6 +22,8 @@ internal sealed record GuestDrawTexture( bool IsStorage, uint MipLevels = 1, uint MipLevel = 0, + uint BaseMipLevel = 0, + uint ResourceMipLevels = 1, uint Pitch = 0, uint TileMode = 0, uint DstSelect = 0xFAC, @@ -36,7 +38,11 @@ internal readonly record struct GuestSampler( internal sealed record GuestMemoryBuffer( ulong BaseAddress, - byte[] Data); + byte[] Data, + int Length, + bool Pooled, + bool Writable = false, + bool WriteBackToGuest = true); /// DataFormat/NumberFormat are raw guest vertex-attribute codes. internal sealed record GuestVertexBuffer( @@ -47,11 +53,15 @@ internal sealed record GuestVertexBuffer( ulong BaseAddress, uint Stride, uint OffsetBytes, - byte[] Data); + byte[] Data, + int Length, + bool Pooled); internal sealed record GuestIndexBuffer( byte[] Data, - bool Is32Bit); + int Length, + bool Is32Bit, + bool Pooled); internal readonly record struct GuestRect( int X, @@ -67,6 +77,25 @@ internal readonly record struct GuestViewport( float MinDepth, float MaxDepth); +internal readonly record struct GuestRasterState( + bool CullFront, + bool CullBack, + bool FrontFaceClockwise, + bool Wireframe) +{ + public static GuestRasterState Default { get; } = new(false, false, false, false); +} + +// CompareOp uses the GCN DB_DEPTH_CONTROL ZFUNC encoding, which matches the +// Vulkan CompareOp ordering (0=Never through 7=Always). +internal readonly record struct GuestDepthState( + bool TestEnable, + bool WriteEnable, + uint CompareOp) +{ + public static GuestDepthState Default { get; } = new(false, false, 7); +} + /// Factors/funcs are raw guest CB_BLEND*_CONTROL register bitfields; the /// defaults (1/0) are the guest ONE/ZERO codes. internal readonly record struct GuestBlendState( @@ -95,12 +124,16 @@ internal readonly record struct GuestBlendState( internal sealed record GuestRenderState( IReadOnlyList Blends, GuestRect? Scissor, - GuestViewport? Viewport) + GuestViewport? Viewport, + GuestRasterState Raster, + GuestDepthState Depth) { public static GuestRenderState Default { get; } = new( [GuestBlendState.Default], Scissor: null, - Viewport: null); + Viewport: null, + GuestRasterState.Default, + GuestDepthState.Default); public GuestBlendState Blend => Blends.Count == 0 ? GuestBlendState.Default : Blends[0]; @@ -114,3 +147,17 @@ internal sealed record GuestRenderTarget( uint Format, uint NumberType, uint MipLevels = 1); + +/// Guest DB surface bound alongside a color render target. +internal sealed record GuestDepthTarget( + ulong ReadAddress, + ulong WriteAddress, + uint Width, + uint Height, + uint GuestFormat, + uint SwizzleMode, + float ClearDepth, + bool ReadOnly) +{ + public ulong Address => WriteAddress != 0 ? WriteAddress : ReadAddress; +} diff --git a/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs b/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs index 0963fc2..371ad82 100644 --- a/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs +++ b/src/SharpEmu.Libs/Gpu/IGuestGpuBackend.cs @@ -33,7 +33,9 @@ internal interface IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1); + int scalarRegisterBufferIndex = -1, + int requiredVertexOutputCount = 0, + ulong storageBufferOffsetAlignment = 1); bool TryCompilePixelShader( Gen5ShaderState state, @@ -44,7 +46,10 @@ internal interface IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1); + int scalarRegisterBufferIndex = -1, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1); bool TryCompileComputeShader( Gen5ShaderState state, @@ -53,7 +58,14 @@ internal interface IGuestGpuBackend uint localSizeY, uint localSizeZ, out IGuestCompiledShader? shader, - out string error); + out string error, + int totalGlobalBufferCount = -1, + int initialScalarBufferIndex = -1, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1); + + /// Returns the backend's no-color-output fragment shader. + IGuestCompiledShader GetDepthOnlyFragmentShader(); void HideSplashScreen(); @@ -78,6 +90,21 @@ internal interface IGuestGpuBackend IReadOnlyList? vertexBuffers = null, GuestRenderState? renderState = null); + void SubmitDepthOnlyTranslatedDraw( + IGuestCompiledShader pixelShader, + IReadOnlyList textures, + IReadOnlyList globalMemoryBuffers, + uint attributeCount, + GuestDepthTarget depthTarget, + IGuestCompiledShader? vertexShader = null, + uint vertexCount = 3, + uint instanceCount = 1, + uint primitiveType = 4, + GuestIndexBuffer? indexBuffer = null, + IReadOnlyList? vertexBuffers = null, + GuestRenderState? renderState = null, + ulong shaderAddress = 0); + void SubmitOffscreenTranslatedDraw( IGuestCompiledShader pixelShader, IReadOnlyList textures, @@ -90,7 +117,9 @@ internal interface IGuestGpuBackend uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null); + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0); void SubmitStorageTranslatedDraw( IGuestCompiledShader pixelShader, @@ -98,16 +127,28 @@ internal interface IGuestGpuBackend IReadOnlyList globalMemoryBuffers, uint attributeCount, uint width, - uint height); + uint height, + ulong shaderAddress = 0); - void SubmitComputeDispatch( + long SubmitComputeDispatch( ulong shaderAddress, IGuestCompiledShader computeShader, IReadOnlyList textures, IReadOnlyList globalMemoryBuffers, uint groupCountX, uint groupCountY, - uint groupCountZ); + uint groupCountZ, + uint baseGroupX, + uint baseGroupY, + uint baseGroupZ, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + bool isIndirect, + bool writesGlobalMemory, + uint threadCountX = uint.MaxValue, + uint threadCountY = uint.MaxValue, + uint threadCountZ = uint.MaxValue); bool TrySubmitGuestImage( ulong address, @@ -115,6 +156,14 @@ internal interface IGuestGpuBackend uint height, uint pitchInPixel); + bool TrySubmitOrderedGuestImageFlip( + int videoOutHandle, + int displayBufferIndex, + ulong address, + uint width, + uint height, + uint pitchInPixel); + /// Registers a display buffer with its guest texture format tag. void RegisterKnownDisplayBuffer(ulong address, uint guestFormat); @@ -126,10 +175,12 @@ internal interface IGuestGpuBackend uint sourceWidth, uint sourceHeight, uint sourceFormat, + uint sourceNumberType, ulong destinationAddress, uint destinationWidth, uint destinationHeight, - uint destinationFormat); + uint destinationFormat, + uint destinationNumberType); /// /// Whether the backend supports the guest render-target format, and how its pixel diff --git a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs index 702d6de..c221fe4 100644 --- a/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs +++ b/src/SharpEmu.Libs/Gpu/Vulkan/VulkanGuestGpuBackend.cs @@ -15,6 +15,9 @@ namespace SharpEmu.Libs.Gpu.Vulkan; /// internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend { + private static readonly IGuestCompiledShader DepthOnlyFragmentShader = + new VulkanCompiledGuestShader(SpirvFixedShaders.CreateDepthOnlyFragment()); + public bool TryCompileVertexShader( Gen5ShaderState state, Gen5ShaderEvaluation evaluation, @@ -23,7 +26,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int scalarRegisterBufferIndex = -1, + int requiredVertexOutputCount = 0, + ulong storageBufferOffsetAlignment = 1) { shader = null; if (!Gen5SpirvTranslator.TryCompileVertexShader( @@ -34,7 +39,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex)) + scalarRegisterBufferIndex, + requiredVertexOutputCount, + storageBufferOffsetAlignment)) { return false; } @@ -52,7 +59,10 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int scalarRegisterBufferIndex = -1, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1) { shader = null; if (!Gen5SpirvTranslator.TryCompilePixelShader( @@ -64,7 +74,10 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex)) + scalarRegisterBufferIndex, + pixelInputEnable, + pixelInputAddress, + storageBufferOffsetAlignment)) { return false; } @@ -80,7 +93,11 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint localSizeY, uint localSizeZ, out IGuestCompiledShader? shader, - out string error) + out string error, + int totalGlobalBufferCount = -1, + int initialScalarBufferIndex = -1, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1) { shader = null; if (!Gen5SpirvTranslator.TryCompileComputeShader( @@ -90,7 +107,11 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend localSizeY, localSizeZ, out var compiled, - out error)) + out error, + totalGlobalBufferCount, + initialScalarBufferIndex, + waveLaneCount, + storageBufferOffsetAlignment)) { return false; } @@ -99,6 +120,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend return true; } + public IGuestCompiledShader GetDepthOnlyFragmentShader() => + DepthOnlyFragmentShader; + public void EnsureStarted(uint width, uint height) => VulkanVideoPresenter.EnsureStarted(width, height); @@ -140,6 +164,35 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend vertexBuffers, renderState); + public void SubmitDepthOnlyTranslatedDraw( + IGuestCompiledShader pixelShader, + IReadOnlyList textures, + IReadOnlyList globalMemoryBuffers, + uint attributeCount, + GuestDepthTarget depthTarget, + IGuestCompiledShader? vertexShader = null, + uint vertexCount = 3, + uint instanceCount = 1, + uint primitiveType = 4, + GuestIndexBuffer? indexBuffer = null, + IReadOnlyList? vertexBuffers = null, + GuestRenderState? renderState = null, + ulong shaderAddress = 0) => + VulkanVideoPresenter.SubmitDepthOnlyTranslatedDraw( + Spirv(pixelShader), + textures, + globalMemoryBuffers, + attributeCount, + depthTarget, + vertexShader is null ? null : Spirv(vertexShader), + vertexCount, + instanceCount, + primitiveType, + indexBuffer, + vertexBuffers, + renderState, + shaderAddress); + public void SubmitOffscreenTranslatedDraw( IGuestCompiledShader pixelShader, IReadOnlyList textures, @@ -152,7 +205,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null) => + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0) => VulkanVideoPresenter.SubmitOffscreenTranslatedDraw( Spirv(pixelShader), textures, @@ -165,7 +220,9 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend primitiveType, indexBuffer, vertexBuffers, - renderState); + renderState, + depthTarget, + shaderAddress); public void SubmitStorageTranslatedDraw( IGuestCompiledShader pixelShader, @@ -173,23 +230,36 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend IReadOnlyList globalMemoryBuffers, uint attributeCount, uint width, - uint height) => + uint height, + ulong shaderAddress = 0) => VulkanVideoPresenter.SubmitStorageTranslatedDraw( Spirv(pixelShader), textures, globalMemoryBuffers, attributeCount, width, - height); + height, + shaderAddress); - public void SubmitComputeDispatch( + public long SubmitComputeDispatch( ulong shaderAddress, IGuestCompiledShader computeShader, IReadOnlyList textures, IReadOnlyList globalMemoryBuffers, uint groupCountX, uint groupCountY, - uint groupCountZ) => + uint groupCountZ, + uint baseGroupX, + uint baseGroupY, + uint baseGroupZ, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + bool isIndirect, + bool writesGlobalMemory, + uint threadCountX = uint.MaxValue, + uint threadCountY = uint.MaxValue, + uint threadCountZ = uint.MaxValue) => VulkanVideoPresenter.SubmitComputeDispatch( shaderAddress, Spirv(computeShader), @@ -197,7 +267,18 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend globalMemoryBuffers, groupCountX, groupCountY, - groupCountZ); + groupCountZ, + baseGroupX, + baseGroupY, + baseGroupZ, + localSizeX, + localSizeY, + localSizeZ, + isIndirect, + writesGlobalMemory, + threadCountX, + threadCountY, + threadCountZ); public bool TrySubmitGuestImage( ulong address, @@ -206,6 +287,21 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint pitchInPixel) => VulkanVideoPresenter.TrySubmitGuestImage(address, width, height, pitchInPixel); + public bool TrySubmitOrderedGuestImageFlip( + int videoOutHandle, + int displayBufferIndex, + ulong address, + uint width, + uint height, + uint pitchInPixel) => + VulkanVideoPresenter.TrySubmitOrderedGuestImageFlip( + videoOutHandle, + displayBufferIndex, + address, + width, + height, + pitchInPixel); + public void RegisterKnownDisplayBuffer(ulong address, uint guestFormat) => VulkanVideoPresenter.RegisterKnownDisplayBuffer(address, guestFormat); @@ -217,19 +313,23 @@ internal sealed class VulkanGuestGpuBackend : IGuestGpuBackend uint sourceWidth, uint sourceHeight, uint sourceFormat, + uint sourceNumberType, ulong destinationAddress, uint destinationWidth, uint destinationHeight, - uint destinationFormat) => + uint destinationFormat, + uint destinationNumberType) => VulkanVideoPresenter.TrySubmitGuestImageBlit( sourceAddress, sourceWidth, sourceHeight, sourceFormat, + sourceNumberType, destinationAddress, destinationWidth, destinationHeight, - destinationFormat); + destinationFormat, + destinationNumberType); public bool TryGetRenderTargetOutputKind(uint dataFormat, uint numberType, out Gen5PixelOutputKind outputKind) { diff --git a/src/SharpEmu.Libs/HostTiming.cs b/src/SharpEmu.Libs/HostTiming.cs new file mode 100644 index 0000000..181a8f1 --- /dev/null +++ b/src/SharpEmu.Libs/HostTiming.cs @@ -0,0 +1,82 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.Libs; + +/// +/// High-resolution host sleeps for guest pacing. +/// routinely overshoots by a scheduler quantum, which turns per-frame waits +/// (flip pacing, usleep-based game loops) into a hard frame-rate cap; these +/// helpers sleep coarsely for the bulk of the interval and yield-spin the +/// remainder so wakeups land within tens of microseconds of the target. +/// +internal static class HostTiming +{ + /// + /// Blocks until reaches + /// . + /// + public static void SleepUntil(long targetTimestamp) + { + while (true) + { + var remainingTicks = targetTimestamp - Stopwatch.GetTimestamp(); + if (remainingTicks <= 0) + { + return; + } + + if (remainingTicks > Stopwatch.Frequency * 60) + { + // Far-future target: coarse sleep avoids overflowing the + // microsecond conversion below and needs no precision. + Thread.Sleep(30_000); + continue; + } + + var remainingMicroseconds = remainingTicks * 1_000_000 / Stopwatch.Frequency; + if (remainingMicroseconds > 2200) + { + // Coarse sleep for the bulk; macOS overshoots ~0.5-1.5 ms. + Thread.Sleep((int)((remainingMicroseconds - 1200) / 1000)); + } + else if (remainingMicroseconds > 1200) + { + // A 1 ms nap typically wakes within the margin and costs no + // CPU, unlike yield-spinning through the whole tail. + Thread.Sleep(1); + } + else if (remainingMicroseconds > 100) + { + Thread.Sleep(0); + } + else + { + Thread.SpinWait(64); + } + } + } + + /// Blocks for the given number of microseconds. + public static void SleepMicroseconds(long microseconds) + { + if (microseconds <= 0) + { + return; + } + + if (microseconds >= 10_000_000) + { + // Long/sentinel sleeps (usleep(-1) parks): sub-millisecond + // precision is irrelevant and the tick conversion below would + // overflow, which used to turn the park into a hot spin. + Thread.Sleep((int)Math.Min(microseconds / 1000, int.MaxValue)); + return; + } + + var ticks = microseconds * Stopwatch.Frequency / 1_000_000; + SleepUntil(Stopwatch.GetTimestamp() + ticks); + } +} diff --git a/src/SharpEmu.Libs/Json/JsonExports.cs b/src/SharpEmu.Libs/Json/JsonExports.cs index c62262b..322351e 100644 --- a/src/SharpEmu.Libs/Json/JsonExports.cs +++ b/src/SharpEmu.Libs/Json/JsonExports.cs @@ -2,12 +2,34 @@ // SPDX-License-Identifier: GPL-2.0-or-later using System; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Text.Json; using SharpEmu.HLE; namespace SharpEmu.Libs.Json; public static class JsonExports { + private const int ValueObjectSize = 0x20; + private const int StringObjectSize = 0x08; + private const ulong MaximumJsonBufferSize = 16 * 1024 * 1024; + private const int SceJsonParserErrorInvalidToken = unchecked((int)0x80920101); + private const int SceJsonParserErrorEmptyBuffer = unchecked((int)0x80920105); + + private sealed record JsonValueState(JsonElement Element); + + private sealed record JsonStringState( + string Value, + ulong GuestBufferAddress = 0, + int GuestBufferCapacity = 0); + + private static readonly ConcurrentDictionary _values = new(); + private static readonly ConcurrentDictionary _strings = new(); + private static readonly JsonElement _nullElement = CreateNullElement(); + [SysAbiExport( Nid = "-hJRce8wn1U", ExportName = "_ZN3sce4Json12MemAllocatorC2Ev", @@ -78,9 +100,6 @@ public static class JsonExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - // sce::Json::Initializer::setGlobalNullAccessCallback(const Value& (*)(ValueType, const Value*, void*), void*) - // Registers the guest hook invoked when a Value is accessed as the wrong type. Quake calls it - // during kexPSNWebAPI::Initialize and treats a non-zero return as a fatal init failure. [SysAbiExport( Nid = "+drDFyAS6u4", ExportName = "_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_", @@ -91,26 +110,38 @@ public static class JsonExports var thisAddress = ctx[CpuRegister.Rdi]; if (thisAddress == 0) { - ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } JsonObjectHeap.GlobalNullAccessCallback = ctx[CpuRegister.Rsi]; JsonObjectHeap.GlobalNullAccessCallbackContext = ctx[CpuRegister.Rdx]; TraceJson("Initializer.setGlobalNullAccessCallback", thisAddress, ctx[CpuRegister.Rsi]); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + return SetReturn(ctx, 0); } [SysAbiExport( Nid = "WSOuge5IsCg", ExportName = "_ZN3sce4Json14InitParameter2C1Ev", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] + Target = Generation.Gen5, + LibraryName = "libSceJson")] public static int InitParameter2Constructor(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("InitParameter2.ctor", thisAddress, ctx[CpuRegister.Rsi]); + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + // The PS5 ABI object occupies 0x28 bytes in the caller's frame. Its + // setters below replace the allocator and file-buffer fields. + Span parameter = stackalloc byte[0x28]; + parameter.Clear(); + if (!ctx.Memory.TryWrite(thisAddress, parameter)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceJson("InitParameter2.ctor", thisAddress, 0); ctx[CpuRegister.Rax] = thisAddress; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -118,12 +149,24 @@ public static class JsonExports [SysAbiExport( Nid = "I2QC8PYhJWY", ExportName = "_ZN3sce4Json14InitParameter212setAllocatorEPNS0_12MemAllocatorEPv", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] + Target = Generation.Gen5, + LibraryName = "libSceJson")] public static int InitParameter2SetAllocator(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("InitParameter2.setAllocator", thisAddress, ctx[CpuRegister.Rsi]); + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + Span fields = stackalloc byte[sizeof(ulong) * 2]; + BinaryPrimitives.WriteUInt64LittleEndian(fields, ctx[CpuRegister.Rsi]); + BinaryPrimitives.WriteUInt64LittleEndian(fields[sizeof(ulong)..], ctx[CpuRegister.Rdx]); + if (!ctx.Memory.TryWrite(thisAddress, fields)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + ctx[CpuRegister.Rax] = thisAddress; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -131,12 +174,16 @@ public static class JsonExports [SysAbiExport( Nid = "Eu95jmqn5Rw", ExportName = "_ZN3sce4Json14InitParameter217setFileBufferSizeEm", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] + Target = Generation.Gen5, + LibraryName = "libSceJson")] public static int InitParameter2SetFileBufferSize(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("InitParameter2.setFileBufferSize", thisAddress, ctx[CpuRegister.Rsi]); + if (thisAddress == 0 || !ctx.TryWriteUInt64(thisAddress + 0x10, ctx[CpuRegister.Rsi])) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + ctx[CpuRegister.Rax] = thisAddress; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -144,14 +191,474 @@ public static class JsonExports [SysAbiExport( Nid = "IXW-z8pggfg", ExportName = "_ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceJson2")] - public static int Initializer2Constructor(CpuContext ctx) + Target = Generation.Gen5, + LibraryName = "libSceJson")] + public static int InitializerInitialize2(CpuContext ctx) { var thisAddress = ctx[CpuRegister.Rdi]; - TraceJson("Initializer2.ctor", thisAddress, ctx[CpuRegister.Rsi]); + var initParameterAddress = ctx[CpuRegister.Rsi]; + if (thisAddress == 0 || initParameterAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + TraceJson("Initializer.initialize2", thisAddress, initParameterAddress); + return SetReturn(ctx, 0); + } + public static int ValueConstructor(CpuContext ctx) + { + _ = ConstructValue(ctx); + return JsonValueExports.ValueDefaultConstructor(ctx); + } + + [SysAbiExport( + Nid = "-wa17B7TGnw", + ExportName = "_ZN3sce4Json5ValueC2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueBaseConstructor(CpuContext ctx) => ConstructValue(ctx); + public static int ValueDestructor(CpuContext ctx) + { + _ = DestroyValue(ctx); + return JsonValueExports.ValueDestructor(ctx); + } + + [SysAbiExport( + Nid = "0eUrW9JAxM0", + ExportName = "_ZN3sce4Json5ValueD2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueBaseDestructor(CpuContext ctx) => DestroyValue(ctx); + + [SysAbiExport( + Nid = "S5JxQnoGF3E", + ExportName = "_ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ParserParseBuffer(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var bufferAddress = ctx[CpuRegister.Rsi]; + var bufferSize = ctx[CpuRegister.Rdx]; + if (valueAddress == 0 || bufferAddress == 0 || bufferSize == 0) + { + return SetReturn(ctx, SceJsonParserErrorEmptyBuffer); + } + + if (bufferSize > MaximumJsonBufferSize || bufferSize > int.MaxValue) + { + return SetReturn(ctx, SceJsonParserErrorInvalidToken); + } + + var buffer = new byte[(int)bufferSize]; + if (!ctx.Memory.TryRead(bufferAddress, buffer)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + try + { + using var document = JsonDocument.Parse(buffer); + var element = document.RootElement.Clone(); + StoreValue(ctx, valueAddress, element); + TraceJsonText("Parser.parse", valueAddress, Encoding.UTF8.GetString(buffer)); + return SetReturn(ctx, 0); + } + catch (JsonException) + { + return SetReturn(ctx, SceJsonParserErrorInvalidToken); + } + } + + [SysAbiExport( + Nid = "SHtAad20YYM", + ExportName = "_ZNK3sce4Json5Value7getTypeEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetType(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var element = GetValue(valueAddress); + ctx[CpuRegister.Rax] = (ulong)GetValueType(element); + return 0; + } + + [SysAbiExport( + Nid = "RBw+4NukeGQ", + ExportName = "_ZNK3sce4Json5Value5countEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueCount(CpuContext ctx) + { + var element = GetValue(ctx[CpuRegister.Rdi]); + ctx[CpuRegister.Rax] = element.ValueKind switch + { + System.Text.Json.JsonValueKind.Array => (ulong)element.GetArrayLength(), + System.Text.Json.JsonValueKind.Object => (ulong)element.EnumerateObject().Count(), + _ => 0, + }; + return 0; + } + + [SysAbiExport( + Nid = "zTwZdI8AZ5Y", + ExportName = "_ZNK3sce4Json5Value10getBooleanEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetBoolean(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "DIxvoy7Ngvk", + ExportName = "_ZNK3sce4Json5Value10getIntegerEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetInteger(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "sn4HNCtNRzY", + ExportName = "_ZNK3sce4Json5Value11getUIntegerEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetUnsignedInteger(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "3qrge7L-AU4", + ExportName = "_ZNK3sce4Json5Value7getRealEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetReal(CpuContext ctx) => ReturnValueStorage(ctx); + + [SysAbiExport( + Nid = "HwDt5lD9Bfo", + ExportName = "_ZNK3sce4Json5ValueixEPKc", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueIndexCString(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var keyAddress = ctx[CpuRegister.Rsi]; + if (!TryReadUtf8CString(ctx, keyAddress, 4096, out var key) || + !TryAllocateGuestObject(ctx, ValueObjectSize, out var childAddress)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + var parent = GetValue(valueAddress); + var child = parent.ValueKind == System.Text.Json.JsonValueKind.Object && + parent.TryGetProperty(key, out var property) + ? property.Clone() + : _nullElement; + StoreValue(ctx, childAddress, child); + ctx[CpuRegister.Rax] = childAddress; + TraceJsonText("Value.index", valueAddress, key); + return 0; + } + + [SysAbiExport( + Nid = "XlWbvieLj2M", + ExportName = "_ZNK3sce4Json5ValueixEm", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueIndexPosition(CpuContext ctx) => ReturnIndexedValue(ctx); + + [SysAbiExport( + Nid = "0YqYAoO-+Uo", + ExportName = "_ZNK3sce4Json5Value8getValueEm", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueGetPosition(CpuContext ctx) => ReturnIndexedValue(ctx); + + [SysAbiExport( + Nid = "4zrm6VrgIAw", + ExportName = "_ZN3sce4Json5ValueaSERKS1_", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueAssignment(CpuContext ctx) + { + var destinationAddress = ctx[CpuRegister.Rdi]; + var sourceAddress = ctx[CpuRegister.Rsi]; + if (destinationAddress != 0) + { + StoreValue(ctx, destinationAddress, GetValue(sourceAddress)); + } + + ctx[CpuRegister.Rax] = destinationAddress; + return 0; + } + public static int StringConstructor(CpuContext ctx) + { + _ = ConstructString(ctx); + return JsonValueExports.StringDefaultConstructor(ctx); + } + + [SysAbiExport( + Nid = "eG9E9M6XvTM", + ExportName = "_ZN3sce4Json6StringC2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int StringBaseConstructor(CpuContext ctx) => ConstructString(ctx); + public static int StringDestructor(CpuContext ctx) + { + _ = DestroyString(ctx); + return JsonValueExports.StringDestructor(ctx); + } + + [SysAbiExport( + Nid = "Ui7YFnSTCBw", + ExportName = "_ZN3sce4Json6StringD2Ev", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int StringBaseDestructor(CpuContext ctx) => DestroyString(ctx); + + [SysAbiExport( + Nid = "Ncel8t2Rrpc", + ExportName = "_ZNK3sce4Json5Value8toStringERNS0_6StringE", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int ValueToString(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var stringAddress = ctx[CpuRegister.Rsi]; + if (stringAddress != 0) + { + var element = GetValue(valueAddress); + var value = element.ValueKind == System.Text.Json.JsonValueKind.String + ? element.GetString() ?? string.Empty + : element.GetRawText(); + _strings[stringAddress] = new JsonStringState(value); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + [SysAbiExport( + Nid = "L1KAkYWml-M", + ExportName = "_ZNK3sce4Json6String5c_strEv", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceJson")] + public static int StringCStr(CpuContext ctx) + { + var stringAddress = ctx[CpuRegister.Rdi]; + if (!_strings.TryGetValue(stringAddress, out var state)) + { + state = new JsonStringState(string.Empty); + } + + var bytes = Encoding.UTF8.GetBytes(state.Value + '\0'); + var guestBufferAddress = state.GuestBufferAddress; + if (guestBufferAddress == 0 || state.GuestBufferCapacity < bytes.Length) + { + if (!TryAllocateGuestObject(ctx, bytes.Length, out guestBufferAddress)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + } + + if (!ctx.Memory.TryWrite(guestBufferAddress, bytes)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + _strings[stringAddress] = state with + { + GuestBufferAddress = guestBufferAddress, + GuestBufferCapacity = bytes.Length, + }; + ctx.TryWriteUInt64(stringAddress, guestBufferAddress); + ctx[CpuRegister.Rax] = guestBufferAddress; + TraceJsonText("String.c_str", stringAddress, state.Value); + return 0; + } + + private static int ConstructValue(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + StoreValue(ctx, thisAddress, _nullElement); ctx[CpuRegister.Rax] = thisAddress; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + return 0; + } + + private static int ReturnIndexedValue(CpuContext ctx) + { + var valueAddress = ctx[CpuRegister.Rdi]; + var position = ctx[CpuRegister.Rsi]; + if (!TryAllocateGuestObject(ctx, ValueObjectSize, out var childAddress)) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + var parent = GetValue(valueAddress); + var child = parent.ValueKind == System.Text.Json.JsonValueKind.Array && + position < (ulong)parent.GetArrayLength() + ? parent[(int)position].Clone() + : _nullElement; + StoreValue(ctx, childAddress, child); + ctx[CpuRegister.Rax] = childAddress; + return 0; + } + + private static int ReturnValueStorage(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + ctx[CpuRegister.Rax] = thisAddress == 0 ? 0 : thisAddress + 0x10; + return 0; + } + + private static int DestroyValue(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + _values.TryRemove(thisAddress, out _); + if (thisAddress != 0) + { + Span empty = stackalloc byte[ValueObjectSize]; + empty.Clear(); + ctx.Memory.TryWrite(thisAddress, empty); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + private static int ConstructString(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + if (thisAddress == 0) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + _strings[thisAddress] = new JsonStringState(string.Empty); + ctx.TryWriteUInt64(thisAddress, 0); + ctx[CpuRegister.Rax] = thisAddress; + return 0; + } + + private static int DestroyString(CpuContext ctx) + { + var thisAddress = ctx[CpuRegister.Rdi]; + _strings.TryRemove(thisAddress, out _); + if (thisAddress != 0) + { + ctx.TryWriteUInt64(thisAddress, 0); + } + + ctx[CpuRegister.Rax] = 0; + return 0; + } + + private static JsonElement CreateNullElement() + { + using var document = JsonDocument.Parse("null"); + return document.RootElement.Clone(); + } + + private static JsonElement GetValue(ulong address) => + address != 0 && _values.TryGetValue(address, out var state) + ? state.Element + : _nullElement; + + private static void StoreValue(CpuContext ctx, ulong address, JsonElement element) + { + if (address == 0) + { + return; + } + + var clone = element.Clone(); + _values[address] = new JsonValueState(clone); + + Span mirror = stackalloc byte[ValueObjectSize]; + mirror.Clear(); + var type = GetValueType(clone); + BinaryPrimitives.WriteInt32LittleEndian(mirror[0x1C..], type); + switch (clone.ValueKind) + { + case System.Text.Json.JsonValueKind.True: + mirror[0x10] = 1; + break; + case System.Text.Json.JsonValueKind.Number when clone.TryGetInt64(out var integer): + BinaryPrimitives.WriteInt64LittleEndian(mirror[0x10..], integer); + break; + case System.Text.Json.JsonValueKind.Number when clone.TryGetUInt64(out var unsignedInteger): + BinaryPrimitives.WriteUInt64LittleEndian(mirror[0x10..], unsignedInteger); + break; + case System.Text.Json.JsonValueKind.Number: + BinaryPrimitives.WriteInt64LittleEndian( + mirror[0x10..], + BitConverter.DoubleToInt64Bits(clone.GetDouble())); + break; + } + + ctx.Memory.TryWrite(address, mirror); + } + + private static int GetValueType(JsonElement element) => element.ValueKind switch + { + System.Text.Json.JsonValueKind.True or System.Text.Json.JsonValueKind.False => 1, + System.Text.Json.JsonValueKind.Number when element.TryGetInt64(out _) => 2, + System.Text.Json.JsonValueKind.Number when element.TryGetUInt64(out _) => 3, + System.Text.Json.JsonValueKind.Number => 4, + System.Text.Json.JsonValueKind.String => 5, + System.Text.Json.JsonValueKind.Array => 6, + System.Text.Json.JsonValueKind.Object => 7, + _ => 0, + }; + + private static bool TryAllocateGuestObject(CpuContext ctx, int size, out ulong address) + { + address = 0; + return size > 0 && + ctx.Memory is IGuestMemoryAllocator allocator && + allocator.TryAllocateGuestMemory((ulong)size, 0x10, out address); + } + + private static bool TryReadUtf8CString( + CpuContext ctx, + ulong address, + int maximumLength, + out string value) + { + value = string.Empty; + if (address == 0 || maximumLength <= 0) + { + return false; + } + + var bytes = new byte[maximumLength]; + Span current = stackalloc byte[1]; + for (var index = 0; index < bytes.Length; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, current)) + { + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, index); + return true; + } + + bytes[index] = current[0]; + } + + return false; + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; } private static void TraceJson(string operation, ulong thisAddress, ulong argument) @@ -164,4 +671,16 @@ public static class JsonExports Console.Error.WriteLine( $"[LOADER][TRACE] json.{operation} this=0x{thisAddress:X16} arg=0x{argument:X16}"); } + + private static void TraceJsonText(string operation, ulong thisAddress, string value) + { + if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_JSON"), "1", StringComparison.Ordinal)) + { + return; + } + + var preview = value.Length <= 128 ? value : value[..128]; + Console.Error.WriteLine( + $"[LOADER][TRACE] json.{operation} this=0x{thisAddress:X16} value={preview}"); + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs index c4febeb..552df52 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventFlagCompatExports.cs @@ -1,8 +1,8 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Buffers.Binary; using System.Collections.Concurrent; -using System.Diagnostics; using System.Text; using SharpEmu.HLE; using SharpEmu.Libs.Fiber; @@ -34,45 +34,6 @@ public static class KernelEventFlagCompatExports public object Gate { get; } = new(); } - private sealed class EventFlagWaiter : IGuestThreadBlockWaiter - { - public required CpuContext Ctx { get; init; } - public required EventFlagState State { get; init; } - public required ulong Pattern { get; init; } - public required uint WaitMode { get; init; } - public required ulong ResultAddress { get; init; } - public bool Timed { get; init; } - - // Timed-wait completion state; unused when Timed is false. - public ulong TimeoutAddress { get; init; } - public long DeadlineTimestamp { get; init; } - - public OrbisGen2Result? Result { get; set; } - - // Untimed waits stash the prepared result here at wake and return it at resume. - private OrbisGen2Result _blockedResult = OrbisGen2Result.ORBIS_GEN2_OK; - - public int Resume() => Timed - ? CompleteBlockedTimedWait(Ctx, State, this, Pattern, WaitMode, ResultAddress, TimeoutAddress, DeadlineTimestamp) - : (int)_blockedResult; - - public bool TryWake() - { - if (Timed) - { - return TryCompleteBlockedTimedWait(Ctx, State, this, Pattern, WaitMode, ResultAddress); - } - - if (!TryPrepareBlockedWait(Ctx, State, Pattern, WaitMode, ResultAddress, out var preparedResult)) - { - return false; - } - - _blockedResult = preparedResult; - return true; - } - } - [SysAbiExport( Nid = "BpFoboUJoZU", ExportName = "sceKernelCreateEventFlag", @@ -91,17 +52,17 @@ public static class KernelEventFlagCompatExports optionAddress != 0 || !IsValidAttributes(attributes)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxEventFlagNameLength + 1, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxEventFlagNameLength + 1, out var name)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (Encoding.UTF8.GetByteCount(name) > MaxEventFlagNameLength) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } var handle = unchecked((ulong)Interlocked.Increment(ref _nextEventFlagHandle)); @@ -115,11 +76,11 @@ public static class KernelEventFlagCompatExports if (!ctx.TryWriteUInt64(outAddress, handle)) { _eventFlags.TryRemove(handle, out _); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceEventFlag($"create handle=0x{handle:X16} name='{name}' attr=0x{attributes:X2} bits=0x{initialPattern:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -132,7 +93,7 @@ public static class KernelEventFlagCompatExports var handle = ctx[CpuRegister.Rdi]; if (!_eventFlags.TryRemove(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) @@ -141,7 +102,7 @@ public static class KernelEventFlagCompatExports } TraceEventFlag($"delete handle=0x{handle:X16} name='{state.Name}'"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -156,7 +117,7 @@ public static class KernelEventFlagCompatExports var returnRip = GetCurrentReturnRip(); if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) @@ -167,7 +128,7 @@ public static class KernelEventFlagCompatExports } _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(GetEventFlagWakeKey(handle)); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -181,7 +142,7 @@ public static class KernelEventFlagCompatExports var pattern = ctx[CpuRegister.Rsi]; if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) @@ -190,7 +151,7 @@ public static class KernelEventFlagCompatExports TraceEventFlag($"clear handle=0x{handle:X16} mask=0x{pattern:X16} bits=0x{state.Bits:X16}"); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -207,29 +168,29 @@ public static class KernelEventFlagCompatExports if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (pattern == 0 || !IsValidWaitMode(waitMode)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (state.Gate) { if (!TryWriteResultPattern(ctx, resultAddress, state.Bits)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (!IsSatisfied(state.Bits, pattern, waitMode)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); } ApplyClearMode(state, pattern, waitMode); TraceEventFlag($"poll handle=0x{handle:X16} pattern=0x{pattern:X16} mode=0x{waitMode:X2} bits=0x{state.Bits:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } } @@ -249,18 +210,18 @@ public static class KernelEventFlagCompatExports if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (pattern == 0 || !IsValidWaitMode(waitMode)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } uint timeoutUsec = 0; - if (timeoutAddress != 0 && !ctx.TryReadUInt32(timeoutAddress, out timeoutUsec)) + if (timeoutAddress != 0 && !TryReadUInt32(ctx, timeoutAddress, out timeoutUsec)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } Monitor.Enter(state.Gate); @@ -268,65 +229,64 @@ public static class KernelEventFlagCompatExports { if (TryCompleteSatisfiedWait(ctx, state, pattern, waitMode, resultAddress, out var immediateWaitResult)) { - return ctx.SetReturn(immediateWaitResult); + return SetReturn(ctx, immediateWaitResult); } - if (timeoutAddress != 0) - { - if (timeoutUsec == 0) - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); - TraceEventFlag($"wait-timeout handle=0x{handle:X16} pattern=0x{pattern:X16} timeout=0 ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); - } - - var deadline = GuestThreadExecution.ComputeDeadlineTimestamp( - TimeSpan.FromTicks((long)timeoutUsec * 10L)); - var timedWaiter = new EventFlagWaiter - { - Ctx = ctx, - State = state, - Pattern = pattern, - WaitMode = waitMode, - ResultAddress = resultAddress, - Timed = true, - TimeoutAddress = timeoutAddress, - DeadlineTimestamp = deadline, - }; - if (GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "sceKernelWaitEventFlag", - GetEventFlagWakeKey(handle), - timedWaiter, - blockDeadlineTimestamp: deadline)) - { - state.WaitingThreads++; - TraceEventFlag($"wait-block-timed handle=0x{handle:X16} pattern=0x{pattern:X16} timeout={timeoutUsec} waiters={state.WaitingThreads} ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); - TraceEventFlag($"wait-timeout-host handle=0x{handle:X16} pattern=0x{pattern:X16} timeout={timeoutUsec} ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); - } + // Timed waits block on a deadline instead of returning TIMED_OUT + // immediately; a zero-microsecond timeout still degrades to an + // instant poll because the deadline is already in the past. + var deadline = timeoutAddress != 0 + ? GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromMicroseconds(timeoutUsec)) + : 0; + var hostDeadlineMs = timeoutAddress != 0 + ? Environment.TickCount64 + (timeoutUsec == 0 + ? 0L + : Math.Max(1L, (timeoutUsec + 999L) / 1000L)) + : long.MaxValue; var currentGuestThread = GuestThreadExecution.CurrentGuestThreadHandle; var currentFiber = FiberExports.GetCurrentFiberAddressForDiagnostics(ctx); var managedThread = Environment.CurrentManagedThreadId; + var blockedWaitResult = OrbisGen2Result.ORBIS_GEN2_OK; + var satisfied = false; var requestedBlock = GuestThreadExecution.RequestCurrentThreadBlock( ctx, "sceKernelWaitEventFlag", GetEventFlagWakeKey(handle), - new EventFlagWaiter + () => { - Ctx = ctx, - State = state, - Pattern = pattern, - WaitMode = waitMode, - ResultAddress = resultAddress, - }); + if (satisfied) + { + return (int)blockedWaitResult; + } + + // Deadline expiry: report timeout with the current bits. + if (timeoutAddress != 0) + { + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + } + + _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + }, + () => + { + if (!TryPrepareBlockedWait( + ctx, + state, + pattern, + waitMode, + resultAddress, + out var preparedResult)) + { + return false; + } + + blockedWaitResult = preparedResult; + satisfied = true; + return true; + }, + deadline); TraceEventFlag($"wait-unsatisfied handle=0x{handle:X16} pattern=0x{pattern:X16} bits=0x{state.Bits:X16} guest_thread=0x{currentGuestThread:X16} fiber=0x{currentFiber:X16} managed={managedThread} block={requestedBlock} ret=0x{returnRip:X16} frames={FormatFrameChain(ctx)}"); TraceEventFlag($"wait-object handle=0x{handle:X16} name='{state.Name}' {FormatGuestWaitObject(ctx)}"); if (!requestedBlock) @@ -334,7 +294,7 @@ public static class KernelEventFlagCompatExports var scheduler = GuestThreadExecution.Scheduler; if (scheduler is null) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); } state.WaitingThreads++; @@ -359,10 +319,21 @@ public static class KernelEventFlagCompatExports state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); releaseWaiter = false; TraceEventFlag($"wait-wake handle=0x{handle:X16} pattern=0x{pattern:X16} bits=0x{state.Bits:X16} waiters={state.WaitingThreads} ret=0x{returnRip:X16}"); - return ctx.SetReturn(pumpedWaitResult); + return SetReturn(ctx, pumpedWaitResult); } - Monitor.Wait(state.Gate, HostWaitPumpMilliseconds); + var remaining = hostDeadlineMs - Environment.TickCount64; + if (timeoutAddress != 0 && remaining <= 0) + { + state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); + releaseWaiter = false; + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + _ = TryWriteResultPattern(ctx, resultAddress, state.Bits); + TraceEventFlag($"wait-timeout handle=0x{handle:X16} pattern=0x{pattern:X16} bits=0x{state.Bits:X16} ret=0x{returnRip:X16}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); + } + + Monitor.Wait(state.Gate, (int)Math.Min(remaining, HostWaitPumpMilliseconds)); } } finally @@ -376,7 +347,7 @@ public static class KernelEventFlagCompatExports state.WaitingThreads++; TraceEventFlag($"wait-block handle=0x{handle:X16} pattern=0x{pattern:X16} waiters={state.WaitingThreads} guest_thread=0x{currentGuestThread:X16} fiber=0x{currentFiber:X16} managed={managedThread} ret=0x{returnRip:X16}"); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } finally { @@ -396,24 +367,26 @@ public static class KernelEventFlagCompatExports var waiterCountAddress = ctx[CpuRegister.Rdx]; if (!_eventFlags.TryGetValue(handle, out var state)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } lock (state.Gate) { if (waiterCountAddress != 0 && - !ctx.TryWriteUInt32(waiterCountAddress, unchecked((uint)state.WaitingThreads))) + !TryWriteUInt32(ctx, waiterCountAddress, unchecked((uint)state.WaitingThreads))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } state.Bits = setPattern; state.WaitingThreads = 0; Monitor.PulseAll(state.Gate); - TraceEventFlag($"cancel handle=0x{handle:X16} bits=0x{setPattern:X16}"); + TraceEventFlag( + $"cancel handle=0x{handle:X16} bits=0x{setPattern:X16} " + + $"guest_thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} ret=0x{GetCurrentReturnRip():X16}"); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } private static bool IsValidAttributes(uint attributes) @@ -509,98 +482,90 @@ public static class KernelEventFlagCompatExports } } - private static bool TryCompleteBlockedTimedWait( - CpuContext ctx, - EventFlagState state, - EventFlagWaiter waiter, - ulong pattern, - uint waitMode, - ulong resultAddress) - { - lock (state.Gate) - { - if (waiter.Result is not null) - { - return true; - } - - if (!IsSatisfied(state.Bits, pattern, waitMode)) - { - return false; - } - - waiter.Result = TryWriteResultPattern(ctx, resultAddress, state.Bits) - ? OrbisGen2Result.ORBIS_GEN2_OK - : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - if (waiter.Result == OrbisGen2Result.ORBIS_GEN2_OK) - { - ApplyClearMode(state, pattern, waitMode); - } - - state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); - return true; - } - } - - private static int CompleteBlockedTimedWait( - CpuContext ctx, - EventFlagState state, - EventFlagWaiter waiter, - ulong pattern, - uint waitMode, - ulong resultAddress, - ulong timeoutAddress, - long deadlineTimestamp) - { - lock (state.Gate) - { - if (waiter.Result is null) - { - if (IsSatisfied(state.Bits, pattern, waitMode)) - { - waiter.Result = TryWriteResultPattern(ctx, resultAddress, state.Bits) - ? OrbisGen2Result.ORBIS_GEN2_OK - : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - if (waiter.Result == OrbisGen2Result.ORBIS_GEN2_OK) - { - ApplyClearMode(state, pattern, waitMode); - } - } - else - { - waiter.Result = TryWriteResultPattern(ctx, resultAddress, state.Bits) - ? OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT - : OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - state.WaitingThreads = Math.Max(0, state.WaitingThreads - 1); - } - } - - if (waiter.Result == OrbisGen2Result.ORBIS_GEN2_OK) - { - var remainingTicks = deadlineTimestamp - Stopwatch.GetTimestamp(); - var remainingMicros = remainingTicks <= 0 - ? 0u - : (uint)Math.Min( - uint.MaxValue, - remainingTicks / (double)Stopwatch.Frequency * 1_000_000d); - _ = ctx.TryWriteUInt32(timeoutAddress, remainingMicros); - } - else - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - } - - return (int)waiter.Result.Value; - } - private static string GetEventFlagWakeKey(ulong handle) => $"event_flag:0x{handle:X16}"; private static bool TryWriteResultPattern(CpuContext ctx, ulong address, ulong bits) => address == 0 || ctx.TryWriteUInt64(address, bits); + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + Span buffer = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(buffer); + return true; + } + + private static bool TryReadByte(CpuContext ctx, ulong address, out byte value) + { + Span buffer = stackalloc byte[1]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = buffer[0]; + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int capacity, out string value) + { + var bytes = new byte[capacity]; + Span current = stackalloc byte[1]; + for (var index = 0; index < bytes.Length; index++) + { + if (!ctx.Memory.TryRead(address + (ulong)index, current)) + { + value = string.Empty; + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, index); + return true; + } + + bytes[index] = current[0]; + } + + value = Encoding.UTF8.GetString(bytes); + return true; + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + var value = (int)result; + ctx[CpuRegister.Rax] = unchecked((ulong)value); + return value; + } + private static void TraceEventFlag(string message) { if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_EVENT_FLAG"), "1", StringComparison.Ordinal)) @@ -684,7 +649,7 @@ public static class KernelEventFlagCompatExports private static void AppendByte(StringBuilder builder, CpuContext ctx, ulong address, string name) { - if (ctx.TryReadByte(address, out var value)) + if (TryReadByte(ctx, address, out var value)) { builder.Append($" {name}=0x{value:X2}"); } @@ -692,7 +657,7 @@ public static class KernelEventFlagCompatExports private static void AppendUInt32(StringBuilder builder, CpuContext ctx, ulong address, string name) { - if (ctx.TryReadUInt32(address, out var value)) + if (TryReadUInt32(ctx, address, out var value)) { builder.Append($" {name}=0x{value:X8}"); } @@ -700,7 +665,7 @@ public static class KernelEventFlagCompatExports private static void AppendUInt64(StringBuilder builder, CpuContext ctx, ulong address, string name) { - if (ctx.TryReadUInt64(address, out var value)) + if (TryReadUInt64(ctx, address, out var value)) { builder.Append($" {name}=0x{value:X16}"); } diff --git a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs index dafdb9c..b7223d7 100644 --- a/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelEventQueueCompatExports.cs @@ -395,13 +395,13 @@ public static class KernelEventQueueCompatExports } uint timeoutUsec = 0; - if (timeoutAddress != 0 && !ctx.TryReadUInt32(timeoutAddress, out timeoutUsec)) + if (timeoutAddress != 0 && !TryReadUInt32(ctx, timeoutAddress, out timeoutUsec)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } var deliveredCount = DequeueEvents(ctx, handle, eventsAddress, eventCapacity); - if (outCountAddress != 0 && !ctx.TryWriteUInt32(outCountAddress, (uint)deliveredCount)) + if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -450,7 +450,7 @@ public static class KernelEventQueueCompatExports } deliveredCount = DequeueEvents(ctx, handle, eventsAddress, eventCapacity); - if (outCountAddress != 0 && !ctx.TryWriteUInt32(outCountAddress, (uint)deliveredCount)) + if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -649,6 +649,60 @@ public static class KernelEventQueueCompatExports return triggeredCount; } + /// + /// Queues one event for every registration using . + /// Unlike , this preserves distinct + /// event identifiers registered on the same queue. AGC driver completion + /// queues use this form because the driver, rather than a packet-provided + /// identifier, announces that the whole submission reached end-of-pipe. + /// + public static int TriggerRegisteredEventsDistinct(short filter) + { + HashSet? wakeHandles = null; + var triggeredCount = 0; + lock (_eventQueueGate) + { + foreach (var (handle, registrations) in _registeredEvents) + { + foreach (var registration in registrations.Values) + { + if (registration.Filter != filter) + { + continue; + } + + if (!_pendingEvents.TryGetValue(handle, out var queue)) + { + queue = new KernelEventDeque(); + _pendingEvents[handle] = queue; + } + + QueueOrUpdateEvent( + queue, + new KernelQueuedEvent( + registration.Ident, + registration.Filter, + 0, + 1, + registration.Ident, + registration.UserData)); + (wakeHandles ??= []).Add(handle); + triggeredCount++; + } + } + } + + if (wakeHandles is not null) + { + foreach (var handle in wakeHandles) + { + WakeEventQueue(handle); + } + } + + return triggeredCount; + } + private static bool TriggerRegisteredEvent( ulong handle, ulong ident, @@ -752,7 +806,7 @@ public static class KernelEventQueueCompatExports ulong outCountAddress) { var deliveredCount = DequeueEvents(ctx, handle, eventsAddress, eventCapacity); - if (outCountAddress != 0 && !ctx.TryWriteUInt32(outCountAddress, (uint)deliveredCount)) + if (outCountAddress != 0 && !TryWriteUInt32(ctx, outCountAddress, (uint)deliveredCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -865,8 +919,29 @@ public static class KernelEventQueueCompatExports return; } - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out ulong returnRip); + var returnRip = 0UL; + _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out returnRip); Console.Error.WriteLine( - $"[LOADER][TRACE] equeue.{operation}: thread=0x{KernelPthreadState.GetCurrentThreadHandle():X16} handle=0x{handle:X16} rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} ret=0x{returnRip:X16}"); + $"[LOADER][TRACE] equeue.{operation}: handle=0x{handle:X16} rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} ret=0x{returnRip:X16}"); + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; } } diff --git a/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs index 417cdd0..4008f7b 100644 --- a/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelExceptionCompatExports.cs @@ -62,4 +62,75 @@ public static class KernelExceptionCompatExports ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + + [SysAbiExport( + Nid = "il03nluKfMk", + ExportName = "sceKernelRaiseException", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int RaiseException(CpuContext ctx) + { + var targetThread = ctx[CpuRegister.Rdi]; + var exceptionType = unchecked((int)ctx[CpuRegister.Rsi]); + if (targetThread == 0 || exceptionType is < 0 or >= 128) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ulong handler; + lock (_gate) + { + if (!_installedHandlers.TryGetValue(exceptionType, out handler)) + { + // The kernel accepts a raise for a type with no process + // handler; there is simply no user callback to deliver. + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_IGNORE_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + + var scheduler = GuestThreadExecution.Scheduler; + string? error = null; + if (scheduler is null || + !scheduler.TryRaiseGuestException( + ctx, + targetThread, + handler, + exceptionType, + out error)) + { + Console.Error.WriteLine( + $"[LOADER][WARN] sceKernelRaiseException delivery failed: " + + $"target=0x{targetThread:X16} type=0x{exceptionType:X2} " + + $"error={error ?? "scheduler unavailable"}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + } + + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_GUEST_EXCEPTIONS"), + "1", + StringComparison.Ordinal)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] guest_exception.raise " + + $"target=0x{targetThread:X16} type=0x{exceptionType:X2} " + + $"handler=0x{handler:X16}"); + } + + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + var value = (int)result; + ctx[CpuRegister.Rax] = unchecked((ulong)value); + return value; + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelExports.cs b/src/SharpEmu.Libs/Kernel/KernelExports.cs index 0235c92..09fb5b2 100644 --- a/src/SharpEmu.Libs/Kernel/KernelExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelExports.cs @@ -13,8 +13,6 @@ public static class KernelExports private static readonly object _coredumpGate = new(); private static ulong _coredumpHandler; private static ulong _coredumpHandlerContext; - private const uint Gen4CompiledSdkVersion = 0x05000000; - private const uint Gen5CompiledSdkVersion = 0x09000000; private readonly record struct CxaDestructorEntry( ulong Function, @@ -28,24 +26,7 @@ public static class KernelExports LibraryName = "libKernel")] public static int KernelGetCompiledSdkVersion(CpuContext ctx) { - var versionAddress = ctx[CpuRegister.Rdi]; - if (versionAddress == 0) - { - ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; - } - - var sdkVersion = ctx.TargetGeneration == Generation.Gen5 - ? Gen5CompiledSdkVersion - : Gen4CompiledSdkVersion; - - if (!ctx.TryWriteUInt32(versionAddress, sdkVersion)) - { - ctx[CpuRegister.Rax] = unchecked((ulong)(int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = 0; + _ = ctx; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -71,20 +52,12 @@ public static class KernelExports ExportName = "exit", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int Exit(CpuContext ctx) => RequestProcessExit(ctx, "exit"); - - [SysAbiExport( - Nid = "L1SBTkC+Cvw", - ExportName = "abort", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Abort(CpuContext ctx) + public static int Exit(CpuContext ctx) { - // Route through the same graceful guest-entry-exit path as exit(): letting the call - // fall through to the host's native abort() does not unwind the guest thread cleanly. - Console.Error.WriteLine("[LOADER][INFO] abort() called by guest - terminating"); - GuestThreadExecution.RequestCurrentEntryExit("abort", -1); - ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); + var status = unchecked((int)ctx[CpuRegister.Rdi]); + Console.Error.WriteLine($"[LOADER][INFO] exit(status={status})"); + GuestThreadExecution.RequestCurrentEntryExit("exit", status); + ctx[CpuRegister.Rax] = unchecked((ulong)status); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -199,7 +172,11 @@ public static class KernelExports ExportName = "_exit", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int UnderscoreExit(CpuContext ctx) => RequestProcessExit(ctx, "_exit"); + public static int UnderscoreExit(CpuContext ctx) + { + _ = ctx; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } [SysAbiExport( Nid = "Ac86z8q7T8A", @@ -218,12 +195,14 @@ public static class KernelExports Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] public static int PthreadCreate(CpuContext ctx) + => PthreadCreateCore(ctx, ctx[CpuRegister.R8]); + + private static int PthreadCreateCore(CpuContext ctx, ulong nameAddress) { var threadIdAddress = ctx[CpuRegister.Rdi]; var attrAddress = ctx[CpuRegister.Rsi]; var entryAddress = ctx[CpuRegister.Rdx]; var argument = ctx[CpuRegister.Rcx]; - var nameAddress = ctx[CpuRegister.R8]; var name = nameAddress == 0 ? string.Empty : ReadCString(ctx, nameAddress, 256); var threadHandle = KernelPthreadState.CreateThreadHandle(name); KernelPthreadExtendedCompatExports.GetThreadStartScheduling( @@ -278,14 +257,20 @@ public static class KernelExports ExportName = "pthread_create", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixPthreadCreate(CpuContext ctx) => PthreadCreate(ctx); + public static int PosixPthreadCreate(CpuContext ctx) + { + return PthreadCreateCore(ctx, nameAddress: 0); + } [SysAbiExport( Nid = "Jmi+9w9u0E4", ExportName = "pthread_create_name_np", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixPthreadCreateNameNp(CpuContext ctx) => PthreadCreate(ctx); + public static int PosixPthreadCreateNameNp(CpuContext ctx) + { + return PthreadCreateCore(ctx, ctx[CpuRegister.R8]); + } [SysAbiExport( Nid = "3kg7rT0NQIs", @@ -295,6 +280,9 @@ public static class KernelExports public static int PthreadExit(CpuContext ctx) { var value = ctx[CpuRegister.Rdi]; + // Run cleanup on the still-executable thread before unwinding it. + KernelPthreadExtendedCompatExports.RunThreadLocalDestructors(ctx); + KernelMemoryCompatExports.RunThreadDtors(ctx); GuestThreadExecution.RequestCurrentEntryExit("scePthreadExit", value); ctx[CpuRegister.Rax] = value; return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -308,6 +296,8 @@ public static class KernelExports public static int PosixPthreadExit(CpuContext ctx) { var value = ctx[CpuRegister.Rdi]; + KernelPthreadExtendedCompatExports.RunThreadLocalDestructors(ctx); + KernelMemoryCompatExports.RunThreadDtors(ctx); GuestThreadExecution.RequestCurrentEntryExit("pthread_exit", value); ctx[CpuRegister.Rax] = value; return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -362,7 +352,10 @@ public static class KernelExports ExportName = "pthread_join", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] - public static int PosixPthreadJoin(CpuContext ctx) => PthreadJoin(ctx); + public static int PosixPthreadJoin(CpuContext ctx) + { + return PthreadJoin(ctx); + } [SysAbiExport( Nid = "wuCroIGjt2g", @@ -437,14 +430,21 @@ public static class KernelExports private static string ReadCString(CpuContext ctx, ulong address, int maxLen) { Span buf = stackalloc byte[maxLen]; - if (!ctx.Memory.TryRead(address, buf)) - return $""; + Span one = stackalloc byte[1]; + var len = 0; + while (len < buf.Length) + { + if (!ctx.Memory.TryRead(address + (ulong)len, one)) + return len == 0 ? $"" : System.Text.Encoding.UTF8.GetString(buf[..len]); - int len = 0; - while (len < buf.Length && buf[len] != 0) len++; + if (one[0] == 0) + break; - try { return System.Text.Encoding.UTF8.GetString(buf.Slice(0, len)); } - catch { return System.Text.Encoding.ASCII.GetString(buf.Slice(0, len)); } + buf[len++] = one[0]; + } + + try { return System.Text.Encoding.UTF8.GetString(buf[..len]); } + catch { return System.Text.Encoding.ASCII.GetString(buf[..len]); } } private static bool ShouldTracePthread() @@ -452,19 +452,18 @@ public static class KernelExports return string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PTHREADS"), "1", StringComparison.Ordinal); } - private static int RequestProcessExit(CpuContext ctx, string syscallName) + [SysAbiExport( + Nid = "L1SBTkC+Cvw", + ExportName = "abort", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Abort(CpuContext ctx) { - var status = unchecked((int)ctx[CpuRegister.Rdi]); - Console.Error.WriteLine($"[LOADER][INFO] {syscallName}(status={status})"); - GuestThreadExecution.RequestCurrentEntryExit(syscallName, status); - ctx[CpuRegister.Rax] = unchecked((ulong)status); + // Route through the same graceful guest-entry-exit path as exit(): letting the call + // fall through to the host's native abort() does not unwind the guest thread cleanly. + Console.Error.WriteLine("[LOADER][INFO] abort() called by guest - terminating"); + GuestThreadExecution.RequestCurrentEntryExit("abort", -1); + ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - - // NOTE: "SHtAad20YYM"/"DIxvoy7Ngvk" are sce::Json::Value::getType/getInteger, and - // "3GPpjQdAMTw"/"9rAeANT2tyE"/"tsvEmnenz48" are __cxa_guard_acquire/__cxa_guard_release/ - // __cxa_atexit (verified by hashing against scripts/ps5_names.txt). Do not register them - // here as kernel mutex functions: the cxa guards are implemented in CxxAbiExports.cs and - // shadowing them breaks every C++ static-init guard in the game. - } diff --git a/src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs b/src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs new file mode 100644 index 0000000..67c468c --- /dev/null +++ b/src/SharpEmu.Libs/Kernel/KernelFileExtendedExports.cs @@ -0,0 +1,662 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Threading; + +namespace SharpEmu.Libs.Kernel; + +/// +/// Positional and scatter/gather file I/O, synchronization, renaming, fcntl, +/// polling and asynchronous I/O (AIO) that complement the base file exports. +/// These share the same guest FD table and path-resolution helpers via the +/// partial class. +/// +public static partial class KernelMemoryCompatExports +{ + // fcntl commands (FreeBSD numbering used by the PS4/PS5 libc). + private const int F_DUPFD = 0; + private const int F_GETFD = 1; + private const int F_SETFD = 2; + private const int F_GETFL = 3; + private const int F_SETFL = 4; + + // poll event bits. + private const short POLLIN = 0x0001; + private const short POLLOUT = 0x0004; + + // AIO request state (SCE_KERNEL_AIO_STATE_*). + private const uint AioStateCompleted = 3; + private const int SizeofAioRwRequest = 40; + + private static long _nextAioSubmitId = 1; + private static readonly ConcurrentDictionary _aioResults = new(); + + private static FileStream? GetOpenFile(int fd) + { + lock (_fdGate) + { + return _openFiles.TryGetValue(fd, out var stream) ? stream : null; + } + } + + // ---- Positional read/write (do not move the file offset) ---- + + [SysAbiExport(Nid = "ezv-RSBNKqI", ExportName = "pread", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixPread(CpuContext ctx) => KernelPreadCore(ctx); + + [SysAbiExport(Nid = "+r3rMFwItV4", ExportName = "sceKernelPread", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelPread(CpuContext ctx) => KernelPreadCore(ctx); + + private static int KernelPreadCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var bufferAddress = ctx[CpuRegister.Rsi]; + var requested = (int)Math.Min(ctx[CpuRegister.Rdx], int.MaxValue); + var offset = unchecked((long)ctx[CpuRegister.Rcx]); + if (requested < 0 || (requested > 0 && bufferAddress == 0) || offset < 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (requested == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var buffer = GC.AllocateUninitializedArray(requested); + int read; + try + { + read = RandomAccess.Read(stream.SafeFileHandle, buffer, offset); + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (read > 0 && !ctx.Memory.TryWrite(bufferAddress, buffer.AsSpan(0, read))) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = unchecked((ulong)read); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "C2kJ-byS5rM", ExportName = "pwrite", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixPwrite(CpuContext ctx) => KernelPwriteCore(ctx); + + [SysAbiExport(Nid = "nKWi-N2HBV4", ExportName = "sceKernelPwrite", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelPwrite(CpuContext ctx) => KernelPwriteCore(ctx); + + private static int KernelPwriteCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var bufferAddress = ctx[CpuRegister.Rsi]; + var requested = (int)Math.Min(ctx[CpuRegister.Rdx], int.MaxValue); + var offset = unchecked((long)ctx[CpuRegister.Rcx]); + if (requested < 0 || (requested > 0 && bufferAddress == 0) || offset < 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (requested == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var payload = GC.AllocateUninitializedArray(requested); + if (!ctx.Memory.TryRead(bufferAddress, payload)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + try + { + RandomAccess.Write(stream.SafeFileHandle, payload, offset); + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + ctx[CpuRegister.Rax] = unchecked((ulong)requested); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Synchronization ---- + + [SysAbiExport(Nid = "juWbTNM+8hw", ExportName = "fsync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFsync(CpuContext ctx) => KernelFsyncCore(ctx); + + [SysAbiExport(Nid = "fTx66l5iWIA", ExportName = "sceKernelFsync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFsync(CpuContext ctx) => KernelFsyncCore(ctx); + + [SysAbiExport(Nid = "KIbJFQ0I1Cg", ExportName = "fdatasync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFdatasync(CpuContext ctx) => KernelFsyncCore(ctx); + + [SysAbiExport(Nid = "30Rh4ixbKy4", ExportName = "sceKernelFdatasync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFdatasync(CpuContext ctx) => KernelFsyncCore(ctx); + + private static int KernelFsyncCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var stream = GetOpenFile(fd); + if (stream is null) + { + if (fd is 0 or 1 or 2) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + try + { + stream.Flush(flushToDisk: true); + } + catch (IOException) + { + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "Y2OqwJQ3lr8", ExportName = "sync", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixSync(CpuContext ctx) + { + lock (_fdGate) + { + foreach (var stream in _openFiles.Values) + { + try { stream.Flush(flushToDisk: true); } catch (IOException) { } + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Truncation ---- + + [SysAbiExport(Nid = "ih4CD9-gghM", ExportName = "ftruncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFtruncate(CpuContext ctx) => KernelFtruncateCore(ctx); + + [SysAbiExport(Nid = "VW3TVZiM4-E", ExportName = "sceKernelFtruncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFtruncate(CpuContext ctx) => KernelFtruncateCore(ctx); + + private static int KernelFtruncateCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var length = unchecked((long)ctx[CpuRegister.Rsi]); + if (length < 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + try + { + stream.SetLength(length); + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "ayrtszI7GBg", ExportName = "truncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixTruncate(CpuContext ctx) => KernelTruncateCore(ctx); + + [SysAbiExport(Nid = "WlyEA-sLDf0", ExportName = "sceKernelTruncate", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelTruncate(CpuContext ctx) => KernelTruncateCore(ctx); + + private static int KernelTruncateCore(CpuContext ctx) + { + var pathAddress = ctx[CpuRegister.Rdi]; + var length = unchecked((long)ctx[CpuRegister.Rsi]); + if (length < 0 || !TryReadNullTerminatedUtf8(ctx, pathAddress, MaxGuestStringLength, out var guestPath)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (IsReadOnlyGuestMutationPath(guestPath)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + + var hostPath = ResolveGuestPath(guestPath); + try + { + using var stream = new FileStream(hostPath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); + stream.SetLength(length); + } + catch (FileNotFoundException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Rename ---- + + [SysAbiExport(Nid = "NN01qLRhiqU", ExportName = "rename", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixRename(CpuContext ctx) => KernelRenameCore(ctx); + + [SysAbiExport(Nid = "52NcYU9+lEo", ExportName = "sceKernelRename", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelRename(CpuContext ctx) => KernelRenameCore(ctx); + + private static int KernelRenameCore(CpuContext ctx) + { + if (!TryReadNullTerminatedUtf8(ctx, ctx[CpuRegister.Rdi], MaxGuestStringLength, out var fromGuest) || + !TryReadNullTerminatedUtf8(ctx, ctx[CpuRegister.Rsi], MaxGuestStringLength, out var toGuest)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (IsReadOnlyGuestMutationPath(fromGuest) || IsReadOnlyGuestMutationPath(toGuest)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + + var fromHost = ResolveGuestPath(fromGuest); + var toHost = ResolveGuestPath(toGuest); + try + { + if (Directory.Exists(fromHost)) + { + Directory.Move(fromHost, toHost); + } + else + { + File.Move(fromHost, toHost, overwrite: true); + } + } + catch (FileNotFoundException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + catch (DirectoryNotFoundException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + catch (IOException) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + InvalidateNegativeStatCacheForPathAndAncestors(toGuest); + AddNegativeStatCacheForGuestPath(fromGuest); + InvalidateAprFileSizeCache(fromHost); + InvalidateAprFileSizeCache(toHost); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Descriptor duplication ---- + + [SysAbiExport(Nid = "iiQjzvfWDq0", ExportName = "dup", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixDup(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + lock (_fdGate) + { + if (!_openFiles.TryGetValue(fd, out var stream)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + // POSIX dup shares the open file description (and offset), which is + // exactly the shared FileStream reference. + var newFd = (int)Interlocked.Increment(ref _nextFileDescriptor); + _openFiles[newFd] = stream; + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + } + + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "wdUufa9g-D8", ExportName = "dup2", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixDup2(CpuContext ctx) + { + var oldFd = unchecked((int)ctx[CpuRegister.Rdi]); + var newFd = unchecked((int)ctx[CpuRegister.Rsi]); + lock (_fdGate) + { + if (!_openFiles.TryGetValue(oldFd, out var stream)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (oldFd == newFd) + { + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // If newFd names an open file, dup2 closes it first. + if (_openFiles.TryGetValue(newFd, out var existing) && !ReferenceEquals(existing, stream)) + { + try { existing.Dispose(); } catch (IOException) { } + } + + _openFiles[newFd] = stream; + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + } + + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- fcntl ---- + + [SysAbiExport(Nid = "8nY19bKoiZk", ExportName = "fcntl", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixFcntl(CpuContext ctx) => KernelFcntlCore(ctx); + + [SysAbiExport(Nid = "SoZkxZkCHaw", ExportName = "sceKernelFcntl", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelFcntl(CpuContext ctx) => KernelFcntlCore(ctx); + + private static int KernelFcntlCore(CpuContext ctx) + { + var fd = unchecked((int)ctx[CpuRegister.Rdi]); + var command = unchecked((int)ctx[CpuRegister.Rsi]); + var argument = unchecked((int)ctx[CpuRegister.Rdx]); + + switch (command) + { + case F_DUPFD: + lock (_fdGate) + { + if (!_openFiles.TryGetValue(fd, out var stream)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + var newFd = Math.Max((int)Interlocked.Increment(ref _nextFileDescriptor), argument); + _openFiles[newFd] = stream; + ctx[CpuRegister.Rax] = unchecked((ulong)newFd); + } + + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + + case F_GETFD: + case F_GETFL: + // No close-on-exec / status flags are tracked; report cleared. + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + + case F_SETFD: + case F_SETFL: + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + + default: + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + } + + // ---- poll / select ---- + // Regular files are always ready for both read and write, so report every + // requested descriptor as immediately ready. + + [SysAbiExport(Nid = "ku7D4q1Y9PI", ExportName = "poll", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixPoll(CpuContext ctx) + { + var fdsAddress = ctx[CpuRegister.Rdi]; + var count = unchecked((uint)ctx[CpuRegister.Rsi]); + if (fdsAddress == 0 || count == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // struct pollfd { int fd; short events; short revents; } == 8 bytes. + var ready = 0; + Span buffer = stackalloc byte[8]; + for (uint i = 0; i < count && i < 4096; i++) + { + var entry = fdsAddress + i * 8; + if (!ctx.Memory.TryRead(entry, buffer)) + { + break; + } + + var events = BinaryPrimitives.ReadInt16LittleEndian(buffer[4..]); + var revents = (short)(events & (POLLIN | POLLOUT)); + BinaryPrimitives.WriteInt16LittleEndian(buffer[6..], revents); + if (revents != 0) + { + ready++; + } + + _ = ctx.Memory.TryWrite(entry, buffer); + } + + ctx[CpuRegister.Rax] = unchecked((ulong)ready); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "T8fER+tIGgk", ExportName = "select", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int PosixSelect(CpuContext ctx) + { + // nfds in Rdi; the fd_sets are left as-is (all reported ready) and the + // ready count returned is nfds so callers proceed without blocking. + var nfds = unchecked((int)ctx[CpuRegister.Rdi]); + ctx[CpuRegister.Rax] = unchecked((ulong)Math.Max(nfds, 0)); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // ---- Asynchronous I/O (executed synchronously) ---- + + [SysAbiExport(Nid = "HgX7+AORI58", ExportName = "sceKernelAioSubmitReadCommands", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioSubmitReadCommands(CpuContext ctx) => KernelAioSubmit(ctx, write: false); + + [SysAbiExport(Nid = "lXT0m3P-vs4", ExportName = "sceKernelAioSubmitReadCommandsMultiple", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioSubmitReadCommandsMultiple(CpuContext ctx) => KernelAioSubmit(ctx, write: false); + + [SysAbiExport(Nid = "XQ8C8y+de+E", ExportName = "sceKernelAioSubmitWriteCommands", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioSubmitWriteCommands(CpuContext ctx) => KernelAioSubmit(ctx, write: true); + + private static int KernelAioSubmit(CpuContext ctx, bool write) + { + var requestsAddress = ctx[CpuRegister.Rdi]; + var count = unchecked((int)ctx[CpuRegister.Rsi]); + var outIdAddress = ctx[CpuRegister.Rcx]; + if (requestsAddress == 0 || count <= 0 || count > 0x10000) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + // Each SceKernelAioRWRequest: offset(8) nbyte(8) buf(8) result(8) fd(4). + Span request = stackalloc byte[SizeofAioRwRequest]; + Span result = stackalloc byte[16]; + for (var i = 0; i < count; i++) + { + var entry = requestsAddress + (ulong)(i * SizeofAioRwRequest); + if (!ctx.Memory.TryRead(entry, request)) + { + break; + } + + var offset = BinaryPrimitives.ReadInt64LittleEndian(request[0..]); + var nbyte = (int)Math.Min(BinaryPrimitives.ReadUInt64LittleEndian(request[8..]), int.MaxValue); + var buf = BinaryPrimitives.ReadUInt64LittleEndian(request[16..]); + var resultPtr = BinaryPrimitives.ReadUInt64LittleEndian(request[24..]); + var fd = BinaryPrimitives.ReadInt32LittleEndian(request[32..]); + + long transferred = KernelAioTransfer(ctx, fd, offset, nbyte, buf, write); + if (resultPtr != 0) + { + BinaryPrimitives.WriteInt64LittleEndian(result[0..], transferred); + BinaryPrimitives.WriteUInt32LittleEndian(result[8..], AioStateCompleted); + _ = ctx.Memory.TryWrite(resultPtr, result); + } + } + + var submitId = unchecked((uint)Interlocked.Increment(ref _nextAioSubmitId)); + _aioResults[submitId] = 0; + if (outIdAddress != 0) + { + Span idBuffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(idBuffer, submitId); + _ = ctx.Memory.TryWrite(outIdAddress, idBuffer); + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + private static long KernelAioTransfer(CpuContext ctx, int fd, long offset, int nbyte, ulong buf, bool write) + { + if (nbyte <= 0 || buf == 0 || offset < 0) + { + return 0; + } + + var stream = GetOpenFile(fd); + if (stream is null) + { + return -1; + } + + var scratch = GC.AllocateUninitializedArray(nbyte); + try + { + if (write) + { + if (!ctx.Memory.TryRead(buf, scratch)) + { + return -1; + } + + RandomAccess.Write(stream.SafeFileHandle, scratch, offset); + return nbyte; + } + + var read = RandomAccess.Read(stream.SafeFileHandle, scratch, offset); + if (read > 0 && !ctx.Memory.TryWrite(buf, scratch.AsSpan(0, read))) + { + return -1; + } + + return read; + } + catch (IOException) + { + return -1; + } + } + + [SysAbiExport(Nid = "o7O4z3jwKzo", ExportName = "sceKernelAioPollRequests", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioPollRequests(CpuContext ctx) => KernelAioComplete(ctx); + + [SysAbiExport(Nid = "lgK+oIWkJyA", ExportName = "sceKernelAioWaitRequests", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioWaitRequests(CpuContext ctx) => KernelAioComplete(ctx); + + private static int KernelAioComplete(CpuContext ctx) + { + // Submission already performed the I/O synchronously, so every request + // reports completed. Rsi points at the state-out array, Rdx = count. + var statesAddress = ctx[CpuRegister.Rsi]; + var count = unchecked((int)ctx[CpuRegister.Rdx]); + if (statesAddress != 0 && count > 0 && count <= 0x10000) + { + Span state = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(state, AioStateCompleted); + for (var i = 0; i < count; i++) + { + _ = ctx.Memory.TryWrite(statesAddress + (ulong)(i * sizeof(uint)), state); + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "fR521KIGgb8", ExportName = "sceKernelAioCancelRequest", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioCancelRequest(CpuContext ctx) + { + // Already completed; report the completed state if an out pointer given. + var stateAddress = ctx[CpuRegister.Rsi]; + if (stateAddress != 0) + { + Span state = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(state, AioStateCompleted); + _ = ctx.Memory.TryWrite(stateAddress, state); + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport(Nid = "5TgME6AYty4", ExportName = "sceKernelAioDeleteRequest", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libKernel")] + public static int KernelAioDeleteRequest(CpuContext ctx) + { + var submitId = unchecked((uint)ctx[CpuRegister.Rdi]); + _aioResults.TryRemove(submitId, out _); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } +} diff --git a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs index 77ba324..81848a7 100644 --- a/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelMemoryCompatExports.cs @@ -2,27 +2,25 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Libs.Ampr; +using SharpEmu.Libs.Bink; using System.Buffers; using System.Buffers.Binary; using System.Collections.Concurrent; using System.Text; +using System.Threading; using System.Runtime.InteropServices; +using System.Linq; using System.Globalization; namespace SharpEmu.Libs.Kernel; -public static class KernelMemoryCompatExports +public static partial class KernelMemoryCompatExports { private const int MaxGuestStringLength = 4096; private const int WideCharSize = sizeof(ushort); private const int MemsetChunkSize = 16 * 1024; - private const int MemcpyChunkSize = 256 * 1024; - - // Shared all-zero scratch for chunked zero-fill loops; never written to. private static readonly byte[] _zeroChunk = new byte[MemsetChunkSize]; - private const int TlsModuleBlockSize = 0x10000; private const int O_WRONLY = 0x1; private const int O_RDWR = 0x2; private const int O_APPEND = 0x8; @@ -57,13 +55,12 @@ public static class KernelMemoryCompatExports private const ulong FlexibleMemorySizeBytes = 448UL * 1024 * 1024; private const int OrbisVirtualQueryInfoSize = 72; private const int OrbisKernelMaximumNameLength = 32; - // Raw Windows PAGE_* values used only against HostRegionInfo.RawProtection, - // which by contract carries the untranslated protection word of the host - // platform in use (see IHostMemory). + private const uint MemCommit = 0x1000; private const uint HostPageNoAccess = 0x01; private const uint HostPageReadOnly = 0x02; private const uint HostPageReadWrite = 0x04; private const uint HostPageWriteCopy = 0x08; + private const uint HostPageExecute = 0x10; private const uint HostPageExecuteRead = 0x20; private const uint HostPageExecuteReadWrite = 0x40; private const uint HostPageExecuteWriteCopy = 0x80; @@ -100,7 +97,6 @@ public static class KernelMemoryCompatExports private static readonly Dictionary _openDirectories = new(); private static readonly object _libcAllocGate = new(); private static readonly object _memoryGate = new(); - private static readonly object _tlsGate = new(); private static readonly object _ioTraceGate = new(); private static readonly object _statCacheGate = new(); private static readonly object _guestMountGate = new(); @@ -108,7 +104,6 @@ public static class KernelMemoryCompatExports private static readonly Dictionary _libcAllocations = new(); private static readonly Dictionary _mappedRegions = new(); private static readonly Dictionary _mappedRegionNames = new(); - private static readonly Dictionary _tlsModuleBlocks = new(); private static readonly Dictionary _guestMounts = new(StringComparer.OrdinalIgnoreCase); private static readonly HashSet _tracedStatResults = new(StringComparer.Ordinal); private static readonly HashSet _negativeStatCache = new(StringComparer.OrdinalIgnoreCase); @@ -147,9 +142,35 @@ public static class KernelMemoryCompatExports private static string? _cachedApp0Root; private static string? _cachedDownload0Root; - // Property (not a cached field) so merely touching this type never resolves - // the platform backend; non-Windows hosts only throw if a call is reached. - private static IHostMemory HostMemory => HostPlatform.Current.Memory; + [StructLayout(LayoutKind.Sequential)] + private struct MemoryBasicInformation + { + public nint BaseAddress; + public nint AllocationBase; + public uint AllocationProtect; + public nuint RegionSize; + public uint State; + public uint Protect; + public uint Type; + } + + private static unsafe nuint VirtualQuery(nint lpAddress, out MemoryBasicInformation lpBuffer, nuint dwLength) + { + _ = dwLength; + var result = HostMemory.Query((void*)lpAddress, out var info); + lpBuffer = default; + lpBuffer.BaseAddress = (nint)info.BaseAddress; + lpBuffer.AllocationBase = (nint)info.AllocationBase; + lpBuffer.AllocationProtect = info.AllocationProtect; + lpBuffer.RegionSize = (nuint)info.RegionSize; + lpBuffer.State = info.State; + lpBuffer.Protect = info.Protect; + lpBuffer.Type = info.Type; + return result; + } + + private static unsafe bool VirtualProtect(nint lpAddress, nuint dwSize, uint flNewProtect, out uint lpflOldProtect) => + HostMemory.Protect((void*)lpAddress, dwSize, flNewProtect, out lpflOldProtect); private sealed class OpenDirectory { @@ -159,7 +180,7 @@ public static class KernelMemoryCompatExports } private readonly record struct DirectAllocation(ulong Start, ulong Length, int MemoryType); - private readonly record struct LibcHeapAllocation(nint BaseAddress, nuint Size, nuint Alignment, bool IsGuarded); + private readonly record struct LibcHeapAllocation(nint BaseAddress, nuint Size, nuint Alignment); private readonly record struct MappedRegion(ulong Address, ulong Length, int Protection, bool IsFlexible, bool IsDirect, ulong DirectStart); private readonly record struct BatchMapEntry(ulong Start, ulong Offset, ulong Length, byte Protection, byte Type, int Operation); @@ -189,53 +210,11 @@ public static class KernelMemoryCompatExports } } - public static bool TryUnregisterGuestPathMount(string guestMountPoint) - { - if (string.IsNullOrWhiteSpace(guestMountPoint)) - { - return false; - } - - var normalizedMountPoint = NormalizeGuestStatCachePath(guestMountPoint); - if (normalizedMountPoint is null || normalizedMountPoint == "/") - { - return false; - } - - var removed = false; - lock (_guestMountGate) - { - removed = _guestMounts.Remove(normalizedMountPoint); - } - - if (!removed) - { - return false; - } - - lock (_statCacheGate) - { - _negativeStatCache.RemoveWhere(path => - string.Equals(path, normalizedMountPoint, StringComparison.OrdinalIgnoreCase) || - path.StartsWith(normalizedMountPoint + "/", StringComparison.OrdinalIgnoreCase)); - } - - return true; - } - internal static bool TryAllocateHleData( CpuContext ctx, ulong length, ulong alignment, out ulong address) - => TryAllocateHleData(ctx, length, alignment, OrbisProtCpuReadWrite, out address); - - internal static bool TryAllocateHleData( - CpuContext ctx, - ulong length, - ulong alignment, - int protection, - out ulong address) { address = 0; if (length == 0 || length > int.MaxValue) @@ -250,7 +229,7 @@ public static class KernelMemoryCompatExports var desiredAddress = AlignUp( _nextVirtualAddress == 0 ? DefaultMapSearchBase : _nextVirtualAddress, effectiveAlignment); - if (!TryReserveGuestVirtualRange(ctx, desiredAddress, mappedLength, protection, effectiveAlignment, out address) || + if (!TryReserveGuestVirtualRange(ctx, desiredAddress, mappedLength, OrbisProtCpuReadWrite, effectiveAlignment, out address) || address == 0) { return false; @@ -260,7 +239,7 @@ public static class KernelMemoryCompatExports _mappedRegions[address] = new MappedRegion( address, mappedLength, - protection, + OrbisProtCpuReadWrite, IsFlexible: false, IsDirect: false, DirectStart: 0); @@ -280,69 +259,6 @@ public static class KernelMemoryCompatExports return true; } - private static ulong _dummyVtableAddress; - private const int DummyVtableSlotCount = 64; - - // Some KexEngine objects (mutex wrappers, NGS2 system/rack/voice handles, etc.) are treated - // by guest code as C++ instances with a vtable pointer at offset 0. When an HLE constructor - // leaves that slot zeroed, the game's own virtual dispatch (call [ [obj]+N ]) reads a null - // vtable and jumps through address N, crashing (e.g. "CALL qword ptr [RAX+0x58]" with RAX=0). - // This allocates one shared, executable no-op stub plus a vtable of pointers to it, so any - // HLE object constructor can make virtual calls on its output land safely. - internal static bool TryWriteDummyVtable(CpuContext ctx, ulong objectAddress) - { - if (objectAddress == 0 || !TryEnsureDummyVtable(ctx, out var vtableAddress)) - { - return false; - } - - return ctx.TryWriteUInt64(objectAddress, vtableAddress); - } - - private static bool TryEnsureDummyVtable(CpuContext ctx, out ulong vtableAddress) - { - lock (_memoryGate) - { - if (_dummyVtableAddress != 0) - { - vtableAddress = _dummyVtableAddress; - return true; - } - - const int executableReadWrite = OrbisProtCpuRead | OrbisProtCpuWrite | OrbisProtCpuExec; - if (!TryAllocateHleData(ctx, 0x1000, 0x1000, executableReadWrite, out var block)) - { - vtableAddress = 0; - return false; - } - - // xor eax, eax; ret - every dummy virtual method just returns 0 and does nothing else. - if (!ctx.Memory.TryWrite(block, new byte[] { 0x31, 0xC0, 0xC3 })) - { - vtableAddress = 0; - return false; - } - - var table = new byte[DummyVtableSlotCount * sizeof(ulong)]; - for (var i = 0; i < DummyVtableSlotCount; i++) - { - BinaryPrimitives.WriteUInt64LittleEndian(table.AsSpan(i * sizeof(ulong), sizeof(ulong)), block); - } - - var tableAddress = block + 0x100; - if (!ctx.Memory.TryWrite(tableAddress, table)) - { - vtableAddress = 0; - return false; - } - - Console.Error.WriteLine($"[LOADER][INFO] Dummy vtable ready: stub=0x{block:X16} vtable=0x{tableAddress:X16} slots={DummyVtableSlotCount}"); - _dummyVtableAddress = tableAddress; - vtableAddress = tableAddress; - return true; - } - } - internal static void RegisterReservedVirtualRange(ulong address, ulong length) { if (address == 0 || length == 0) @@ -393,18 +309,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - // Longer null-dst memsets are unrecoverable, but RAX must still be set - leaving it - // stale here previously let callers that do `buf = memset(...)` carry on with a - // garbage "buffer" pointer instead of a clean NULL, causing a *different*, - // confusingly-located crash further downstream. - var largeRecoveryIndex = Interlocked.Increment(ref _nullMemsetRecoveryCount); - if (largeRecoveryIndex <= 8) - { - Console.Error.WriteLine( - $"[LOADER][WARNING] memset null-dst (len>0x20) recovery#{largeRecoveryIndex}: rip=0x{ctx.Rip:X16} len=0x{length:X} val=0x{value:X2}"); - } - - ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } @@ -422,13 +326,16 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - const ulong MaxSane = 512UL * 1024 * 1024; + // Only obviously-bogus destinations or absurd sizes are rejected. Large + // but plausible clears (engines zero multi-hundred-MB resource pools in + // one call) are allowed; the write loop below clamps to the mapped range + // if the request runs off the end of a region instead of hard-faulting. + const ulong MaxSane = 2UL * 1024 * 1024 * 1024; if (destination < 0x1000 || destination >= CanonicalUserUpper || length > MaxSane) { Console.WriteLine("!!! CRITICAL: Bad Memset Call !!!"); Console.WriteLine($"Called from RIP: 0x{ctx.Rip:X}"); Console.WriteLine($"dst=0x{destination:X} val=0x{value:X2} len=0x{length:X}"); - ctx[CpuRegister.Rax] = destination; return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -450,6 +357,9 @@ public static class KernelMemoryCompatExports var take = (int)Math.Min((ulong)chunkLength, remaining); if (!TryWriteCompat(ctx, cursor, chunk.AsSpan(0, take))) { + // Clamp oversized clears to the valid mapped prefix. Small + // inaccessible writes are tolerated for compatibility with + // titles that probe optional state during startup. if (length <= 0x40) { var recoveryIndex = Interlocked.Increment(ref _inaccessibleMemsetRecoveryCount); @@ -463,7 +373,16 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + var clampIndex = Interlocked.Increment(ref _inaccessibleMemsetRecoveryCount); + if (clampIndex <= 8) + { + Console.Error.WriteLine( + $"[LOADER][WARNING] memset clamped to mapped range#{clampIndex}: rip=0x{ctx.Rip:X16} " + + $"dst=0x{destination:X16} len=0x{length:X} written=0x{cursor - destination:X} val=0x{value:X2}"); + } + + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } cursor += (ulong)take; @@ -678,80 +597,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "AV6ipCNa4Rw", - ExportName = "strcasecmp", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strcasecmp(CpuContext ctx) - { - var left = ctx[CpuRegister.Rdi]; - var right = ctx[CpuRegister.Rsi]; - if (left == 0 || right == 0) - { - var recoveryIndex = Interlocked.Increment(ref _nullStrcasecmpRecoveryCount); - if (recoveryIndex <= 16) - { - var otherAddress = left == 0 ? right : left; - var otherText = otherAddress != 0 && TryReadNullTerminatedUtf8(ctx, otherAddress, 256, out var text) - ? text - : ""; - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); - Console.Error.WriteLine( - $"[LOADER][WARNING] strcasecmp null-arg recovery#{recoveryIndex}: ret=0x{returnRip:X16} left=0x{left:X16} right=0x{right:X16} other=\"{otherText}\""); - } - - // Real strcasecmp(NULL, x) is undefined behaviour and previously crashed inside the - // LLE-routed implementation. Treat it as "not equal" instead so callers doing - // `if (strcasecmp(a, b) == 0)` degrade gracefully rather than taking down the guest. - ctx[CpuRegister.Rax] = left == right ? 0uL : 1uL; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - if (!TryCompareStringsCaseInsensitive(ctx, left, right, limit: ulong.MaxValue, out var compare)) - { - ctx[CpuRegister.Rax] = 1; - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = unchecked((ulong)compare); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - private static bool TryCompareStringsCaseInsensitive(CpuContext ctx, ulong left, ulong right, ulong limit, out int compare) - { - compare = 0; - if (left == 0 || right == 0) - { - return false; - } - - var max = limit == ulong.MaxValue ? 1_048_576UL : Math.Min(limit, 1_048_576UL); - Span leftByte = stackalloc byte[1]; - Span rightByte = stackalloc byte[1]; - for (ulong i = 0; i < max; i++) - { - if (!TryReadCompat(ctx, left + i, leftByte) || - !TryReadCompat(ctx, right + i, rightByte)) - { - return false; - } - - var leftLower = ToAsciiLower(leftByte[0]); - var rightLower = ToAsciiLower(rightByte[0]); - compare = leftLower - rightLower; - if (compare != 0 || leftByte[0] == 0 || rightByte[0] == 0) - { - return true; - } - } - - compare = 0; - return true; - } - - private static byte ToAsciiLower(byte value) => value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; - [SysAbiExport( Nid = "0nV21JjYCH8", ExportName = "wcsncpy", @@ -803,54 +648,6 @@ public static class KernelMemoryCompatExports return VsnprintfCore(ctx); } - // sprintf/vsprintf are served from the same HLE formatting engine as snprintf/vsnprintf - // instead of falling through to the game's bundled libc. - [SysAbiExport( - Nid = "tcVi5SivF7Q", - ExportName = "sprintf", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Sprintf(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var formatAddress = ctx[CpuRegister.Rsi]; - - if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var format = Encoding.UTF8.GetString(formatBytes); - var rendered = FormatStringFromVarArgs(ctx, format, firstGpArgIndex: 2); - return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); - } - - [SysAbiExport( - Nid = "jbz9I9vkqkk", - ExportName = "vsprintf", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Vsprintf(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var formatAddress = ctx[CpuRegister.Rsi]; - var vaListAddress = ctx[CpuRegister.Rdx]; - - if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var format = Encoding.UTF8.GetString(formatBytes); - if (!TryCreateVaListCursor(ctx, vaListAddress, out var vaCursor)) - { - return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, formatBytes); - } - - var rendered = FormatString(ctx, format, ref vaCursor); - return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); - } - [SysAbiExport( Nid = "nJz16JE1txM", ExportName = "swprintf", @@ -1017,20 +814,59 @@ public static class KernelMemoryCompatExports } var payload = new byte[count * WideCharSize]; - for (var copied = 0; copied < count; copied++) + if (count == 0) { - if (!TryReadUInt16Compat(ctx, source + ((ulong)copied * WideCharSize), out var unit)) + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // Keep host-pointer reads page-bounded and copy several UTF-16 code + // units per validation. Large scratch strings otherwise pay the host + // address-validation and temporary-buffer cost once per character. + const int maxReadBytes = 4096; + var readBuffer = GC.AllocateUninitializedArray( + Math.Min(maxReadBytes, payload.Length)); + var copied = 0; + while (copied < count) + { + var sourceAddress = source + ((ulong)copied * WideCharSize); + if (sourceAddress < source) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - BinaryPrimitives.WriteUInt16LittleEndian( - payload.AsSpan(copied * WideCharSize, WideCharSize), - unit); - - if (unit == 0) + var pageBytesRemaining = maxReadBytes - + (int)(sourceAddress & (maxReadBytes - 1)); + var remainingBytes = (count - copied) * WideCharSize; + var readBytes = Math.Min( + readBuffer.Length, + Math.Min(pageBytesRemaining, remainingBytes)); + readBytes &= ~(WideCharSize - 1); + if (readBytes == 0 || + !TryReadCompat(ctx, sourceAddress, readBuffer.AsSpan(0, readBytes))) { - break; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + for (var offset = 0; offset < readBytes; offset += WideCharSize) + { + var unit = BinaryPrimitives.ReadUInt16LittleEndian( + readBuffer.AsSpan(offset, WideCharSize)); + if (unit == 0) + { + // payload is zero-initialized, supplying wcsncpy padding. + copied = count; + break; + } + + BinaryPrimitives.WriteUInt16LittleEndian( + payload.AsSpan((copied * WideCharSize) + offset, WideCharSize), + unit); + } + + if (copied != count) + { + copied += readBytes / WideCharSize; } } @@ -1055,23 +891,48 @@ public static class KernelMemoryCompatExports private static int WcschrCore(CpuContext ctx, ulong address, ushort needle) { - for (ulong index = 0; index < 1_048_576; index++) + const int maxReadBytes = 4096; + var readBuffer = GC.AllocateUninitializedArray(maxReadBytes); + const ulong maxUnits = 1_048_576; + for (ulong index = 0; index < maxUnits;) { - if (!TryReadUInt16Compat(ctx, address + (index * WideCharSize), out var unit)) + var unitAddress = address + (index * WideCharSize); + if (unitAddress < address) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (unit == needle) + var remainingBytes = (maxUnits - index) * WideCharSize; + var pageBytesRemaining = maxReadBytes - + (int)(unitAddress & (maxReadBytes - 1)); + var readBytes = (int)Math.Min( + (ulong)Math.Min(readBuffer.Length, pageBytesRemaining), + remainingBytes); + readBytes &= ~(WideCharSize - 1); + if (readBytes == 0 || + !TryReadCompat(ctx, unitAddress, readBuffer.AsSpan(0, readBytes))) { - ctx[CpuRegister.Rax] = address + (index * WideCharSize); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (unit == 0) + for (var offset = 0; offset < readBytes; offset += WideCharSize) { - break; + var unit = BinaryPrimitives.ReadUInt16LittleEndian( + readBuffer.AsSpan(offset, WideCharSize)); + if (unit == needle) + { + ctx[CpuRegister.Rax] = unitAddress + (ulong)offset; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + if (unit == 0) + { + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } } + + index += (ulong)(readBytes / WideCharSize); } ctx[CpuRegister.Rax] = 0; @@ -1209,65 +1070,16 @@ public static class KernelMemoryCompatExports { var destination = ctx[CpuRegister.Rdi]; var source = ctx[CpuRegister.Rsi]; - var rawCount = ctx[CpuRegister.Rdx]; - - // A garbage/absurd count (observed as e.g. 0xA7560035 from the same still-unidentified - // upstream bug that also feeds bad lengths to memset) must not turn into a multi-GB - // copy attempt from a guest-thread call context, which corrupted the CLR outright - // ("Invalid Program: attempted to call a UnmanagedCallersOnly method from managed - // code") instead of throwing a normal exception. Reject anything above a sane bound. - const ulong maxSaneCount = 512UL * 1024 * 1024; - if (rawCount > maxSaneCount) + var count = (int)Math.Min(ctx[CpuRegister.Rdx], int.MaxValue); + if (count < 0) { - Console.Error.WriteLine($"[LOADER][WARNING] memcpy oversized count rejected: dst=0x{destination:X16} src=0x{source:X16} count=0x{rawCount:X}"); - ctx[CpuRegister.Rax] = destination; return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - ctx[CpuRegister.Rax] = destination; - if (rawCount == 0) + var payload = GC.AllocateUninitializedArray(count); + if (count > 0 && (!TryReadCompat(ctx, source, payload) || !TryWriteCompat(ctx, destination, payload))) { - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - // Cap iterations at the requested chunk size, not chunk.Length: Rent may hand - // back a larger array, and the copy granularity should not depend on pool - // bucketing internals. - var chunkLength = (int)Math.Min(rawCount, (ulong)MemcpyChunkSize); - var chunk = ArrayPool.Shared.Rent(chunkLength); - try - { - // memmove aliases this export, so overlapping ranges must survive the chunked - // copy: when the destination starts inside the source range, copy high-to-low - // so no source byte is overwritten before it has been read. - var copyBackward = destination > source && destination - source < rawCount; - var remaining = rawCount; - ulong offset = copyBackward ? rawCount : 0; - while (remaining > 0) - { - var take = (int)Math.Min((ulong)chunkLength, remaining); - if (copyBackward) - { - offset -= (ulong)take; - } - - var span = chunk.AsSpan(0, take); - if (!TryReadCompat(ctx, source + offset, span) || !TryWriteCompat(ctx, destination + offset, span)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!copyBackward) - { - offset += (ulong)take; - } - - remaining -= (ulong)take; - } - } - finally - { - ArrayPool.Shared.Return(chunk); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -1508,24 +1320,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - // Was unresolved (returning the 0x80020002 sentinel, then crashing when the guest - // dereferenced it) - the game's own heap instrumentation calls this hook when it - // detects a corrupted/invalid block, not the emulator's allocator, so this is purely - // a diagnostic sink: log what was reported and return success so the caller continues. - [SysAbiExport( - Nid = "al3JzFI9MQ0", - ExportName = "sceLibcInternalHeapErrorReportForGame", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceLibcInternal")] - public static int LibcInternalHeapErrorReportForGame(CpuContext ctx) - { - Console.Error.WriteLine( - $"[LOADER][WARN] sceLibcInternalHeapErrorReportForGame: rdi=0x{ctx[CpuRegister.Rdi]:X16} " + - $"rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} rcx=0x{ctx[CpuRegister.Rcx]:X16}"); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "DfivPArhucg", ExportName = "memcmp", @@ -1563,214 +1357,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "ob5xAW4ln-0", - ExportName = "strchr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strchr(CpuContext ctx) - { - var address = ctx[CpuRegister.Rdi]; - var needle = unchecked((byte)ctx[CpuRegister.Rsi]); - if (address == 0) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - // The terminator counts as part of the scanned range, so strchr(s, '\0') - // returns a pointer to the string's null byte just like a native libc. - Span current = stackalloc byte[1]; - for (ulong index = 0; index < 1_048_576; index++) - { - if (!TryReadCompat(ctx, address + index, current)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (current[0] == needle) - { - ctx[CpuRegister.Rax] = address + index; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - if (current[0] == 0) - { - break; - } - } - - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "9yDWMxEFdJU", - ExportName = "strrchr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strrchr(CpuContext ctx) - { - var address = ctx[CpuRegister.Rdi]; - var needle = unchecked((byte)ctx[CpuRegister.Rsi]); - if (address == 0) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ulong match = 0; - var found = false; - Span current = stackalloc byte[1]; - for (ulong index = 0; index < 1_048_576; index++) - { - if (!TryReadCompat(ctx, address + index, current)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (current[0] == needle) - { - match = address + index; - found = true; - } - - if (current[0] == 0) - { - break; - } - } - - ctx[CpuRegister.Rax] = found ? match : 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "8u8lPzUEq+U", - ExportName = "memchr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Memchr(CpuContext ctx) - { - var address = ctx[CpuRegister.Rdi]; - var needle = unchecked((byte)ctx[CpuRegister.Rsi]); - var count = ctx[CpuRegister.Rdx]; - - Span current = stackalloc byte[1]; - for (ulong index = 0; index < count; index++) - { - if (!TryReadCompat(ctx, address + index, current)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (current[0] == needle) - { - ctx[CpuRegister.Rax] = address + index; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - } - - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "Ls4tzzhimqQ", - ExportName = "strcat", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strcat(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var source = ctx[CpuRegister.Rsi]; - if (!TryReadCString(ctx, source, 1_048_576, out var sourceBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - // Overwrite the destination terminator and re-terminate after the copied bytes. - var appendAddress = destination + (ulong)destinationBytes.Length; - var payload = new byte[sourceBytes.Length + 1]; - sourceBytes.CopyTo(payload.AsSpan()); - if (!TryWriteCompat(ctx, appendAddress, payload)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = destination; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "kHg45qPC6f0", - ExportName = "strncat", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strncat(CpuContext ctx) - { - var destination = ctx[CpuRegister.Rdi]; - var source = ctx[CpuRegister.Rsi]; - var limit = ctx[CpuRegister.Rdx]; - - // Bounding the source read by the count yields strncat's "at most n bytes" - // semantics while still stopping early at the source terminator. - if (!TryReadCString(ctx, source, limit, out var sourceBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var appendAddress = destination + (ulong)destinationBytes.Length; - var payload = new byte[sourceBytes.Length + 1]; - sourceBytes.CopyTo(payload.AsSpan()); - if (!TryWriteCompat(ctx, appendAddress, payload)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - ctx[CpuRegister.Rax] = destination; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - [SysAbiExport( - Nid = "viiwFMaNamA", - ExportName = "strstr", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libc")] - public static int Strstr(CpuContext ctx) - { - var haystack = ctx[CpuRegister.Rdi]; - var needle = ctx[CpuRegister.Rsi]; - if (!TryReadCString(ctx, haystack, 1_048_576, out var haystackBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (!TryReadCString(ctx, needle, 1_048_576, out var needleBytes)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - // An empty needle matches at the start of the haystack. - if (needleBytes.Length == 0) - { - ctx[CpuRegister.Rax] = haystack; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - var matchIndex = haystackBytes.AsSpan().IndexOf(needleBytes.AsSpan()); - ctx[CpuRegister.Rax] = matchIndex >= 0 ? haystack + (ulong)matchIndex : 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "QrZZdJ8XsX0", ExportName = "fputs", @@ -1828,6 +1414,17 @@ public static class KernelMemoryCompatExports var mode = ResolveOpenMode(flags, access); try { + if (Bink2MovieBridge.ShouldSkipGuestMovie(hostPath)) + { + LogOpenTrace( + "_open bink-skip path='" + guestPath + "' host='" + hostPath + + "' flags=0x" + flags.ToString("X8")); + Console.Error.WriteLine( + "[LOADER][INFO] Skipping Bink movie without a decoder: " + + Path.GetFileName(hostPath)); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + if (IsMutatingOpen(flags) && IsReadOnlyGuestMutationPath(guestPath)) { LogOpenTrace($"_open readonly path='{guestPath}' host='{hostPath}' flags=0x{flags:X8}"); @@ -1849,10 +1446,9 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - int directoryFd; + var directoryFd = (int)Interlocked.Increment(ref _nextFileDescriptor); lock (_fdGate) { - directoryFd = AllocateGuestFileDescriptor(); _openDirectories[directoryFd] = new OpenDirectory { Path = hostPath, @@ -1873,13 +1469,18 @@ public static class KernelMemoryCompatExports stream.Seek(0, SeekOrigin.End); } - int fd; + var fd = (int)Interlocked.Increment(ref _nextFileDescriptor); lock (_fdGate) { - fd = AllocateGuestFileDescriptor(); _openFiles[fd] = stream; } + // Bink is linked directly into some games, so there is no media + // import for the HLE codec layer to intercept. The successful + // guest file open is the stable boundary at which the optional + // host Bink bridge can attach to the same movie. + Bink2MovieBridge.ObserveGuestMovie(hostPath); + if (IsMutatingOpen(flags)) { InvalidateNegativeStatCacheForPathAndAncestors(guestPath); @@ -1899,45 +1500,6 @@ public static class KernelMemoryCompatExports } } - [SysAbiExport( - Nid = "fgIsQ10xYVA", - ExportName = "sceKernelChmod", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int KernelChmod(CpuContext ctx) - { - var pathAddress = ctx[CpuRegister.Rdi]; - var mode = unchecked((uint)ctx[CpuRegister.Rsi]); - if (pathAddress == 0) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; - } - - if (!TryReadNullTerminatedUtf8(ctx, pathAddress, MaxGuestStringLength, out var guestPath)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - var hostPath = ResolveGuestPath(guestPath); - if (IsReadOnlyGuestMutationPath(guestPath)) - { - LogOpenTrace($"chmod readonly path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; - } - - if (!File.Exists(hostPath) && !Directory.Exists(hostPath)) - { - AddNegativeStatCacheForGuestPath(guestPath); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - } - - // POSIX permission bits have no host equivalent on Windows; accept the call - // so guests that chmod their freshly created files/directories can proceed. - LogOpenTrace($"chmod path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "NNtFaKJbPt0", ExportName = "_close", @@ -2011,25 +1573,8 @@ public static class KernelMemoryCompatExports Nid = "E6ao34wPw+U", ExportName = "stat", Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixStat(CpuContext ctx) - { - var result = KernelStat(ctx); - if (result == (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return 0; - } - - var errno = result switch - { - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => Einval, - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => Efault, - _ => 2, - }; - KernelRuntimeCompatExports.TrySetErrno(ctx, errno); - ctx[CpuRegister.Rax] = ulong.MaxValue; - return -1; - } + LibraryName = "libc")] + public static int PosixStat(CpuContext ctx) => KernelStat(ctx); [SysAbiExport( Nid = "gEpBkcwxUjw", @@ -2048,15 +1593,15 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var entryCount = (int)count; - Span localIds = count <= 256 ? stackalloc uint[entryCount] : new uint[entryCount]; - Span localSizes = count <= 128 ? stackalloc ulong[entryCount] : new ulong[entryCount]; - var resolvedGuestPaths = new string[entryCount]; - var resolvedHostPaths = new string[entryCount]; - for (ulong i = 0; i < count; i++) { - var index = (int)i; + if (idsAddress != 0 && + !TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), uint.MaxValue)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + if (!TryResolveAprFilepath(ctx, pathListAddress, i, out var guestPath)) { KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); @@ -2066,52 +1611,137 @@ public static class KernelMemoryCompatExports var hostPath = ResolveGuestPath(guestPath); if (!TryGetAprFileSize(hostPath, out var fileSize)) { + // Per-file resolve: a missing entry gets an invalid id + // (0xFFFFFFFF, already written above) and size 0, and the batch + // CONTINUES. Aborting the whole batch on the first miss left the + // remaining paths unresolved and could stall the guest's asset + // streaming when a batch happens to include an absent (e.g. + // patch/DLC) file; the caller checks per-file id/size. LogIoTrace("apr_resolve", guestPath, $"host='{hostPath}' index={i} count={count} result=not_found"); - KernelRuntimeCompatExports.TrySetErrno(ctx, 2); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + if (sizesAddress != 0 && + !TryWriteUInt64Compat(ctx, sizesAddress + (i * sizeof(ulong)), 0)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + continue; } - var fileId = AmprFileRegistry.ComputeFileId(guestPath); + var fileId = AmprFileRegistry.Register(guestPath, hostPath); LogIoTrace("apr_resolve", guestPath, $"host='{hostPath}' index={i} count={count} id=0x{fileId:X8} size={fileSize}"); - localIds[index] = fileId; - localSizes[index] = fileSize; - resolvedGuestPaths[index] = guestPath; - resolvedHostPaths[index] = hostPath; - } - - Span sizePayload = count <= 64 ? stackalloc byte[entryCount * sizeof(ulong)] : new byte[entryCount * sizeof(ulong)]; - for (ulong i = 0; i < count; i++) - { - BinaryPrimitives.WriteUInt64LittleEndian(sizePayload[(int)(i * sizeof(ulong))..], localSizes[(int)i]); - } - - if (!TryWriteCompat(ctx, sizesAddress, sizePayload)) - { - KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - if (idsAddress != 0) - { - Span idPayload = count <= 128 ? stackalloc byte[entryCount * sizeof(uint)] : new byte[entryCount * sizeof(uint)]; - for (ulong i = 0; i < count; i++) + if (idsAddress != 0 && + !TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), fileId)) { - BinaryPrimitives.WriteUInt32LittleEndian(idPayload[(int)(i * sizeof(uint))..], localIds[(int)i]); + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (!TryWriteCompat(ctx, idsAddress, idPayload)) + if (!TryWriteUInt64Compat(ctx, sizesAddress + (i * sizeof(ulong)), fileSize)) { KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } } - for (var i = 0; i < entryCount; i++) + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // The IDs-only sibling of sceKernelAprResolveFilepathsToIdsAndFileSizes. + // Games that stream via AMPR APR call this to turn asset paths into file + // IDs, then hand those IDs to sceAmprAprCommandBufferReadFile. Without it + // the paths never register in AmprFileRegistry, so every subsequent + // ReadFile fails with NOT_FOUND and the streaming pipeline stalls forever. + // Signature: (const char* const* paths, size_t count, uint32_t* ids). + [SysAbiExport( + Nid = "WT-5NKy42fw", + ExportName = "sceKernelAprResolveFilepathsToIds", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelAprResolveFilepathsToIds(CpuContext ctx) + { + var pathListAddress = ctx[CpuRegister.Rdi]; + var count = ctx[CpuRegister.Rsi]; + var idsAddress = ctx[CpuRegister.Rdx]; + if (pathListAddress == 0 || count == 0 || idsAddress == 0 || count > 1024) { - AmprFileRegistry.Register(resolvedGuestPaths[i], resolvedHostPaths[i]); + KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + for (ulong i = 0; i < count; i++) + { + if (!TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), uint.MaxValue)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryResolveAprFilepath(ctx, pathListAddress, i, out var guestPath)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var hostPath = ResolveGuestPath(guestPath); + if (!TryGetAprFileSize(hostPath, out _)) + { + LogIoTrace("apr_resolve_ids", guestPath, $"host='{hostPath}' index={i} count={count} result=not_found"); + KernelRuntimeCompatExports.TrySetErrno(ctx, 2); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + var fileId = AmprFileRegistry.Register(guestPath, hostPath); + LogIoTrace("apr_resolve_ids", guestPath, $"host='{hostPath}' index={i} count={count} id=0x{fileId:X8}"); + + if (!TryWriteUInt32Compat(ctx, idsAddress + (i * sizeof(uint)), fileId)) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // Stat an AMPR APR file by the id returned from sceKernelAprResolveFilepathsToIds. + // Games call this after resolving ids to learn each asset's size before + // issuing the streaming read. When it is missing the guest gets no size back + // and dereferences a null result pointer (observed SIGSEGV write to 0x0 in + // Void Terrarium). Signature: (SceKernelAprFileId id, SceKernelStat* stat). + [SysAbiExport( + Nid = "ApkYaHb8Sek", + ExportName = "sceKernelAprGetFileStat", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelAprGetFileStat(CpuContext ctx) + { + var fileId = unchecked((uint)ctx[CpuRegister.Rdi]); + var statAddress = ctx[CpuRegister.Rsi]; + if (statAddress == 0) + { + KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!AmprFileRegistry.TryGetHostPath(fileId, out var hostPath)) + { + LogIoTrace("apr_get_file_stat", $"id=0x{fileId:X8}", "result=id_not_registered"); + KernelRuntimeCompatExports.TrySetErrno(ctx, 2); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!TryWriteHostPathStat(ctx, statAddress, hostPath)) + { + LogIoTrace("apr_get_file_stat", hostPath, $"id=0x{fileId:X8} result=not_found"); + KernelRuntimeCompatExports.TrySetErrno(ctx, 2); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + LogIoTrace("apr_get_file_stat", hostPath, $"id=0x{fileId:X8}"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -2312,12 +1942,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (KernelSocketCompatExports.TryCloseSocketFd(fd)) - { - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - FileStream? stream; lock (_fdGate) { @@ -2361,17 +1985,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (KernelSocketCompatExports.TryReadSocketFd(ctx, fd, bufferAddress, requested, out var socketBytesRead, out var socketError)) - { - if (socketError != OrbisGen2Result.ORBIS_GEN2_OK) - { - return (int)socketError; - } - - ctx[CpuRegister.Rax] = socketBytesRead; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - FileStream? stream; lock (_fdGate) { @@ -2601,17 +2214,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (KernelSocketCompatExports.TryWriteSocketFd(ctx, fd, payload, out var socketError)) - { - if (socketError != OrbisGen2Result.ORBIS_GEN2_OK) - { - return (int)socketError; - } - - ctx[CpuRegister.Rax] = unchecked((ulong)requested); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - FileStream? stream; lock (_fdGate) { @@ -2650,37 +2252,23 @@ public static class KernelMemoryCompatExports LibraryName = "libKernel")] public static int ClockGettime(CpuContext ctx) { - var clockId = unchecked((int)ctx[CpuRegister.Rdi]); var timespecAddress = ctx[CpuRegister.Rsi]; if (timespecAddress == 0) { - KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); - ctx[CpuRegister.Rax] = unchecked((ulong)-1); - return -1; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - long seconds; - long nanoseconds; - if (!KernelRuntimeCompatExports.ResolveClockTime(clockId, out seconds, out nanoseconds)) + var now = DateTimeOffset.UtcNow; + var seconds = now.ToUnixTimeSeconds(); + var nanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; + if (!ctx.TryWriteUInt64(timespecAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timespecAddress + sizeof(long), unchecked((ulong)nanoseconds))) { - KernelRuntimeCompatExports.TrySetErrno(ctx, Einval); - ctx[CpuRegister.Rax] = unchecked((ulong)-1); - return -1; - } - - Span timespecBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer[sizeof(long)..], nanoseconds); - - if (!ctx.Memory.TryWrite(timespecAddress, timespecBuffer)) - { - KernelRuntimeCompatExports.TrySetErrno(ctx, Efault); - ctx[CpuRegister.Rax] = unchecked((ulong)-1); - return -1; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } ctx[CpuRegister.Rax] = 0; - return 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } [SysAbiExport( @@ -2708,31 +2296,7 @@ public static class KernelMemoryCompatExports private static ulong ResolveTlsAddress(CpuContext ctx, ulong moduleId, ulong offset) { - if (ctx.FsBase == 0) - { - return 0; - } - - if (moduleId <= 1) - { - return unchecked(ctx.FsBase + offset); - } - - var key = (ctx.FsBase << 16) ^ (moduleId & 0xFFFFUL); - ulong moduleBase; - lock (_tlsGate) - { - if (!_tlsModuleBlocks.TryGetValue(key, out moduleBase)) - { - var block = Marshal.AllocHGlobal(TlsModuleBlockSize); - Marshal.Copy(new byte[TlsModuleBlockSize], 0, block, TlsModuleBlockSize); - - moduleBase = unchecked((ulong)block); - _tlsModuleBlocks[key] = moduleBase; - } - } - - return unchecked(moduleBase + offset); + return SharpEmu.HLE.GuestTlsTemplate.ResolveAddress(ctx, moduleId, offset); } [SysAbiExport( @@ -2768,6 +2332,31 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + /// + /// Invokes the registered runtime thread-destructor callback (set via + /// _sceKernelSetThreadDtors) on the exiting guest thread. C++ + /// runtimes register this to flush per-thread cleanup that would + /// otherwise leak. + /// + public static void RunThreadDtors(CpuContext ctx) + { + var callback = _threadDtorsCallback; + if (callback == 0) + { + return; + } + + _ = GuestThreadExecution.Scheduler?.TryCallGuestFunction( + ctx, + callback, + 0, + 0, + 0, + 0, + "kernel_thread_dtors", + out _); + } + [SysAbiExport( Nid = "Tz4RNUCBbGI", ExportName = "_sceKernelRtldThreadAtexitIncrement", @@ -2960,7 +2549,10 @@ public static class KernelMemoryCompatExports searchStart = 0; } - var align = alignment == 0 ? 0x1000UL : alignment; + // PS5 direct memory is allocated in 16 KiB pages; when the guest does + // not care about alignment, default to that granularity rather than the + // host 4 KiB page so physical offsets stay on true page boundaries. + var align = alignment == 0 ? OrbisPageSize : alignment; ulong selectedAddress; lock (_memoryGate) { @@ -3029,7 +2621,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; ulong aligned; lock (_memoryGate) { @@ -3108,20 +2700,21 @@ public static class KernelMemoryCompatExports { var start = ctx[CpuRegister.Rdi]; var length = ctx[CpuRegister.Rsi]; - if (length == 0) + if (!IsAligned(start, OrbisPageSize) || !IsAligned(length, OrbisPageSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + if (length == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + lock (_memoryGate) { - if (!_directAllocations.TryGetValue(start, out var allocation) || allocation.Length != length) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; - } - - _directAllocations.Remove(start); - _nextPhysicalAddress = GetDirectMemoryHighWaterMarkLocked(); + // The unchecked API ignores an unallocated range, matching the + // kernel contract used by guest pool allocators during teardown. + _ = TryReleaseDirectMemoryRangeLocked(start, length); } return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -3137,21 +2730,22 @@ public static class KernelMemoryCompatExports var start = ctx[CpuRegister.Rdi]; var length = ctx[CpuRegister.Rsi]; - if (length == 0) + if (!IsAligned(start, OrbisPageSize) || !IsAligned(length, OrbisPageSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + if (length == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + lock (_memoryGate) { - if (!_directAllocations.TryGetValue(start, out var allocation) || - allocation.Length != length) + if (!TryReleaseDirectMemoryRangeLocked(start, length)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - - _directAllocations.Remove(start); - _nextPhysicalAddress = GetDirectMemoryHighWaterMarkLocked(); } return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -3188,7 +2782,7 @@ public static class KernelMemoryCompatExports ulong mappedAddress; lock (_memoryGate) { - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; var fixedMapping = (flags & 0x10UL) != 0; var desiredAddress = requestedAddress != 0 ? requestedAddress @@ -3209,7 +2803,34 @@ public static class KernelMemoryCompatExports if (!reserved) { - mappedAddress = 0; + // Rosetta places the x86-64 process stack in the low + // address window used by some fixed PS5 mappings. Do not + // clobber that host memory: relocate the mapping and + // return the actual address through the in/out pointer. + if (OperatingSystem.IsMacOS()) + { + var fallbackAddress = AlignUp( + _nextVirtualAddress == 0 ? DefaultMapSearchBase : _nextVirtualAddress, + effectiveAlignment); + reserved = TryReserveGuestVirtualRange( + ctx, + fallbackAddress, + length, + protection, + effectiveAlignment, + out mappedAddress); + + if (reserved && ShouldTraceDirectMemory()) + { + Console.Error.WriteLine( + $"[LOADER][WARN] map_direct relocated fixed mapping: requested=0x{requestedAddress:X16} mapped=0x{mappedAddress:X16} len=0x{length:X16}"); + } + } + + if (!reserved) + { + mappedAddress = 0; + } } } else @@ -3389,6 +3010,31 @@ public static class KernelMemoryCompatExports return KernelBatchMapCore(ctx, unchecked((int)ctx[CpuRegister.Rcx])); } + [SysAbiExport( + Nid = "yDBwVAolDgg", + ExportName = "sceKernelIsStack", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelIsStack(CpuContext ctx) + { + _ = ctx[CpuRegister.Rdi]; + var startAddress = ctx[CpuRegister.Rsi]; + var endAddress = ctx[CpuRegister.Rdx]; + + // The queried ranges used by libc's VM allocator are ordinary heap + // mappings. The kernel still initializes both outputs when a mapping + // is not a pthread stack; leaving them untouched makes libc consume + // stale stack values and issue invalid fixed-range reservations. + if ((startAddress != 0 && !ctx.TryWriteUInt64(startAddress, 0)) || + (endAddress != 0 && !ctx.TryWriteUInt64(endAddress, 0))) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "cQke9UuBQOk", ExportName = "sceKernelMunmap", @@ -3398,27 +3044,45 @@ public static class KernelMemoryCompatExports { var address = ctx[CpuRegister.Rdi]; var length = ctx[CpuRegister.Rsi]; - if (address == 0 || length == 0) + if (address == 0 || length == 0 || ulong.MaxValue - address < length) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } + var rangeEnd = address + length; + var physicallyBacked = IsGuestRangeBacked(ctx, address, length); + var removedAny = false; lock (_memoryGate) { - if (!_mappedRegions.TryGetValue(address, out var mappedRegion) || mappedRegion.Length != length) + var removedRegions = _mappedRegions.Values + .Where(region => + region.Address >= address && + region.Address < rangeEnd && + region.Length <= rangeEnd - region.Address) + .ToArray(); + + if (removedRegions.Length == 0 && !physicallyBacked) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - _mappedRegions.Remove(address); - if (mappedRegion.IsFlexible) + foreach (var mappedRegion in removedRegions) { - _allocatedFlexibleBytes = mappedRegion.Length >= _allocatedFlexibleBytes - ? 0 - : _allocatedFlexibleBytes - mappedRegion.Length; + removedAny |= _mappedRegions.Remove(mappedRegion.Address); + if (mappedRegion.IsFlexible) + { + _allocatedFlexibleBytes = mappedRegion.Length >= _allocatedFlexibleBytes + ? 0 + : _allocatedFlexibleBytes - mappedRegion.Length; + } } } + if (physicallyBacked || removedAny) + { + KernelRuntimeCompatExports.RegisterReleasedVirtualRange(address, length); + } + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -3574,7 +3238,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (protectionOut != 0 && !ctx.TryWriteInt32(protectionOut, region.Protection)) + if (protectionOut != 0 && !TryWriteInt32(ctx, protectionOut, region.Protection)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -3613,7 +3277,7 @@ public static class KernelMemoryCompatExports if (!ctx.TryWriteUInt64(infoAddress, block.Start) || !ctx.TryWriteUInt64(infoAddress + sizeof(ulong), block.Start + block.Length) || - !ctx.TryWriteInt32(infoAddress + (sizeof(ulong) * 2), block.MemoryType)) + !TryWriteInt32(ctx, infoAddress + (sizeof(ulong) * 2), block.MemoryType)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -3645,7 +3309,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryProtectHostRange(ctx, alignedAddress, alignedLength, protection)) + if (!TryProtectHostRange(alignedAddress, alignedLength, protection)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } @@ -3679,7 +3343,7 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryProtectHostRange(ctx, alignedAddress, alignedLength, protection)) + if (!TryProtectHostRange(alignedAddress, alignedLength, protection)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } @@ -3755,7 +3419,6 @@ public static class KernelMemoryCompatExports } var rendered = FormatString(ctx, format, ref vaCursor); - return WriteSnprintfOutput(ctx, destination, bufferSize, rendered); } @@ -3828,7 +3491,11 @@ public static class KernelMemoryCompatExports return true; } - internal static bool TryFormatStringFromVaList(CpuContext ctx, string format, ulong vaListAddress, out string rendered) + internal static bool TryFormatStringFromVaList( + CpuContext ctx, + string format, + ulong vaListAddress, + out string rendered) { if (!TryCreateVaListCursor(ctx, vaListAddress, out var vaCursor)) { @@ -4181,19 +3848,48 @@ public static class KernelMemoryCompatExports return FormatStringFromVarArgs(ctx, format, firstGpArgIndex: 3); } + [ThreadStatic] + private static StringBuilder? _formatBuilder; + + // printf length modifier collapsed to the widths our conversions care about. + // Kept as an enum rather than a per-argument substring so the hot format + // loop avoids a string allocation and string comparisons on every spec. + private enum PrintfLength + { + None, + Char, // hh + Short, // h + Long, // l / ll / j / z / t / L + } + private static string FormatString( CpuContext ctx, string format, ref TArgumentSource argumentSource) where TArgumentSource : struct, IPrintfArgumentSource { - var sb = new StringBuilder(format.Length + 32); + // printf formatting is one of the hottest HLE paths during loading, so + // reuse a per-thread builder and append literal runs as spans instead of + // allocating a builder and appending char-by-char on every call. + var sb = _formatBuilder ??= new StringBuilder(256); + sb.Clear(); + if (sb.Capacity < format.Length + 32) + { + sb.EnsureCapacity(format.Length + 32); + } for (var i = 0; i < format.Length; i++) { if (format[i] != '%') { - sb.Append(format[i]); + var literalStart = i; + while (i < format.Length && format[i] != '%') + { + i++; + } + + sb.Append(format.AsSpan(literalStart, i - literalStart)); + i--; continue; } @@ -4267,19 +3963,27 @@ public static class KernelMemoryCompatExports } } - var lengthMod = ""; + var lengthMod = PrintfLength.None; if (i < format.Length) { - if (i + 1 < format.Length && - ((format[i] == 'h' && format[i + 1] == 'h') || - (format[i] == 'l' && format[i + 1] == 'l'))) + if (i + 1 < format.Length && format[i] == 'h' && format[i + 1] == 'h') { - lengthMod = format.Substring(i, 2); + lengthMod = PrintfLength.Char; i += 2; } - else if (format[i] is 'h' or 'l' or 'j' or 'z' or 't' or 'L') + else if (i + 1 < format.Length && format[i] == 'l' && format[i + 1] == 'l') { - lengthMod = format[i].ToString(); + lengthMod = PrintfLength.Long; + i += 2; + } + else if (format[i] == 'h') + { + lengthMod = PrintfLength.Short; + i++; + } + else if (format[i] is 'l' or 'j' or 'z' or 't' or 'L') + { + lengthMod = PrintfLength.Long; i++; } } @@ -4303,17 +4007,13 @@ public static class KernelMemoryCompatExports { long value = lengthMod switch { - "hh" => unchecked((sbyte)argumentSource.NextGpArg()), - "h" => unchecked((short)argumentSource.NextGpArg()), - "l" => unchecked((long)argumentSource.NextGpArg()), - "ll" => unchecked((long)argumentSource.NextGpArg()), - "j" => unchecked((long)argumentSource.NextGpArg()), - "z" => unchecked((long)argumentSource.NextGpArg()), - "t" => unchecked((long)argumentSource.NextGpArg()), + PrintfLength.Char => unchecked((sbyte)argumentSource.NextGpArg()), + PrintfLength.Short => unchecked((short)argumentSource.NextGpArg()), + PrintfLength.Long => unchecked((long)argumentSource.NextGpArg()), _ => unchecked((int)argumentSource.NextGpArg()) }; - var formatted = value.ToString(CultureInfo.InvariantCulture); + var formatted = value.ToString(); if (showSign && value >= 0) formatted = "+" + formatted; else if (spaceForSign && value >= 0) @@ -4327,17 +4027,13 @@ public static class KernelMemoryCompatExports { ulong value = lengthMod switch { - "hh" => (byte)argumentSource.NextGpArg(), - "h" => (ushort)argumentSource.NextGpArg(), - "l" => argumentSource.NextGpArg(), - "ll" => argumentSource.NextGpArg(), - "j" => argumentSource.NextGpArg(), - "z" => argumentSource.NextGpArg(), - "t" => argumentSource.NextGpArg(), + PrintfLength.Char => (byte)argumentSource.NextGpArg(), + PrintfLength.Short => (ushort)argumentSource.NextGpArg(), + PrintfLength.Long => argumentSource.NextGpArg(), _ => (uint)argumentSource.NextGpArg() }; - var formatted = value.ToString(CultureInfo.InvariantCulture); + var formatted = value.ToString(); sb.Append(PadString(formatted, width, leftAlign, padWithZero && !leftAlign)); } break; @@ -4347,19 +4043,15 @@ public static class KernelMemoryCompatExports { ulong value = lengthMod switch { - "hh" => (byte)argumentSource.NextGpArg(), - "h" => (ushort)argumentSource.NextGpArg(), - "l" => argumentSource.NextGpArg(), - "ll" => argumentSource.NextGpArg(), - "j" => argumentSource.NextGpArg(), - "z" => argumentSource.NextGpArg(), - "t" => argumentSource.NextGpArg(), + PrintfLength.Char => (byte)argumentSource.NextGpArg(), + PrintfLength.Short => (ushort)argumentSource.NextGpArg(), + PrintfLength.Long => argumentSource.NextGpArg(), _ => (uint)argumentSource.NextGpArg() }; var formatted = specifier == 'x' - ? value.ToString("x", CultureInfo.InvariantCulture) - : value.ToString("X", CultureInfo.InvariantCulture); + ? value.ToString("x") + : value.ToString("X"); if (alternateForm && value != 0) formatted = specifier == 'x' ? "0x" + formatted : "0X" + formatted; @@ -4372,13 +4064,9 @@ public static class KernelMemoryCompatExports { ulong value = lengthMod switch { - "hh" => (byte)argumentSource.NextGpArg(), - "h" => (ushort)argumentSource.NextGpArg(), - "l" => argumentSource.NextGpArg(), - "ll" => argumentSource.NextGpArg(), - "j" => argumentSource.NextGpArg(), - "z" => argumentSource.NextGpArg(), - "t" => argumentSource.NextGpArg(), + PrintfLength.Char => (byte)argumentSource.NextGpArg(), + PrintfLength.Short => (ushort)argumentSource.NextGpArg(), + PrintfLength.Long => argumentSource.NextGpArg(), _ => (uint)argumentSource.NextGpArg() }; @@ -4403,12 +4091,12 @@ public static class KernelMemoryCompatExports case 's': { var strAddr = argumentSource.NextGpArg(); - TracePrintfStringArgument(ctx, lengthMod, strAddr); + TracePrintfStringArgument(ctx, lengthMod == PrintfLength.Long ? "l" : "", strAddr); if (strAddr == 0) { sb.Append("(null)"); } - else if (lengthMod == "l") + else if (lengthMod == PrintfLength.Long) { if (TryReadWideCString(ctx, strAddr, 1_048_576, out var wideUnits)) { @@ -4439,7 +4127,7 @@ public static class KernelMemoryCompatExports case 'c': { string renderedChar; - if (lengthMod == "l") + if (lengthMod == PrintfLength.Long) { var scalar = unchecked((ushort)argumentSource.NextGpArg()); renderedChar = TryConvertWideScalarToString(scalar, out var wideCharText) @@ -4467,10 +4155,7 @@ public static class KernelMemoryCompatExports var formatStr = precision >= 0 ? $"{{0:{specifier}{precision}}}" : $"{{0:{specifier}}}"; - var formatted = string.Format( - CultureInfo.InvariantCulture, - formatStr, - value); + var formatted = string.Format(formatStr, value); if (showSign && value >= 0) formatted = "+" + formatted; @@ -4486,7 +4171,7 @@ public static class KernelMemoryCompatExports var addr = argumentSource.NextGpArg(); if (addr != 0) { - _ = ctx.TryWriteInt32(addr, sb.Length); + _ = TryWriteInt32(ctx, addr, sb.Length); } } break; @@ -4532,59 +4217,33 @@ public static class KernelMemoryCompatExports private struct RegisterPrintfArgumentSource : IPrintfArgumentSource { - // SysV AMD64: variadic integer/pointer args use the GP registers (rdi..r9, up to - // 6) and variadic float/double args use xmm0..xmm7 (8) — INDEPENDENT counters. - // Args beyond the registers spill to the stack in source order, so GP-overflow - // and FP-overflow share one stack cursor. private readonly CpuContext _ctx; private int _gpIndex; - private int _fpIndex; - private int _stackIndex; public RegisterPrintfArgumentSource(CpuContext ctx, int gpIndex) { _ctx = ctx; _gpIndex = gpIndex; - _fpIndex = 0; - _stackIndex = 0; } public ulong NextGpArg() { - var index = _gpIndex; - if (index < 6) + var index = _gpIndex++; + return index switch { - _gpIndex++; - return index switch - { - 0 => _ctx[CpuRegister.Rdi], - 1 => _ctx[CpuRegister.Rsi], - 2 => _ctx[CpuRegister.Rdx], - 3 => _ctx[CpuRegister.Rcx], - 4 => _ctx[CpuRegister.R8], - _ => _ctx[CpuRegister.R9], - }; - } - - return ReadStackArg(_ctx, (ulong)(_stackIndex++) * 8); + 0 => _ctx[CpuRegister.Rdi], + 1 => _ctx[CpuRegister.Rsi], + 2 => _ctx[CpuRegister.Rdx], + 3 => _ctx[CpuRegister.Rcx], + 4 => _ctx[CpuRegister.R8], + 5 => _ctx[CpuRegister.R9], + _ => ReadStackArg(_ctx, (ulong)(index - 6) * 8) + }; } public double NextFloatArg() { - // Float/double variadic args live in xmm0..xmm7 (low 64 bits), NOT the GP - // registers. Reading them from GP prints garbage and desynchronizes every - // subsequent argument. - ulong bits; - if (_fpIndex < 8) - { - _ctx.GetXmmRegister(_fpIndex++, out bits, out _); - } - else - { - bits = ReadStackArg(_ctx, (ulong)(_stackIndex++) * 8); - } - - return BitConverter.Int64BitsToDouble(unchecked((long)bits)); + return BitConverter.Int64BitsToDouble(unchecked((long)NextGpArg())); } } @@ -4668,7 +4327,7 @@ public static class KernelMemoryCompatExports return 0; } - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; if (_nextVirtualAddress == 0) { _nextVirtualAddress = 0x0100_0000UL; @@ -4741,7 +4400,7 @@ public static class KernelMemoryCompatExports out _); } - private static bool IsGuestRangeBacked(CpuContext ctx, ulong address, ulong length) + internal static bool IsGuestRangeBacked(CpuContext ctx, ulong address, ulong length) { if (address == 0 || length == 0 || ulong.MaxValue - address < length - 1) { @@ -5206,6 +4865,7 @@ public static class KernelMemoryCompatExports writer.Advance(firstReadLength); } + Span one = stackalloc byte[1]; while (offset < (ulong)limit) { var current = address + offset; @@ -5237,7 +4897,6 @@ public static class KernelMemoryCompatExports continue; } - var one = chunk.AsSpan(0, 1); if (!TryReadCompat(ctx, current, one)) { return false; @@ -5290,35 +4949,20 @@ public static class KernelMemoryCompatExports private static bool TryReadWideCString(CpuContext ctx, ulong address, ulong maxLength, out ushort[] units) { - units = Array.Empty(); - if (address == 0) - { - return false; - } - - var limit = (int)Math.Min(maxLength, 1_048_576UL); - var buffer = new List(Math.Min(limit, 256)); - for (var i = 0; i < limit; i++) - { - if (!TryReadUInt16Compat(ctx, address + ((ulong)i * WideCharSize), out var unit)) - { - return false; - } - - if (unit == 0) - { - units = buffer.ToArray(); - return true; - } - - buffer.Add(unit); - } - - units = buffer.ToArray(); - return true; + return TryReadWideCStringCore(ctx, address, maxLength, out units, out _); } private static bool TryReadWideCStringBounded(CpuContext ctx, ulong address, ulong maxLength, out ushort[] units, out bool terminated) + { + return TryReadWideCStringCore(ctx, address, maxLength, out units, out terminated); + } + + private static bool TryReadWideCStringCore( + CpuContext ctx, + ulong address, + ulong maxLength, + out ushort[] units, + out bool terminated) { units = Array.Empty(); terminated = false; @@ -5329,21 +4973,45 @@ public static class KernelMemoryCompatExports var limit = (int)Math.Min(maxLength, 1_048_576UL); var buffer = new List(Math.Min(limit, 256)); - for (var i = 0; i < limit; i++) + const int maxReadBytes = 4096; + var readBuffer = GC.AllocateUninitializedArray( + Math.Min(maxReadBytes, Math.Max(WideCharSize, limit * WideCharSize))); + var index = 0; + while (index < limit) { - if (!TryReadUInt16Compat(ctx, address + ((ulong)i * WideCharSize), out var unit)) + var unitAddress = address + ((ulong)index * WideCharSize); + if (unitAddress < address) { return false; } - if (unit == 0) + var remainingBytes = (limit - index) * WideCharSize; + var pageBytesRemaining = maxReadBytes - + (int)(unitAddress & (maxReadBytes - 1)); + var readBytes = Math.Min( + readBuffer.Length, + Math.Min(pageBytesRemaining, remainingBytes)); + readBytes &= ~(WideCharSize - 1); + if (readBytes == 0 || + !TryReadCompat(ctx, unitAddress, readBuffer.AsSpan(0, readBytes))) { - terminated = true; - units = buffer.ToArray(); - return true; + return false; } - buffer.Add(unit); + for (var offset = 0; offset < readBytes; offset += WideCharSize) + { + var unit = BinaryPrimitives.ReadUInt16LittleEndian( + readBuffer.AsSpan(offset, WideCharSize)); + if (unit == 0) + { + terminated = true; + units = buffer.ToArray(); + return true; + } + + buffer.Add(unit); + index++; + } } units = buffer.ToArray(); @@ -5676,7 +5344,7 @@ public static class KernelMemoryCompatExports processedCount++; } - if (processedOutAddress != 0 && !ctx.TryWriteInt32(processedOutAddress, processedCount)) + if (processedOutAddress != 0 && !TryWriteInt32(ctx, processedOutAddress, processedCount)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -5879,40 +5547,42 @@ public static class KernelMemoryCompatExports return alignedLength != 0; } - private static bool TryProtectHostRange(CpuContext ctx, ulong address, ulong length, int orbisProtection) + private static bool TryProtectHostRange(ulong address, ulong length, int orbisProtection) { if (length == 0 || length > nuint.MaxValue) { return false; } - if (!KernelVirtualRangeAllocator.TryResolveAddressSpace(ctx.Memory, out var addressSpace)) + var hostProtection = ResolveHostProtection(orbisProtection); + if (!VirtualProtect((nint)address, (nuint)length, hostProtection, out _)) { return false; } - return addressSpace.TryProtect(address, length, ResolveGuestProtection(orbisProtection)); + return true; } - private static GuestPageProtection ResolveGuestProtection(int orbisProtection) + private static uint ResolveHostProtection(int orbisProtection) { - var protection = GuestPageProtection.None; - if ((orbisProtection & (OrbisProtCpuRead | OrbisProtGpuRead)) != 0) + var read = (orbisProtection & (OrbisProtCpuRead | OrbisProtGpuRead)) != 0; + var write = (orbisProtection & (OrbisProtCpuWrite | OrbisProtGpuWrite)) != 0; + var execute = (orbisProtection & OrbisProtCpuExec) != 0; + + if (execute) { - protection |= GuestPageProtection.Read; + return write + ? HostPageExecuteReadWrite + : read + ? HostPageExecuteRead + : HostPageExecute; } - if ((orbisProtection & (OrbisProtCpuWrite | OrbisProtGpuWrite)) != 0) - { - protection |= GuestPageProtection.Write; - } - - if ((orbisProtection & OrbisProtCpuExec) != 0) - { - protection |= GuestPageProtection.Execute; - } - - return protection; + return write + ? HostPageReadWrite + : read + ? HostPageReadOnly + : HostPageNoAccess; } private static bool TryFindVirtualQueryRegionLocked(ulong queryAddress, bool findNext, out MappedRegion region) @@ -5975,6 +5645,56 @@ public static class KernelMemoryCompatExports return string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_DIRECT_MEMORY"), "1", StringComparison.Ordinal); } + private static bool TryReleaseDirectMemoryRangeLocked(ulong start, ulong length) + { + if (!TryAddU64(start, length, out var releaseEnd)) + { + return false; + } + + DirectAllocation? owner = null; + ulong ownerEnd = 0; + foreach (var allocation in _directAllocations.Values) + { + if (TryAddU64(allocation.Start, allocation.Length, out var allocationEnd) && + start >= allocation.Start && + releaseEnd <= allocationEnd) + { + owner = allocation; + ownerEnd = allocationEnd; + break; + } + } + + if (owner is not { } releasedFrom) + { + return false; + } + + _directAllocations.Remove(releasedFrom.Start); + if (start > releasedFrom.Start) + { + _directAllocations[releasedFrom.Start] = releasedFrom with + { + Length = start - releasedFrom.Start, + }; + } + + if (releaseEnd < ownerEnd) + { + _directAllocations[releaseEnd] = new DirectAllocation( + releaseEnd, + ownerEnd - releaseEnd, + releasedFrom.MemoryType); + } + + _nextPhysicalAddress = GetDirectMemoryHighWaterMarkLocked(); + return true; + } + + private static bool IsAligned(ulong value, ulong alignment) => + alignment != 0 && value % alignment == 0; + private static bool TryAllocateDirectMemoryLocked( ulong searchStart, ulong searchEnd, @@ -5990,7 +5710,7 @@ public static class KernelMemoryCompatExports return false; } - var effectiveAlignment = alignment == 0 ? 0x1000UL : alignment; + var effectiveAlignment = alignment == 0 ? OrbisPageSize : alignment; if (!TryFindAllocatableDirectMemoryRangeLocked(searchStart, searchEnd, length, effectiveAlignment, allocationLimit, out var freePosition) || !TryAddU64(freePosition, length, out var endAddress)) { @@ -6262,26 +5982,6 @@ public static class KernelMemoryCompatExports alignment = NormalizeLibcAlignment(alignment); var actualSize = requestedSize == 0 ? 1u : requestedSize; - // This intentionally uses host guard pages for allocations made through the libc HLE. - // It turns an overrun that reaches the next page into an immediate access violation at - // the offending guest instruction, rather than allowing it to poison a later import. - var guardHeap = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_GUARD_HEAP"), - "1", - StringComparison.Ordinal); - - if (guardHeap && TryAllocateGuardedLibcHeap(actualSize, alignment, zeroFill, out address)) - { - return true; - } - - // Do not silently fall back to an unguarded allocation when guard heap was explicitly - // requested: a failed setup must remain visible to the caller/reproducer. - if (guardHeap) - { - return false; - } - nuint totalSize; try { @@ -6317,7 +6017,7 @@ public static class KernelMemoryCompatExports var alignedAddress = AlignUp(unchecked((ulong)baseAddress) + (ulong)IntPtr.Size, (ulong)alignment); lock (_libcAllocGate) { - _libcAllocations[alignedAddress] = new LibcHeapAllocation(baseAddress, actualSize, alignment, IsGuarded: false); + _libcAllocations[alignedAddress] = new LibcHeapAllocation(baseAddress, actualSize, alignment); } try @@ -6337,51 +6037,6 @@ public static class KernelMemoryCompatExports return true; } - private static unsafe bool TryAllocateGuardedLibcHeap(nuint actualSize, nuint alignment, bool zeroFill, out ulong address) - { - address = 0; - const nuint pageSize = 0x1000; - - try - { - var effectiveAlignment = Math.Max(alignment, pageSize); - var usableSize = checked((nuint)AlignUp((ulong)actualSize, (ulong)pageSize)); - var reservationSize = checked(pageSize + effectiveAlignment - 1 + usableSize + pageSize); - var baseAddress = unchecked((nint)HostMemory.Allocate(0, reservationSize, HostPageProtection.ReadWrite)); - if (baseAddress == 0) - { - return false; - } - - var alignedAddress = AlignUp(unchecked((ulong)baseAddress) + (ulong)pageSize, (ulong)effectiveAlignment); - var guardAddress = alignedAddress + (ulong)usableSize; - if (!HostMemory.Protect(guardAddress, pageSize, HostPageProtection.NoAccess, out _)) - { - _ = HostMemory.Free(unchecked((ulong)baseAddress)); - return false; - } - - lock (_libcAllocGate) - { - _libcAllocations[alignedAddress] = new LibcHeapAllocation(baseAddress, actualSize, alignment, IsGuarded: true); - } - - if (zeroFill) - { - NativeMemory.Clear((void*)alignedAddress, actualSize); - } - - Console.Error.WriteLine( - $"[LOADER][TRACE] guard-heap alloc: ptr=0x{alignedAddress:X16} size=0x{actualSize:X} guard=0x{guardAddress:X16}"); - address = alignedAddress; - return true; - } - catch (OverflowException) - { - return false; - } - } - private static unsafe bool TryReallocateLibcHeap(ulong existingAddress, ulong requestedSize, out ulong resizedAddress) { resizedAddress = 0; @@ -6490,14 +6145,7 @@ public static class KernelMemoryCompatExports } } - if (allocation.IsGuarded) - { - _ = HostMemory.Free(unchecked((ulong)allocation.BaseAddress)); - } - else - { - Marshal.FreeHGlobal(allocation.BaseAddress); - } + Marshal.FreeHGlobal(allocation.BaseAddress); } private static bool TryMultiplyAllocationSize(ulong left, ulong right, out nuint size) @@ -6588,7 +6236,7 @@ public static class KernelMemoryCompatExports return false; } - if (!TryQueryHostPage(address, out var startInfo) || !HasRequiredProtection(startInfo.RawProtection, writeAccess)) + if (!TryQueryHostPage(address, out var startInfo) || !HasRequiredProtection(startInfo.Protect, writeAccess)) { return false; } @@ -6599,7 +6247,7 @@ public static class KernelMemoryCompatExports return true; } - if (!TryQueryHostPage(endAddress, out var endInfo) || !HasRequiredProtection(endInfo.RawProtection, writeAccess)) + if (!TryQueryHostPage(endAddress, out var endInfo) || !HasRequiredProtection(endInfo.Protect, writeAccess)) { return false; } @@ -6607,14 +6255,16 @@ public static class KernelMemoryCompatExports return true; } - private static bool TryQueryHostPage(ulong address, out HostRegionInfo info) + private static bool TryQueryHostPage(ulong address, out MemoryBasicInformation info) { - if (!HostMemory.Query(address, out info)) + info = default; + var size = (nuint)Marshal.SizeOf(); + if (VirtualQuery((nint)address, out info, size) == 0) { return false; } - return info.State == HostRegionState.Committed; + return info.State == MemCommit; } private static bool HasRequiredProtection(uint protect, bool writeAccess) @@ -6630,6 +6280,13 @@ public static class KernelMemoryCompatExports return (protect & expected) != 0; } + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BitConverter.TryWriteBytes(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + private static bool TryWriteOpenDescriptorStat(CpuContext ctx, int fd, ulong statAddress) { if (fd is 0 or 1 or 2) @@ -6821,19 +6478,19 @@ public static class KernelMemoryCompatExports } var currentIndex = directory.NextIndex; + if (basePointerAddress != 0 && !TryWriteUInt64Compat(ctx, basePointerAddress, (ulong)currentIndex)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + if (currentIndex >= directory.Entries.Length) { - if (basePointerAddress != 0 && - !TryWriteUInt64Compat(ctx, basePointerAddress, (ulong)currentIndex)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } var entryName = directory.Entries[currentIndex]; + directory.NextIndex = currentIndex + 1; var entryBytes = Encoding.UTF8.GetBytes(entryName); var nameLength = Math.Min(entryBytes.Length, 255); @@ -6852,13 +6509,6 @@ public static class KernelMemoryCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (basePointerAddress != 0 && - !TryWriteUInt64Compat(ctx, basePointerAddress, (ulong)currentIndex)) - { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - } - - directory.NextIndex = currentIndex + 1; ctx[CpuRegister.Rax] = 512; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -7108,4 +6758,396 @@ public static class KernelMemoryCompatExports sum = left + right; return sum >= left; } + + [SysAbiExport( + Nid = "AV6ipCNa4Rw", + ExportName = "strcasecmp", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strcasecmp(CpuContext ctx) + { + var left = ctx[CpuRegister.Rdi]; + var right = ctx[CpuRegister.Rsi]; + if (left == 0 || right == 0) + { + var recoveryIndex = Interlocked.Increment(ref _nullStrcasecmpRecoveryCount); + if (recoveryIndex <= 16) + { + var otherAddress = left == 0 ? right : left; + var otherText = otherAddress != 0 && TryReadNullTerminatedUtf8(ctx, otherAddress, 256, out var text) + ? text + : ""; + _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); + Console.Error.WriteLine( + $"[LOADER][WARNING] strcasecmp null-arg recovery#{recoveryIndex}: ret=0x{returnRip:X16} left=0x{left:X16} right=0x{right:X16} other=\"{otherText}\""); + } + + // Real strcasecmp(NULL, x) is undefined behaviour and previously crashed inside the + // LLE-routed implementation. Treat it as "not equal" instead so callers doing + // `if (strcasecmp(a, b) == 0)` degrade gracefully rather than taking down the guest. + ctx[CpuRegister.Rax] = left == right ? 0uL : 1uL; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + if (!TryCompareStringsCaseInsensitive(ctx, left, right, limit: ulong.MaxValue, out var compare)) + { + ctx[CpuRegister.Rax] = 1; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = unchecked((ulong)compare); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + // sprintf/vsprintf are served from the same HLE formatting engine as snprintf/vsnprintf + // instead of falling through to the game's bundled libc. + [SysAbiExport( + Nid = "tcVi5SivF7Q", + ExportName = "sprintf", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Sprintf(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var formatAddress = ctx[CpuRegister.Rsi]; + + if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var format = Encoding.UTF8.GetString(formatBytes); + var rendered = FormatStringFromVarArgs(ctx, format, firstGpArgIndex: 2); + return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); + } + + [SysAbiExport( + Nid = "jbz9I9vkqkk", + ExportName = "vsprintf", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Vsprintf(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var formatAddress = ctx[CpuRegister.Rsi]; + var vaListAddress = ctx[CpuRegister.Rdx]; + + if (!TryReadCString(ctx, formatAddress, 1_048_576, out var formatBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var format = Encoding.UTF8.GetString(formatBytes); + if (!TryCreateVaListCursor(ctx, vaListAddress, out var vaCursor)) + { + return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, formatBytes); + } + + var rendered = FormatString(ctx, format, ref vaCursor); + return WriteSnprintfOutput(ctx, destination, ulong.MaxValue, rendered); + } + + // Was unresolved (returning the 0x80020002 sentinel, then crashing when the guest + // dereferenced it) - the game's own heap instrumentation calls this hook when it + // detects a corrupted/invalid block, not the emulator's allocator, so this is purely + // a diagnostic sink: log what was reported and return success so the caller continues. + [SysAbiExport( + Nid = "al3JzFI9MQ0", + ExportName = "sceLibcInternalHeapErrorReportForGame", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceLibcInternal")] + public static int LibcInternalHeapErrorReportForGame(CpuContext ctx) + { + Console.Error.WriteLine( + $"[LOADER][WARN] sceLibcInternalHeapErrorReportForGame: rdi=0x{ctx[CpuRegister.Rdi]:X16} " + + $"rsi=0x{ctx[CpuRegister.Rsi]:X16} rdx=0x{ctx[CpuRegister.Rdx]:X16} rcx=0x{ctx[CpuRegister.Rcx]:X16}"); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "ob5xAW4ln-0", + ExportName = "strchr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strchr(CpuContext ctx) + { + var address = ctx[CpuRegister.Rdi]; + var needle = unchecked((byte)ctx[CpuRegister.Rsi]); + if (address == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + // The terminator counts as part of the scanned range, so strchr(s, '\0') + // returns a pointer to the string's null byte just like a native libc. + Span current = stackalloc byte[1]; + for (ulong index = 0; index < 1_048_576; index++) + { + if (!TryReadCompat(ctx, address + index, current)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (current[0] == needle) + { + ctx[CpuRegister.Rax] = address + index; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + if (current[0] == 0) + { + break; + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "9yDWMxEFdJU", + ExportName = "strrchr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strrchr(CpuContext ctx) + { + var address = ctx[CpuRegister.Rdi]; + var needle = unchecked((byte)ctx[CpuRegister.Rsi]); + if (address == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ulong match = 0; + var found = false; + Span current = stackalloc byte[1]; + for (ulong index = 0; index < 1_048_576; index++) + { + if (!TryReadCompat(ctx, address + index, current)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (current[0] == needle) + { + match = address + index; + found = true; + } + + if (current[0] == 0) + { + break; + } + } + + ctx[CpuRegister.Rax] = found ? match : 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "8u8lPzUEq+U", + ExportName = "memchr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Memchr(CpuContext ctx) + { + var address = ctx[CpuRegister.Rdi]; + var needle = unchecked((byte)ctx[CpuRegister.Rsi]); + var count = ctx[CpuRegister.Rdx]; + + Span current = stackalloc byte[1]; + for (ulong index = 0; index < count; index++) + { + if (!TryReadCompat(ctx, address + index, current)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (current[0] == needle) + { + ctx[CpuRegister.Rax] = address + index; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "Ls4tzzhimqQ", + ExportName = "strcat", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strcat(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var source = ctx[CpuRegister.Rsi]; + if (!TryReadCString(ctx, source, 1_048_576, out var sourceBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + // Overwrite the destination terminator and re-terminate after the copied bytes. + var appendAddress = destination + (ulong)destinationBytes.Length; + var payload = new byte[sourceBytes.Length + 1]; + sourceBytes.CopyTo(payload.AsSpan()); + if (!TryWriteCompat(ctx, appendAddress, payload)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "kHg45qPC6f0", + ExportName = "strncat", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strncat(CpuContext ctx) + { + var destination = ctx[CpuRegister.Rdi]; + var source = ctx[CpuRegister.Rsi]; + var limit = ctx[CpuRegister.Rdx]; + + // Bounding the source read by the count yields strncat's "at most n bytes" + // semantics while still stopping early at the source terminator. + if (!TryReadCString(ctx, source, limit, out var sourceBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryReadCString(ctx, destination, 1_048_576, out var destinationBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var appendAddress = destination + (ulong)destinationBytes.Length; + var payload = new byte[sourceBytes.Length + 1]; + sourceBytes.CopyTo(payload.AsSpan()); + if (!TryWriteCompat(ctx, appendAddress, payload)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = destination; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "viiwFMaNamA", + ExportName = "strstr", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libc")] + public static int Strstr(CpuContext ctx) + { + var haystack = ctx[CpuRegister.Rdi]; + var needle = ctx[CpuRegister.Rsi]; + if (!TryReadCString(ctx, haystack, 1_048_576, out var haystackBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (!TryReadCString(ctx, needle, 1_048_576, out var needleBytes)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + // An empty needle matches at the start of the haystack. + if (needleBytes.Length == 0) + { + ctx[CpuRegister.Rax] = haystack; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var matchIndex = haystackBytes.AsSpan().IndexOf(needleBytes.AsSpan()); + ctx[CpuRegister.Rax] = matchIndex >= 0 ? haystack + (ulong)matchIndex : 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "fgIsQ10xYVA", + ExportName = "sceKernelChmod", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelChmod(CpuContext ctx) + { + var pathAddress = ctx[CpuRegister.Rdi]; + var mode = unchecked((uint)ctx[CpuRegister.Rsi]); + if (pathAddress == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryReadNullTerminatedUtf8(ctx, pathAddress, MaxGuestStringLength, out var guestPath)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var hostPath = ResolveGuestPath(guestPath); + if (IsReadOnlyGuestMutationPath(guestPath)) + { + LogOpenTrace($"chmod readonly path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + + if (!File.Exists(hostPath) && !Directory.Exists(hostPath)) + { + AddNegativeStatCacheForGuestPath(guestPath); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + // POSIX permission bits have no host equivalent on Windows; accept the call + // so guests that chmod their freshly created files/directories can proceed. + LogOpenTrace($"chmod path='{guestPath}' host='{hostPath}' mode=0x{mode:X}"); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + private static bool TryCompareStringsCaseInsensitive( + CpuContext ctx, + ulong left, + ulong right, + ulong limit, + out int compare) + { + compare = 0; + Span leftBuffer = stackalloc byte[1]; + Span rightBuffer = stackalloc byte[1]; + for (ulong index = 0; index < limit; index++) + { + if (!TryReadCompat(ctx, left + index, leftBuffer) || + !TryReadCompat(ctx, right + index, rightBuffer)) + { + return false; + } + + var leftValue = leftBuffer[0]; + var rightValue = rightBuffer[0]; + var leftLower = ToAsciiLower(leftValue); + var rightLower = ToAsciiLower(rightValue); + if (leftLower != rightLower) + { + compare = leftLower - rightLower; + return true; + } + + if (leftValue == 0) + { + return true; + } + } + + return true; + } + + private static byte ToAsciiLower(byte value) => + value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; } diff --git a/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs b/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs index 1e89323..f4eac33 100644 --- a/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs +++ b/src/SharpEmu.Libs/Kernel/KernelModuleRegistry.cs @@ -9,8 +9,16 @@ namespace SharpEmu.Libs.Kernel; public static class KernelModuleRegistry { + public enum ModuleStartState + { + NotStarted, + Starting, + Started, + } + private static readonly object _gate = new(); private static readonly Dictionary _modulesByHandle = new(); + private static readonly Dictionary> _symbolsByHandle = new(); private static readonly Dictionary _handleByPath = new(StringComparer.OrdinalIgnoreCase); private static readonly Dictionary _handleByName = new(StringComparer.OrdinalIgnoreCase); private static readonly Dictionary _sysmoduleHandleById = new(); @@ -32,6 +40,11 @@ public static class KernelModuleRegistry ulong BaseAddress, ulong EndAddress, ulong EntryPoint, + ulong InitEntryPoint, + ulong EhFrameHeaderAddress, + ulong EhFrameAddress, + ulong EhFrameSize, + ModuleStartState StartState, bool IsMain, bool IsSystemModule); @@ -40,6 +53,7 @@ public static class KernelModuleRegistry lock (_gate) { _modulesByHandle.Clear(); + _symbolsByHandle.Clear(); _handleByPath.Clear(); _handleByName.Clear(); _sysmoduleHandleById.Clear(); @@ -52,6 +66,10 @@ public static class KernelModuleRegistry ulong baseAddress, ulong size, ulong entryPoint, + ulong initEntryPoint, + ulong ehFrameHeaderAddress, + ulong ehFrameAddress, + ulong ehFrameSize, bool isMain, bool isSystemModule = false) { @@ -67,6 +85,10 @@ public static class KernelModuleRegistry BaseAddress = baseAddress, EndAddress = ComputeEnd(baseAddress, size), EntryPoint = entryPoint, + InitEntryPoint = initEntryPoint, + EhFrameHeaderAddress = ehFrameHeaderAddress, + EhFrameAddress = ehFrameAddress, + EhFrameSize = ehFrameSize, IsMain = existing.IsMain || isMain, IsSystemModule = existing.IsSystemModule || isSystemModule, }; @@ -84,6 +106,11 @@ public static class KernelModuleRegistry BaseAddress: baseAddress, EndAddress: ComputeEnd(baseAddress, size), EntryPoint: entryPoint, + InitEntryPoint: initEntryPoint, + EhFrameHeaderAddress: ehFrameHeaderAddress, + EhFrameAddress: ehFrameAddress, + EhFrameSize: ehFrameSize, + StartState: ModuleStartState.NotStarted, IsMain: isMain, IsSystemModule: isSystemModule); _modulesByHandle[handle] = entry; @@ -119,6 +146,11 @@ public static class KernelModuleRegistry BaseAddress: 0, EndAddress: 0, EntryPoint: 0, + InitEntryPoint: 0, + EhFrameHeaderAddress: 0, + EhFrameAddress: 0, + EhFrameSize: 0, + StartState: ModuleStartState.Started, IsMain: false, IsSystemModule: isSystemModule); _modulesByHandle[handle] = entry; @@ -203,6 +235,97 @@ public static class KernelModuleRegistry } } + /// + /// Atomically claims a module initializer. A module can be observed while + /// it is starting (for recursive loader calls), but its DT_INIT routine is + /// executed at most once after a successful start. + /// + public static bool TryBeginModuleStart(int handle, out ModuleEntry module) + { + lock (_gate) + { + if (!_modulesByHandle.TryGetValue(handle, out module)) + { + return false; + } + + if (module.StartState != ModuleStartState.NotStarted) + { + return false; + } + + if (module.InitEntryPoint < 0x10000) + { + module = module with { StartState = ModuleStartState.Started }; + _modulesByHandle[handle] = module; + return false; + } + + module = module with { StartState = ModuleStartState.Starting }; + _modulesByHandle[handle] = module; + return true; + } + } + + public static void CompleteModuleStart(int handle, bool succeeded) + { + lock (_gate) + { + if (!_modulesByHandle.TryGetValue(handle, out var module) || + module.StartState != ModuleStartState.Starting) + { + return; + } + + _modulesByHandle[handle] = module with + { + StartState = succeeded ? ModuleStartState.Started : ModuleStartState.NotStarted, + }; + } + } + + public static void RegisterModuleSymbols(int handle, IReadOnlyDictionary symbols) + { + ArgumentNullException.ThrowIfNull(symbols); + lock (_gate) + { + if (!_modulesByHandle.ContainsKey(handle)) + { + return; + } + + if (!_symbolsByHandle.TryGetValue(handle, out var destination)) + { + destination = new Dictionary(StringComparer.Ordinal); + _symbolsByHandle[handle] = destination; + } + + foreach (var (name, address) in symbols) + { + if (!string.IsNullOrWhiteSpace(name) && address >= 0x10000) + { + destination.TryAdd(name, address); + } + } + } + } + + public static bool TryResolveModuleSymbol(int handle, string symbolName, out ulong address) + { + address = 0; + if (string.IsNullOrWhiteSpace(symbolName)) + { + return false; + } + + lock (_gate) + { + return _symbolsByHandle.TryGetValue(handle, out var symbols) && + symbols.TryGetValue(symbolName, out address) && + address >= 0x10000; + } + } + public static bool TryFindByPathOrName(string? modulePathOrName, out ModuleEntry module) { module = default; diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs index c2aa476..1d8fc45 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadCompatExports.cs @@ -3,7 +3,9 @@ using System.Collections.Concurrent; using SharpEmu.HLE; -using SharpEmu.Libs.Fiber; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Threading; using System.Diagnostics.CodeAnalysis; namespace SharpEmu.Libs.Kernel; @@ -18,15 +20,12 @@ public static class KernelPthreadCompatExports private const int MutexObjectSize = 0x100; private const int MutexAttrObjectSize = 0x40; private const int CondObjectSize = 0x100; - private const int DefaultSpuriousCondWakeMilliseconds = 1; private const int PthreadOnceUninitialized = 0; private const int PthreadOnceInProgress = 1; private const int PthreadOnceDone = 2; private static readonly object _stateGate = new(); private static readonly ConcurrentDictionary _mutexStates = new(); - private static readonly ConcurrentDictionary _mutexWakeKeys = new(); - private static readonly ConcurrentDictionary _condWakeKeys = new(); private static readonly Dictionary _mutexAttrStates = new(); private static readonly Dictionary _condStates = new(); private static readonly Dictionary _onceGates = new(); @@ -38,71 +37,55 @@ public static class KernelPthreadCompatExports string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_PTHREAD_CONDS"), "1", StringComparison.Ordinal); private static readonly HashSet? _tracePthreadMutexFilter = ParseTraceAddressFilter( Environment.GetEnvironmentVariable("SHARPEMU_LOG_PTHREAD_MUTEX_FILTER")); - private static readonly bool _enableMutexLockBlocking = - string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_MUTEX_LOCK_BLOCKING"), "1", StringComparison.Ordinal); - - // On the outer class deliberately: a static on the nested state classes gives them a - // type initializer that first runs on a guest thread and fail-fasts the CLR. - private static long _nextMutexWakeId; - private static long _nextCondWakeId; - private static readonly ConcurrentDictionary _posixTimedWaitCounts = new(); - private static readonly ConcurrentDictionary _reportedContendedMutexes = new(); + private static long _nextSynchronizationWaiterId; private sealed class PthreadMutexState { - public SemaphoreSlim Semaphore { get; } = new(1, 1); public ulong OwnerThreadId { get; set; } public int RecursionCount { get; set; } public int Type { get; set; } = MutexTypeErrorCheck; public int Protocol { get; set; } - - // Keyed on state identity, not the resolved address (which can differ between a - // lock and its unlock for the same state, stranding the waiter). - public string WakeKey { get; } = "pthread_mutex#" + Interlocked.Increment(ref _nextMutexWakeId).ToString("X"); + public LinkedList Waiters { get; } = new(); } - private sealed class PthreadMutexWaiter : IGuestThreadBlockWaiter + private sealed class PthreadMutexWaiter { public required ulong ThreadId { get; init; } - public required CpuContext Ctx { get; init; } - public required ulong MutexAddress { get; init; } - public required ulong ResolvedAddress { get; init; } - public required PthreadMutexState State { get; init; } - public int Reserved; - - public int Resume() => CompleteBlockedMutexLock(Ctx, MutexAddress, ResolvedAddress, State, this); - - public bool TryWake() => TryReserveBlockedMutexLock(Ctx, MutexAddress, ResolvedAddress, State, this); - } - - private sealed class PthreadCondWaiter : IGuestThreadBlockWaiter - { - public required CpuContext Ctx { get; init; } - public required ulong CondAddress { get; init; } - public required ulong MutexAddress { get; init; } - public required PthreadCondState State { get; init; } - public required ulong ObservedEpoch { get; init; } - public required bool Timed { get; init; } - public required int ReleasedRecursion { get; init; } - public required bool PosixResult { get; init; } - - public int Resume() => ResumePthreadCondWait(Ctx, CondAddress, MutexAddress, State, ObservedEpoch, Timed, ReleasedRecursion, PosixResult); - - public bool TryWake() => State.SignalEpoch != ObservedEpoch; + public required string WakeKey { get; init; } + public required bool Cooperative { get; init; } + public LinkedListNode? Node { get; set; } + public int Granted; } private sealed class PthreadCondState { public object SyncRoot { get; } = new(); + public LinkedList WaiterQueue { get; } = new(); public ulong SignalEpoch { get; set; } public int Waiters { get; set; } + } - // See PthreadMutexState.WakeKey. - public string WakeKey { get; } = "pthread_cond#" + Interlocked.Increment(ref _nextCondWakeId).ToString("X"); + private sealed class PthreadCondWaiter + { + public required ulong ThreadId { get; init; } + public required PthreadMutexState MutexState { get; init; } + public required string WakeKey { get; init; } + public required bool Cooperative { get; init; } + public bool PosixErrors { get; init; } + public LinkedListNode? Node { get; set; } + public PthreadMutexWaiter? MutexWaiter { get; set; } + public Timer? TimeoutTimer { get; set; } + // 0 = waiting, 1 = signaled, 2 = timed out. + public int CompletionState { get; set; } } private readonly record struct PthreadMutexAttrState(int Type, int Protocol); + static KernelPthreadCompatExports() + { + RunSynchronizationSelfChecks(); + } + [SysAbiExport( Nid = "aI+OeCz8xrQ", ExportName = "scePthreadSelf", @@ -111,6 +94,7 @@ public static class KernelPthreadCompatExports public static int PthreadSelf(CpuContext ctx) { var currentThreadHandle = KernelPthreadState.GetCurrentThreadHandle(); + GuestThreadExecution.Scheduler?.RegisterGuestThreadContext(currentThreadHandle, ctx); ctx[CpuRegister.Rax] = currentThreadHandle; TracePthreadSelf(ctx, currentThreadHandle); return (int)OrbisGen2Result.ORBIS_GEN2_OK; @@ -150,7 +134,7 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PthreadYield(CpuContext ctx) { - GuestThreadExecution.Scheduler?.Pump(ctx, "scePthreadYield"); + _ = ctx; Thread.Yield(); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -165,10 +149,13 @@ public static class KernelPthreadCompatExports if (_tracePthreads) { var nameAddress = ctx[CpuRegister.Rsi]; - var name = nameAddress != 0 && - KernelMemoryCompatExports.TryReadNullTerminatedUtf8(ctx, nameAddress, 64, out var value) - ? value - : ""; + Span nameBytes = stackalloc byte[64]; + var name = ""; + if (nameAddress != 0 && ctx.Memory.TryRead(nameAddress, nameBytes)) + { + var length = nameBytes.IndexOf((byte)0); + name = System.Text.Encoding.UTF8.GetString(length >= 0 ? nameBytes[..length] : nameBytes); + } Console.Error.WriteLine( $"[LOADER][TRACE] pthread.rename thread=0x{ctx[CpuRegister.Rdi]:X16} name=\"{name}\""); } @@ -343,6 +330,13 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PthreadCondDestroy(CpuContext ctx) => PthreadCondDestroyCore(ctx, ctx[CpuRegister.Rdi]); + [SysAbiExport( + Nid = "RXXqi4CtF8w", + ExportName = "pthread_cond_destroy", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixPthreadCondDestroy(CpuContext ctx) => PthreadCondDestroyCore(ctx, ctx[CpuRegister.Rdi]); + [SysAbiExport( Nid = "WKAXJ4XBPQ4", ExportName = "scePthreadCondWait", @@ -357,104 +351,6 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PthreadCondTimedwait(CpuContext ctx) => PthreadCondWaitCore(ctx, ctx[CpuRegister.Rdi], ctx[CpuRegister.Rsi], timed: true, timeoutUsec: unchecked((uint)ctx[CpuRegister.Rdx])); - [SysAbiExport( - Nid = "27bAgiJmOh0", - ExportName = "pthread_cond_timedwait", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixPthreadCondTimedwait(CpuContext ctx) - { - var absoluteTimeAddress = ctx[CpuRegister.Rdx]; - if (absoluteTimeAddress == 0 || - !ctx.TryReadUInt64(absoluteTimeAddress, out var secondsRaw) || - !ctx.TryReadUInt64(absoluteTimeAddress + sizeof(ulong), out var nanosecondsRaw)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - var deadlineSeconds = unchecked((long)secondsRaw); - var deadlineNanoseconds = unchecked((long)nanosecondsRaw); - if (deadlineSeconds < 0 || deadlineNanoseconds is < 0 or >= 1_000_000_000) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - long nowSeconds; - long nowNanoseconds; - if (deadlineSeconds >= 1_000_000_000) - { - var now = DateTimeOffset.UtcNow; - nowSeconds = now.ToUnixTimeSeconds(); - nowNanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; - } - else - { - // Engine condition variables use monotonic deadlines. - KernelRuntimeCompatExports.GetProcessMonotonicTime(out nowSeconds, out nowNanoseconds); - } - - var remainingNanoseconds = - ((Int128)deadlineSeconds * 1_000_000_000 + deadlineNanoseconds) - - ((Int128)nowSeconds * 1_000_000_000 + nowNanoseconds); - var timeoutUsec = remainingNanoseconds <= 0 - ? 0u - : (uint)Int128.Min(uint.MaxValue, (remainingNanoseconds + 999) / 1_000); - - TracePosixTimedWait( - ctx, - deadlineSeconds, - deadlineNanoseconds, - nowSeconds, - nowNanoseconds, - remainingNanoseconds, - timeoutUsec); - - return PthreadCondWaitCore( - ctx, - ctx[CpuRegister.Rdi], - ctx[CpuRegister.Rsi], - timed: true, - timeoutUsec, - posixResult: true); - } - - private static void TracePosixTimedWait( - CpuContext ctx, - long deadlineSeconds, - long deadlineNanoseconds, - long nowSeconds, - long nowNanoseconds, - Int128 remainingNanoseconds, - uint timeoutUsec) - { - var condAddress = ctx[CpuRegister.Rdi]; - var count = _posixTimedWaitCounts.AddOrUpdate(condAddress, 1, static (_, current) => current + 1); - if (count != 1 && count % 1_000_000 != 0) - { - return; - } - - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); - Console.Error.WriteLine( - $"[LOADER][DIAG] pthread_cond_timedwait#{count}: thread=0x{KernelPthreadState.GetCurrentThreadHandle():X16} " + - $"cond=0x{condAddress:X16} mutex=0x{ctx[CpuRegister.Rsi]:X16} abstime=0x{ctx[CpuRegister.Rdx]:X16} " + - $"deadline={deadlineSeconds}.{deadlineNanoseconds:D9} now={nowSeconds}.{nowNanoseconds:D9} " + - $"remaining_ns={remainingNanoseconds} timeout_us={timeoutUsec} ret=0x{returnRip:X16}"); - - if (GuestThreadExecution.Scheduler is not { } scheduler) - { - return; - } - - foreach (var snapshot in scheduler.SnapshotThreads()) - { - Console.Error.WriteLine( - $"[LOADER][DIAG] guest_thread handle=0x{snapshot.ThreadHandle:X16} name='{snapshot.Name}' " + - $"state={snapshot.State} imports={snapshot.ImportCount} nid={snapshot.LastImportNid ?? "none"} " + - $"ret=0x{snapshot.LastReturnRip:X16} block={snapshot.BlockReason ?? "none"}"); - } - } - [SysAbiExport( Nid = "kDh-NfxgMtE", ExportName = "scePthreadCondSignal", @@ -476,6 +372,62 @@ public static class KernelPthreadCompatExports LibraryName = "libKernel")] public static int PosixPthreadCondWait(CpuContext ctx) => PthreadCondWaitCore(ctx, ctx[CpuRegister.Rdi], ctx[CpuRegister.Rsi], timed: false); + [SysAbiExport( + Nid = "27bAgiJmOh0", + ExportName = "pthread_cond_timedwait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixPthreadCondTimedwait(CpuContext ctx) + { + var deadlineAddress = ctx[CpuRegister.Rdx]; + if (deadlineAddress == 0 || + !KernelMemoryCompatExports.TryReadUInt64Compat(ctx, deadlineAddress, out var rawSeconds) || + !KernelMemoryCompatExports.TryReadUInt64Compat( + ctx, + deadlineAddress + sizeof(long), + out var rawNanoseconds)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + var seconds = unchecked((long)rawSeconds); + var nanoseconds = unchecked((long)rawNanoseconds); + if (seconds < 0 || nanoseconds is < 0 or >= 1_000_000_000) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + var now = DateTimeOffset.UtcNow; + var deltaSeconds = seconds - now.ToUnixTimeSeconds(); + var nowNanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100L; + uint timeoutUsec; + if (deltaSeconds < 0) + { + timeoutUsec = 0; + } + else if (deltaSeconds > uint.MaxValue / 1_000_000L + 1) + { + timeoutUsec = uint.MaxValue; + } + else + { + var remainingNanoseconds = + deltaSeconds * 1_000_000_000L + nanoseconds - nowNanoseconds; + var remainingUsec = remainingNanoseconds <= 0 + ? 0 + : (remainingNanoseconds + 999L) / 1_000L; + timeoutUsec = (uint)Math.Min(remainingUsec, uint.MaxValue); + } + + return PthreadCondWaitCore( + ctx, + ctx[CpuRegister.Rdi], + ctx[CpuRegister.Rsi], + timed: true, + timeoutUsec, + posixErrors: true); + } + [SysAbiExport( Nid = "mkx2fVhNMsg", ExportName = "pthread_cond_broadcast", @@ -543,42 +495,42 @@ public static class KernelPthreadCompatExports var initRoutine = ctx[CpuRegister.Rsi]; if (onceAddress == 0 || initRoutine == 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadInt32(onceAddress, out var onceValue)) + if (!TryReadInt32(ctx, onceAddress, out var onceValue)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (onceValue == PthreadOnceDone) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } var gate = GetPthreadOnceGate(onceAddress); var shouldCall = false; lock (gate) { - if (!ctx.TryReadInt32(onceAddress, out onceValue)) + if (!TryReadInt32(ctx, onceAddress, out onceValue)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } while (onceValue == PthreadOnceInProgress) { Monitor.Wait(gate, TimeSpan.FromMilliseconds(1)); - if (!ctx.TryReadInt32(onceAddress, out onceValue)) + if (!TryReadInt32(ctx, onceAddress, out onceValue)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } if (onceValue != PthreadOnceDone) { - if (!ctx.TryWriteInt32(onceAddress, PthreadOnceInProgress)) + if (!TryWriteInt32(ctx, onceAddress, PthreadOnceInProgress)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } shouldCall = true; @@ -594,21 +546,21 @@ public static class KernelPthreadCompatExports { lock (gate) { - _ = ctx.TryWriteInt32(onceAddress, PthreadOnceUninitialized); + _ = TryWriteInt32(ctx, onceAddress, PthreadOnceUninitialized); Monitor.PulseAll(gate); } TracePthreadOnce(onceAddress, initRoutine, "failed", error); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); } lock (gate) { - if (!ctx.TryWriteInt32(onceAddress, PthreadOnceDone)) + if (!TryWriteInt32(ctx, onceAddress, PthreadOnceDone)) { - _ = ctx.TryWriteInt32(onceAddress, PthreadOnceUninitialized); + _ = TryWriteInt32(ctx, onceAddress, PthreadOnceUninitialized); Monitor.PulseAll(gate); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } Monitor.PulseAll(gate); @@ -616,7 +568,7 @@ public static class KernelPthreadCompatExports } TracePthreadOnce(onceAddress, initRoutine, shouldCall ? "call" : "done", null); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } private static int PthreadMutexInitCore(CpuContext ctx, ulong mutexAddress, ulong attrAddress) @@ -639,8 +591,6 @@ public static class KernelPthreadCompatExports } if (!InitializeMutexObject(ctx, handle, state)) { - TryFreeOpaqueObject(ctx, handle); - state.Semaphore.Dispose(); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -652,8 +602,6 @@ public static class KernelPthreadCompatExports _mutexStates.TryRemove(mutexAddress, out _); _mutexStates.TryRemove(handle, out _); - TryFreeOpaqueObject(ctx, handle); - state.Semaphore.Dispose(); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -668,20 +616,26 @@ public static class KernelPthreadCompatExports } var resolvedAddress = ResolveMutexHandle(ctx, mutexAddress); - _mutexStates.TryRemove(resolvedAddress, out var state); - if (resolvedAddress != mutexAddress) - { - _mutexStates.TryRemove(mutexAddress, out _); - } - - if (state is null) + if (!_mutexStates.TryGetValue(resolvedAddress, out var state)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } + lock (state) + { + if (state.OwnerThreadId != 0 || state.RecursionCount != 0 || state.Waiters.Count != 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + } + + _mutexStates.TryRemove(resolvedAddress, out _); + if (resolvedAddress != mutexAddress) + { + _mutexStates.TryRemove(mutexAddress, out _); + } + } + _ = KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, mutexAddress, 0); - TryFreeOpaqueObject(ctx, resolvedAddress); - state.Semaphore.Dispose(); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -699,6 +653,10 @@ public static class KernelPthreadCompatExports } var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); + var canCooperativelyBlock = !tryOnly && + GuestThreadExecution.IsGuestThread && + GuestThreadExecution.TryGetCurrentImportCallFrame(out _); + PthreadMutexWaiter? waiter = null; lock (state) { if (state.OwnerThreadId == currentThreadId) @@ -718,9 +676,10 @@ public static class KernelPthreadCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; } - state.RecursionCount++; - TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + // FreeBSD maps NORMAL to checked non-recursive behavior: + // self-lock is an error, never implicit recursion. + TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK; } else { @@ -731,105 +690,39 @@ public static class KernelPthreadCompatExports return ownedResult; } } - } - var acquired = state.Semaphore.Wait(0); - if (!acquired) - { - TraceContendedMutex(ctx, mutexAddress, resolvedAddress, state, currentThreadId); - // Fibers retain the synchronous fallback to preserve switch state. - var currentFiber = FiberExports.GetCurrentFiberAddressForDiagnostics(ctx); - var canCooperativelyBlock = _enableMutexLockBlocking || currentFiber == 0; - if (canCooperativelyBlock && - !tryOnly && - GuestThreadExecution.IsGuestThread && - GuestThreadExecution.TryGetCurrentImportCallFrame(out _) && - GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "pthread_mutex_lock", - state.WakeKey, - new PthreadMutexWaiter - { - ThreadId = currentThreadId, - Ctx = ctx, - MutexAddress = mutexAddress, - ResolvedAddress = resolvedAddress, - State = state, - })) + if (state.OwnerThreadId == 0 && state.Waiters.Count == 0) { - TracePthreadMutex(ctx, "lock-block", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); + state.OwnerThreadId = currentThreadId; + state.RecursionCount = 1; + TracePthreadMutex(ctx, tryOnly ? "trylock" : "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (!tryOnly) + if (tryOnly) { - state.Semaphore.Wait(); - acquired = true; - } - } - - if (!acquired) - { - TracePthreadMutex(ctx, tryOnly ? "trylock" : "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; - } - - lock (state) - { - DetectMutexDoubleOwner(resolvedAddress, state, currentThreadId, "lock"); - state.OwnerThreadId = currentThreadId; - state.RecursionCount = 1; - } - - TracePthreadMutex(ctx, tryOnly ? "trylock" : "lock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - private static void TraceContendedMutex( - CpuContext ctx, - ulong mutexAddress, - ulong resolvedAddress, - PthreadMutexState state, - ulong currentThreadId) - { - if (!_reportedContendedMutexes.TryAdd(resolvedAddress, 0)) - { - return; - } - - ulong ownerThreadId; - int recursionCount; - int type; - lock (state) - { - ownerThreadId = state.OwnerThreadId; - recursionCount = state.RecursionCount; - type = state.Type; - } - - _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnRip); - Console.Error.WriteLine( - $"[LOADER][DIAG] pthread_mutex_contended: waiter=0x{currentThreadId:X16} owner=0x{ownerThreadId:X16} " + - $"mutex=0x{mutexAddress:X16} resolved=0x{resolvedAddress:X16} recursion={recursionCount} " + - $"type={type} semaphore_count={state.Semaphore.CurrentCount} ret=0x{returnRip:X16}"); - - if (GuestThreadExecution.Scheduler is not { } scheduler) - { - return; - } - - foreach (var snapshot in scheduler.SnapshotThreads()) - { - if (snapshot.ThreadHandle != currentThreadId && snapshot.ThreadHandle != ownerThreadId) - { - continue; + TracePthreadMutex(ctx, "trylock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; } - Console.Error.WriteLine( - $"[LOADER][DIAG] pthread_mutex_party handle=0x{snapshot.ThreadHandle:X16} name='{snapshot.Name}' " + - $"state={snapshot.State} imports={snapshot.ImportCount} nid={snapshot.LastImportNid ?? "none"} " + - $"ret=0x{snapshot.LastReturnRip:X16} block={snapshot.BlockReason ?? "none"}"); + waiter = EnqueueMutexWaiterLocked(state, currentThreadId, canCooperativelyBlock); } + + if (canCooperativelyBlock && waiter is not null && + GuestThreadExecution.RequestCurrentThreadBlock( + ctx, + "pthread_mutex_lock", + waiter.WakeKey, + () => CompleteBlockedMutexLock(ctx, mutexAddress, resolvedAddress, state, waiter), + () => TryGrantBlockedMutexLock(ctx, mutexAddress, resolvedAddress, state, waiter))) + { + TracePthreadMutex(ctx, "lock-block", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + var hostResult = WaitForHostMutexLock(state, waiter!); + TracePthreadMutex(ctx, "lock", mutexAddress, resolvedAddress, state, currentThreadId, hostResult); + return hostResult; } private static int PthreadMutexUnlockCore(CpuContext ctx, ulong mutexAddress, bool requireOwner) @@ -846,24 +739,16 @@ public static class KernelPthreadCompatExports } var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); - var lenientOwnership = state.Type is MutexTypeNormal or MutexTypeAdaptiveNp; - - var shouldRelease = false; + string? nextWakeKey = null; lock (state) { if (state.RecursionCount <= 0) { - if (lenientOwnership) - { - TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (requireOwner && !lenientOwnership && state.OwnerThreadId != currentThreadId) + if (requireOwner && state.OwnerThreadId != currentThreadId) { TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; @@ -873,25 +758,16 @@ public static class KernelPthreadCompatExports if (state.RecursionCount == 0) { state.OwnerThreadId = 0; - shouldRelease = true; + nextWakeKey = state.Waiters.First?.Value.Cooperative == true + ? state.Waiters.First.Value.WakeKey + : null; + Monitor.PulseAll(state); } } - if (shouldRelease) + if (nextWakeKey is not null) { - try - { - state.Semaphore.Release(); - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(state.WakeKey, 1); - } - catch (SemaphoreFullException) - { - if (!lenientOwnership) - { - TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; - } - } + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(nextWakeKey, 1); } TracePthreadMutex(ctx, "unlock", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); @@ -913,7 +789,6 @@ public static class KernelPthreadCompatExports var initialState = new PthreadMutexAttrState(MutexTypeErrorCheck, 0); if (!WriteMutexAttrObject(ctx, handle, initialState)) { - TryFreeOpaqueObject(ctx, handle); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -931,7 +806,6 @@ public static class KernelPthreadCompatExports _mutexAttrStates.Remove(handle); } - TryFreeOpaqueObject(ctx, handle); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -955,7 +829,6 @@ public static class KernelPthreadCompatExports } } - TryFreeOpaqueObject(ctx, resolvedAddress); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1120,7 +993,7 @@ public static class KernelPthreadCompatExports { if (attrAddress == 0) { - return default; + return new PthreadMutexAttrState(MutexTypeErrorCheck, 0); } var resolvedAddress = ResolveMutexAttrHandle(ctx, attrAddress); @@ -1214,14 +1087,6 @@ public static class KernelPthreadCompatExports lock (_stateGate) { - if (_condStates.TryGetValue(condAddress, out var raced)) - { - TryFreeOpaqueObject(ctx, handle); - resolvedAddress = condAddress; - state = raced; - return true; - } - _condStates[condAddress] = createdState; _condStates[handle] = createdState; } @@ -1234,7 +1099,6 @@ public static class KernelPthreadCompatExports _condStates.Remove(handle); } - TryFreeOpaqueObject(ctx, handle); return false; } @@ -1254,31 +1118,23 @@ public static class KernelPthreadCompatExports Span initialData = stackalloc byte[size]; initialData.Clear(); - if (ctx.Memory.TryWrite(address, initialData)) - { - return true; - } - - allocator.TryFreeGuestMemory(address); - address = 0; - return false; - } - - private static void TryFreeOpaqueObject(CpuContext ctx, ulong address) - { - if (ctx.Memory is IGuestMemoryAllocator allocator) - { - allocator.TryFreeGuestMemory(address); - } + return ctx.Memory.TryWrite(address, initialData); } private static bool InitializeMutexObject(CpuContext ctx, ulong address, PthreadMutexState state) => - ctx.TryWriteUInt32(address + 0x20, unchecked((uint)state.Type)) && - ctx.TryWriteUInt32(address + 0x3C, unchecked((uint)state.Protocol)); + TryWriteUInt32(ctx, address + 0x20, unchecked((uint)state.Type)) && + TryWriteUInt32(ctx, address + 0x3C, unchecked((uint)state.Protocol)); private static bool WriteMutexAttrObject(CpuContext ctx, ulong address, PthreadMutexAttrState state) => - ctx.TryWriteUInt32(address, unchecked((uint)state.Type)) && - ctx.TryWriteUInt32(address + 4, unchecked((uint)state.Protocol)); + TryWriteUInt32(ctx, address, unchecked((uint)state.Type)) && + TryWriteUInt32(ctx, address + 4, unchecked((uint)state.Protocol)); + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BitConverter.TryWriteBytes(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } private static int PthreadCondInitCore(CpuContext ctx, ulong condAddress) { @@ -1307,7 +1163,6 @@ public static class KernelPthreadCompatExports _condStates.Remove(handle); } - TryFreeOpaqueObject(ctx, handle); return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1324,6 +1179,19 @@ public static class KernelPthreadCompatExports var resolvedAddress = ResolveCondHandle(ctx, condAddress); lock (_stateGate) { + if (!_condStates.TryGetValue(resolvedAddress, out var state)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + lock (state.SyncRoot) + { + if (state.WaiterQueue.Count != 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + } + } + _condStates.Remove(resolvedAddress); if (resolvedAddress != condAddress) { @@ -1332,7 +1200,6 @@ public static class KernelPthreadCompatExports } _ = KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, condAddress, 0); - TryFreeOpaqueObject(ctx, resolvedAddress); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1342,141 +1209,123 @@ public static class KernelPthreadCompatExports ulong mutexAddress, bool timed, uint timeoutUsec = 0, - bool posixResult = false) + bool posixErrors = false) { if (condAddress == 0 || mutexAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out var resolvedCondAddress, out var state)) + if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out _, out var state)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - var waitResult = (int)OrbisGen2Result.ORBIS_GEN2_OK; - var spuriousWake = false; - var releasedRecursion = 0; + if (!TryResolveMutexState(ctx, mutexAddress, createIfZero: true, out _, out var mutexState)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); + lock (mutexState) + { + if (mutexState.OwnerThreadId != currentThreadId || mutexState.RecursionCount != 1) + { + return mutexState.OwnerThreadId == currentThreadId + ? (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED; + } + } + + var cooperative = GuestThreadExecution.IsGuestThread && + GuestThreadExecution.TryGetCurrentImportCallFrame(out _); + var waiter = new PthreadCondWaiter + { + ThreadId = currentThreadId, + MutexState = mutexState, + Cooperative = cooperative, + PosixErrors = posixErrors, + WakeKey = cooperative + ? $"pthread_cond_waiter:{Interlocked.Increment(ref _nextSynchronizationWaiterId)}" + : string.Empty, + }; + lock (state.SyncRoot) { + waiter.Node = state.WaiterQueue.AddLast(waiter); state.Waiters++; - var observedEpoch = state.SignalEpoch; - TracePthreadCond( - "wait-enter", - condAddress, - mutexAddress, - state, - timed, - waitResult); + TracePthreadCond("wait-enter", condAddress, mutexAddress, state, timed, (int)OrbisGen2Result.ORBIS_GEN2_OK); - var unlockResult = PthreadMutexFullUnlockForCondWait(ctx, mutexAddress, out releasedRecursion); + var unlockResult = PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); if (unlockResult != (int)OrbisGen2Result.ORBIS_GEN2_OK) { - state.Waiters--; + RemoveCondWaiterLocked(state, waiter); TracePthreadCond("wait-unlock-fail", condAddress, mutexAddress, state, timed, unlockResult); return unlockResult; } - var scheduler = GuestThreadExecution.Scheduler; - if (GuestThreadExecution.IsGuestThread && - GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - timed ? "pthread_cond_timedwait" : "pthread_cond_wait", - state.WakeKey, - new PthreadCondWaiter + if (cooperative && timed) + { + waiter.TimeoutTimer = new Timer( + static callbackState => { - Ctx = ctx, - CondAddress = condAddress, - MutexAddress = mutexAddress, - State = state, - ObservedEpoch = observedEpoch, - Timed = timed, - ReleasedRecursion = releasedRecursion, - PosixResult = posixResult, + var (condState, condWaiter) = ((PthreadCondState, PthreadCondWaiter))callbackState!; + CompleteCondWaiter(condState, condWaiter, timedOut: true); }, - timed ? GuestThreadExecution.ComputeDeadlineTimestamp(GetCondWaitTimeout(timeoutUsec)) : 0)) - { - TracePthreadCond(timed ? "wait-block-timed" : "wait-block", condAddress, mutexAddress, state, timed, waitResult); - return waitResult; + (state, waiter), + GetCondWaitTimeout(timeoutUsec), + Timeout.InfiniteTimeSpan); } + } - if (scheduler is not null) - { - Monitor.Exit(state.SyncRoot); - try - { - scheduler.Pump(ctx, "pthread_cond_wait"); - } - finally - { - Monitor.Enter(state.SyncRoot); - } - } + if (cooperative && + GuestThreadExecution.RequestCurrentThreadBlock( + ctx, + timed ? "pthread_cond_timedwait" : "pthread_cond_wait", + waiter.WakeKey, + () => CompleteBlockedCondWait(ctx, condAddress, mutexAddress, state, waiter), + () => TryGrantCondWaiterMutex(waiter))) + { + TracePthreadCond("wait-block", condAddress, mutexAddress, state, timed, (int)OrbisGen2Result.ORBIS_GEN2_OK); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } - while (state.SignalEpoch == observedEpoch) + // Non-guest callers have no resumable CPU continuation. Park only + // those host-side compatibility callers, preserving the same FIFO + // mutex reacquisition rules as cooperative guest waiters. + lock (state.SyncRoot) + { + var deadline = timed + ? GuestThreadExecution.ComputeDeadlineTimestamp(GetCondWaitTimeout(timeoutUsec)) + : long.MaxValue; + while (waiter.CompletionState == 0) { if (!timed) { - if (!Monitor.Wait(state.SyncRoot, GetCondSpuriousWakeTimeout())) - { - if (scheduler is not null && !GuestThreadExecution.IsGuestThread) - { - Monitor.Exit(state.SyncRoot); - try - { - scheduler.Pump(ctx, "pthread_cond_wait"); - } - finally - { - Monitor.Enter(state.SyncRoot); - } - - continue; - } - - spuriousWake = true; - break; - } - + Monitor.Wait(state.SyncRoot); continue; } - if (!Monitor.Wait(state.SyncRoot, GetCondWaitTimeout(timeoutUsec))) + var remaining = GetRemainingTimeout(deadline); + if (remaining <= TimeSpan.Zero || !Monitor.Wait(state.SyncRoot, remaining)) { - waitResult = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + CompleteCondWaiterLocked(state, waiter, timedOut: true); break; } } - - state.Waiters = Math.Max(0, state.Waiters - 1); - TracePthreadCond( - spuriousWake - ? "wait-spurious" - : (waitResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? "wait-wake" : "wait-timeout"), - condAddress, - mutexAddress, - state, - timed, - waitResult); } - var lockResult = PthreadMutexRelockForCondWait(ctx, mutexAddress, releasedRecursion); - if (lockResult != (int)OrbisGen2Result.ORBIS_GEN2_OK) + if (waiter.MutexWaiter is null) { - TracePthreadCond("wait-relock-fail", condAddress, mutexAddress, state, timed, lockResult); - return lockResult; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - TracePthreadCond( - spuriousWake - ? "wait-exit-spurious" - : (waitResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? "wait-exit" : "wait-exit-timeout"), - condAddress, - mutexAddress, - state, - timed, - waitResult); - return TranslatePthreadResult(waitResult, posixResult); + _ = WaitForHostMutexLock(mutexState, waiter.MutexWaiter); + var waitResult = waiter.CompletionState == 2 + ? CondTimedOutResult(waiter) + : (int)OrbisGen2Result.ORBIS_GEN2_OK; + TracePthreadCond(waiter.CompletionState == 2 ? "wait-exit-timeout" : "wait-exit", condAddress, mutexAddress, state, timed, waitResult); + return waitResult; } private static int PthreadCondSignalCore(CpuContext ctx, ulong condAddress, bool broadcast) @@ -1486,209 +1335,157 @@ public static class KernelPthreadCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out var resolvedCondAddress, out var state)) + if (!TryResolveCondState(ctx, condAddress, createIfZero: true, out _, out var state)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - var shouldWakeScheduler = false; + List? completedWaiters = null; lock (state.SyncRoot) { - // Advance the epoch even with no waiter registered — it's the wait predicate. state.SignalEpoch++; - if (state.Waiters > 0) + for (var node = state.WaiterQueue.First; node is not null;) { - shouldWakeScheduler = true; - if (broadcast) + var next = node.Next; + var waiter = node.Value; + if (waiter.CompletionState == 0 && CompleteCondWaiterLocked(state, waiter, timedOut: false)) { - Monitor.PulseAll(state.SyncRoot); - } - else - { - Monitor.Pulse(state.SyncRoot); + (completedWaiters ??= new List()).Add(waiter); + if (!broadcast) + { + break; + } } + + node = next; } TracePthreadCond(broadcast ? "broadcast" : "signal", condAddress, mutexAddress: 0, state, timed: false, (int)OrbisGen2Result.ORBIS_GEN2_OK); } - if (shouldWakeScheduler) + if (completedWaiters is not null) { - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads( - state.WakeKey, - broadcast ? int.MaxValue : 1); + foreach (var waiter in completedWaiters) + { + WakeCooperativeWaiter(waiter); + } } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static int ResumePthreadCondWait( - CpuContext ctx, - ulong condAddress, - ulong mutexAddress, - PthreadCondState state, - ulong observedEpoch, - bool timed, - int releasedRecursion, - bool posixResult) + private static PthreadMutexWaiter EnqueueMutexWaiterLocked( + PthreadMutexState state, + ulong threadId, + bool cooperative, + string? wakeKey = null) { - var waitResult = (int)OrbisGen2Result.ORBIS_GEN2_OK; - lock (state.SyncRoot) + var waiter = new PthreadMutexWaiter { - state.Waiters = Math.Max(0, state.Waiters - 1); - if (timed && state.SignalEpoch == observedEpoch) - { - waitResult = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; - } - - TracePthreadCond( - waitResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? "wait-resume" : "wait-resume-timeout", - condAddress, - mutexAddress, - state, - timed, - waitResult); - } - - var lockResult = PthreadMutexRelockForCondWait(ctx, mutexAddress, releasedRecursion); - return TranslatePthreadResult( - lockResult == (int)OrbisGen2Result.ORBIS_GEN2_OK ? waitResult : lockResult, - posixResult); - } - - private static int TranslatePthreadResult(int result, bool posixResult) - { - if (!posixResult || result == (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return result; - } - - return result switch - { - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_PERMISSION_DENIED => 1, // EPERM - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND => 2, // ENOENT - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT => 22, // EINVAL - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DEADLOCK => 11, // EDEADLK - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DELETED => 13, // EACCES - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY => 16, // EBUSY - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN => 35, // EAGAIN - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT => 60, // ETIMEDOUT - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_CANCELED => 85, // ECANCELED - (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT => 14, // EFAULT - _ => result + ThreadId = threadId, + Cooperative = cooperative, + WakeKey = cooperative + ? wakeKey ?? $"pthread_mutex_waiter:{Interlocked.Increment(ref _nextSynchronizationWaiterId)}" + : string.Empty, }; + waiter.Node = state.Waiters.AddLast(waiter); + return waiter; } - private static string GetCondWakeKey(ulong resolvedCondAddress) => - _condWakeKeys.GetOrAdd(resolvedCondAddress, static address => string.Create( - 31, - address, - static (destination, value) => - { - "pthread_cond:0x".AsSpan().CopyTo(destination); - _ = value.TryFormat(destination[15..], out _, "X16"); - })); - - private static int PthreadMutexFullUnlockForCondWait(CpuContext ctx, ulong mutexAddress, out int releasedRecursion) + [Conditional("DEBUG")] + private static void RunSynchronizationSelfChecks() { - releasedRecursion = 0; - if (!TryResolveMutexState(ctx, mutexAddress, createIfZero: false, out _, out var state)) + var mutex = new PthreadMutexState(); + PthreadMutexWaiter first; + PthreadMutexWaiter second; + lock (mutex) { - return PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); + first = EnqueueMutexWaiterLocked(mutex, 0x101, cooperative: false); + second = EnqueueMutexWaiterLocked(mutex, 0x202, cooperative: false); + Debug.Assert(!TryGrantMutexWaiterLocked(mutex, second), "A mutex waiter bypassed FIFO order."); + Debug.Assert(TryGrantMutexWaiterLocked(mutex, first), "The FIFO mutex head was not granted."); + Debug.Assert(mutex.OwnerThreadId == first.ThreadId && mutex.RecursionCount == 1, "Mutex ownership was not transferred atomically."); + mutex.OwnerThreadId = 0; + mutex.RecursionCount = 0; + Debug.Assert(TryGrantMutexWaiterLocked(mutex, second), "The second mutex waiter was not granted after release."); } - var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); + var cond = new PthreadCondState(); + var condMutex = new PthreadMutexState(); + var condWaiter = new PthreadCondWaiter + { + ThreadId = 0x303, + MutexState = condMutex, + WakeKey = string.Empty, + Cooperative = false, + }; + lock (cond.SyncRoot) + { + condWaiter.Node = cond.WaiterQueue.AddLast(condWaiter); + cond.Waiters++; + Debug.Assert(CompleteCondWaiterLocked(cond, condWaiter, timedOut: false), "A condition waiter was not completed."); + Debug.Assert(cond.WaiterQueue.Count == 0 && cond.Waiters == 0 && condWaiter.MutexWaiter is not null, "Condition completion did not atomically queue mutex reacquisition."); + } + } + + private static bool TryGrantMutexWaiterLocked(PthreadMutexState state, PthreadMutexWaiter waiter) + { + if (Volatile.Read(ref waiter.Granted) != 0) + { + return true; + } + + if (state.OwnerThreadId != 0 || + waiter.Node is null || + !ReferenceEquals(state.Waiters.First, waiter.Node)) + { + return false; + } + + state.Waiters.Remove(waiter.Node); + waiter.Node = null; + state.OwnerThreadId = waiter.ThreadId; + state.RecursionCount = 1; + Volatile.Write(ref waiter.Granted, 1); + Monitor.PulseAll(state); + return true; + } + + private static int WaitForHostMutexLock(PthreadMutexState state, PthreadMutexWaiter waiter) + { lock (state) { - if (state.RecursionCount <= 0 || state.OwnerThreadId != currentThreadId) + while (!TryGrantMutexWaiterLocked(state, waiter)) { - return PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); - } - - releasedRecursion = state.RecursionCount; - state.RecursionCount = 1; - } - - return PthreadMutexUnlockCore(ctx, mutexAddress, requireOwner: true); - } - - private static int PthreadMutexRelockForCondWait(CpuContext ctx, ulong mutexAddress, int releasedRecursion) - { - var result = PthreadMutexLockCore(ctx, mutexAddress, tryOnly: false); - if (result != (int)OrbisGen2Result.ORBIS_GEN2_OK) - { - return result; - } - - if (releasedRecursion > 1 && - TryResolveMutexState(ctx, mutexAddress, createIfZero: false, out _, out var state)) - { - lock (state) - { - if (state.OwnerThreadId == KernelPthreadState.GetCurrentThreadHandle()) - { - state.RecursionCount = releasedRecursion; - } + Monitor.Wait(state); } } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - public static string? DumpMutexStateForStall(ulong mutexAddress) - { - if (!_mutexStates.TryGetValue(mutexAddress, out var state)) - { - return null; - } - - var waiterThreads = string.Join(",", _mutexStates - .Where(pair => ReferenceEquals(pair.Value, state)) - .Select(pair => $"0x{pair.Key:X}")); - - return $"mutex=0x{mutexAddress:X16} owner=0x{state.OwnerThreadId:X} recursion={state.RecursionCount} " + - $"type={state.Type} semaphore_count={state.Semaphore.CurrentCount} aliases=[{waiterThreads}]"; - } - - private static string GetMutexWakeKey(ulong resolvedMutexAddress) => - _mutexWakeKeys.GetOrAdd(resolvedMutexAddress, static address => string.Create( - 32, - address, - static (destination, value) => - { - "pthread_mutex:0x".AsSpan().CopyTo(destination); - _ = value.TryFormat(destination[16..], out _, "X16"); - })); - - private static bool TryReserveBlockedMutexLock( + private static bool TryGrantBlockedMutexLock( CpuContext ctx, ulong mutexAddress, ulong resolvedAddress, PthreadMutexState state, PthreadMutexWaiter waiter) { + var granted = false; lock (state) { - if (state.OwnerThreadId != 0 || state.RecursionCount != 0) - { - TracePthreadMutex(ctx, "lock-reserve-busy", mutexAddress, resolvedAddress, state, waiter.ThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return false; - } - - if (!state.Semaphore.Wait(0)) - { - TracePthreadMutex(ctx, "lock-reserve-busy", mutexAddress, resolvedAddress, state, waiter.ThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return false; - } - - DetectMutexDoubleOwner(resolvedAddress, state, waiter.ThreadId, "reserve"); - state.OwnerThreadId = waiter.ThreadId; - state.RecursionCount = 1; - Interlocked.Exchange(ref waiter.Reserved, 1); + granted = TryGrantMutexWaiterLocked(state, waiter); } - TracePthreadMutex(ctx, "lock-reserve", mutexAddress, resolvedAddress, state, waiter.ThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return true; + TracePthreadMutex( + ctx, + granted ? "lock-reserve" : "lock-reserve-busy", + mutexAddress, + resolvedAddress, + state, + waiter.ThreadId, + granted ? (int)OrbisGen2Result.ORBIS_GEN2_OK : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return granted; } private static int CompleteBlockedMutexLock( @@ -1699,39 +1496,121 @@ public static class KernelPthreadCompatExports PthreadMutexWaiter waiter) { var currentThreadId = KernelPthreadState.GetCurrentThreadHandle(); - if (Interlocked.Exchange(ref waiter.Reserved, 0) == 1) + if (Volatile.Read(ref waiter.Granted) == 1) { TracePthreadMutex(ctx, "lock-resume", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - if (!state.Semaphore.Wait(0)) - { - TracePthreadMutex(ctx, "lock-resume-busy", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; - } - - lock (state) - { - DetectMutexDoubleOwner(resolvedAddress, state, currentThreadId, "resume"); - state.OwnerThreadId = currentThreadId; - state.RecursionCount = 1; - } - - TracePthreadMutex(ctx, "lock-resume", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_OK); - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + TracePthreadMutex(ctx, "lock-resume-ungranted", mutexAddress, resolvedAddress, state, currentThreadId, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; } - // A nonzero foreign prior owner means two guest threads are in the same critical - // section. Call while holding lock(state). - private static void DetectMutexDoubleOwner(ulong resolvedAddress, PthreadMutexState state, ulong currentThreadId, string site) + private static bool CompleteCondWaiterLocked( + PthreadCondState state, + PthreadCondWaiter waiter, + bool timedOut) { - var priorOwner = state.OwnerThreadId; - if (priorOwner != 0 && priorOwner != currentThreadId) + if (waiter.CompletionState != 0) { - Console.Error.WriteLine( - $"[LOADER][ERROR] MUTEX DOUBLE-OWNER at {site}: resolved=0x{resolvedAddress:X} acquirer=0x{currentThreadId:X} " + - $"prior_owner=0x{priorOwner:X} recursion={state.RecursionCount} type={state.Type} sem_count={state.Semaphore.CurrentCount}"); + return false; + } + + waiter.CompletionState = timedOut ? 2 : 1; + RemoveCondWaiterLocked(state, waiter); + waiter.TimeoutTimer?.Dispose(); + waiter.TimeoutTimer = null; + + lock (waiter.MutexState) + { + waiter.MutexWaiter = EnqueueMutexWaiterLocked( + waiter.MutexState, + waiter.ThreadId, + waiter.Cooperative, + waiter.WakeKey); + } + + Monitor.PulseAll(state.SyncRoot); + return true; + } + + private static void CompleteCondWaiter( + PthreadCondState state, + PthreadCondWaiter waiter, + bool timedOut) + { + var completed = false; + lock (state.SyncRoot) + { + completed = CompleteCondWaiterLocked(state, waiter, timedOut); + } + + if (completed) + { + WakeCooperativeWaiter(waiter); + } + } + + private static void RemoveCondWaiterLocked(PthreadCondState state, PthreadCondWaiter waiter) + { + if (waiter.Node is not null) + { + state.WaiterQueue.Remove(waiter.Node); + waiter.Node = null; + state.Waiters = Math.Max(0, state.Waiters - 1); + } + } + + private static bool TryGrantCondWaiterMutex(PthreadCondWaiter waiter) + { + var mutexWaiter = waiter.MutexWaiter; + if (waiter.CompletionState == 0 || mutexWaiter is null) + { + return false; + } + + lock (waiter.MutexState) + { + return TryGrantMutexWaiterLocked(waiter.MutexState, mutexWaiter); + } + } + + private static int CompleteBlockedCondWait( + CpuContext ctx, + ulong condAddress, + ulong mutexAddress, + PthreadCondState state, + PthreadCondWaiter waiter) + { + waiter.TimeoutTimer?.Dispose(); + waiter.TimeoutTimer = null; + var result = waiter.MutexWaiter is not null && + Volatile.Read(ref waiter.MutexWaiter.Granted) == 1 + ? (waiter.CompletionState == 2 + ? CondTimedOutResult(waiter) + : (int)OrbisGen2Result.ORBIS_GEN2_OK) + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY; + TracePthreadCond( + waiter.CompletionState == 2 ? "wait-resume-timeout" : "wait-resume", + condAddress, + mutexAddress, + state, + waiter.CompletionState == 2, + result); + _ = ctx; + return result; + } + + private static int CondTimedOutResult(PthreadCondWaiter waiter) => + waiter.PosixErrors + ? 60 // ETIMEDOUT on Orbis/FreeBSD; pthread APIs return errno directly. + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; + + private static void WakeCooperativeWaiter(PthreadCondWaiter waiter) + { + if (waiter.Cooperative) + { + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(waiter.WakeKey, 1); } } @@ -1742,20 +1621,18 @@ public static class KernelPthreadCompatExports return TimeSpan.Zero; } - // Round positive sub-millisecond waits to the host sleep quantum. - return timeoutUsec < 1_000 - ? TimeSpan.FromMilliseconds(1) - : TimeSpan.FromTicks((long)timeoutUsec * 10L); + return TimeSpan.FromTicks((long)timeoutUsec * 10L); } - private static TimeSpan GetCondSpuriousWakeTimeout() + private static TimeSpan GetRemainingTimeout(long deadlineTimestamp) { - if (int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_PTHREAD_COND_SPURIOUS_WAKE_MS"), out var milliseconds)) + var remainingTicks = deadlineTimestamp - Stopwatch.GetTimestamp(); + if (remainingTicks <= 0) { - return TimeSpan.FromMilliseconds(Math.Max(1, milliseconds)); + return TimeSpan.Zero; } - return TimeSpan.FromMilliseconds(DefaultSpuriousCondWakeMilliseconds); + return TimeSpan.FromSeconds(remainingTicks / (double)Stopwatch.Frequency); } private static int NormalizeMutexType(int type) @@ -1785,6 +1662,32 @@ public static class KernelPthreadCompatExports } } + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)(int)result); + return (int)result; + } + + private static bool TryReadInt32(CpuContext ctx, ulong address, out int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(bytes); + return true; + } + + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + private static bool CreateImplicitMutexState(CpuContext ctx, ulong mutexAddress, int type, out ulong resolvedAddress, [NotNullWhen(true)] out PthreadMutexState? state) { var createdState = new PthreadMutexState @@ -1800,7 +1703,6 @@ public static class KernelPthreadCompatExports } if (!InitializeMutexObject(ctx, handle, createdState)) { - TryFreeOpaqueObject(ctx, handle); resolvedAddress = 0; state = null; return false; @@ -1810,14 +1712,12 @@ public static class KernelPthreadCompatExports { if (_mutexStates.TryGetValue(mutexAddress, out state)) { - TryFreeOpaqueObject(ctx, handle); resolvedAddress = mutexAddress; return true; } if (_mutexStates.TryGetValue(handle, out state)) { - TryFreeOpaqueObject(ctx, handle); resolvedAddress = handle; return true; } @@ -1831,7 +1731,6 @@ public static class KernelPthreadCompatExports _mutexStates.TryRemove(mutexAddress, out _); _mutexStates.TryRemove(handle, out _); - TryFreeOpaqueObject(ctx, handle); resolvedAddress = 0; state = null; return false; @@ -1879,8 +1778,7 @@ public static class KernelPthreadCompatExports $"[LOADER][TRACE] pthread_{operation}: mutex=0x{mutexAddress:X16} resolved=0x{resolvedAddress:X16} " + $"guest[0]=0x{guestWord0:X16} guest[8]=0x{guestWord1:X16} " + $"current=0x{currentThreadId:X16} owner=0x{(state?.OwnerThreadId ?? 0):X16} " + - $"recursion={(state?.RecursionCount ?? 0)} type={(state?.Type ?? 0)} result=0x{unchecked((uint)result):X8} " + - $"guest_thread=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} managed={Environment.CurrentManagedThreadId}"); + $"recursion={(state?.RecursionCount ?? 0)} type={(state?.Type ?? 0)} result=0x{unchecked((uint)result):X8}"); } private static void TracePthreadCond(string operation, ulong condAddress, ulong mutexAddress, PthreadCondState? state, bool timed, int result) @@ -1891,8 +1789,7 @@ public static class KernelPthreadCompatExports } Console.Error.WriteLine( - $"[LOADER][TRACE] pthread_cond_{operation}: thread=0x{KernelPthreadState.GetCurrentThreadHandle():X16} " + - $"cond=0x{condAddress:X16} key={state?.WakeKey ?? "none"} mutex=0x{mutexAddress:X16} " + + $"[LOADER][TRACE] pthread_cond_{operation}: cond=0x{condAddress:X16} mutex=0x{mutexAddress:X16} " + $"waiters={(state?.Waiters ?? 0)} epoch=0x{(state?.SignalEpoch ?? 0):X} timed={timed} result=0x{unchecked((uint)result):X8}"); } diff --git a/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs index b744201..4785bb4 100644 --- a/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelPthreadExtendedCompatExports.cs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using System.Buffers.Binary; using System.Collections.Concurrent; using System.Text; +using System.Threading; using System.Diagnostics.CodeAnalysis; namespace SharpEmu.Libs.Kernel; @@ -15,6 +17,8 @@ public static class KernelPthreadExtendedCompatExports private const int DefaultDetachState = 0; private const ulong DefaultGuardSize = 0x1000UL; private const ulong DefaultStackSize = 0x1_00000UL; + private const ulong NativeGuestStackSize = 0x20_0000UL; + private const ulong NativeGuestStackStride = 0x100_0000UL; private const int DefaultInheritSched = 4; private const int DefaultSchedPolicy = 1; private const int DefaultSchedPriority = DefaultThreadPriority; @@ -223,6 +227,13 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + [SysAbiExport( + Nid = "+U1R4WtXvoc", + ExportName = "pthread_detach", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixPthreadDetach(CpuContext ctx) => PthreadDetach(ctx); + [SysAbiExport( Nid = "How7B8Oet6k", ExportName = "scePthreadGetname", @@ -273,6 +284,7 @@ public static class KernelPthreadExtendedCompatExports state.Attributes = state.Attributes with { AffinityMask = mask }; } + _ = GuestThreadExecution.Scheduler?.TrySetGuestThreadAffinity(thread, mask); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -326,7 +338,7 @@ public static class KernelPthreadExtendedCompatExports priority = GetOrCreateThreadStateLocked(thread).Priority; } - if (!ctx.TryWriteInt32(outPriorityAddress, priority)) + if (!TryWriteInt32(ctx, outPriorityAddress, priority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -354,6 +366,9 @@ public static class KernelPthreadExtendedCompatExports GetOrCreateThreadStateLocked(thread).Priority = priority; } + // Apply to the live scheduler thread so runtime priority changes take + // effect, not just the local bookkeeping snapshot. + _ = GuestThreadExecution.Scheduler?.TrySetGuestThreadPriority(thread, priority); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -373,7 +388,7 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!ctx.TryReadInt32(schedParamAddress, out var schedPriority)) + if (!TryReadInt32(ctx, schedParamAddress, out var schedPriority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -408,9 +423,9 @@ public static class KernelPthreadExtendedCompatExports public static int PthreadGetschedparam(CpuContext ctx) { var thread = ctx[CpuRegister.Rdi]; - var outPolicyAddress = ctx[CpuRegister.Rsi]; - var outSchedParamAddress = ctx[CpuRegister.Rdx]; - if (thread == 0 || outPolicyAddress == 0 || outSchedParamAddress == 0) + var policyAddress = ctx[CpuRegister.Rsi]; + var schedParamAddress = ctx[CpuRegister.Rdx]; + if (thread == 0 || policyAddress == 0 || schedParamAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } @@ -424,8 +439,8 @@ public static class KernelPthreadExtendedCompatExports priority = state.Priority; } - if (!ctx.TryWriteInt32(outPolicyAddress, policy) || - !ctx.TryWriteInt32(outSchedParamAddress, priority)) + if (!TryWriteInt32(ctx, policyAddress, policy) || + !TryWriteInt32(ctx, schedParamAddress, priority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -528,6 +543,22 @@ public static class KernelPthreadExtendedCompatExports lock (_stateGate) { var threadState = GetOrCreateThreadStateLocked(thread); + + // The native executor maps guest pthread stacks itself, after the + // kernel-facing thread object has been created. Report that live + // mapping when a thread asks for its own attributes. IL2CPP's + // conservative collector uses these two fields to register the stack; + // returning the default null address lets it recycle objects that are + // still reachable only from guest registers/stack frames. + if (thread == KernelPthreadState.GetCurrentThreadHandle() && + TryInferNativeGuestStack(ctx[CpuRegister.Rsp], out var stackAddress)) + { + threadState.Attributes = threadState.Attributes with + { + StackAddress = stackAddress, + StackSize = NativeGuestStackSize, + }; + } _attrStates[outAttrAddress] = threadState.Attributes; } @@ -535,6 +566,28 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + private static bool TryInferNativeGuestStack(ulong stackPointer, out ulong stackAddress) + { + stackAddress = 0; + var candidate = stackPointer & ~(NativeGuestStackStride - 1); + if (stackPointer - candidate >= NativeGuestStackSize) + { + return false; + } + + var highestStack = OperatingSystem.IsWindows() + ? 0x00007FFF_F000_0000UL + : 0x00006FFF_F000_0000UL; + var lowestStack = highestStack - (63 * NativeGuestStackStride); + if (candidate < lowestStack || candidate > highestStack) + { + return false; + } + + stackAddress = candidate; + return true; + } + [SysAbiExport( Nid = "8+s5BzZjxSg", ExportName = "scePthreadAttrGetaffinity", @@ -584,7 +637,7 @@ public static class KernelPthreadExtendedCompatExports state = GetOrCreateAttrStateLocked(attrAddress); } - if (!ctx.TryWriteInt32(outStateAddress, state.DetachState)) + if (!TryWriteInt32(ctx, outStateAddress, state.DetachState)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -827,7 +880,7 @@ public static class KernelPthreadExtendedCompatExports state = GetOrCreateAttrStateLocked(attrAddress); } - if (!ctx.TryWriteInt32(schedParamAddress, state.SchedPriority)) + if (!TryWriteInt32(ctx, schedParamAddress, state.SchedPriority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -850,7 +903,7 @@ public static class KernelPthreadExtendedCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - if (!ctx.TryReadInt32(schedParamAddress, out var schedPriority)) + if (!TryReadInt32(ctx, schedParamAddress, out var schedPriority)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1207,7 +1260,7 @@ public static class KernelPthreadExtendedCompatExports } } - if (!ctx.TryWriteInt32(outKeyAddress, key)) + if (!TryWriteInt32(ctx, outKeyAddress, key)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1315,6 +1368,72 @@ public static class KernelPthreadExtendedCompatExports LibraryName = "libKernel")] public static int OrbisPthreadGetspecific(CpuContext ctx) => PosixPthreadGetspecific(ctx); + private const int PthreadDestructorIterations = 4; + + /// + /// Runs the current thread's pthread TLS-key destructors, as POSIX + /// requires on thread exit. Each key holding a non-null value with a + /// registered destructor has its value cleared first and the destructor + /// then invoked with the previous value; this repeats up to + /// PTHREAD_DESTRUCTOR_ITERATIONS times so destructors that set new + /// thread-local values are themselves cleaned up. Called on the exiting + /// guest thread while it is still executable. + /// + public static void RunThreadLocalDestructors(CpuContext ctx) + { + var scheduler = GuestThreadExecution.Scheduler; + if (scheduler is null) + { + return; + } + + var threadHandle = KernelPthreadState.GetCurrentThreadHandle(); + if (!_threadLocalSpecific.TryGetValue(threadHandle, out var values)) + { + return; + } + + for (var iteration = 0; iteration < PthreadDestructorIterations; iteration++) + { + var ranAny = false; + foreach (var entry in values) + { + var value = entry.Value; + if (value == 0 || + !_tlsKeys.TryGetValue(entry.Key, out var keyState) || + keyState.Destructor == 0) + { + continue; + } + + // Clear before invoking, per POSIX, so a destructor that + // re-sets the key is handled on the next iteration. + if (!values.TryUpdate(entry.Key, 0, value)) + { + continue; + } + + ranAny = true; + _ = scheduler.TryCallGuestFunction( + ctx, + keyState.Destructor, + value, + 0, + 0, + 0, + "pthread_tls_destructor", + out _); + } + + if (!ranAny) + { + break; + } + } + + _threadLocalSpecific.TryRemove(threadHandle, out _); + } + private static int PthreadRwlockLockCore(CpuContext ctx, ulong rwlockAddress, bool write) { if (rwlockAddress == 0) @@ -1680,4 +1799,24 @@ public static class KernelPthreadExtendedCompatExports payload[^1] = 0; return ctx.Memory.TryWrite(address, payload); } + + private static bool TryReadInt32(CpuContext ctx, ulong address, out int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(bytes); + return true; + } + + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs index a59a639..b63881f 100644 --- a/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelRuntimeCompatExports.cs @@ -2,35 +2,21 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; -using SharpEmu.HLE.Host; using SharpEmu.Libs.Fiber; using System.Buffers.Binary; +using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Runtime.Intrinsics.X86; using System.Security.Cryptography; using System.Text; +using System.Threading; namespace SharpEmu.Libs.Kernel; public static class KernelRuntimeCompatExports { - internal const int ClockRealtime = 0; - internal const int ClockVirtual = 1; - internal const int ClockProf = 2; - internal const int ClockMonotonic = 4; - internal const int ClockUptime = 5; - internal const int ClockUptimePrecise = 7; - internal const int ClockUptimeFast = 8; - internal const int ClockRealtimePrecise = 9; - internal const int ClockRealtimeFast = 10; - internal const int ClockMonotonicPrecise = 11; - internal const int ClockMonotonicFast = 12; - internal const int ClockSecond = 13; - internal const int ClockThreadCputimeId = 14; - internal const int ClockProcTime = 15; - private const int Efault = 14; - private const int Einval = 22; private const ulong TlsErrnoOffset = 0x40; private const ulong TlsStackChkGuardBaseOffset = 0x800; private const ulong StackChkGuardFieldOffset = 0x10; @@ -40,20 +26,32 @@ public static class KernelRuntimeCompatExports private const int MallocReplaceSize = 0x70; private const int NewReplaceSize = 0x68; private const int OrbisTimesecSize = sizeof(long) + sizeof(uint) + sizeof(uint); - private const ulong ModuleInfoHandleOffset = 0x108; - private const ulong ModuleInfoNameOffset = 0x10; - private const int ModuleInfoNameMaxBytes = 64; + // PS4/PS5 libkernel module-info ABI. The extended form is consumed by + // sceKernelGetModuleInfoFromAddr and ends at byte 0x1A8; libc commonly + // places its stack canary immediately after that caller-owned buffer. + private const int ModuleInfoNameMaxBytes = 256; + private const int ModuleInfoSize = 0x160; + private const int ModuleInfoExSize = 0x1A8; + private const ulong ModuleInfoNameOffset = 0x08; + private const ulong ModuleInfoExHandleOffset = 0x108; + private const ulong ModuleInfoExInitProcOffset = 0x128; + private const ulong ModuleInfoExSegmentsOffset = 0x160; + private const ulong ModuleInfoExSegmentCountOffset = 0x1A0; + private const int ModuleInfoSegmentSize = 16; private const ulong DefaultKernelTscFrequency = 10_000_000UL; private const ulong PrtAreaStartAddress = 0x0000001000000000UL; private const ulong PrtAreaSize = 0x000000EC00000000UL; private const int MapFlagFixed = 0x10; private const ulong DefaultVirtualRangeAlignment = 0x4000UL; private const int AioInitParamSize = 0x3C; + private const uint MemCommit = 0x1000; + private const uint MemReserve = 0x2000; + private const uint PageExecuteReadWrite = 0x40; private static readonly object _stateGate = new(); private static readonly long _processStartCounter = Stopwatch.GetTimestamp(); private static readonly RdtscDelegate? _rdtscReader = CreateRdtscReader(); private static readonly ulong _kernelTscFrequency = ResolveKernelTscFrequency(); - private static readonly ulong _stackChkGuardValue = 0xC0DEC0DECAFEBABEUL; + private static readonly ulong _stackChkGuardValue = 0xC0DEC0DECAFEBA00UL; private static readonly nint _stackChkGuardObjectAddress = HleDataSymbols.TryGetAddress("f7uOxY9mM1U", out var stackChkGuardAddress) ? unchecked((nint)stackChkGuardAddress) @@ -61,6 +59,7 @@ public static class KernelRuntimeCompatExports private static ulong _applicationHeapApiAddress; private static ulong _processProcParamAddress; private static ulong _nextReservedVirtualBase = 0x6000_0000_0UL; + private static readonly List _releasedVirtualRanges = new(); private static uint _gpoStateBits; private static readonly HashSet _loadedSysmodules = new(); private static readonly object _prtApertureGate = new(); @@ -75,101 +74,14 @@ public static class KernelRuntimeCompatExports [ThreadStatic] private static int _shortUsleepCount; + private static readonly bool _stopwatchTicksAreNanoseconds = + Stopwatch.Frequency == 1_000_000_000L; + + private readonly record struct ReleasedVirtualRange(ulong Address, ulong Length); + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate ulong RdtscDelegate(); - [SysAbiExport( - Nid = "QvsZxomvUHs", - ExportName = "sceKernelNanosleep", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int KernelNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: false); - - [SysAbiExport( - Nid = "yS8U2TGCe1A", - ExportName = "nanosleep", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: true); - - private static int NanosleepCore(CpuContext ctx, bool posix) - { - var requestAddress = ctx[CpuRegister.Rdi]; - var remainAddress = ctx[CpuRegister.Rsi]; - - if (requestAddress == 0) - { - return NanosleepFailure(ctx, posix, Einval, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - Span timespecBuffer = stackalloc byte[16]; - if (!ctx.Memory.TryRead(requestAddress, timespecBuffer)) - { - return NanosleepFailure(ctx, posix, Efault, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - var tvSec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer); - var tvNsec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer[sizeof(long)..]); - - if (tvSec < 0 || tvNsec < 0 || tvNsec >= 1_000_000_000L) - { - return NanosleepFailure(ctx, posix, Einval, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - if (tvSec == 0 && tvNsec == 0) - { - WriteRemainingTime(ctx, remainAddress, 0, 0); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - GuestThreadExecution.Scheduler?.Pump(ctx, posix ? "nanosleep" : "sceKernelNanosleep"); - - // TimeSpan resolution is 100 ns ticks, so sub-100 ns requests round up to - // a single tick rather than collapsing to a zero-length (no-op) sleep. - var totalTicks = tvSec * TimeSpan.TicksPerSecond + Math.Max(tvNsec / 100L, 1L); - try - { - Thread.Sleep(TimeSpan.FromTicks(totalTicks)); - } - catch (ArgumentOutOfRangeException) - { - Thread.Sleep(TimeSpan.FromMilliseconds(int.MaxValue)); - } - - WriteRemainingTime(ctx, remainAddress, 0, 0); - ctx[CpuRegister.Rax] = 0; - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - - private static int NanosleepFailure(CpuContext ctx, bool posix, int errnoValue, OrbisGen2Result sceResult) - { - if (posix) - { - TrySetErrno(ctx, errnoValue); - ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); - } - else - { - ctx[CpuRegister.Rax] = unchecked((ulong)errnoValue); - } - - return (int)sceResult; - } - - private static void WriteRemainingTime(CpuContext ctx, ulong remainAddress, long seconds, long nanoseconds) - { - if (remainAddress == 0) - { - return; - } - - Span remainBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(remainBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(remainBuffer[sizeof(long)..], nanoseconds); - ctx.Memory.TryWrite(remainAddress, remainBuffer); - } - [SysAbiExport( Nid = "1jfXLRVzisc", ExportName = "sceKernelUsleep", @@ -203,21 +115,16 @@ public static class KernelRuntimeCompatExports else { _shortUsleepCount = 0; - var sleepMilliseconds = (int)Math.Min((micros + 999UL) / 1000UL, int.MaxValue); - Thread.Sleep(sleepMilliseconds); + // Precise sleep: rounding microseconds up to Thread.Sleep + // milliseconds (which itself overshoots by a scheduler quantum) + // hard-caps games that pace their frame loop with usleep. + HostTiming.SleepMicroseconds((long)Math.Min(micros, long.MaxValue)); } ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "QcteRwbsnV0", - ExportName = "usleep", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int PosixUsleep(CpuContext ctx) => KernelUsleep(ctx); - private static void TraceUsleepSpin(CpuContext ctx, ulong micros) { if (!_traceUsleep) @@ -299,16 +206,31 @@ public static class KernelRuntimeCompatExports long seconds; long nanoseconds; - if (!ResolveClockTime(clockId, out seconds, out nanoseconds)) + if (clockId == 0) { - return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + var now = DateTimeOffset.UtcNow; + seconds = now.ToUnixTimeSeconds(); + nanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; + } + else + { + var elapsedTicks = Stopwatch.GetTimestamp() - _processStartCounter; + if (_stopwatchTicksAreNanoseconds) + { + // Constant divisors let the JIT strength-reduce the division; + // games call this thousands of times per second. + seconds = elapsedTicks / 1_000_000_000L; + nanoseconds = elapsedTicks - seconds * 1_000_000_000L; + } + else + { + seconds = elapsedTicks / Stopwatch.Frequency; + nanoseconds = (elapsedTicks % Stopwatch.Frequency) * 1_000_000_000L / Stopwatch.Frequency; + } } - Span timespecBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timespecBuffer[sizeof(long)..], nanoseconds); - - if (!ctx.Memory.TryWrite(timeAddress, timespecBuffer)) + if (!ctx.TryWriteUInt64(timeAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timeAddress + sizeof(long), unchecked((ulong)nanoseconds))) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -333,11 +255,8 @@ public static class KernelRuntimeCompatExports var now = DateTimeOffset.UtcNow; var seconds = now.ToUnixTimeSeconds(); var microseconds = (now.Ticks % TimeSpan.TicksPerSecond) / 10; - - Span timevalBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer[sizeof(long)..], microseconds); - if (!ctx.Memory.TryWrite(timeAddress, timevalBuffer)) + if (!ctx.TryWriteUInt64(timeAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timeAddress + sizeof(long), unchecked((ulong)microseconds))) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -359,28 +278,18 @@ public static class KernelRuntimeCompatExports var seconds = now.ToUnixTimeSeconds(); var microseconds = (now.Ticks % TimeSpan.TicksPerSecond) / 10; - if (timeAddress != 0) + if (timeAddress != 0 && + (!ctx.TryWriteUInt64(timeAddress, unchecked((ulong)seconds)) || + !ctx.TryWriteUInt64(timeAddress + sizeof(long), unchecked((ulong)microseconds)))) { - Span timevalBuffer = stackalloc byte[16]; - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer, seconds); - BinaryPrimitives.WriteInt64LittleEndian(timevalBuffer[sizeof(long)..], microseconds); - if (!ctx.Memory.TryWrite(timeAddress, timevalBuffer)) - { - TrySetErrno(ctx, Efault); - return -1; - } + return -1; } - if (timezoneAddress != 0) + if (timezoneAddress != 0 && + (!TryWriteInt32(ctx, timezoneAddress, 0) || + !TryWriteInt32(ctx, timezoneAddress + sizeof(int), 0))) { - Span timezoneBuffer = stackalloc byte[8]; - BinaryPrimitives.WriteInt32LittleEndian(timezoneBuffer, 0); - BinaryPrimitives.WriteInt32LittleEndian(timezoneBuffer[sizeof(int)..], 0); - if (!ctx.Memory.TryWrite(timezoneAddress, timezoneBuffer)) - { - TrySetErrno(ctx, Efault); - return -1; - } + return -1; } ctx[CpuRegister.Rax] = 0; @@ -429,18 +338,6 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "HoLVWNanBBc", - ExportName = "getpid", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libKernel")] - public static int GetProcessId(CpuContext ctx) - { - var processId = Environment.ProcessId; - ctx[CpuRegister.Rax] = unchecked((uint)processId); - return processId; - } - [SysAbiExport( Nid = "fgxnMeTNUtY", ExportName = "sceKernelGetProcessTimeCounter", @@ -747,58 +644,7 @@ public static class KernelRuntimeCompatExports internal static bool TrySetErrno(CpuContext ctx, int value) { var address = GetTlsScratchAddress(ctx, TlsErrnoOffset); - return address != 0 && ctx.TryWriteInt32(address, value); - } - - internal static bool ResolveClockTime(int clockId, out long seconds, out long nanoseconds) - { - switch (clockId) - { - case ClockRealtime: - case ClockRealtimePrecise: - case ClockRealtimeFast: - case ClockVirtual: - case ClockProf: - { - var now = DateTimeOffset.UtcNow; - seconds = now.ToUnixTimeSeconds(); - nanoseconds = (now.Ticks % TimeSpan.TicksPerSecond) * 100; - return true; - } - - // CLOCK_SECOND is FreeBSD's cached whole-second realtime clock (Quake's - // audio_output_thread polls it and treated the previous EINVAL as a fatal - // init failure, exiting and getting respawned in a loop). - case ClockSecond: - seconds = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - nanoseconds = 0; - return true; - - case ClockMonotonic: - case ClockMonotonicPrecise: - case ClockMonotonicFast: - case ClockUptime: - case ClockUptimePrecise: - case ClockUptimeFast: - // Per-thread/process CPU time approximated with the monotonic clock; games - // use these for profiling deltas where monotonicity matters, not absolutes. - case ClockThreadCputimeId: - case ClockProcTime: - GetProcessMonotonicTime(out seconds, out nanoseconds); - return true; - - default: - seconds = 0; - nanoseconds = 0; - return false; - } - } - - internal static void GetProcessMonotonicTime(out long seconds, out long nanoseconds) - { - var elapsedTicks = Stopwatch.GetTimestamp() - _processStartCounter; - seconds = elapsedTicks / Stopwatch.Frequency; - nanoseconds = (elapsedTicks % Stopwatch.Frequency) * 1_000_000_000L / Stopwatch.Frequency; + return address != 0 && TryWriteInt32(ctx, address, value); } [SysAbiExport( @@ -914,7 +760,27 @@ public static class KernelRuntimeCompatExports : AlignUp(_nextReservedVirtualBase, effectiveAlignment); } - if (!TryReserveVirtualRange(ctx, desiredAddress, length, effectiveAlignment, allowSearch: !fixedMapping, out var mappedAddress)) + ulong releasedAddress = 0; + var reusedReleasedRange = !fixedMapping && requestedAddress == 0 && + TryTakeReleasedVirtualRange(length, effectiveAlignment, out releasedAddress); + var alreadyBacked = fixedMapping && requestedAddress != 0 && + KernelMemoryCompatExports.IsGuestRangeBacked(ctx, requestedAddress, length); + ulong mappedAddress; + if (reusedReleasedRange) + { + mappedAddress = releasedAddress; + } + else if (alreadyBacked) + { + mappedAddress = requestedAddress; + } + else if (!TryReserveVirtualRange( + ctx, + desiredAddress, + length, + effectiveAlignment, + allowSearch: !fixedMapping, + out mappedAddress)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } @@ -922,7 +788,7 @@ public static class KernelRuntimeCompatExports if (ShouldTraceVirtualMemory()) { Console.Error.WriteLine( - $"[LOADER][TRACE] reserve_virtual_range: req=0x{requestedAddress:X16} desired=0x{desiredAddress:X16} mapped=0x{mappedAddress:X16} len=0x{length:X16} flags=0x{flags:X8} align=0x{effectiveAlignment:X16}"); + $"[LOADER][TRACE] reserve_virtual_range: req=0x{requestedAddress:X16} desired=0x{desiredAddress:X16} mapped=0x{mappedAddress:X16} len=0x{length:X16} flags=0x{flags:X8} align=0x{effectiveAlignment:X16} already_backed={alreadyBacked} reused={reusedReleasedRange}"); } if (!ctx.TryWriteUInt64(inOutAddressPointer, mappedAddress)) @@ -939,6 +805,75 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + internal static void RegisterReleasedVirtualRange(ulong address, ulong length) + { + if (address == 0 || length == 0 || ulong.MaxValue - address < length) + { + return; + } + + lock (_stateGate) + { + var start = address; + var end = address + length; + for (var i = _releasedVirtualRanges.Count - 1; i >= 0; i--) + { + var existing = _releasedVirtualRanges[i]; + var existingEnd = existing.Address + existing.Length; + if (end < existing.Address || existingEnd < start) + { + continue; + } + + start = Math.Min(start, existing.Address); + end = Math.Max(end, existingEnd); + _releasedVirtualRanges.RemoveAt(i); + } + + _releasedVirtualRanges.Add(new ReleasedVirtualRange(start, end - start)); + } + } + + private static bool TryTakeReleasedVirtualRange( + ulong length, + ulong alignment, + out ulong address) + { + address = 0; + lock (_stateGate) + { + for (var i = 0; i < _releasedVirtualRanges.Count; i++) + { + var range = _releasedVirtualRanges[i]; + var rangeEnd = range.Address + range.Length; + var aligned = AlignUp(range.Address, alignment); + if (aligned >= rangeEnd || length > rangeEnd - aligned) + { + continue; + } + + _releasedVirtualRanges.RemoveAt(i); + if (aligned > range.Address) + { + _releasedVirtualRanges.Add( + new ReleasedVirtualRange(range.Address, aligned - range.Address)); + } + + var allocationEnd = aligned + length; + if (allocationEnd < rangeEnd) + { + _releasedVirtualRanges.Add( + new ReleasedVirtualRange(allocationEnd, rangeEnd - allocationEnd)); + } + + address = aligned; + return true; + } + } + + return false; + } + [SysAbiExport( Nid = "BohYr-F7-is", ExportName = "sceKernelSetPrtAperture", @@ -1000,22 +935,36 @@ public static class KernelRuntimeCompatExports public static int KernelGetModuleInfoFromAddr(CpuContext ctx) { var queriedAddress = ctx[CpuRegister.Rdi]; - _ = ctx[CpuRegister.Rsi]; // mode + var flags = unchecked((int)ctx[CpuRegister.Rsi]); var outInfoAddress = ctx[CpuRegister.Rdx]; if (outInfoAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var moduleHandle = ResolveModuleHandleByAddress(queriedAddress); - if (!ctx.TryWriteInt32(outInfoAddress + ModuleInfoHandleOffset, moduleHandle)) + if (flags is < 0 or >= 3) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (KernelModuleRegistry.TryGetModuleByHandle(moduleHandle, out var module)) + if (callerSize != ModuleInfoExSize) { - _ = TryWriteModuleName(ctx, outInfoAddress, module.Name); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!KernelModuleRegistry.TryGetModuleByAddress(queriedAddress, out var module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!TryWriteModuleInfoEx(ctx, outInfoAddress, module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } ctx[CpuRegister.Rax] = 0; @@ -1030,22 +979,36 @@ public static class KernelRuntimeCompatExports public static int KernelGetModuleInfoForUnwind(CpuContext ctx) { var queriedAddress = ctx[CpuRegister.Rdi]; - _ = ctx[CpuRegister.Rsi]; // flags + var flags = unchecked((int)ctx[CpuRegister.Rsi]); var outInfoAddress = ctx[CpuRegister.Rdx]; if (outInfoAddress == 0) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; } - var moduleHandle = ResolveModuleHandleByAddress(queriedAddress); - if (!ctx.TryWriteInt32(outInfoAddress + ModuleInfoHandleOffset, moduleHandle)) + if (flags is < 0 or >= 3) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (KernelModuleRegistry.TryGetModuleByHandle(moduleHandle, out var module)) + if (callerSize < 0x130) { - _ = TryWriteModuleName(ctx, outInfoAddress, module.Name); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!KernelModuleRegistry.TryGetModuleByAddress(queriedAddress, out var module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!TryWriteModuleInfoForUnwind(ctx, outInfoAddress, module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } ctx[CpuRegister.Rax] = 0; @@ -1085,7 +1048,7 @@ public static class KernelRuntimeCompatExports LibraryName = "libKernel")] public static int KernelGetModuleInfoInternal(CpuContext ctx) { - return KernelGetModuleInfoByHandleCore( + return KernelGetModuleInfoExByHandleCore( ctx, unchecked((int)ctx[CpuRegister.Rdi]), ctx[CpuRegister.Rsi]); @@ -1239,7 +1202,7 @@ public static class KernelRuntimeCompatExports public static int KernelStopUnloadModule(CpuContext ctx) { var resultAddress = ctx[CpuRegister.R9]; - if (resultAddress != 0 && !ctx.TryWriteInt32(resultAddress, 0)) + if (resultAddress != 0 && !TryWriteInt32(ctx, resultAddress, 0)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1256,8 +1219,10 @@ public static class KernelRuntimeCompatExports public static int KernelLoadStartModule(CpuContext ctx) { var modulePathAddress = ctx[CpuRegister.Rdi]; + var argumentSize = ctx[CpuRegister.Rsi]; + var argumentAddress = ctx[CpuRegister.Rdx]; var resultAddress = ctx[CpuRegister.R9]; - if (resultAddress != 0 && !ctx.TryWriteInt32(resultAddress, 0)) + if (resultAddress != 0 && !TryWriteInt32(ctx, resultAddress, 0)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1283,6 +1248,40 @@ public static class KernelRuntimeCompatExports handle = KernelModuleRegistry.RegisterSyntheticModule("module.sprx", isSystemModule: false); } + if (KernelModuleRegistry.TryBeginModuleStart(handle, out var moduleToStart)) + { + var scheduler = GuestThreadExecution.Scheduler; + string? startError = null; + var started = scheduler is not null && scheduler.TryCallGuestFunction( + ctx, + moduleToStart.InitEntryPoint, + argumentSize, + argumentAddress, + 0, + 0, + $"sceKernelLoadStartModule:{moduleToStart.Name}", + out startError); + KernelModuleRegistry.CompleteModuleStart(handle, started); + if (!started) + { + Console.Error.WriteLine( + $"[LOADER][ERROR] sceKernelLoadStartModule failed to start '{moduleToStart.Name}' " + + $"at 0x{moduleToStart.InitEntryPoint:X16}: {startError ?? "guest scheduler unavailable"}"); + var error = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_CPU_TRAP; + if (resultAddress != 0) + { + _ = TryWriteInt32(ctx, resultAddress, error); + } + + ctx[CpuRegister.Rax] = unchecked((ulong)(long)error); + return error; + } + + Console.Error.WriteLine( + $"[LOADER][INFO] sceKernelLoadStartModule started '{moduleToStart.Name}' " + + $"at 0x{moduleToStart.InitEntryPoint:X16}"); + } + ctx[CpuRegister.Rax] = unchecked((uint)handle); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1414,8 +1413,8 @@ public static class KernelRuntimeCompatExports : 0; var minutesWest = unchecked((int)-offset.TotalMinutes); - if (!ctx.TryWriteInt32(timezoneAddress, minutesWest) || - !ctx.TryWriteInt32(timezoneAddress + sizeof(int), dstSeconds / 60)) + if (!TryWriteInt32(ctx, timezoneAddress, minutesWest) || + !TryWriteInt32(ctx, timezoneAddress + sizeof(int), dstSeconds / 60)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1425,7 +1424,7 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (dstSecondsAddress != 0 && !ctx.TryWriteInt32(dstSecondsAddress, dstSeconds)) + if (dstSecondsAddress != 0 && !TryWriteInt32(ctx, dstSecondsAddress, dstSeconds)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1485,7 +1484,7 @@ public static class KernelRuntimeCompatExports _loadedSysmodules.Add(moduleId); } - if (resultAddress != 0 && !ctx.TryWriteInt32(resultAddress, 0)) + if (resultAddress != 0 && !TryWriteInt32(ctx, resultAddress, 0)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1547,22 +1546,6 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static int ResolveModuleHandleByAddress(ulong queriedAddress) - { - if (queriedAddress != 0 && - KernelModuleRegistry.TryGetModuleByAddress(queriedAddress, out var moduleFromAddress)) - { - return moduleFromAddress.Handle; - } - - if (KernelModuleRegistry.TryGetFirstModule(out var firstModule)) - { - return firstModule.Handle; - } - - return 1; - } - private static int KernelGetModuleInfoByHandleCore(CpuContext ctx, int handle, ulong outInfoAddress) { if (outInfoAddress == 0) @@ -1575,12 +1558,52 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; } - if (!ctx.TryWriteInt32(outInfoAddress + ModuleInfoHandleOffset, module.Handle)) + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (callerSize != ModuleInfoSize) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryWriteModuleInfo(ctx, outInfoAddress, module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + private static int KernelGetModuleInfoExByHandleCore(CpuContext ctx, int handle, ulong outInfoAddress) + { + if (outInfoAddress == 0) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!KernelModuleRegistry.TryGetModuleByHandle(handle, out var module)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND; + } + + if (!ctx.TryReadUInt64(outInfoAddress, out var callerSize)) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; + } + + if (callerSize != ModuleInfoExSize) + { + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT; + } + + if (!TryWriteModuleInfoEx(ctx, outInfoAddress, module)) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - _ = TryWriteModuleName(ctx, outInfoAddress, module.Name); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1612,7 +1635,7 @@ public static class KernelRuntimeCompatExports var writableCount = (int)Math.Min(Math.Min(capacity, (ulong)int.MaxValue), (ulong)handles.Length); for (var i = 0; i < writableCount; i++) { - if (!ctx.TryWriteInt32(handlesAddress + (ulong)(i * sizeof(int)), handles[i])) + if (!TryWriteInt32(ctx, handlesAddress + (ulong)(i * sizeof(int)), handles[i])) { return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } @@ -1627,22 +1650,88 @@ public static class KernelRuntimeCompatExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - private static bool TryWriteModuleName(CpuContext ctx, ulong outInfoAddress, string moduleName) + private static bool TryWriteModuleInfo( + CpuContext ctx, + ulong outInfoAddress, + KernelModuleRegistry.ModuleEntry module) { - if (outInfoAddress == 0 || string.IsNullOrWhiteSpace(moduleName)) + var payload = new byte[ModuleInfoSize]; + BinaryPrimitives.WriteUInt64LittleEndian(payload, ModuleInfoSize); + WriteModuleName(payload, module.Name); + WriteModuleSegment(payload, 0x108, module); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0x148), 1); + return ctx.Memory.TryWrite(outInfoAddress, payload); + } + + private static bool TryWriteModuleInfoEx( + CpuContext ctx, + ulong outInfoAddress, + KernelModuleRegistry.ModuleEntry module) + { + var payload = new byte[ModuleInfoExSize]; + BinaryPrimitives.WriteUInt64LittleEndian(payload, ModuleInfoExSize); + WriteModuleName(payload, module.Name); + BinaryPrimitives.WriteInt32LittleEndian( + payload.AsSpan((int)ModuleInfoExHandleOffset), + module.Handle); + BinaryPrimitives.WriteUInt64LittleEndian( + payload.AsSpan((int)ModuleInfoExInitProcOffset), + module.EntryPoint); + WriteModuleSegment(payload, (int)ModuleInfoExSegmentsOffset, module); + BinaryPrimitives.WriteUInt32LittleEndian( + payload.AsSpan((int)ModuleInfoExSegmentCountOffset), + 1); + return ctx.Memory.TryWrite(outInfoAddress, payload); + } + + private static bool TryWriteModuleInfoForUnwind( + CpuContext ctx, + ulong outInfoAddress, + KernelModuleRegistry.ModuleEntry module) + { + const int unwindInfoSize = 0x130; + var payload = new byte[unwindInfoSize]; + BinaryPrimitives.WriteUInt64LittleEndian(payload, unwindInfoSize); + WriteModuleName(payload, module.Name); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x108), module.EhFrameHeaderAddress); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x110), module.EhFrameAddress); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x118), module.EhFrameSize); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(0x120), module.BaseAddress); + BinaryPrimitives.WriteUInt64LittleEndian( + payload.AsSpan(0x128), + module.EndAddress - module.BaseAddress); + return ctx.Memory.TryWrite(outInfoAddress, payload); + } + + private static void WriteModuleName(Span payload, string moduleName) + { + if (string.IsNullOrWhiteSpace(moduleName)) { - return false; + return; } var encoded = Encoding.UTF8.GetBytes(moduleName); var payloadLength = Math.Min(encoded.Length, ModuleInfoNameMaxBytes - 1); - var buffer = new byte[ModuleInfoNameMaxBytes]; if (payloadLength > 0) { - Array.Copy(encoded, 0, buffer, 0, payloadLength); + encoded.AsSpan(0, payloadLength).CopyTo( + payload.Slice((int)ModuleInfoNameOffset, payloadLength)); } + } - return ctx.Memory.TryWrite(outInfoAddress + ModuleInfoNameOffset, buffer); + private static void WriteModuleSegment( + Span payload, + int offset, + KernelModuleRegistry.ModuleEntry module) + { + Debug.Assert(offset >= 0 && offset + ModuleInfoSegmentSize <= payload.Length); + BinaryPrimitives.WriteUInt64LittleEndian(payload.Slice(offset), module.BaseAddress); + BinaryPrimitives.WriteUInt32LittleEndian( + payload.Slice(offset + sizeof(ulong)), + (uint)Math.Min(module.EndAddress - module.BaseAddress, uint.MaxValue)); + // Loaded modules are represented as a single aggregate readable and + // executable range until per-program-header registry data is exposed. + BinaryPrimitives.WriteInt32LittleEndian(payload.Slice(offset + 12), 5); } private static bool TryReadUtf8Z(CpuContext ctx, ulong address, int maxLength, out string value) @@ -1653,42 +1742,26 @@ public static class KernelRuntimeCompatExports return false; } - const int pageSize = 4096; - const int inlineChunkSize = 64; - Span buffer = stackalloc byte[Math.Min(maxLength, 512)]; - var length = 0; - - for (var offset = 0; offset < maxLength && length < buffer.Length;) + var bytes = new List(Math.Min(maxLength, 64)); + Span one = stackalloc byte[1]; + for (var i = 0; i < maxLength; i++) { - var current = address + (ulong)offset; - var pageRemaining = pageSize - (int)(current & (pageSize - 1)); - var chunkSize = Math.Min( - buffer.Length - length, - Math.Min(maxLength - offset, Math.Min(inlineChunkSize, pageRemaining))); - - if (chunkSize <= 0) + if (!ctx.Memory.TryRead(address + (ulong)i, one)) { return false; } - var chunk = buffer.Slice(length, chunkSize); - if (!ctx.Memory.TryRead(current, chunk)) + if (one[0] == 0) { - return false; - } - - var nulIndex = chunk.IndexOf((byte)0); - if (nulIndex >= 0) - { - value = Encoding.UTF8.GetString(buffer[..(length + nulIndex)]); + value = Encoding.UTF8.GetString(bytes.ToArray()); return true; } - length += chunkSize; - offset += chunkSize; + bytes.Add(one[0]); } - return false; + value = Encoding.UTF8.GetString(bytes.ToArray()); + return true; } private static ulong GetTlsScratchAddress(CpuContext ctx, ulong offset) @@ -1701,6 +1774,13 @@ public static class KernelRuntimeCompatExports return unchecked(ctx.FsBase + offset); } + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + private static nint AllocateStackChkGuardObject() { try @@ -1902,20 +1982,21 @@ public static class KernelRuntimeCompatExports private static RdtscDelegate? CreateRdtscReader() { - if (!OperatingSystem.IsWindows() || !Environment.Is64BitProcess) + if (!Environment.Is64BitProcess || + RuntimeInformation.ProcessArchitecture != Architecture.X64) { return null; } try { - nint stubAddress = unchecked((nint)HostPlatform.Current.Memory.Allocate(0, 16, HostPageProtection.ReadWriteExecute)); + nint stubAddress = VirtualAlloc(nint.Zero, (nuint)16, MemCommit | MemReserve, PageExecuteReadWrite); if (stubAddress == 0) { return null; } - ReadOnlySpan stub = stackalloc byte[] + Span stub = stackalloc byte[] { 0x0F, 0x31, 0x48, 0xC1, 0xE2, 0x20, @@ -1923,14 +2004,7 @@ public static class KernelRuntimeCompatExports 0xC3, }; - unsafe - { - fixed (byte* src = stub) - { - Buffer.MemoryCopy(src, (void*)stubAddress, stub.Length, stub.Length); - } - } - + Marshal.Copy(stub.ToArray(), 0, stubAddress, stub.Length); return Marshal.GetDelegateForFunctionPointer(stubAddress); } catch @@ -1939,6 +2013,9 @@ public static class KernelRuntimeCompatExports } } + private static unsafe nint VirtualAlloc(nint lpAddress, nuint dwSize, uint flAllocationType, uint flProtect) => + (nint)HostMemory.Alloc((void*)lpAddress, dwSize, flAllocationType, flProtect); + private static bool TryReserveVirtualRange( CpuContext ctx, ulong desiredAddress, @@ -1974,4 +2051,116 @@ public static class KernelRuntimeCompatExports { return string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIRTUAL_MEMORY"), "1", StringComparison.Ordinal); } + [SysAbiExport( + Nid = "QvsZxomvUHs", + ExportName = "sceKernelNanosleep", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int KernelNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: false); + + [SysAbiExport( + Nid = "yS8U2TGCe1A", + ExportName = "nanosleep", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixNanosleep(CpuContext ctx) => NanosleepCore(ctx, posix: true); + + private static int NanosleepCore(CpuContext ctx, bool posix) + { + const int eFault = 14; + const int eInvalid = 22; + var requestAddress = ctx[CpuRegister.Rdi]; + var remainAddress = ctx[CpuRegister.Rsi]; + + if (requestAddress == 0) + { + return NanosleepFailure(ctx, posix, eInvalid, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + Span timespecBuffer = stackalloc byte[16]; + if (!ctx.Memory.TryRead(requestAddress, timespecBuffer)) + { + return NanosleepFailure(ctx, posix, eFault, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + var tvSec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer); + var tvNsec = BinaryPrimitives.ReadInt64LittleEndian(timespecBuffer[sizeof(long)..]); + if (tvSec < 0 || tvNsec < 0 || tvNsec >= 1_000_000_000L) + { + return NanosleepFailure(ctx, posix, eInvalid, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + if (tvSec == 0 && tvNsec == 0) + { + WriteRemainingTime(ctx, remainAddress, 0, 0); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + GuestThreadExecution.Scheduler?.Pump(ctx, posix ? "nanosleep" : "sceKernelNanosleep"); + var totalTicks = tvSec * TimeSpan.TicksPerSecond + Math.Max(tvNsec / 100L, 1L); + try + { + Thread.Sleep(TimeSpan.FromTicks(totalTicks)); + } + catch (ArgumentOutOfRangeException) + { + Thread.Sleep(TimeSpan.FromMilliseconds(int.MaxValue)); + } + + WriteRemainingTime(ctx, remainAddress, 0, 0); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + private static int NanosleepFailure( + CpuContext ctx, + bool posix, + int errnoValue, + OrbisGen2Result sceResult) + { + if (posix) + { + TrySetErrno(ctx, errnoValue); + ctx[CpuRegister.Rax] = unchecked((ulong)(-1L)); + } + else + { + ctx[CpuRegister.Rax] = unchecked((ulong)errnoValue); + } + + return (int)sceResult; + } + + private static void WriteRemainingTime(CpuContext ctx, ulong remainAddress, long seconds, long nanoseconds) + { + if (remainAddress == 0) + { + return; + } + + Span remainBuffer = stackalloc byte[16]; + BinaryPrimitives.WriteInt64LittleEndian(remainBuffer, seconds); + BinaryPrimitives.WriteInt64LittleEndian(remainBuffer[sizeof(long)..], nanoseconds); + ctx.Memory.TryWrite(remainAddress, remainBuffer); + } + + [SysAbiExport( + Nid = "QcteRwbsnV0", + ExportName = "usleep", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixUsleep(CpuContext ctx) => KernelUsleep(ctx); + + [SysAbiExport( + Nid = "HoLVWNanBBc", + ExportName = "getpid", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int GetProcessId(CpuContext ctx) + { + var processId = Environment.ProcessId; + ctx[CpuRegister.Rax] = unchecked((uint)processId); + return processId; + } } diff --git a/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs b/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs index 4fc12f8..097a550 100644 --- a/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs +++ b/src/SharpEmu.Libs/Kernel/KernelSemaphoreCompatExports.cs @@ -1,8 +1,9 @@ // Copyright (C) 2026 SharpEmu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +using System.Buffers.Binary; using System.Collections.Concurrent; -using System.Diagnostics; +using System.Text; using SharpEmu.HLE; namespace SharpEmu.Libs.Kernel; @@ -13,9 +14,6 @@ public static class KernelSemaphoreCompatExports private static readonly ConcurrentDictionary _semaphores = new(); private static int _nextSemaphoreHandle = 1; - [ThreadStatic] - private static int _semaPollBackoffCount; - private sealed class KernelSemaphoreState { public required string Name { get; init; } @@ -25,35 +23,9 @@ public static class KernelSemaphoreCompatExports public required int MaxCount { get; init; } public int Count { get; set; } public int WaitingThreads { get; set; } - public int CancelEpoch { get; set; } - public bool Deleted { get; set; } public object Gate { get; } = new(); } - private sealed class SemaphoreWaiter : IGuestThreadBlockWaiter - { - public required KernelSemaphoreState Semaphore { get; init; } - public required int NeedCount { get; init; } - public required int CancelEpochAtBlock { get; init; } - public bool Timed { get; init; } - - // Timed-wait completion state; unused when Timed is false. - public CpuContext? Ctx { get; init; } - public ulong TimeoutAddress { get; init; } - public long DeadlineTimestamp { get; init; } - - // Written and read only under the owning semaphore's Gate. - public int? Result { get; set; } - - public int Resume() => Timed - ? CompleteBlockedTimedSemaWait(Ctx!, Semaphore, this, TimeoutAddress, DeadlineTimestamp) - : CompleteBlockedSemaWait(Semaphore, this); - - public bool TryWake() => TryConsumeBlockedSemaWait(Semaphore, this); - } - - private static string GetSemaphoreWakeKey(uint handle) => $"kernel_sema:0x{handle:X8}"; - [SysAbiExport( Nid = "188x57JYp0g", ExportName = "sceKernelCreateSema", @@ -76,12 +48,12 @@ public static class KernelSemaphoreCompatExports initialCount > maxCount || optionAddress != 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (!ctx.TryReadNullTerminatedUtf8(nameAddress, MaxSemaphoreNameLength, out var name)) + if (!TryReadNullTerminatedUtf8(ctx, nameAddress, MaxSemaphoreNameLength, out var name)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } var handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); @@ -90,7 +62,7 @@ public static class KernelSemaphoreCompatExports handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); } - var state = new KernelSemaphoreState + _semaphores[handle] = new KernelSemaphoreState { Name = name, WakeKey = GetSemaphoreWakeKey(handle), @@ -98,28 +70,15 @@ public static class KernelSemaphoreCompatExports MaxCount = maxCount, Count = initialCount, }; - _semaphores[handle] = state; - if (!ctx.TryWriteUInt32(semaphoreAddress, handle)) + if (!TryWriteUInt32(ctx, semaphoreAddress, handle)) { _semaphores.TryRemove(handle, out _); - // Handles are sequential and guest-predictable, so a hostile guest can - // race a WaitSema onto the handle between publication above and this - // rollback. Strand-proof that waiter exactly like DeleteSema does. - lock (state.Gate) - { - state.Deleted = true; - } - - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(state.WakeKey); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - if (_traceSema) - { - TraceSemaphore($"create handle=0x{handle:X8} name='{name}' attr=0x{attr:X} init={initialCount} max={maxCount}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"create handle=0x{handle:X8} name='{name}' attr=0x{attr:X} init={initialCount} max={maxCount}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -135,125 +94,144 @@ public static class KernelSemaphoreCompatExports if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (needCount < 1 || needCount > semaphore.MaxCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + uint timeoutUsec = 0; + if (timeoutAddress != 0 && !TryReadUInt32(ctx, timeoutAddress, out timeoutUsec)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } - var pollTimedOut = false; lock (semaphore.Gate) { if (semaphore.Count >= needCount) { semaphore.Count -= needCount; - if (_traceSema) + if (timeoutAddress != 0) { - TraceSemaphore($"wait handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + _ = TryWriteUInt32(ctx, timeoutAddress, timeoutUsec); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + + TraceSemaphore($"wait handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} timeout={(timeoutAddress == 0 ? "infinite" : timeoutUsec)}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } + semaphore.WaitingThreads++; + } + + // Block cooperatively: the wake predicate atomically acquires the + // tokens (so a wake commits the acquisition), while the resume + // handler distinguishes a real acquisition from a deadline expiry. + var acquired = false; + var deadline = timeoutAddress != 0 + ? GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromMicroseconds(timeoutUsec)) + : 0; + + bool WakePredicate() + { + lock (semaphore.Gate) + { + if (semaphore.Count >= needCount) + { + semaphore.Count -= needCount; + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + acquired = true; + return true; + } + + return false; + } + } + + int ResumeWait() + { if (timeoutAddress != 0) { - if (!ctx.TryReadUInt32(timeoutAddress, out var timeoutMicros)) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - if (timeoutMicros == 0) - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - if (_traceSema) - { - TraceSemaphore($"wait-timeout handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - pollTimedOut = true; - } - else - { - var deadline = GuestThreadExecution.ComputeDeadlineTimestamp(TimeSpan.FromTicks((long)timeoutMicros * 10L)); - var timedWaiter = new SemaphoreWaiter - { - Semaphore = semaphore, - NeedCount = needCount, - CancelEpochAtBlock = semaphore.CancelEpoch, - Timed = true, - Ctx = ctx, - TimeoutAddress = timeoutAddress, - DeadlineTimestamp = deadline, - }; - if (GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "sceKernelWaitSema", - semaphore.WakeKey, - timedWaiter, - blockDeadlineTimestamp: deadline)) - { - semaphore.WaitingThreads++; - if (_traceSema) - { - TraceSemaphore($"wait-block-timed handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} timeout_us={timeoutMicros} waiters={semaphore.WaitingThreads}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); - } - - // Host-owned threads cannot park in the guest scheduler; degrade to the - // immediate-timeout poll the callers already tolerate. - _ = ctx.TryWriteUInt32(timeoutAddress, 0); - if (_traceSema) - { - TraceSemaphore($"wait-timeout handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - pollTimedOut = true; - } + _ = TryWriteUInt32(ctx, timeoutAddress, 0); } - if (!pollTimedOut) + if (acquired) { - var waiter = new SemaphoreWaiter - { - Semaphore = semaphore, - NeedCount = needCount, - CancelEpochAtBlock = semaphore.CancelEpoch, - }; - if (!GuestThreadExecution.RequestCurrentThreadBlock( - ctx, - "sceKernelWaitSema", - semaphore.WakeKey, - waiter)) - { - if (_traceSema) - { - TraceSemaphore($"wait-would-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN); - } - - semaphore.WaitingThreads++; - if (_traceSema) - { - TraceSemaphore($"wait-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"wait-wake handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + + lock (semaphore.Gate) + { + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + } + + TraceSemaphore($"wait-timeout handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; } - GuestThreadExecution.Scheduler?.Pump(ctx, "sceKernelWaitSema"); - if ((++_semaPollBackoffCount & 255) == 0) + if (GuestThreadExecution.RequestCurrentThreadBlock( + ctx, + "sceKernelWaitSema", + GetSemaphoreWakeKey(handle), + ResumeWait, + WakePredicate, + deadline)) { - Thread.Sleep(0); - } - else - { - Thread.Yield(); + TraceSemaphore($"wait-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} timeout={(timeoutAddress == 0 ? "infinite" : timeoutUsec)} waiters={semaphore.WaitingThreads} {FormatCallSite(ctx)}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); + // Not a guest thread (or no scheduler): fall back to a host-thread + // wait so the semantics still hold on non-cooperative callers. + return WaitSemaphoreOnHostThread(ctx, semaphore, handle, needCount, timeoutAddress, timeoutUsec); } + private static int WaitSemaphoreOnHostThread( + CpuContext ctx, + KernelSemaphoreState semaphore, + uint handle, + int needCount, + ulong timeoutAddress, + uint timeoutUsec) + { + var deadlineMs = timeoutAddress != 0 + ? Environment.TickCount64 + Math.Max(1L, timeoutUsec / 1000L) + : long.MaxValue; + lock (semaphore.Gate) + { + TraceSemaphore( + $"wait-host-block handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} " + + $"count={semaphore.Count} timeout={(timeoutAddress == 0 ? "infinite" : timeoutUsec)} {FormatCallSite(ctx)}"); + while (semaphore.Count < needCount) + { + var remaining = deadlineMs - Environment.TickCount64; + if (timeoutAddress != 0 && remaining <= 0) + { + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT); + } + + Monitor.Wait(semaphore.Gate, (int)Math.Min(remaining, 100)); + } + + semaphore.Count -= needCount; + semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); + TraceSemaphore( + $"wait-host-wake handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count} {FormatCallSite(ctx)}"); + if (timeoutAddress != 0) + { + _ = TryWriteUInt32(ctx, timeoutAddress, 0); + } + + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); + } + } + + private static string GetSemaphoreWakeKey(uint handle) => $"sceKernelWaitSema:{handle:X8}"; + [SysAbiExport( Nid = "12wOHk8ywb0", ExportName = "sceKernelPollSema", @@ -263,31 +241,25 @@ public static class KernelSemaphoreCompatExports { if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (needCount < 1 || needCount > semaphore.MaxCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (semaphore.Gate) { if (semaphore.Count < needCount) { - if (_traceSema) - { - TraceSemaphore($"poll-busy handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); + TraceSemaphore($"poll-busy handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_BUSY); } semaphore.Count -= needCount; - if (_traceSema) - { - TraceSemaphore($"poll handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"poll handle=0x{handle:X8} name='{semaphore.Name}' need={needCount} count={semaphore.Count}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } } @@ -300,34 +272,31 @@ public static class KernelSemaphoreCompatExports { if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (signalCount <= 0) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (semaphore.Gate) { if (semaphore.Count > semaphore.MaxCount - signalCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } semaphore.Count += signalCount; - if (_traceSema) - { - TraceSemaphore($"signal handle=0x{handle:X8} name='{semaphore.Name}' signal={signalCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } + // Wake host-thread waiters parked in the fallback path. + Monitor.PulseAll(semaphore.Gate); + TraceSemaphore($"signal handle=0x{handle:X8} name='{semaphore.Name}' signal={signalCount} count={semaphore.Count} waiters={semaphore.WaitingThreads} {FormatCallSite(ctx)}"); } - // Wake after releasing the gate (lock order: scheduler gate -> semaphore gate). - // Wake everyone; the wake handler consumes the count per waiter, so a waiter - // whose needCount exceeds the remaining count stays parked while a smaller - // waiter can proceed. - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(semaphore.WakeKey); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + // Wake cooperatively-blocked guest threads; their wake predicate + // acquires the tokens atomically, so this respects the new count. + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(GetSemaphoreWakeKey(handle)); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -339,35 +308,29 @@ public static class KernelSemaphoreCompatExports { if (!_semaphores.TryGetValue(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } if (setCount > semaphore.MaxCount) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } lock (semaphore.Gate) { - if (waitingThreadsAddress != 0 && !ctx.TryWriteUInt32(waitingThreadsAddress, unchecked((uint)semaphore.WaitingThreads))) + if (waitingThreadsAddress != 0 && !TryWriteUInt32(ctx, waitingThreadsAddress, unchecked((uint)semaphore.WaitingThreads))) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } semaphore.Count = setCount < 0 ? semaphore.InitialCount : setCount; - semaphore.CancelEpoch++; - // WaitingThreads is NOT zeroed here: each canceled waiter decrements it - // exactly once in its wake handler. Zeroing here as well would double-count - // and silently absorb the increment of a waiter that parks between this - // gate release and the wake-all below. - if (_traceSema) - { - TraceSemaphore($"cancel handle=0x{handle:X8} name='{semaphore.Name}' set={setCount} count={semaphore.Count}"); - } + semaphore.WaitingThreads = 0; + Monitor.PulseAll(semaphore.Gate); + TraceSemaphore($"cancel handle=0x{handle:X8} name='{semaphore.Name}' set={setCount} count={semaphore.Count}"); } - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(semaphore.WakeKey); - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(GetSemaphoreWakeKey(handle)); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } [SysAbiExport( @@ -380,138 +343,238 @@ public static class KernelSemaphoreCompatExports var handle = unchecked((uint)ctx[CpuRegister.Rdi]); if (!_semaphores.TryRemove(handle, out var semaphore)) { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_NOT_FOUND); } - // Delete succeeds even with blocked waiters; they wake with the deleted - // result (the SCE kernel wakes waiters with the EACCES-class code). - lock (semaphore.Gate) - { - semaphore.Deleted = true; - } - - _ = GuestThreadExecution.Scheduler?.WakeBlockedThreads(semaphore.WakeKey); - if (_traceSema) - { - TraceSemaphore($"delete handle=0x{handle:X8} name='{semaphore.Name}'"); - } - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); + TraceSemaphore($"delete handle=0x{handle:X8} name='{semaphore.Name}'"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } - // Wake handler: runs under the scheduler's guest-thread gate (lock order: - // scheduler gate -> semaphore gate). Returns true iff the waiter has a final - // result and should be re-readied; false leaves it parked. - private static bool TryConsumeBlockedSemaWait(KernelSemaphoreState semaphore, SemaphoreWaiter waiter) + [SysAbiExport( + Nid = "pDuPEf3m4fI", + ExportName = "sem_init", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemInit(CpuContext ctx) { - lock (semaphore.Gate) + var semaphoreAddress = ctx[CpuRegister.Rdi]; + var initialCountValue = ctx[CpuRegister.Rdx]; + if (semaphoreAddress == 0 || initialCountValue > int.MaxValue) { - return TryConsumeBlockedSemaWaitLocked(semaphore, waiter); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } + + var handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); + if (handle == 0) + { + handle = unchecked((uint)Interlocked.Increment(ref _nextSemaphoreHandle)); + } + + var initialCount = unchecked((int)initialCountValue); + _semaphores[handle] = new KernelSemaphoreState + { + Name = $"posix@0x{semaphoreAddress:X16}", + WakeKey = GetSemaphoreWakeKey(handle), + InitialCount = initialCount, + MaxCount = int.MaxValue, + Count = initialCount, + }; + if (!TryWriteUInt32(ctx, semaphoreAddress, handle)) + { + _semaphores.TryRemove(handle, out _); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + TraceSemaphore($"posix-init address=0x{semaphoreAddress:X16} handle=0x{handle:X8} count={initialCount}"); + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK); } - private static bool TryConsumeBlockedSemaWaitLocked(KernelSemaphoreState semaphore, SemaphoreWaiter waiter) + [SysAbiExport( + Nid = "YCV5dGGBcCo", + ExportName = "sem_wait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemWait(CpuContext ctx) { - if (waiter.Result is not null) + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) { - return true; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } - if (semaphore.Deleted) - { - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_DELETED; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-deleted name='{semaphore.Name}' need={waiter.NeedCount}"); - } - return true; - } - - if (semaphore.CancelEpoch != waiter.CancelEpochAtBlock) - { - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_CANCELED; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-canceled name='{semaphore.Name}' need={waiter.NeedCount}"); - } - return true; - } - - if (semaphore.Count >= waiter.NeedCount) - { - semaphore.Count -= waiter.NeedCount; - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_OK; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-consume name='{semaphore.Name}' need={waiter.NeedCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } - return true; - } - - return false; + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = 0; + return KernelWaitSema(ctx); } - // Resume handler: runs on the woken guest thread outside the scheduler gate; - // its return value becomes the guest's RAX for the resumed sceKernelWaitSema. - private static int CompleteBlockedSemaWait(KernelSemaphoreState semaphore, SemaphoreWaiter waiter) + [SysAbiExport( + Nid = "WBWzsRifCEA", + ExportName = "sem_trywait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemTryWait(CpuContext ctx) { - lock (semaphore.Gate) + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) { - if (waiter.Result is null && !TryConsumeBlockedSemaWaitLocked(semaphore, waiter)) - { - // Nothing readies a parked semaphore waiter without the wake handler - // resolving it, so reaching here means the scheduler contract changed. - Console.Error.WriteLine( - $"[LOADER][GAP] sema.resume-no-outcome name='{semaphore.Name}' need={waiter.NeedCount} count={semaphore.Count}"); - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - } - - return waiter.Result!.Value; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } + + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + return KernelPollSema(ctx, handle, 1); } - private static int CompleteBlockedTimedSemaWait( - CpuContext ctx, - KernelSemaphoreState semaphore, - SemaphoreWaiter waiter, - ulong timeoutAddress, - long deadlineTimestamp) + [SysAbiExport( + Nid = "w5IHyvahg-o", + ExportName = "sem_timedwait", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemTimedWait(CpuContext ctx) { - int result; - lock (semaphore.Gate) + var timeoutAddress = ctx[CpuRegister.Rsi]; + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) { - if (waiter.Result is null && !TryConsumeBlockedSemaWaitLocked(semaphore, waiter)) - { - waiter.Result = (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TIMED_OUT; - semaphore.WaitingThreads = Math.Max(0, semaphore.WaitingThreads - 1); - if (_traceSema) - { - TraceSemaphore($"wake-timeout name='{semaphore.Name}' need={waiter.NeedCount} count={semaphore.Count} waiters={semaphore.WaitingThreads}"); - } - } - - result = waiter.Result!.Value; + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + ctx[CpuRegister.Rdx] = timeoutAddress; + return KernelWaitSema(ctx); + } + + [SysAbiExport( + Nid = "IKP8typ0QUk", + ExportName = "sem_post", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemPost(CpuContext ctx) + { + if (!TryGetPosixSemaphoreHandle(ctx, ctx[CpuRegister.Rdi], out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ctx[CpuRegister.Rdi] = handle; + ctx[CpuRegister.Rsi] = 1; + return KernelSignalSema(ctx, handle, 1); + } + + [SysAbiExport( + Nid = "Bq+LRV-N6Hk", + ExportName = "sem_getvalue", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemGetValue(CpuContext ctx) + { + var semaphoreAddress = ctx[CpuRegister.Rdi]; + var valueAddress = ctx[CpuRegister.Rsi]; + if (valueAddress == 0 || + !TryGetPosixSemaphoreHandle(ctx, semaphoreAddress, out var handle) || + !_semaphores.TryGetValue(handle, out var semaphore)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + int count; + lock (semaphore.Gate) + { + count = semaphore.Count; + } + + return TryWriteUInt32(ctx, valueAddress, unchecked((uint)count)) + ? SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "cDW233RAwWo", + ExportName = "sem_destroy", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libKernel")] + public static int PosixSemDestroy(CpuContext ctx) + { + var semaphoreAddress = ctx[CpuRegister.Rdi]; + if (!TryGetPosixSemaphoreHandle(ctx, semaphoreAddress, out var handle)) + { + return SetReturn(ctx, OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + } + + ctx[CpuRegister.Rdi] = handle; + var result = KernelDeleteSema(ctx); if (result == (int)OrbisGen2Result.ORBIS_GEN2_OK) { - var remainingTicks = deadlineTimestamp - Stopwatch.GetTimestamp(); - var remainingMicros = remainingTicks <= 0 - ? 0u - : (uint)Math.Min(uint.MaxValue, remainingTicks / (double)Stopwatch.Frequency * 1_000_000d); - _ = ctx.TryWriteUInt32(timeoutAddress, remainingMicros); - } - else - { - _ = ctx.TryWriteUInt32(timeoutAddress, 0); + _ = TryWriteUInt32(ctx, semaphoreAddress, 0); } return result; } + private static bool TryGetPosixSemaphoreHandle(CpuContext ctx, ulong semaphoreAddress, out uint handle) + { + handle = 0; + return semaphoreAddress != 0 && + TryReadUInt32(ctx, semaphoreAddress, out handle) && + handle != 0; + } + + private static int SetReturn(CpuContext ctx, OrbisGen2Result result) + { + var value = (int)result; + ctx[CpuRegister.Rax] = unchecked((ulong)value); + return value; + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + return ctx.Memory.TryWrite(address, buffer); + } + + private static bool TryReadNullTerminatedUtf8(CpuContext ctx, ulong address, int maxLength, out string value) + { + value = string.Empty; + if (address == 0 || maxLength <= 0) + { + return false; + } + + var bytes = new byte[Math.Min(maxLength, 4096)]; + Span current = stackalloc byte[1]; + for (var i = 0; i < bytes.Length; i++) + { + if (!ctx.Memory.TryRead(address + (ulong)i, current)) + { + return false; + } + + if (current[0] == 0) + { + value = Encoding.UTF8.GetString(bytes, 0, i); + return true; + } + + bytes[i] = current[0]; + } + + value = Encoding.UTF8.GetString(bytes); + return true; + } + // Call sites must check this before building the interpolated message; the trace // strings would otherwise be allocated on every semaphore op even with tracing off. private static readonly bool _traceSema = @@ -521,4 +584,10 @@ public static class KernelSemaphoreCompatExports { Console.Error.WriteLine($"[LOADER][TRACE] sema.{message}"); } + + private static string FormatCallSite(CpuContext ctx) + { + _ = ctx.TryReadUInt64(ctx[CpuRegister.Rsp], out var returnAddress); + return $"guest=0x{GuestThreadExecution.CurrentGuestThreadHandle:X16} ret=0x{returnAddress:X16}"; + } } diff --git a/src/SharpEmu.Libs/LibcStdioExports.cs b/src/SharpEmu.Libs/LibcStdioExports.cs index b49c5db..b9a17af 100644 --- a/src/SharpEmu.Libs/LibcStdioExports.cs +++ b/src/SharpEmu.Libs/LibcStdioExports.cs @@ -14,9 +14,9 @@ public static class LibcStdioExports private const int MaxPathLength = 4096; private const int MaxModeLength = 16; private const int ReadChunkSize = 1024 * 1024; + private const ulong GuestFileObjectSize = 0x100; private static readonly ConcurrentDictionary _fileHandles = new(); - private static long _nextHandle = 0x1000 - 8; private static readonly bool _traceStdio = string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_STDIO"), "1", StringComparison.Ordinal); @@ -96,7 +96,21 @@ public static class LibcStdioExports stream.Seek(0, SeekOrigin.End); } - var handle = unchecked((ulong)Interlocked.Add(ref _nextHandle, 8)); + // A FILE* can cross between HLE stdio and a title's bundled libc. + // Back the handle with zeroed guest memory so native helpers such + // as _Lockfilelock can safely inspect the FILE object instead of + // dereferencing the old small integer token (0x1000, 0x1008...). + if (!KernelMemoryCompatExports.TryAllocateHleData( + ctx, + GuestFileObjectSize, + alignment: 0x10, + out var handle)) + { + stream.Dispose(); + ctx[CpuRegister.Rax] = 0; + return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_TRY_AGAIN; + } + _fileHandles[handle] = stream; if (_traceStdio) diff --git a/src/SharpEmu.Libs/Network/NetExports.cs b/src/SharpEmu.Libs/Network/NetExports.cs index 65b2dc3..c80a7fc 100644 --- a/src/SharpEmu.Libs/Network/NetExports.cs +++ b/src/SharpEmu.Libs/Network/NetExports.cs @@ -3,6 +3,9 @@ using System.Buffers.Binary; using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Runtime.InteropServices; using System.Text; using SharpEmu.HLE; @@ -12,13 +15,29 @@ public static class NetExports { private const int NetErrorBadFileDescriptor = unchecked((int)0x80410109); private const int NetErrorInvalidArgument = unchecked((int)0x80410116); + private const int NetErrorWouldBlock = unchecked((int)0x80410123); + private const int NetErrorAddressInUse = unchecked((int)0x80410130); + private const int NetErrorNotInitialized = unchecked((int)0x804101C8); + private const int NetErrnoBadFileDescriptor = 9; + private const int NetErrnoInvalidArgument = 22; + private const int NetErrnoWouldBlock = 35; + private const int NetErrnoAddressInUse = 48; + private const int NetErrnoNotInitialized = 200; private const int MaxNameLength = 256; private static readonly ConcurrentDictionary _pools = new(); private static readonly ConcurrentDictionary _resolvers = new(); + private static readonly ConcurrentDictionary _sockets = new(); private static int _nextPoolId; private static int _nextResolverId = 0x2000; - private static bool _initialized; + private static int _nextSocketId = 0x4000; + // The platform networking module is usable immediately after it is loaded. + // Games and middleware (notably FMOD) can create internal sockets before an + // explicit sceNetInit call reaches application code. + private static bool _initialized = true; + + [ThreadStatic] + private static nint _errnoAddress; private sealed record NetPool(string Name, int Size, int Flags); @@ -46,10 +65,232 @@ public static class NetExports _initialized = false; _pools.Clear(); _resolvers.Clear(); + foreach (var socket in _sockets.Values) + { + socket.Dispose(); + } + _sockets.Clear(); TraceNet("term", 0, 0, 0, 0); return ctx.SetReturn(0); } + [SysAbiExport( + Nid = "Q4qBuN-c0ZM", + ExportName = "sceNetSocket", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetSocket(CpuContext ctx) + { + if (!_initialized) + { + return SetNetError(ctx, NetErrorNotInitialized, NetErrnoNotInitialized); + } + + var nameAddress = ctx[CpuRegister.Rdi]; + var family = unchecked((int)ctx[CpuRegister.Rsi]); + var type = unchecked((int)ctx[CpuRegister.Rdx]); + var protocol = unchecked((int)ctx[CpuRegister.Rcx]); + var name = TryReadUtf8Z(ctx, nameAddress, MaxNameLength, out var value) + ? value + : string.Empty; + + if (!TryTranslateSocketParameters(family, type, protocol, out var addressFamily, out var socketType, out var protocolType)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + try + { + var socket = new Socket(addressFamily, socketType, protocolType); + var id = Interlocked.Increment(ref _nextSocketId); + _sockets[id] = socket; + TraceNet("socket.create", id, unchecked((ulong)family), unchecked((ulong)type), unchecked((ulong)protocol)); + ctx[CpuRegister.Rax] = unchecked((ulong)id); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "45ggEzakPJQ", + ExportName = "sceNetSocketClose", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetSocketClose(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryRemove(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + socket.Dispose(); + TraceNet("socket.close", id, 0, 0, 0); + return ctx.SetReturn(0); + } + + [SysAbiExport( + Nid = "2mKX2Spso7I", + ExportName = "sceNetSetsockopt", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetSetsockopt(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + var level = unchecked((int)ctx[CpuRegister.Rsi]); + var option = unchecked((int)ctx[CpuRegister.Rdx]); + var valueAddress = ctx[CpuRegister.Rcx]; + var valueLength = unchecked((int)ctx[CpuRegister.R8]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + // ORBIS_NET_SOL_SOCKET / ORBIS_NET_SO_NBIO. This is the first option + // used by FMOD's discovery socket and maps directly to host blocking. + if (level == 0xFFFF && option == 0x1200) + { + Span value = stackalloc byte[sizeof(int)]; + if (valueLength < value.Length || valueAddress == 0 || !ctx.Memory.TryRead(valueAddress, value)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + socket.Blocking = BinaryPrimitives.ReadInt32LittleEndian(value) == 0; + TraceNet("socket.nonblocking", id, socket.Blocking ? 0UL : 1UL, 0, 0); + return ctx.SetReturn(0); + } + + // ORBIS_NET_SO_REUSEADDR uses the BSD value 0x0004. + if (level == 0xFFFF && option == 0x0004) + { + Span value = stackalloc byte[sizeof(int)]; + if (valueLength < value.Length || valueAddress == 0 || !ctx.Memory.TryRead(valueAddress, value)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + socket.SetSocketOption( + SocketOptionLevel.Socket, + SocketOptionName.ReuseAddress, + BinaryPrimitives.ReadInt32LittleEndian(value) != 0); + TraceNet("socket.reuseaddr", id, BinaryPrimitives.ReadUInt32LittleEndian(value), 0, 0); + return ctx.SetReturn(0); + } + + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + [SysAbiExport( + Nid = "bErx49PgxyY", + ExportName = "sceNetBind", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetBind(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + if (!TryReadSocketAddress(ctx, ctx[CpuRegister.Rsi], unchecked((int)ctx[CpuRegister.Rdx]), out var endpoint)) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + + try + { + socket.Bind(endpoint); + TraceNet("socket.bind", id, unchecked((ulong)endpoint.Port), 0, 0); + return ctx.SetReturn(0); + } + catch (SocketException ex) when (ex.SocketErrorCode == SocketError.AddressAlreadyInUse) + { + return SetNetError(ctx, NetErrorAddressInUse, NetErrnoAddressInUse); + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "kOj1HiAGE54", + ExportName = "sceNetListen", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetListen(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + try + { + socket.Listen(Math.Max(0, unchecked((int)ctx[CpuRegister.Rsi]))); + TraceNet("socket.listen", id, ctx[CpuRegister.Rsi], 0, 0); + return ctx.SetReturn(0); + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "PIWqhn9oSxc", + ExportName = "sceNetAccept", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetAccept(CpuContext ctx) + { + var id = unchecked((int)ctx[CpuRegister.Rdi]); + if (!_sockets.TryGetValue(id, out var socket)) + { + return SetNetError(ctx, NetErrorBadFileDescriptor, NetErrnoBadFileDescriptor); + } + + try + { + var accepted = socket.Accept(); + var acceptedId = Interlocked.Increment(ref _nextSocketId); + _sockets[acceptedId] = accepted; + TraceNet("socket.accept", acceptedId, unchecked((ulong)id), 0, 0); + ctx[CpuRegister.Rax] = unchecked((ulong)acceptedId); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + catch (SocketException ex) when (ex.SocketErrorCode is SocketError.WouldBlock or SocketError.IOPending) + { + return SetNetError(ctx, NetErrorWouldBlock, NetErrnoWouldBlock); + } + catch (SocketException) + { + return SetNetError(ctx, NetErrorInvalidArgument, NetErrnoInvalidArgument); + } + } + + [SysAbiExport( + Nid = "HQOwnfMGipQ", + ExportName = "sceNetErrnoLoc", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNet")] + public static int NetErrnoLoc(CpuContext ctx) + { + if (_errnoAddress == 0) + { + _errnoAddress = Marshal.AllocHGlobal(sizeof(int)); + Marshal.WriteInt32(_errnoAddress, 0); + } + + ctx[CpuRegister.Rax] = unchecked((ulong)_errnoAddress); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "dgJBaeJnGpo", ExportName = "sceNetPoolCreate", @@ -207,6 +448,69 @@ public static class NetExports : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + private static int SetNetError(CpuContext ctx, int result, int errno) + { + if (_errnoAddress == 0) + { + _errnoAddress = Marshal.AllocHGlobal(sizeof(int)); + } + Marshal.WriteInt32(_errnoAddress, errno); + return ctx.SetReturn(result); + } + + private static bool TryTranslateSocketParameters( + int family, + int type, + int protocol, + out AddressFamily addressFamily, + out SocketType socketType, + out ProtocolType protocolType) + { + addressFamily = family switch + { + 2 => AddressFamily.InterNetwork, + 28 => AddressFamily.InterNetworkV6, + _ => AddressFamily.Unspecified, + }; + socketType = type switch + { + 1 => SocketType.Stream, + 2 => SocketType.Dgram, + _ => SocketType.Unknown, + }; + protocolType = protocol switch + { + 0 when socketType == SocketType.Stream => ProtocolType.Tcp, + 0 when socketType == SocketType.Dgram => ProtocolType.Udp, + 6 => ProtocolType.Tcp, + 17 => ProtocolType.Udp, + _ => ProtocolType.Unknown, + }; + + return addressFamily != AddressFamily.Unspecified && + socketType != SocketType.Unknown && + protocolType != ProtocolType.Unknown; + } + + private static bool TryReadSocketAddress(CpuContext ctx, ulong address, int length, out IPEndPoint endpoint) + { + endpoint = new IPEndPoint(IPAddress.Any, 0); + if (address == 0 || length < 16) + { + return false; + } + + Span bytes = stackalloc byte[16]; + if (!ctx.Memory.TryRead(address, bytes) || bytes[1] != 2) + { + return false; + } + + var port = BinaryPrimitives.ReadUInt16BigEndian(bytes[2..4]); + endpoint = new IPEndPoint(new IPAddress(bytes[4..8]), port); + return true; + } + private static bool TryReadUtf8Z(CpuContext ctx, ulong address, int maxLength, out string value) { value = string.Empty; diff --git a/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs b/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs index dc3af01..f93c047 100644 --- a/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs +++ b/src/SharpEmu.Libs/Ngs2/Ngs2Exports.cs @@ -4,6 +4,7 @@ using SharpEmu.HLE; using SharpEmu.Libs.Kernel; using System.Buffers.Binary; +using System.Threading; namespace SharpEmu.Libs.Ngs2; @@ -13,7 +14,7 @@ public static class Ngs2Exports private const int OrbisNgs2ErrorInvalidSystemHandle = unchecked((int)0x804A0230); private const int OrbisNgs2ErrorInvalidRackHandle = unchecked((int)0x804A0261); private const int OrbisNgs2ErrorInvalidVoiceHandle = unchecked((int)0x804A0300); - private const ulong HandleStorageSize = 0x1000; + private const ulong HandleStorageSize = 0x20; private const int RenderBufferInfoSize = 0x18; private const ulong MaximumRenderBufferSize = 16 * 1024 * 1024; @@ -38,13 +39,13 @@ public static class Ngs2Exports var outHandleAddress = ctx[CpuRegister.Rdx]; if (outHandleAddress == 0) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidOutAddress); + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); } if (!TryCreateHandle(ctx, type: 1, ownerHandle: 0, out var handle) || !ctx.TryWriteUInt64(outHandleAddress, handle)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (StateGate) @@ -52,7 +53,7 @@ public static class Ngs2Exports Systems[handle] = new SystemState(unchecked((uint)Interlocked.Increment(ref _nextUid))); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -67,7 +68,7 @@ public static class Ngs2Exports { if (!Systems.Remove(handle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidSystemHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidSystemHandle); } var rackHandles = Racks @@ -80,7 +81,7 @@ public static class Ngs2Exports } } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -97,19 +98,19 @@ public static class Ngs2Exports { if (!Systems.ContainsKey(systemHandle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidSystemHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidSystemHandle); } } if (outHandleAddress == 0) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidOutAddress); + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); } if (!TryCreateHandle(ctx, type: 2, systemHandle, out var handle) || !ctx.TryWriteUInt64(outHandleAddress, handle)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (StateGate) @@ -117,7 +118,7 @@ public static class Ngs2Exports Racks[handle] = new RackState(systemHandle, rackId); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -132,13 +133,13 @@ public static class Ngs2Exports { if (!Racks.ContainsKey(handle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidRackHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidRackHandle); } RemoveRackLocked(handle); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -155,7 +156,7 @@ public static class Ngs2Exports { if (!Racks.ContainsKey(rackHandle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidRackHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidRackHandle); } var existing = Voices.FirstOrDefault( @@ -163,20 +164,20 @@ public static class Ngs2Exports if (existing.Key != 0) { return ctx.TryWriteUInt64(outHandleAddress, existing.Key) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } if (outHandleAddress == 0) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidOutAddress); + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); } if (!TryCreateHandle(ctx, type: 4, rackHandle, out var handle) || !ctx.TryWriteUInt64(outHandleAddress, handle)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } lock (StateGate) @@ -184,7 +185,7 @@ public static class Ngs2Exports Voices[handle] = new VoiceState(rackHandle, voiceIndex); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -196,19 +197,12 @@ public static class Ngs2Exports { lock (StateGate) { - return ctx.SetReturn( - Voices.ContainsKey(ctx[CpuRegister.Rdi]) - ? 0 - : OrbisNgs2ErrorInvalidVoiceHandle); + return SetReturn( + ctx, + Voices.ContainsKey(ctx[CpuRegister.Rdi]) ? 0 : OrbisNgs2ErrorInvalidVoiceHandle); } } - // The earlier "alt NID" guesses here were wrong: an NID is the aerolib hash of one - // symbol name, and hashing scripts/ps5_names.txt shows the previous alt bindings - // actually belonged to sceImeUpdate (-4GCfYdNF1s), sceMouseRead (x8qnXqh-tiM) and - // sceSystemGestureUpdateAllTouchRecognizer (wPJGwI2RM2I) — Quake's audio thread was - // receiving an NGS2 error from what it thought was sceImeUpdate. Those now live in - // their real libraries; the NIDs below are verified against the hash. [SysAbiExport( Nid = "AbYvTOZ8Pts", ExportName = "sceNgs2VoiceRunCommands", @@ -216,48 +210,6 @@ public static class Ngs2Exports LibraryName = "libSceNgs2")] public static int Ngs2VoiceRunCommands(CpuContext ctx) => Ngs2VoiceControl(ctx); - [SysAbiExport( - Nid = "-TOuuAQ-buE", - ExportName = "sceNgs2VoiceGetState", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2VoiceGetState(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "rEh728kXk3w", - ExportName = "sceNgs2VoiceGetStateFlags", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2VoiceGetStateFlags(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "xa8oL9dmXkM", - ExportName = "sceNgs2PanInit", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2PanInit(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "1WsleK-MTkE", - ExportName = "sceNgs2GeomCalcListener", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2GeomCalcListener(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "0lbbayqDNoE", - ExportName = "sceNgs2GeomResetSourceParam", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2GeomResetSourceParam(CpuContext ctx) => ctx.SetReturn(0); - - [SysAbiExport( - Nid = "7Lcfo8SmpsU", - ExportName = "sceNgs2GeomResetListenerParam", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNgs2")] - public static int Ngs2GeomResetListenerParam(CpuContext ctx) => ctx.SetReturn(0); - [SysAbiExport( Nid = "i0VnXM-C9fc", ExportName = "sceNgs2SystemRender", @@ -272,13 +224,13 @@ public static class Ngs2Exports { if (!Systems.ContainsKey(systemHandle)) { - return ctx.SetReturn(OrbisNgs2ErrorInvalidSystemHandle); + return SetReturn(ctx, OrbisNgs2ErrorInvalidSystemHandle); } } if (bufferInfoCount != 0 && bufferInfoAddress == 0) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); } for (uint i = 0; i < bufferInfoCount; i++) @@ -287,14 +239,14 @@ public static class Ngs2Exports if (!ctx.TryReadUInt64(entryAddress, out var bufferAddress) || !ctx.TryReadUInt64(entryAddress + 8, out var bufferSize)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (bufferAddress != 0 && bufferSize != 0) { if (bufferSize > MaximumRenderBufferSize || !TryClearGuestBuffer(ctx, bufferAddress, bufferSize)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } } @@ -306,11 +258,138 @@ public static class Ngs2Exports $"[LOADER][TRACE] ngs2.render#{count} system=0x{systemHandle:X16} buffers={bufferInfoCount}"); } - return ctx.SetReturn(0); + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "pgFAiLR5qT4", + ExportName = "sceNgs2SystemQueryBufferSize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemQueryBufferSize(CpuContext ctx) => WriteBufferSize(ctx, ctx[CpuRegister.Rsi]); + + [SysAbiExport( + Nid = "0eFLVCfWVds", + ExportName = "sceNgs2RackQueryBufferSize", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2RackQueryBufferSize(CpuContext ctx) => WriteBufferSize(ctx, ctx[CpuRegister.Rdx]); + + // Report a fixed working-memory footprint for the requested object. The + // out struct (SceNgs2BufferAllocator-style) begins with the size field. + private static int WriteBufferSize(CpuContext ctx, ulong outAddress) + { + if (outAddress == 0) + { + return SetReturn(ctx, OrbisNgs2ErrorInvalidOutAddress); + } + + Span info = stackalloc byte[RenderBufferInfoSize]; + info.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(info[0..8], 0x10000); + BinaryPrimitives.WriteUInt64LittleEndian(info[8..16], 0x100); + return ctx.Memory.TryWrite(outAddress, info) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "l4Q2dWEH6UM", + ExportName = "sceNgs2SystemSetGrainSamples", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemSetGrainSamples(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "-tbc2SxQD60", + ExportName = "sceNgs2SystemSetSampleRate", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemSetSampleRate(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "gThZqM5PYlQ", + ExportName = "sceNgs2SystemLock", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemLock(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "JXRC5n0RQls", + ExportName = "sceNgs2SystemUnlock", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2SystemUnlock(CpuContext ctx) => ValidateSystem(ctx); + + [SysAbiExport( + Nid = "-TOuuAQ-buE", + ExportName = "sceNgs2VoiceGetState", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2VoiceGetState(CpuContext ctx) + { + var voiceHandle = ctx[CpuRegister.Rdi]; + var stateAddress = ctx[CpuRegister.Rsi]; + var stateSize = (int)Math.Min(ctx[CpuRegister.Rdx], 0x400); + lock (StateGate) + { + if (!Voices.ContainsKey(voiceHandle)) + { + return SetReturn(ctx, OrbisNgs2ErrorInvalidVoiceHandle); + } + } + + // Report an idle (not-in-use) voice: all-zero state block. + if (stateAddress != 0 && stateSize > 0) + { + if (!TryClearGuestBuffer(ctx, stateAddress, (ulong)stateSize)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + } + + return SetReturn(ctx, 0); + } + + [SysAbiExport( + Nid = "rEh728kXk3w", + ExportName = "sceNgs2VoiceGetStateFlags", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2VoiceGetStateFlags(CpuContext ctx) + { + var voiceHandle = ctx[CpuRegister.Rdi]; + var flagsAddress = ctx[CpuRegister.Rsi]; + lock (StateGate) + { + if (!Voices.ContainsKey(voiceHandle)) + { + return SetReturn(ctx, OrbisNgs2ErrorInvalidVoiceHandle); + } + } + + // No flags set: voice is idle. + if (flagsAddress != 0 && !ctx.TryWriteUInt64(flagsAddress, 0)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetReturn(ctx, 0); + } + + private static int ValidateSystem(CpuContext ctx) + { + lock (StateGate) + { + return SetReturn( + ctx, + Systems.ContainsKey(ctx[CpuRegister.Rdi]) ? 0 : OrbisNgs2ErrorInvalidSystemHandle); + } } private static bool TryCreateHandle(CpuContext ctx, uint type, ulong ownerHandle, out ulong handle) { + handle = 0; if (!KernelMemoryCompatExports.TryAllocateHleData(ctx, HandleStorageSize, 16, out handle)) { return false; @@ -318,18 +397,11 @@ public static class Ngs2Exports Span data = stackalloc byte[(int)HandleStorageSize]; data.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(data[0..8], handle); BinaryPrimitives.WriteUInt64LittleEndian(data[8..16], ownerHandle); BinaryPrimitives.WriteUInt32LittleEndian(data[16..20], 1); BinaryPrimitives.WriteUInt32LittleEndian(data[24..28], type); - if (!ctx.Memory.TryWrite(handle, data)) - { - return false; - } - - // Offset 0 doubles as a vtable slot for guest code that treats this handle as a C++ - // object; stamp it with the shared dummy vtable instead of a self-referential value. - KernelMemoryCompatExports.TryWriteDummyVtable(ctx, handle); - return true; + return ctx.Memory.TryWrite(handle, data); } private static bool TryClearGuestBuffer(CpuContext ctx, ulong address, ulong length) @@ -367,4 +439,37 @@ public static class Ngs2Exports Environment.GetEnvironmentVariable("SHARPEMU_LOG_NGS2"), "1", StringComparison.Ordinal); + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + [SysAbiExport( + Nid = "xa8oL9dmXkM", + ExportName = "sceNgs2PanInit", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2PanInit(CpuContext ctx) => ctx.SetReturn(0); + + [SysAbiExport( + Nid = "1WsleK-MTkE", + ExportName = "sceNgs2GeomCalcListener", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2GeomCalcListener(CpuContext ctx) => ctx.SetReturn(0); + + [SysAbiExport( + Nid = "0lbbayqDNoE", + ExportName = "sceNgs2GeomResetSourceParam", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2GeomResetSourceParam(CpuContext ctx) => ctx.SetReturn(0); + + [SysAbiExport( + Nid = "7Lcfo8SmpsU", + ExportName = "sceNgs2GeomResetListenerParam", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNgs2")] + public static int Ngs2GeomResetListenerParam(CpuContext ctx) => ctx.SetReturn(0); } diff --git a/src/SharpEmu.Libs/Np/NpManagerExports.cs b/src/SharpEmu.Libs/Np/NpManagerExports.cs index f4e6c08..e5d44fe 100644 --- a/src/SharpEmu.Libs/Np/NpManagerExports.cs +++ b/src/SharpEmu.Libs/Np/NpManagerExports.cs @@ -10,7 +10,7 @@ public static class NpManagerExports { private const int NpTitleIdSize = 16; private const int NpTitleSecretSize = 128; - private const int NpReachabilityStateUnavailable = 0; + private const int NpErrorInvalidArgument = unchecked((int)0x80550003); [SysAbiExport( Nid = "3Zl8BePTh9Y", @@ -58,35 +58,6 @@ public static class NpManagerExports return WriteOfflineOnlineId(ctx, ctx[CpuRegister.Rsi]); } - [SysAbiExport( - Nid = "rbknaUjpqWo", - ExportName = "sceNpGetAccountIdA", - Target = Generation.Gen5, - LibraryName = "libSceNpManager")] - public static int NpGetOnlineIdA(CpuContext ctx) - { - // The A variant takes a user ID before OnlineId. - return WriteOfflineOnlineId(ctx, ctx[CpuRegister.Rsi]); - } - - [SysAbiExport( - Nid = "e-ZuhGEoeC4", - ExportName = "sceNpGetNpReachabilityState", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceNpManager")] - public static int NpGetNpReachabilityState(CpuContext ctx) - { - var stateAddress = ctx[CpuRegister.Rsi]; - if (stateAddress == 0) - { - return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_INVALID_ARGUMENT); - } - - return ctx.TryWriteInt32(stateAddress, NpReachabilityStateUnavailable) - ? ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK) - : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - [SysAbiExport( Nid = "VfRSmPmj8Q8", ExportName = "sceNpRegisterStateCallback", @@ -140,6 +111,77 @@ public static class NpManagerExports : ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "rbknaUjpqWo", + ExportName = "sceNpGetAccountIdA", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNpManager")] + public static int NpGetAccountIdA(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var accountIdAddress = ctx[CpuRegister.Rsi]; + if (userId == -1 || accountIdAddress == 0) + { + return SetReturn(ctx, NpErrorInvalidArgument); + } + + // The offline profile exposed by sceNpGetState is signed in. Keep the + // account query consistent with that state: Unity's PSN integration + // treats SIGNED_OUT as an exceptional state and retries it every frame. + // A stable local-only id is sufficient for titles which only use the + // value as a profile key. + Span accountId = stackalloc byte[sizeof(ulong)]; + BinaryPrimitives.WriteUInt64LittleEndian(accountId, 1); + return ctx.Memory.TryWrite(accountIdAddress, accountId) + ? SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "JT+t00a3TxA", + ExportName = "sceNpGetAccountCountryA", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNpManager")] + public static int NpGetAccountCountryA(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var countryAddress = ctx[CpuRegister.Rsi]; + if (userId == -1 || countryAddress == 0) + { + return SetReturn(ctx, NpErrorInvalidArgument); + } + + Span country = stackalloc byte[4]; + country[0] = (byte)'U'; + country[1] = (byte)'S'; + country[2] = 0; + country[3] = 0; + return ctx.Memory.TryWrite(countryAddress, country) + ? SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "e-ZuhGEoeC4", + ExportName = "sceNpGetNpReachabilityState", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceNpManager")] + public static int NpGetNpReachabilityState(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var stateAddress = ctx[CpuRegister.Rsi]; + if (userId == -1 || stateAddress == 0) + { + return SetReturn(ctx, NpErrorInvalidArgument); + } + + Span state = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(state, 0); // Unavailable while offline. + return ctx.Memory.TryWrite(stateAddress, state) + ? SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_OK) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + [SysAbiExport( Nid = "Ec63y59l9tw", ExportName = "sceNpSetNpTitleId", @@ -166,6 +208,12 @@ public static class NpManagerExports return ctx.SetReturn(OrbisGen2Result.ORBIS_GEN2_OK); } + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + private static string ReadTitleId(ReadOnlySpan bytes) { var length = 0; diff --git a/src/SharpEmu.Libs/Pad/HostWindowInput.cs b/src/SharpEmu.Libs/Pad/HostWindowInput.cs index 9a609b6..7494cba 100644 --- a/src/SharpEmu.Libs/Pad/HostWindowInput.cs +++ b/src/SharpEmu.Libs/Pad/HostWindowInput.cs @@ -40,6 +40,11 @@ public static class HostWindowInput { keyboard.KeyDown += (_, key, _) => { + if (key == Key.F1) + { + VideoOut.PerfOverlay.Toggle(); + } + lock (Gate) { Pressed.Add(key); diff --git a/src/SharpEmu.Libs/Pad/ImeExports.cs b/src/SharpEmu.Libs/Pad/ImeExports.cs new file mode 100644 index 0000000..9cbee88 --- /dev/null +++ b/src/SharpEmu.Libs/Pad/ImeExports.cs @@ -0,0 +1,41 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Pad; + +public static class ImeExports +{ + private const int PrimaryUserId = 0x10000000; + private const int ImeErrorInvalidAddress = unchecked((int)0x80BC0001); + private const int ImeErrorInvalidUserId = unchecked((int)0x80BC0010); + private const int ImeErrorNotOpened = unchecked((int)0x80BC0005); + + private static bool _keyboardOpen; + public static int ImeKeyboardOpen(CpuContext ctx) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var parameterAddress = ctx[CpuRegister.Rsi]; + if (parameterAddress == 0) + { + return SetReturn(ctx, ImeErrorInvalidAddress); + } + + if (userId != PrimaryUserId) + { + return SetReturn(ctx, ImeErrorInvalidUserId); + } + + _keyboardOpen = true; + return SetReturn(ctx, 0); + } + public static int ImeUpdate(CpuContext ctx) => + SetReturn(ctx, _keyboardOpen ? 0 : ImeErrorNotOpened); + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/Pad/MouseExports.cs b/src/SharpEmu.Libs/Pad/MouseExports.cs new file mode 100644 index 0000000..f87ace5 --- /dev/null +++ b/src/SharpEmu.Libs/Pad/MouseExports.cs @@ -0,0 +1,100 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Buffers.Binary; +using System.Diagnostics; +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Pad; + +public static class MouseExports +{ + private const int PrimaryUserId = 0x10000000; + private const int MouseDataSize = 0x28; + private const int MouseErrorInvalidArgument = unchecked((int)0x80020002); + private const int MouseErrorInvalidHandle = unchecked((int)0x80020003); + private const int MouseErrorNotInitialized = unchecked((int)0x80020005); + private const int MouseErrorAlreadyOpened = unchecked((int)0x80020008); + + private static readonly bool[] OpenHandles = new bool[2]; + private static bool _initialized; + public static int MouseInit(CpuContext ctx) + { + _initialized = true; + Array.Clear(OpenHandles); + return SetReturn(ctx, 0); + } + public static int MouseOpen(CpuContext ctx) + { + if (!_initialized) + { + return SetReturn(ctx, MouseErrorNotInitialized); + } + + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var type = unchecked((int)ctx[CpuRegister.Rsi]); + var index = unchecked((int)ctx[CpuRegister.Rdx]); + if (userId != PrimaryUserId || type != 0 || index is < 0 or > 1) + { + return SetReturn(ctx, MouseErrorInvalidArgument); + } + + if (OpenHandles[index]) + { + return SetReturn(ctx, MouseErrorAlreadyOpened); + } + + OpenHandles[index] = true; + return SetReturn(ctx, index); + } + public static int MouseRead(CpuContext ctx) + { + var handle = unchecked((int)ctx[CpuRegister.Rdi]); + var dataAddress = ctx[CpuRegister.Rsi]; + var count = unchecked((int)ctx[CpuRegister.Rdx]); + if (dataAddress == 0 || count is < 1 or > 64) + { + return SetReturn(ctx, MouseErrorInvalidArgument); + } + + if (handle is < 0 or > 1 || !OpenHandles[handle]) + { + return SetReturn(ctx, MouseErrorInvalidHandle); + } + + Span data = stackalloc byte[MouseDataSize]; + data.Clear(); + var ticks = Stopwatch.GetTimestamp(); + var timestamp = + ((ulong)(ticks / Stopwatch.Frequency) * 1_000_000UL) + + ((ulong)(ticks % Stopwatch.Frequency) * 1_000_000UL / (ulong)Stopwatch.Frequency); + BinaryPrimitives.WriteUInt64LittleEndian(data, timestamp); + data[0x08] = 0; // No host mouse is exposed to the guest yet. + return ctx.Memory.TryWrite(dataAddress, data) + ? SetReturn(ctx, 1) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + [SysAbiExport( + Nid = "cAnT0Rw-IwU", + ExportName = "sceMouseClose", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceMouse")] + public static int MouseClose(CpuContext ctx) + { + var handle = unchecked((int)ctx[CpuRegister.Rdi]); + if (handle is < 0 or > 1 || !OpenHandles[handle]) + { + return SetReturn(ctx, MouseErrorInvalidHandle); + } + + OpenHandles[handle] = false; + return SetReturn(ctx, 0); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } +} diff --git a/src/SharpEmu.Libs/Pad/PadExports.cs b/src/SharpEmu.Libs/Pad/PadExports.cs index 54a595f..f5253ac 100644 --- a/src/SharpEmu.Libs/Pad/PadExports.cs +++ b/src/SharpEmu.Libs/Pad/PadExports.cs @@ -14,7 +14,11 @@ public static class PadExports private const int OrbisPadErrorNotInitialized = unchecked((int)0x80920005); private const int OrbisPadErrorDeviceNotConnected = unchecked((int)0x80920007); private const int OrbisPadErrorDeviceNoHandle = unchecked((int)0x80920008); - private const int PrimaryUserId = 1000; + // Keep the pad session on the same retail user id returned by + // libSceUserService. A mismatched emulator-local id makes games pass a + // valid 0x10000000 user to scePadOpen and receive DEVICE_NOT_CONNECTED, + // leaving every later keyboard/gamepad read on an invalid handle. + private const int PrimaryUserId = 0x10000000; private const int StandardPortType = 0; private const int PrimaryPadHandle = 1; private const int ControllerInformationSize = 0x1C; diff --git a/src/SharpEmu.Libs/SaveData/SaveDataExports.cs b/src/SharpEmu.Libs/SaveData/SaveDataExports.cs index b05a3ab..9d06104 100644 --- a/src/SharpEmu.Libs/SaveData/SaveDataExports.cs +++ b/src/SharpEmu.Libs/SaveData/SaveDataExports.cs @@ -30,19 +30,15 @@ public static class SaveDataExports private const uint MountModeCreate2 = 1u << 5; private const int MountResultSize = 0x40; private static readonly object _stateGate = new(); - private static readonly HashSet _transactionResources = []; private static readonly HashSet _preparedTransactionResources = []; private static string? _titleId; - private static int _nextTransactionResource; public static void ConfigureApplicationInfo(string? titleId) { lock (_stateGate) { _titleId = string.IsNullOrWhiteSpace(titleId) ? null : SanitizePathSegment(titleId.Trim()); - _transactionResources.Clear(); _preparedTransactionResources.Clear(); - _nextTransactionResource = 0; } } @@ -56,15 +52,15 @@ public static class SaveDataExports try { Directory.CreateDirectory(ResolveSaveDataRoot()); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } catch (IOException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (UnauthorizedAccessException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } } @@ -79,18 +75,18 @@ public static class SaveDataExports var resultAddress = ctx[CpuRegister.Rsi]; if (condAddress == 0 || resultAddress == 0) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } if (!TryReadSearchCond(ctx, condAddress, out var cond) || !TryReadSearchResult(ctx, resultAddress, out var result)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (cond.UserId < 0 || cond.SortKey > SortKeyFreeBlocks || cond.SortOrder > SortOrderDescent) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } try @@ -102,7 +98,7 @@ public static class SaveDataExports } else if (!TryReadFixedAscii(ctx, cond.TitleIdAddress, SaveDataTitleIdSize, out titleId)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } var root = ResolveTitleSaveRoot(cond.UserId, titleId); @@ -114,21 +110,21 @@ public static class SaveDataExports var setNum = result.DirNamesNum == 0 ? 0 : Math.Min(result.DirNamesNum, entries.Count); - if (!ctx.TryWriteUInt32(resultAddress + ResultHitNumOffset, checked((uint)entries.Count)) || - !ctx.TryWriteUInt32(resultAddress + ResultSetNumOffset, checked((uint)setNum))) + if (!TryWriteUInt32(ctx, resultAddress + ResultHitNumOffset, checked((uint)entries.Count)) || + !TryWriteUInt32(ctx, resultAddress + ResultSetNumOffset, checked((uint)setNum))) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (setNum == 0) { TraceSaveData($"dir_name_search user={cond.UserId} title={titleId} hits={entries.Count} set=0 root='{root}'"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } if (result.DirNamesAddress == 0) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } for (var i = 0; i < setNum; i++) @@ -144,20 +140,20 @@ public static class SaveDataExports (result.InfosAddress != 0 && !TryWriteSearchInfo(ctx, result.InfosAddress + ((ulong)i * SaveDataSearchInfoSize), entry))) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } } TraceSaveData($"dir_name_search user={cond.UserId} title={titleId} hits={entries.Count} set={setNum} root='{root}'"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } catch (IOException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (UnauthorizedAccessException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } } @@ -172,25 +168,25 @@ public static class SaveDataExports var resultAddress = ctx[CpuRegister.Rsi]; if (mountAddress == 0 || resultAddress == 0) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } - if (!ctx.TryReadInt32(mountAddress, out var userId) || + if (!TryReadInt32(ctx, mountAddress, out var userId) || !ctx.TryReadUInt64(mountAddress + 0x08, out var dirNameAddress) || !ctx.TryReadUInt64(mountAddress + 0x10, out var blocks) || !ctx.TryReadUInt64(mountAddress + 0x18, out var systemBlocks) || - !ctx.TryReadUInt32(mountAddress + 0x20, out var mountMode) || - !ctx.TryReadUInt32(mountAddress + 0x24, out var resource) || - !ctx.TryReadUInt32(mountAddress + 0x28, out var mode) || + !TryReadUInt32(ctx, mountAddress + 0x20, out var mountMode) || + !TryReadUInt32(ctx, mountAddress + 0x24, out var resource) || + !TryReadUInt32(ctx, mountAddress + 0x28, out var mode) || dirNameAddress == 0 || !TryReadFixedAscii(ctx, dirNameAddress, SaveDataDirNameSize, out var dirName)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } if (userId < 0 || string.IsNullOrWhiteSpace(dirName)) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } try @@ -205,12 +201,12 @@ public static class SaveDataExports if (!existed && !create && !createIfMissing) { - return ctx.SetReturn(OrbisSaveDataErrorNotFound); + return SetReturn(ctx, OrbisSaveDataErrorNotFound); } if (existed && create) { - return ctx.SetReturn(OrbisSaveDataErrorExists); + return SetReturn(ctx, OrbisSaveDataErrorExists); } if (!existed) @@ -227,29 +223,30 @@ public static class SaveDataExports BinaryPrimitives.WriteUInt32LittleEndian(result[0x1C..], createIfMissing && !existed ? 1u : 0u); if (!ctx.Memory.TryWrite(resultAddress, result)) { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } TraceSaveData( $"mount3 user={userId} title={titleId} dir={dirName} blocks={blocks} " + $"system_blocks={systemBlocks} mount_mode=0x{mountMode:X} resource={resource} mode={mode} " + $"mount_point={mountPoint} created={!existed} root='{savePath}'"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } catch (IOException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (UnauthorizedAccessException) { - return ctx.SetReturn(OrbisSaveDataErrorInternal); + return SetReturn(ctx, OrbisSaveDataErrorInternal); } catch (ArgumentException) { - return ctx.SetReturn(OrbisSaveDataErrorParameter); + return SetReturn(ctx, OrbisSaveDataErrorParameter); } } + private static int _nextTransactionResource; [SysAbiExport( Nid = "gjRZNnw0JPE", ExportName = "sceSaveDataCreateTransactionResource", @@ -257,16 +254,39 @@ public static class SaveDataExports LibraryName = "libSceSaveData")] public static int SaveDataCreateTransactionResource(CpuContext ctx) { - var memorySize = ctx[CpuRegister.Rdi]; - int resource; - lock (_stateGate) + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var reserved = ctx[CpuRegister.Rsi]; + + var id = (uint)Interlocked.Increment(ref _nextTransactionResource); + + // The resource-out pointer's argument slot varies by SDK revision: some + // callers pass it in rdx, others in rcx (a 4-arg form where rdx holds a + // count/flag). Void Terrarium passes rdx=0x1 (not a pointer) and the + // real out-pointer in rcx. Probe the plausible candidates and write the + // handle to the first writable one instead of faulting on a bad rdx. + // This is a stub-level create (matches shadPS4's return-OK semantics); + // never return MEMORY_FAULT for it, or the guest treats savedata init as + // failed and never advances. + var resourceAddress = 0UL; + foreach (var candidate in new[] + { + ctx[CpuRegister.Rdx], + ctx[CpuRegister.Rcx], + ctx[CpuRegister.R8], + ctx[CpuRegister.R9], + }) { - resource = ++_nextTransactionResource; - _transactionResources.Add(resource); + if (candidate != 0 && TryWriteUInt32(ctx, candidate, id)) + { + resourceAddress = candidate; + break; + } } - TraceSaveData($"create_transaction_resource memory_size=0x{memorySize:X} resource={resource}"); - return ctx.SetReturn(resource); + TraceSaveData( + $"create_transaction_resource user={userId} reserved=0x{reserved:X} resource_addr=0x{resourceAddress:X} id={id}"); + + return SetReturn(ctx, 0); } [SysAbiExport( @@ -279,70 +299,11 @@ public static class SaveDataExports var resource = unchecked((int)ctx[CpuRegister.Rdi]); lock (_stateGate) { - _transactionResources.Remove(resource); _preparedTransactionResources.Remove(resource); } TraceSaveData($"delete_transaction_resource resource={resource}"); - return ctx.SetReturn(0); - } - - [SysAbiExport( - Nid = "sDCBrmc61XU", - ExportName = "sceSaveDataPrepare", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceSaveData")] - public static int SaveDataPrepare(CpuContext ctx) - { - var mountPointAddress = ctx[CpuRegister.Rdi]; - var resource = unchecked((int)ctx[CpuRegister.Rdx]); - if (mountPointAddress == 0) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - if (!TryReadFixedAscii(ctx, mountPointAddress, 16, out var mountPoint)) - { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - if (string.IsNullOrWhiteSpace(mountPoint)) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - lock (_stateGate) - { - if (resource != 0) - { - _preparedTransactionResources.Add(resource); - } - } - - TraceSaveData($"prepare mount_point={mountPoint} resource={resource}"); - return ctx.SetReturn(0); - } - - [SysAbiExport( - Nid = "ie7qhZ4X0Cc", - ExportName = "sceSaveDataCommit", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceSaveData")] - public static int SaveDataCommit(CpuContext ctx) - { - var commitAddress = ctx[CpuRegister.Rdi]; - if (commitAddress == 0) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - lock (_stateGate) - { - _preparedTransactionResources.Clear(); - } - - TraceSaveData($"commit commit=0x{commitAddress:X16}"); - return ctx.SetReturn(0); + return SetReturn(ctx, 0); } [SysAbiExport( @@ -352,40 +313,21 @@ public static class SaveDataExports LibraryName = "libSceSaveData")] public static int SaveDataUmount2(CpuContext ctx) { - var mountPointAddress = ctx[CpuRegister.Rdi]; - if (mountPointAddress == 0) - { - mountPointAddress = ctx[CpuRegister.Rsi]; - } - - if (mountPointAddress == 0) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - if (!TryReadFixedAscii(ctx, mountPointAddress, 16, out var mountPoint)) - { - return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); - } - - if (string.IsNullOrWhiteSpace(mountPoint)) - { - return ctx.SetReturn(OrbisSaveDataErrorParameter); - } - - var unmounted = KernelMemoryCompatExports.TryUnregisterGuestPathMount(mountPoint); - TraceSaveData($"umount2 mount_point={mountPoint} unregistered={unmounted}"); - return ctx.SetReturn(0); + // Unmounting a save directory always succeeds in the stub filesystem; + // returning an error here makes the game's save flow stall before it + // hands control to the title/gameplay state. + TraceSaveData($"umount2 user={unchecked((int)ctx[CpuRegister.Rdi])}"); + return SetReturn(ctx, 0); } private static bool TryReadSearchCond(CpuContext ctx, ulong address, out SearchCond cond) { cond = default; - if (!ctx.TryReadInt32(address, out var userId) || + if (!TryReadInt32(ctx, address, out var userId) || !ctx.TryReadUInt64(address + 0x08, out var titleIdAddress) || !ctx.TryReadUInt64(address + 0x10, out var dirNameAddress) || - !ctx.TryReadUInt32(address + 0x18, out var sortKey) || - !ctx.TryReadUInt32(address + 0x1C, out var sortOrder)) + !TryReadUInt32(ctx, address + 0x18, out var sortKey) || + !TryReadUInt32(ctx, address + 0x1C, out var sortOrder)) { return false; } @@ -408,7 +350,7 @@ public static class SaveDataExports { result = default; if (!ctx.TryReadUInt64(address + ResultDirNamesOffset, out var dirNamesAddress) || - !ctx.TryReadUInt32(address + ResultDirNamesNumOffset, out var dirNamesNum) || + !TryReadUInt32(ctx, address + ResultDirNamesNumOffset, out var dirNamesNum) || !ctx.TryReadUInt64(address + ResultParamsOffset, out var paramsAddress) || !ctx.TryReadUInt64(address + ResultInfosOffset, out var infosAddress)) { @@ -609,6 +551,45 @@ public static class SaveDataExports } } + private static bool TryReadInt32(CpuContext ctx, ulong address, out int value) + { + Span bytes = stackalloc byte[sizeof(int)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadInt32LittleEndian(bytes); + return true; + } + + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + + private static bool TryWriteUInt32(CpuContext ctx, ulong address, uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + private static void TraceSaveData(string message) { if (string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_SAVEDATA"), "1", StringComparison.Ordinal)) @@ -631,4 +612,62 @@ public static class SaveDataExports ulong InfosAddress); private readonly record struct SaveEntry(string Name, string Path, DateTime LastWriteUtc); + + [SysAbiExport( + Nid = "sDCBrmc61XU", + ExportName = "sceSaveDataPrepare", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSaveData")] + public static int SaveDataPrepare(CpuContext ctx) + { + var mountPointAddress = ctx[CpuRegister.Rdi]; + var resource = unchecked((int)ctx[CpuRegister.Rdx]); + if (mountPointAddress == 0) + { + return ctx.SetReturn(OrbisSaveDataErrorParameter); + } + + if (!TryReadFixedAscii(ctx, mountPointAddress, 16, out var mountPoint)) + { + return ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + if (string.IsNullOrWhiteSpace(mountPoint)) + { + return ctx.SetReturn(OrbisSaveDataErrorParameter); + } + + lock (_stateGate) + { + if (resource != 0) + { + _preparedTransactionResources.Add(resource); + } + } + + TraceSaveData($"prepare mount_point={mountPoint} resource={resource}"); + return ctx.SetReturn(0); + } + + [SysAbiExport( + Nid = "ie7qhZ4X0Cc", + ExportName = "sceSaveDataCommit", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSaveData")] + public static int SaveDataCommit(CpuContext ctx) + { + var commitAddress = ctx[CpuRegister.Rdi]; + if (commitAddress == 0) + { + return ctx.SetReturn(OrbisSaveDataErrorParameter); + } + + lock (_stateGate) + { + _preparedTransactionResources.Clear(); + } + + TraceSaveData($"commit commit=0x{commitAddress:X16}"); + return ctx.SetReturn(0); + } } diff --git a/src/SharpEmu.Libs/Stubs/GameServiceStubs.cs b/src/SharpEmu.Libs/Stubs/GameServiceStubs.cs new file mode 100644 index 0000000..4915bcd --- /dev/null +++ b/src/SharpEmu.Libs/Stubs/GameServiceStubs.cs @@ -0,0 +1,62 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; + +namespace SharpEmu.Libs.Stubs; + +/// +/// Success stubs for trophy, character-encoding and telemetry ABI calls that +/// Void Terrarium (and other titles) invoke during startup. They were +/// previously unresolved and returned NOT_FOUND, and a title that gates its +/// UI/text initialization on these succeeding then skips ahead and never draws +/// its content (a black screen with only a clear pass). These return success +/// (and a non-zero handle where an out pointer is expected) so init proceeds. +/// +public static class GameServiceStubs +{ + private static int Ok(CpuContext ctx) + { + ctx[CpuRegister.Rax] = 0; + return 0; + } + + // Writes a small non-zero handle to the pointer in the given register so + // the caller treats the object as created; returns success. + private static int OkWithHandle(CpuContext ctx, CpuRegister outPointerRegister) + { + var outAddress = ctx[outPointerRegister]; + if (outAddress != 0) + { + Span handle = stackalloc byte[sizeof(int)]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(handle, 1); + _ = ctx.Memory.TryWrite(outAddress, handle); + } + + return Ok(ctx); + } + + // ---- NpTrophy2: trophy context/handle registration at boot ---- + public static int NpTrophy2CreateContext(CpuContext ctx) => OkWithHandle(ctx, CpuRegister.Rdi); + public static int NpTrophy2CreateHandle(CpuContext ctx) => OkWithHandle(ctx, CpuRegister.Rdi); + public static int NpTrophy2RegisterContext(CpuContext ctx) => Ok(ctx); + + [SysAbiExport(Nid = "4IzqhhUQ3nk", ExportName = "sceNpTrophy2GetGameInfo", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceNpTrophy2")] + public static int NpTrophy2GetGameInfo(CpuContext ctx) => Ok(ctx); + + // ---- CES: Shift-JIS <-> Unicode conversion setup (Japanese text) ---- + + [SysAbiExport(Nid = "ZiDCxUUGbec", ExportName = "sceCesUcsProfileInitSJis1997Cp932", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceLibcInternal")] + public static int CesUcsProfileInitSJis1997Cp932(CpuContext ctx) => Ok(ctx); + + [SysAbiExport(Nid = "538bRGc6Zo8", ExportName = "sceCesMbcsUcsContextInit", + Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceLibcInternal")] + public static int CesMbcsUcsContextInit(CpuContext ctx) => Ok(ctx); + + // ---- NpUniversalDataSystem: gameplay telemetry events ---- + public static int NpUniversalDataSystemCreateEvent(CpuContext ctx) => OkWithHandle(ctx, CpuRegister.Rdi); + public static int NpUniversalDataSystemPostEvent(CpuContext ctx) => Ok(ctx); + public static int NpUniversalDataSystemDestroyEvent(CpuContext ctx) => Ok(ctx); +} diff --git a/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs b/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs index f172c48..d0a35b0 100644 --- a/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs +++ b/src/SharpEmu.Libs/SystemService/SystemServiceExports.cs @@ -13,6 +13,7 @@ public static class SystemServiceExports private const int OrbisSystemServiceErrorParameter = unchecked((int)0x80A10003); private const int SystemServiceStatusSize = 0x0C; private const int DisplaySafeAreaInfoSize = sizeof(float) + 128; + private const int HdrToneMapLuminanceSize = sizeof(float) * 3; [SysAbiExport( Nid = "fZo48un7LK4", @@ -42,6 +43,35 @@ public static class SystemServiceExports : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "SsC-m-S9JTA", + ExportName = "sceSystemServiceParamGetString", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSystemService")] + public static int SystemServiceParamGetString(CpuContext ctx) + { + _ = unchecked((int)ctx[CpuRegister.Rdi]); // parameter id (nickname, etc.) + var bufferAddress = ctx[CpuRegister.Rsi]; + var bufferSize = unchecked((int)ctx[CpuRegister.Rdx]); + if (bufferAddress == 0 || bufferSize <= 0) + { + return ctx.SetReturn(OrbisSystemServiceErrorParameter); + } + + // String params are typically the user nickname. Callers that gate UI + // or text setup on a successful read (and skip it on failure) stall on + // a black screen when this returns NOT_FOUND, so return a neutral + // non-empty default and success. + var value = System.Text.Encoding.UTF8.GetBytes("SharpEmu"); + var writeLength = Math.Min(value.Length, bufferSize - 1); + Span output = stackalloc byte[writeLength + 1]; + value.AsSpan(0, writeLength).CopyTo(output); + output[writeLength] = 0; + return ctx.Memory.TryWrite(bufferAddress, output) + ? ctx.SetReturn(0) + : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + [SysAbiExport( Nid = "rPo6tV8D9bM", ExportName = "sceSystemServiceGetStatus", @@ -87,6 +117,28 @@ public static class SystemServiceExports : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } + [SysAbiExport( + Nid = "mPpPxv5CZt4", + ExportName = "sceSystemServiceGetHdrToneMapLuminance", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceSystemService")] + public static int SystemServiceGetHdrToneMapLuminance(CpuContext ctx) + { + var luminanceAddress = ctx[CpuRegister.Rdi]; + if (luminanceAddress == 0) + { + return ctx.SetReturn(OrbisSystemServiceErrorParameter); + } + + Span luminance = stackalloc byte[HdrToneMapLuminanceSize]; + BinaryPrimitives.WriteSingleLittleEndian(luminance, 1000.0f); + BinaryPrimitives.WriteSingleLittleEndian(luminance[sizeof(float)..], 1000.0f); + BinaryPrimitives.WriteSingleLittleEndian(luminance[(sizeof(float) * 2)..], 0.01f); + return ctx.Memory.TryWrite(luminanceAddress, luminance) + ? ctx.SetReturn(0) + : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + [SysAbiExport( Nid = "Vo5V8KAwCmk", ExportName = "sceSystemServiceHideSplashScreen", diff --git a/src/SharpEmu.Libs/UserService/UserServiceExports.cs b/src/SharpEmu.Libs/UserService/UserServiceExports.cs index a8c5ce1..2a4724d 100644 --- a/src/SharpEmu.Libs/UserService/UserServiceExports.cs +++ b/src/SharpEmu.Libs/UserService/UserServiceExports.cs @@ -4,6 +4,7 @@ using SharpEmu.HLE; using System.Buffers.Binary; using System.Text; +using System.Threading; namespace SharpEmu.Libs.UserService; @@ -13,15 +14,14 @@ public static class UserServiceExports private const int OrbisUserServiceErrorNoEvent = unchecked((int)0x80960007); private const int OrbisUserServiceErrorInvalidParameter = unchecked((int)0x80960009); private const int OrbisUserServiceErrorBufferTooShort = unchecked((int)0x8096000A); - // Retail user-service IDs begin at 0x3E8. - private const int PrimaryUserId = 1000; + // Retail user ids encode their local user slot in the 0x10000000 range; + // the first signed-in user maps to slot 0. Small emulator-local ids do + // not pass Unity's user-id-to-slot conversion and become slot -1. + private const int PrimaryUserId = 0x10000000; private const int InvalidUserId = -1; private const string PrimaryUserName = "SharpEmu"; private static int _loginEventDelivered; - private static readonly bool _traceUserService = - string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_USER_SERVICE"), "1", StringComparison.Ordinal); - [SysAbiExport( Nid = "j3YMu1MVNNo", ExportName = "sceUserServiceInitialize", @@ -29,6 +29,7 @@ public static class UserServiceExports LibraryName = "libSceUserService")] public static int UserServiceInitialize(CpuContext ctx) { + Trace("initialize"); ctx[CpuRegister.Rax] = 0; return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -43,14 +44,12 @@ public static class UserServiceExports var userIdAddress = ctx[CpuRegister.Rdi]; if (userIdAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } - var result = ctx.TryWriteInt32(userIdAddress, PrimaryUserId) - ? 0 - : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - TraceUserService($"get_initial_user out=0x{userIdAddress:X16} value={PrimaryUserId} result=0x{result:X8}"); - return ctx.SetReturn(result); + return TryWriteInt32(ctx, userIdAddress, PrimaryUserId) + ? SetReturnWithTrace(ctx, 0, $"get_initial_user user={PrimaryUserId} out=0x{userIdAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -63,7 +62,7 @@ public static class UserServiceExports var userIdListAddress = ctx[CpuRegister.Rdi]; if (userIdListAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } Span userIds = stackalloc byte[sizeof(int) * 4]; @@ -72,8 +71,12 @@ public static class UserServiceExports BinaryPrimitives.WriteInt32LittleEndian(userIds[0x08..], InvalidUserId); BinaryPrimitives.WriteInt32LittleEndian(userIds[0x0C..], InvalidUserId); return ctx.Memory.TryWrite(userIdListAddress, userIds) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturnWithTrace( + ctx, + 0, + $"get_login_user_id_list users=[{PrimaryUserId},{InvalidUserId},{InvalidUserId},{InvalidUserId}] " + + $"out=0x{userIdListAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -86,20 +89,23 @@ public static class UserServiceExports var eventAddress = ctx[CpuRegister.Rdi]; if (eventAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } if (Interlocked.Exchange(ref _loginEventDelivered, 1) != 0) { - return ctx.SetReturn(OrbisUserServiceErrorNoEvent); + return SetReturn(ctx, OrbisUserServiceErrorNoEvent); } Span payload = stackalloc byte[sizeof(int) * 2]; BinaryPrimitives.WriteInt32LittleEndian(payload[0..], 0); BinaryPrimitives.WriteInt32LittleEndian(payload[sizeof(int)..], PrimaryUserId); return ctx.Memory.TryWrite(eventAddress, payload) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + ? SetReturnWithTrace( + ctx, + 0, + $"get_event type=login user={PrimaryUserId} out=0x{eventAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } [SysAbiExport( @@ -112,53 +118,30 @@ public static class UserServiceExports var userId = unchecked((int)ctx[CpuRegister.Rdi]); var nameAddress = ctx[CpuRegister.Rsi]; var capacity = ctx[CpuRegister.Rdx]; - // Zero selects the current user. - if (userId != 0 && userId != PrimaryUserId) + if (userId != PrimaryUserId) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidParameter); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); } if (nameAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } var nameBytes = Encoding.UTF8.GetBytes(PrimaryUserName); if (capacity <= (ulong)nameBytes.Length) { - return ctx.SetReturn(OrbisUserServiceErrorBufferTooShort); + return SetReturn(ctx, OrbisUserServiceErrorBufferTooShort); } Span output = stackalloc byte[nameBytes.Length + 1]; nameBytes.CopyTo(output); - var result = ctx.Memory.TryWrite(nameAddress, output) - ? 0 - : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; - TraceUserService( - $"get_user_name user={userId} out=0x{nameAddress:X16} capacity=0x{capacity:X} value='{PrimaryUserName}' result=0x{result:X8}"); - return ctx.SetReturn(result); - } - - [SysAbiExport( - Nid = "-sD02mFDBh4", - ExportName = "sceUserServiceGetGamePresets", - Target = Generation.Gen5, - LibraryName = "libSceUserService")] - public static int UserServiceGetGamePresets(CpuContext ctx) - { - // Return deterministic defaults for the offline profile. - var userId = unchecked((int)ctx[CpuRegister.Rdi]); - var resultAddress = ctx[CpuRegister.Rcx]; - Span defaults = stackalloc byte[0x18]; - if (resultAddress == 0 || !ctx.Memory.TryWrite(resultAddress, defaults)) - { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); - } - - TraceUserService( - $"get_game_presets user={userId} title=0x{ctx[CpuRegister.Rsi]:X16} " + - $"key=0x{ctx[CpuRegister.R9]:X} out=0x{resultAddress:X16} result=0x00000000"); - return ctx.SetReturn(0); + return ctx.Memory.TryWrite(nameAddress, output) + ? SetReturnWithTrace( + ctx, + 0, + $"get_user_name user={userId} name='{PrimaryUserName}' out=0x{nameAddress:X16}") + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } // Name not yet in ps5_names.txt and the NID was captured from titles; revisit when the symbol is catalogued. @@ -174,50 +157,165 @@ public static class UserServiceExports var valueAddress = ctx[CpuRegister.Rsi]; if (parameterId != 1000) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidParameter); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); } if (valueAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } - return ctx.TryWriteInt32(valueAddress, 0) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + return TryWriteInt32(ctx, valueAddress, 0) + ? SetReturn(ctx, 0) + : SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); } #pragma warning restore SHEM006 [SysAbiExport( Nid = "woNpu+45RLk", ExportName = "sceUserServiceGetAgeLevel", - Target = Generation.Gen4 | Generation.Gen5, + Target = Generation.Gen5, LibraryName = "libSceUserService")] - public static int UserServiceGetAgeLevel(CpuContext ctx) + public static int UserServiceGetAgeLevel(CpuContext ctx) => + WriteUserSettingInt32(ctx, 18, "get_age_level"); + + [SysAbiExport( + Nid = "-sD02mFDBh4", + ExportName = "sceUserServiceGetGamePresets", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetGamePresets(CpuContext ctx) { var userId = unchecked((int)ctx[CpuRegister.Rdi]); - var ageLevelAddress = ctx[CpuRegister.Rsi]; - if (userId != 1000) + var presetsAddress = ctx[CpuRegister.Rsi]; + if (userId != PrimaryUserId) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidParameter); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); } - if (ageLevelAddress == 0) + if (presetsAddress == 0) { - return ctx.SetReturn(OrbisUserServiceErrorInvalidArgument); + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); } - // Report an adult account so titles skip parental-restriction paths. - return ctx.TryWriteInt32(ageLevelAddress, 21) - ? ctx.SetReturn(0) - : ctx.SetReturn((int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + Span presets = stackalloc byte[0x28]; + presets.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(presets, (ulong)presets.Length); + if (!ctx.Memory.TryWrite(presetsAddress, presets)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetReturnWithTrace( + ctx, + 0, + $"get_game_presets user={userId} out=0x{presetsAddress:X16}"); } - private static void TraceUserService(string message) + [SysAbiExport( + Nid = "rnEhHqG-4xo", + ExportName = "sceUserServiceGetAccessibilityChatTranscription", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityChatTranscription(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_chat_transcription"); + + [SysAbiExport( + Nid = "ZKJtxdgvzwg", + ExportName = "sceUserServiceGetAccessibilityPressAndHoldDelay", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityPressAndHoldDelay(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_press_and_hold_delay"); + + [SysAbiExport( + Nid = "-3Y5GO+-i78", + ExportName = "sceUserServiceGetAccessibilityTriggerEffect", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityTriggerEffect(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_trigger_effect"); + + [SysAbiExport( + Nid = "qWYHOFwqCxY", + ExportName = "sceUserServiceGetAccessibilityVibration", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityVibration(CpuContext ctx) => + WriteUserSettingInt32(ctx, 1, "get_accessibility_vibration"); + + [SysAbiExport( + Nid = "hD-H81EN9Vg", + ExportName = "sceUserServiceGetAccessibilityZoomEnabled", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityZoomEnabled(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_zoom_enabled"); + + [SysAbiExport( + Nid = "O6IW1-Dwm-w", + ExportName = "sceUserServiceGetAccessibilityZoomFollowFocus", + Target = Generation.Gen5, + LibraryName = "libSceUserService")] + public static int UserServiceGetAccessibilityZoomFollowFocus(CpuContext ctx) => + WriteUserSettingInt32(ctx, 0, "get_accessibility_zoom_follow_focus"); + + private static bool TryWriteInt32(CpuContext ctx, ulong address, int value) { - if (_traceUserService) + Span bytes = stackalloc byte[sizeof(int)]; + BinaryPrimitives.WriteInt32LittleEndian(bytes, value); + return ctx.Memory.TryWrite(address, bytes); + } + + private static int WriteUserSettingInt32(CpuContext ctx, int value, string operation) + { + var userId = unchecked((int)ctx[CpuRegister.Rdi]); + var valueAddress = ctx[CpuRegister.Rsi]; + if (userId != PrimaryUserId) { - Console.Error.WriteLine($"[LOADER][TRACE] user_service.{message}"); + return SetReturn(ctx, OrbisUserServiceErrorInvalidParameter); + } + + if (valueAddress == 0) + { + return SetReturn(ctx, OrbisUserServiceErrorInvalidArgument); + } + + if (!TryWriteInt32(ctx, valueAddress, value)) + { + return SetReturn(ctx, (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT); + } + + return SetReturnWithTrace( + ctx, + 0, + $"{operation} user={userId} value={value} out=0x{valueAddress:X16}"); + } + + private static int SetReturn(CpuContext ctx, int result) + { + ctx[CpuRegister.Rax] = unchecked((ulong)result); + return result; + } + + private static int SetReturnWithTrace(CpuContext ctx, int result, string message) + { + Trace(message); + return SetReturn(ctx, result); + } + + private static void Trace(string message) + { + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_USER_SERVICE"), + "1", + StringComparison.Ordinal)) + { + var returnRip = GuestThreadExecution.TryGetCurrentImportCallFrame(out var frame) + ? frame.ReturnRip + : 0; + Console.Error.WriteLine( + $"[LOADER][TRACE] user_service.{message} ret=0x{returnRip:X16}"); } } } diff --git a/src/SharpEmu.Libs/VideoOut/PerfOverlay.cs b/src/SharpEmu.Libs/VideoOut/PerfOverlay.cs new file mode 100644 index 0000000..fa7d8a2 --- /dev/null +++ b/src/SharpEmu.Libs/VideoOut/PerfOverlay.cs @@ -0,0 +1,363 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using System.Diagnostics; + +namespace SharpEmu.Libs.VideoOut; + +/// +/// In-window performance HUD. The panel is rasterized on the CPU into a +/// small BGRA buffer each frame (embedded 5x7 font, no assets) and the +/// presenter blits it onto the swapchain image, so it needs no pipelines, +/// descriptors or blending state. Toggled with F1; SHARPEMU_OVERLAY=0 +/// starts it hidden. +/// +public static class PerfOverlay +{ + public const int PanelWidth = 376; + public const int PanelHeight = 176; + + private const int GlyphColumns = 5; + private const int GlyphRows = 7; + private const int Scale = 2; + private const int CellWidth = (GlyphColumns + 1) * Scale; + private const int LineHeight = (GlyphRows + 2) * Scale; + private const int FrameHistorySize = 128; + + private static volatile bool _enabled = !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_OVERLAY"), + "0", + StringComparison.Ordinal); + + private static long _lastPresentTimestamp; + private static readonly double[] _frameMilliseconds = new double[FrameHistorySize]; + private static int _frameHistoryIndex; + private static long _presentedInWindow; + private static long _submittedInWindow; + private static long _drawsInWindow; + + // Refreshed once per second so per-frame fills never allocate. + private static long _statsWindowStart = Stopwatch.GetTimestamp(); + private static double _fps; + private static double _submittedFps; + private static double _drawsPerSecond; + private static double _averageFrameMs; + private static double _allocatedMbPerSecond; + private static long _lastAllocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + private static int _lastGen0 = GC.CollectionCount(0); + private static int _lastGen1 = GC.CollectionCount(1); + private static int _lastGen2 = GC.CollectionCount(2); + private static int _gen0PerWindow; + private static int _gen1PerWindow; + private static int _gen2PerWindow; + private static double _cpuPercent; + private static TimeSpan _lastCpuTime = GetProcessCpuTime(); + private static string _line1 = string.Empty; + private static string _line2 = string.Empty; + private static string _line3 = string.Empty; + private static string _line4 = string.Empty; + + public static bool Enabled => _enabled; + + public static void Toggle() => _enabled = !_enabled; + + /// Called by the presenter after each successful present. + public static void RecordPresent() + { + var now = Stopwatch.GetTimestamp(); + var last = _lastPresentTimestamp; + _lastPresentTimestamp = now; + Interlocked.Increment(ref _presentedInWindow); + if (last != 0) + { + var milliseconds = (now - last) * 1000.0 / Stopwatch.Frequency; + if (milliseconds < 1000.0) + { + _frameMilliseconds[_frameHistoryIndex] = milliseconds; + _frameHistoryIndex = (_frameHistoryIndex + 1) % FrameHistorySize; + } + } + } + + /// Called on every guest flip submission. + public static void RecordSubmit() => Interlocked.Increment(ref _submittedInWindow); + + /// Called per translated draw/dispatch executed. + public static void RecordDraw() => Interlocked.Increment(ref _drawsInWindow); + + /// + /// Rasterizes the panel into a BGRA byte span of PanelWidth x PanelHeight. + /// Runs on the render thread. + /// + public static void Fill(Span bgra, int pendingWork, int inFlightSubmissions) + { + RefreshStatsIfDue(pendingWork, inFlightSubmissions); + + // Background: opaque dark slate. + for (var i = 0; i < bgra.Length; i += 4) + { + bgra[i] = 0x20; + bgra[i + 1] = 0x18; + bgra[i + 2] = 0x14; + bgra[i + 3] = 0xFF; + } + + var y = 6; + DrawString(bgra, 8, y, _line1, 0x60, 0xFF, 0x60); + y += LineHeight; + DrawString(bgra, 8, y, _line2, 0xFF, 0xFF, 0xFF); + y += LineHeight; + DrawString(bgra, 8, y, _line3, 0xB0, 0xD0, 0xFF); + y += LineHeight; + DrawString(bgra, 8, y, _line4, 0xB0, 0xB0, 0xB0); + y += LineHeight + 4; + DrawFrameGraph(bgra, 8, y, PanelWidth - 16, PanelHeight - y - 6); + } + + private static void RefreshStatsIfDue(int pendingWork, int inFlightSubmissions) + { + var now = Stopwatch.GetTimestamp(); + var elapsedTicks = now - _statsWindowStart; + if (elapsedTicks >= Stopwatch.Frequency) + { + var seconds = (double)elapsedTicks / Stopwatch.Frequency; + _statsWindowStart = now; + _fps = Interlocked.Exchange(ref _presentedInWindow, 0) / seconds; + _submittedFps = Interlocked.Exchange(ref _submittedInWindow, 0) / seconds; + _drawsPerSecond = Interlocked.Exchange(ref _drawsInWindow, 0) / seconds; + + double totalMs = 0; + var samples = 0; + foreach (var ms in _frameMilliseconds) + { + if (ms > 0) + { + totalMs += ms; + samples++; + } + } + + _averageFrameMs = samples > 0 ? totalMs / samples : 0; + + var allocated = GC.GetTotalAllocatedBytes(precise: false); + _allocatedMbPerSecond = (allocated - _lastAllocatedBytes) / seconds / (1024.0 * 1024.0); + _lastAllocatedBytes = allocated; + + var gen0 = GC.CollectionCount(0); + var gen1 = GC.CollectionCount(1); + var gen2 = GC.CollectionCount(2); + _gen0PerWindow = gen0 - _lastGen0; + _gen1PerWindow = gen1 - _lastGen1; + _gen2PerWindow = gen2 - _lastGen2; + _lastGen0 = gen0; + _lastGen1 = gen1; + _lastGen2 = gen2; + + var cpuTime = GetProcessCpuTime(); + _cpuPercent = (cpuTime - _lastCpuTime).TotalSeconds / seconds * 100.0; + _lastCpuTime = cpuTime; + + var drawsPerFrame = _fps > 0.5 ? _drawsPerSecond / _fps : 0; + _line1 = $"FPS {_fps:0.0} FLIP {_submittedFps:0.0} {_averageFrameMs:0.0} MS"; + _line2 = $"DRAWS {_drawsPerSecond:0}/S {drawsPerFrame:0}/F Q {pendingWork}+{inFlightSubmissions}"; + _line3 = $"ALLOC {_allocatedMbPerSecond:0.0} MB/S GC {_gen0PerWindow}/{_gen1PerWindow}/{_gen2PerWindow}"; + _line4 = $"CPU {_cpuPercent:0}% HEAP {GC.GetTotalMemory(false) / (1024 * 1024)} MB F1 HIDE"; + } + } + + private static TimeSpan GetProcessCpuTime() + { + var usage = Environment.CpuUsage; + return usage.UserTime + usage.PrivilegedTime; + } + + private static void DrawFrameGraph(Span bgra, int x, int y, int width, int height) + { + if (height < 8) + { + return; + } + + // Reference lines at 16.7ms and 33.3ms of a 40ms-tall scale. + const double maxMs = 40.0; + DrawHorizontalLine(bgra, x, y + height - (int)(16.7 / maxMs * height), width, 0x30, 0x60, 0x30); + DrawHorizontalLine(bgra, x, y + height - (int)(33.3 / maxMs * height), width, 0x30, 0x30, 0x60); + + var barWidth = Math.Max(1, width / FrameHistorySize); + for (var i = 0; i < FrameHistorySize; i++) + { + var ms = _frameMilliseconds[(_frameHistoryIndex + i) % FrameHistorySize]; + if (ms <= 0) + { + continue; + } + + var barHeight = (int)Math.Min(height, ms / maxMs * height); + var barX = x + i * barWidth; + if (barX + barWidth > x + width) + { + break; + } + + byte r, g, b; + if (ms <= 17.5) + { + (r, g, b) = ((byte)0x40, (byte)0xE0, (byte)0x40); + } + else if (ms <= 34.0) + { + (r, g, b) = ((byte)0xE0, (byte)0xC0, (byte)0x30); + } + else + { + (r, g, b) = ((byte)0xE8, (byte)0x40, (byte)0x40); + } + + for (var py = 0; py < barHeight; py++) + { + var rowY = y + height - 1 - py; + for (var px = 0; px < barWidth; px++) + { + SetPixel(bgra, barX + px, rowY, r, g, b); + } + } + } + } + + private static void DrawHorizontalLine(Span bgra, int x, int y, int width, byte r, byte g, byte b) + { + if (y < 0 || y >= PanelHeight) + { + return; + } + + for (var px = 0; px < width; px++) + { + SetPixel(bgra, x + px, y, r, g, b); + } + } + + private static void DrawString(Span bgra, int x, int y, string text, byte r, byte g, byte b) + { + var penX = x; + foreach (var rawChar in text) + { + var c = char.ToUpperInvariant(rawChar); + if (c < ' ' || c > 'Z') + { + c = '?'; + } + + var glyph = Font.Slice((c - ' ') * GlyphColumns, GlyphColumns); + for (var column = 0; column < GlyphColumns; column++) + { + var bits = glyph[column]; + for (var row = 0; row < GlyphRows; row++) + { + if ((bits & (1 << row)) == 0) + { + continue; + } + + for (var sy = 0; sy < Scale; sy++) + { + for (var sx = 0; sx < Scale; sx++) + { + SetPixel( + bgra, + penX + column * Scale + sx, + y + row * Scale + sy, + r, + g, + b); + } + } + } + } + + penX += CellWidth; + if (penX + CellWidth > PanelWidth) + { + break; + } + } + } + + private static void SetPixel(Span bgra, int x, int y, byte r, byte g, byte b) + { + if ((uint)x >= PanelWidth || (uint)y >= PanelHeight) + { + return; + } + + var offset = (y * PanelWidth + x) * 4; + bgra[offset] = b; + bgra[offset + 1] = g; + bgra[offset + 2] = r; + bgra[offset + 3] = 0xFF; + } + + // Classic 5x7 column-encoded font (bit 0 = top row), ASCII 32..90. + private static ReadOnlySpan Font => + [ + 0x00, 0x00, 0x00, 0x00, 0x00, // ' ' + 0x00, 0x00, 0x5F, 0x00, 0x00, // '!' + 0x00, 0x07, 0x00, 0x07, 0x00, // '"' + 0x14, 0x7F, 0x14, 0x7F, 0x14, // '#' + 0x24, 0x2A, 0x7F, 0x2A, 0x12, // '$' + 0x23, 0x13, 0x08, 0x64, 0x62, // '%' + 0x36, 0x49, 0x55, 0x22, 0x50, // '&' + 0x00, 0x05, 0x03, 0x00, 0x00, // ''' + 0x00, 0x1C, 0x22, 0x41, 0x00, // '(' + 0x00, 0x41, 0x22, 0x1C, 0x00, // ')' + 0x08, 0x2A, 0x1C, 0x2A, 0x08, // '*' + 0x08, 0x08, 0x3E, 0x08, 0x08, // '+' + 0x00, 0x50, 0x30, 0x00, 0x00, // ',' + 0x08, 0x08, 0x08, 0x08, 0x08, // '-' + 0x00, 0x60, 0x60, 0x00, 0x00, // '.' + 0x20, 0x10, 0x08, 0x04, 0x02, // '/' + 0x3E, 0x51, 0x49, 0x45, 0x3E, // '0' + 0x00, 0x42, 0x7F, 0x40, 0x00, // '1' + 0x42, 0x61, 0x51, 0x49, 0x46, // '2' + 0x21, 0x41, 0x45, 0x4B, 0x31, // '3' + 0x18, 0x14, 0x12, 0x7F, 0x10, // '4' + 0x27, 0x45, 0x45, 0x45, 0x39, // '5' + 0x3C, 0x4A, 0x49, 0x49, 0x30, // '6' + 0x01, 0x71, 0x09, 0x05, 0x03, // '7' + 0x36, 0x49, 0x49, 0x49, 0x36, // '8' + 0x06, 0x49, 0x49, 0x29, 0x1E, // '9' + 0x00, 0x36, 0x36, 0x00, 0x00, // ':' + 0x00, 0x56, 0x36, 0x00, 0x00, // ';' + 0x00, 0x08, 0x14, 0x22, 0x41, // '<' + 0x14, 0x14, 0x14, 0x14, 0x14, // '=' + 0x41, 0x22, 0x14, 0x08, 0x00, // '>' + 0x02, 0x01, 0x51, 0x09, 0x06, // '?' + 0x32, 0x49, 0x79, 0x41, 0x3E, // '@' + 0x7E, 0x11, 0x11, 0x11, 0x7E, // 'A' + 0x7F, 0x49, 0x49, 0x49, 0x36, // 'B' + 0x3E, 0x41, 0x41, 0x41, 0x22, // 'C' + 0x7F, 0x41, 0x41, 0x22, 0x1C, // 'D' + 0x7F, 0x49, 0x49, 0x49, 0x41, // 'E' + 0x7F, 0x09, 0x09, 0x09, 0x01, // 'F' + 0x3E, 0x41, 0x49, 0x49, 0x7A, // 'G' + 0x7F, 0x08, 0x08, 0x08, 0x7F, // 'H' + 0x00, 0x41, 0x7F, 0x41, 0x00, // 'I' + 0x20, 0x40, 0x41, 0x3F, 0x01, // 'J' + 0x7F, 0x08, 0x14, 0x22, 0x41, // 'K' + 0x7F, 0x40, 0x40, 0x40, 0x40, // 'L' + 0x7F, 0x02, 0x0C, 0x02, 0x7F, // 'M' + 0x7F, 0x04, 0x08, 0x10, 0x7F, // 'N' + 0x3E, 0x41, 0x41, 0x41, 0x3E, // 'O' + 0x7F, 0x09, 0x09, 0x09, 0x06, // 'P' + 0x3E, 0x41, 0x51, 0x21, 0x5E, // 'Q' + 0x7F, 0x09, 0x19, 0x29, 0x46, // 'R' + 0x46, 0x49, 0x49, 0x49, 0x31, // 'S' + 0x01, 0x01, 0x7F, 0x01, 0x01, // 'T' + 0x3F, 0x40, 0x40, 0x40, 0x3F, // 'U' + 0x1F, 0x20, 0x40, 0x20, 0x1F, // 'V' + 0x3F, 0x40, 0x38, 0x40, 0x3F, // 'W' + 0x63, 0x14, 0x08, 0x14, 0x63, // 'X' + 0x07, 0x08, 0x70, 0x08, 0x07, // 'Y' + 0x61, 0x51, 0x49, 0x45, 0x43, // 'Z' + ]; +} diff --git a/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs b/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs index 290a169..f01f8f2 100644 --- a/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs +++ b/src/SharpEmu.Libs/VideoOut/VideoOutExports.cs @@ -6,7 +6,6 @@ using SharpEmu.HLE.Host; using SharpEmu.Libs.Gpu; using SharpEmu.Libs.Audio; using SharpEmu.Libs.Kernel; -using SharpEmu.Logging; using System.Buffers; using System.Buffers.Binary; using System.Diagnostics; @@ -36,221 +35,63 @@ public static class VideoOutExports private const int VideoOutBufferAttribute2Size = 0x50; private const int VideoOutBuffersEntrySize = 0x20; private const int VideoOutOutputStatusSize = 0x30; + private const int VideoOutVblankStatusSize = 0x28; private const ulong SceVideoOutPixelFormatA8R8G8B8Srgb = 0x80000000; private const ulong SceVideoOutPixelFormatA8B8G8R8Srgb = 0x80002200; - private const ulong SceVideoOutPixelFormatB8G8R8A8Unorm = 0x8100000000000000; - private const ulong SceVideoOutPixelFormatR8G8B8A8Unorm = 0x8100000022000000; private const ulong SceVideoOutPixelFormatA2R10G10B10 = 0x88060000; private const ulong SceVideoOutPixelFormatA2R10G10B10Srgb = 0x88000000; private const ulong SceVideoOutPixelFormatA2R10G10B10Bt2020Pq = 0x88740000; - private const ulong SceVideoOutInternalEventVblank = 0x5; + // Prospero/PS5 format2 values are 64-bit encodings. The 0x22000000 field + // selects R-first component order; notably, the 0x81000000... family is + // packed 10:10:10:2 and must not be mistaken for an 8-bit RGBA format. + private const ulong SceVideoOutPixelFormat2R8G8B8A8Srgb = 0x8000000022000000; + private const ulong SceVideoOutPixelFormat2B8G8R8A8Srgb = 0x8000000000000000; + private const ulong SceVideoOutPixelFormat2R10G10B10A2 = 0x8100000622000000; + private const ulong SceVideoOutPixelFormat2B10G10R10A2 = 0x8100000600000000; + private const ulong SceVideoOutPixelFormat2R10G10B10A2Srgb = 0x8100000022000000; + private const ulong SceVideoOutPixelFormat2B10G10R10A2Srgb = 0x8100000000000000; + private const ulong SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq = 0x8100070422000000; + private const ulong SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq = 0x8100070400000000; private const ulong SceVideoOutInternalEventFlip = 0x6; + // Distinct internal ident for vblank events. Games interpret events through + // sceVideoOutGetEventId (mapped below), so the exact value is internal; only + // its distinctness from the flip ident matters for GetEventId/GetEventData. + private const ulong SceVideoOutInternalEventVblank = 0x40; private const short OrbisKernelEventFilterVideoOut = -13; private static readonly object _stateGate = new(); private static readonly object _frameDumpGate = new(); private static readonly Dictionary _ports = new(); - - // Hardware raises vblank autonomously; UE blocks its frame loop on it. - private const double VblankHz = 60.0; - private const int VblankWaitTimeoutMilliseconds = 100; - private static Thread? _vblankPumpThread; - private static int _vblankPumpStarted; - private static volatile int _vblankPumpStopRequested; - private static volatile int _presentationWindowCloseNotified; - - private static readonly object _vblankEdgeGate = new(); - private static ulong _vblankEdgeSequence; - private static long _vblankMissedEdges; - - private static void EnsureVblankPumpStarted() - { - if (Interlocked.Exchange(ref _vblankPumpStarted, 1) != 0) - { - return; - } - - HostPlatform.Current.Threading.RequestTimerResolution(); - - _vblankPumpThread = new Thread(VblankPumpLoop) - { - IsBackground = true, - Name = "SharpEmu VideoOut vblank", - Priority = ThreadPriority.AboveNormal, - }; - _vblankPumpThread.Start(); - } - - private static void VblankPumpLoop() - { - var intervalTicks = Math.Max(1L, (long)(Stopwatch.Frequency / VblankHz)); - var nextEdge = Stopwatch.GetTimestamp() + intervalTicks; - - while (_vblankPumpStopRequested == 0) - { - WaitUntilTimestamp(nextEdge); - PumpVblanks(); - - nextEdge += intervalTicks; - - var now = Stopwatch.GetTimestamp(); - if (nextEdge < now) - { - var missed = (now - nextEdge) / intervalTicks + 1; - Interlocked.Add(ref _vblankMissedEdges, missed); - nextEdge = now + intervalTicks; - } - } - } - - public static void NotifyPresentationWindowClosed() - { - if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) - { - return; - } - - Console.Error.WriteLine("[LOADER][INFO] VideoOut presentation window closed"); - RequestHostShutdown("videoout-window-closed"); - } - - public static void NotifyHostInterrupt() - { - if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) - { - return; - } - - Console.Error.WriteLine("[LOADER][INFO] Host interrupt requested"); - RequestHostShutdown("host-interrupt"); - } - - private static void RequestHostShutdown(string reason) - { - AudioOutExports.ShutdownAllPorts(); - StopVblankPump(); - HostSessionControl.RequestShutdown(reason); - ScheduleProcessExitIfGuestDoesNotStop(); - } - - private static void ScheduleProcessExitIfGuestDoesNotStop() - { - ThreadPool.QueueUserWorkItem(static _ => - { - Thread.Sleep(2000); - Environment.Exit(0); - }); - } - - public static void StopVblankPump() - { - if (Interlocked.Exchange(ref _vblankPumpStopRequested, 1) != 0) - { - return; - } - - var thread = _vblankPumpThread; - if (thread is { IsAlive: true }) - { - thread.Join(TimeSpan.FromSeconds(2)); - } - } - - private static void WaitUntilTimestamp(long deadlineTicks) - { - var spinThresholdTicks = Stopwatch.Frequency * 2L / 1000L; - - while (true) - { - var remaining = deadlineTicks - Stopwatch.GetTimestamp(); - if (remaining <= 0) - { - return; - } - - if (remaining > spinThresholdTicks) - { - var sleepMilliseconds = - (int)((remaining - spinThresholdTicks) * 1000L / Stopwatch.Frequency); - if (sleepMilliseconds > 0) - { - Thread.Sleep(sleepMilliseconds); - continue; - } - } - - Thread.SpinWait(64); - } - } - - // Only ever touched by the vblank pump thread; reused across edges so the 60 Hz - // pump does not allocate a fresh snapshot per edge. - private static readonly List _vblankPumpPorts = new(); - - private static void PumpVblanks() - { - lock (_vblankEdgeGate) - { - _vblankEdgeSequence++; - Monitor.PulseAll(_vblankEdgeGate); - } - - _vblankPumpPorts.Clear(); - lock (_stateGate) - { - if (_ports.Count == 0) - { - return; - } - - // Signalling reaches WakeBlockedThreads -> Pump(), which serialises on one global - // flag. Waking an unwatched queue would hold it 60x/sec and starve guest threads. - foreach (var port in _ports.Values) - { - if (port.VblankEvents.Count != 0) - { - _vblankPumpPorts.Add(port); - } - } - } - - foreach (var port in _vblankPumpPorts) - { - SignalVblank(port); - } - } - + private static int _presentationWindowCloseNotified; + private static int _vblankStopRequested; private static readonly Dictionary<(int Handle, int BufferIndex, ulong Address), ulong> _lastFrameFingerprints = new(); private static int _nextHandle = 1; private static int _frameDumpCount; private static long _nextFrameDumpIndex; - private static string _windowTitleBase = "SharpEmu VideoOut"; private static string _windowTitle = "SharpEmu VideoOut"; private static readonly bool _logFrameRate = string.Equals( Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT_FPS"), "1", StringComparison.Ordinal); - private static readonly bool _logVideoOutSync = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT_SYNC"), - "1", - StringComparison.Ordinal); - // Call sites must check this before building the interpolated message; the trace - // strings would otherwise be allocated on the per-frame flip path even with tracing off. - private static readonly bool _logVideoOut = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT"), - "1", - StringComparison.Ordinal); - private static readonly bool _dumpVideoOut = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_DUMP_VIDEOOUT"), - "1", - StringComparison.Ordinal); private static long _frameRateWindowStart = Stopwatch.GetTimestamp(); private static long _submittedFrameCount; + private static int _diagnosticFlipCount; + private static readonly int _holdFirstFlipMilliseconds = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_HOLD_FIRST_FLIP_MS"), out var holdMs) + ? Math.Clamp(holdMs, 0, 60_000) + : 0; + private static readonly int _holdFlipNumber = + int.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_HOLD_FLIP_NUMBER"), out var holdFlip) + ? Math.Max(1, holdFlip) + : 1; private static long _presentedFrameCount; - private static long _vblankSignalCount; - private static long _flipSubmitCount; - public static void ConfigureApplicationInfo(string? title, string? titleId, string? version, string? emulatorCommitSha) + static VideoOutExports() + { + RunPixelFormatSelfChecks(); + } + + public static void ConfigureApplicationInfo(string? title, string? titleId, string? version) { var parts = new List(); if (!string.IsNullOrWhiteSpace(title)) @@ -265,12 +106,17 @@ public static class VideoOutExports var application = parts.Count == 0 ? "VideoOut" : string.Join(' ', parts); var versionSuffix = string.IsNullOrWhiteSpace(version) ? string.Empty : $" v{version.Trim()}"; - var commitSuffix = string.IsNullOrWhiteSpace(emulatorCommitSha) ? string.Empty : $" \u00b7 {emulatorCommitSha.Trim()}"; - var hardwareSuffix = $" \u00b7 {HostSystemInfo.CpuName}"; lock (_stateGate) { - _windowTitleBase = $"SharpEmu{commitSuffix} - {application}{versionSuffix}{hardwareSuffix}"; - _windowTitle = $"{_windowTitleBase} \u00b7 {HostSystemInfo.GpuName}"; + _windowTitle = $"SharpEmu - {application}{versionSuffix}"; + } + } + + internal static string GetWindowTitle() + { + lock (_stateGate) + { + return _windowTitle; } } @@ -283,16 +129,42 @@ public static class VideoOutExports lock (_stateGate) { - _windowTitle = $"{_windowTitleBase} \u00b7 {gpuName.Trim()}"; + _windowTitle = $"{_windowTitle} · {gpuName.Trim()}"; } } - internal static string GetWindowTitle() + public static void NotifyPresentationWindowClosed() { - lock (_stateGate) + if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) { - return _windowTitle; + return; } + + RequestHostShutdown("videoout-window-closed"); + } + + public static void NotifyHostInterrupt() + { + if (Interlocked.Exchange(ref _presentationWindowCloseNotified, 1) != 0) + { + return; + } + + RequestHostShutdown("host-interrupt"); + } + + private static void RequestHostShutdown(string reason) + { + Console.Error.WriteLine($"[LOADER][INFO] Host shutdown requested: {reason}"); + VulkanVideoPresenter.RequestClose(); + AudioOutExports.ShutdownAllPorts(); + Interlocked.Exchange(ref _vblankStopRequested, 1); + HostSessionControl.RequestShutdown(reason); + ThreadPool.QueueUserWorkItem(static _ => + { + Thread.Sleep(2000); + Environment.Exit(0); + }); } private sealed class VideoOutPortState @@ -308,8 +180,10 @@ public static class VideoOutExports public float Gamma { get; set; } = 1.0f; public VideoOutBufferGroup?[] Groups { get; } = new VideoOutBufferGroup?[MaxDisplayBufferGroups]; public VideoOutBufferSlot[] BufferSlots { get; } = CreateBufferSlots(); - public List VblankEvents { get; } = new(); public List FlipEvents { get; } = new(); + public List VblankEvents { get; } = new(); + public long OpenTimestamp; + public long LastVblankTimestamp; } private sealed class VideoOutBufferGroup @@ -374,11 +248,13 @@ public static class VideoOutExports } var handle = _nextHandle++; + var openedAt = Stopwatch.GetTimestamp(); _ports[handle] = new VideoOutPortState { Handle = handle, + OpenTimestamp = openedAt, + LastVblankTimestamp = openedAt, }; - EnsureVblankPumpStarted(); return handle; } } @@ -422,16 +298,6 @@ public static class VideoOutExports return (int)OrbisGen2Result.ORBIS_GEN2_OK; } - [SysAbiExport( - Nid = "+I4K03i3EL0", - ExportName = "sceVideoOutInitializeOutputOptions", - Target = Generation.Gen4 | Generation.Gen5, - LibraryName = "libSceVideoOut")] - public static int VideoOutInitializeOutputOptions(CpuContext ctx) - { - return (int)OrbisGen2Result.ORBIS_GEN2_OK; - } - [SysAbiExport( Nid = "w0hLuNarQxY", ExportName = "sceVideoOutConfigureOutput", @@ -439,14 +305,19 @@ public static class VideoOutExports LibraryName = "libSceVideoOut")] public static int VideoOutConfigureOutput(CpuContext ctx) { - // Accept the requested output configuration; the presenter always renders - // at the display buffer's native size. var handle = unchecked((int)ctx[CpuRegister.Rdi]); - if (!TryGetPort(handle, out _)) - { - return OrbisVideoOutErrorInvalidHandle; - } + return TryGetPort(handle, out _) + ? (int)OrbisGen2Result.ORBIS_GEN2_OK + : OrbisVideoOutErrorInvalidHandle; + } + [SysAbiExport( + Nid = "+I4K03i3EL0", + ExportName = "sceVideoOutInitializeOutputOptions", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceVideoOut")] + public static int VideoOutInitializeOutputOptions(CpuContext ctx) + { return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -550,66 +421,75 @@ public static class VideoOutExports return OrbisVideoOutErrorInvalidHandle; } - EnsureVblankPumpStarted(); - - lock (_vblankEdgeGate) + // Wait to the next boundary of the emulated display refresh rather + // than a raw Thread.Sleep(1): coarse sleeps overshoot to the + // scheduler quantum, which mis-paces games that spin on vblank. A + // caller that arrives past the boundary already missed the vblank: + // report it immediately instead of charging a full extra interval. + var intervalTicks = Stopwatch.Frequency / Math.Max(1, (long)port.RefreshRate); + var now = Stopwatch.GetTimestamp(); + var last = Interlocked.Read(ref port.LastVblankTimestamp); + var target = last + intervalTicks; + if (target <= now || target > now + intervalTicks) { - var entryEdge = _vblankEdgeSequence; - while (_vblankEdgeSequence == entryEdge) - { - if (!Monitor.Wait(_vblankEdgeGate, VblankWaitTimeoutMilliseconds)) - { - break; - } - } + Interlocked.CompareExchange(ref port.LastVblankTimestamp, now, last); + } + else + { + HostTiming.SleepUntil(target); + Interlocked.CompareExchange(ref port.LastVblankTimestamp, target, last); + } + lock (_stateGate) + { + port.VblankCount++; } - - SignalVblank(port); return (int)OrbisGen2Result.ORBIS_GEN2_OK; } [SysAbiExport( - Nid = "Xru92wHJRmg", - ExportName = "sceVideoOutAddVblankEvent", + Nid = "1FZBKy8HeNU", + ExportName = "sceVideoOutGetVblankStatus", Target = Generation.Gen4 | Generation.Gen5, LibraryName = "libSceVideoOut")] - public static int VideoOutAddVblankEvent(CpuContext ctx) + public static int VideoOutGetVblankStatus(CpuContext ctx) { - var equeue = ctx[CpuRegister.Rdi]; - var handle = unchecked((int)ctx[CpuRegister.Rsi]); + var handle = unchecked((int)ctx[CpuRegister.Rdi]); + var statusAddress = ctx[CpuRegister.Rsi]; + if (statusAddress == 0) + { + return OrbisVideoOutErrorInvalidAddress; + } + if (!TryGetPort(handle, out var port)) { return OrbisVideoOutErrorInvalidHandle; } - if (!KernelEventQueueCompatExports.IsValidEqueue(equeue)) - { - return OrbisVideoOutErrorInvalidEventQueue; - } - - var userData = ctx[CpuRegister.Rdx]; + var now = Stopwatch.GetTimestamp(); + ulong count; + long openedAt; lock (_stateGate) { - var existingIndex = port.VblankEvents.FindIndex(registration => registration.Equeue == equeue); - if (existingIndex >= 0) - { - port.VblankEvents[existingIndex] = new FlipEventRegistration(equeue, userData); - } - else - { - port.VblankEvents.Add(new FlipEventRegistration(equeue, userData)); - } + openedAt = port.OpenTimestamp; + var elapsedTicks = Math.Max(now - openedAt, 0); + var elapsedCount = unchecked((ulong)(elapsedTicks * + Math.Max(1L, (long)port.RefreshRate) / Stopwatch.Frequency)); + port.VblankCount = Math.Max(port.VblankCount, elapsedCount); + count = port.VblankCount; } - // Some engines wait on this queue before issuing their first flip. Provide a first - // edge now; later calls to WaitVblank advance the same notification sequence. - SignalVblank(port); - if (_logVideoOut) - { - TraceVideoOut($"videoout.add_vblank_event eq=0x{equeue:X16} handle={handle} udata=0x{userData:X16}"); - } - return (int)OrbisGen2Result.ORBIS_GEN2_OK; + var elapsedMicroseconds = unchecked((ulong)(Math.Max(now - openedAt, 0) * + 1_000_000L / Stopwatch.Frequency)); + Span status = stackalloc byte[VideoOutVblankStatusSize]; + status.Clear(); + BinaryPrimitives.WriteUInt64LittleEndian(status, count); + BinaryPrimitives.WriteUInt64LittleEndian(status[0x08..], elapsedMicroseconds); + BinaryPrimitives.WriteUInt64LittleEndian(status[0x10..], unchecked((ulong)now)); + status[0x20] = 0; + return ctx.Memory.TryWrite(statusAddress, status) + ? (int)OrbisGen2Result.ORBIS_GEN2_OK + : (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } [SysAbiExport( @@ -645,13 +525,76 @@ public static class VideoOutExports } } - if (_logVideoOut) + if (_traceVideoOut) { TraceVideoOut($"videoout.add_flip_event eq=0x{equeue:X16} handle={handle} udata=0x{userData:X16}"); } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } + [SysAbiExport( + Nid = "Xru92wHJRmg", + ExportName = "sceVideoOutAddVblankEvent", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceVideoOut")] + public static int VideoOutAddVblankEvent(CpuContext ctx) + { + var equeue = ctx[CpuRegister.Rdi]; + var handle = unchecked((int)ctx[CpuRegister.Rsi]); + var userData = ctx[CpuRegister.Rdx]; + if (!TryGetPort(handle, out var port)) + { + return OrbisVideoOutErrorInvalidHandle; + } + + if (!KernelEventQueueCompatExports.IsValidEqueue(equeue)) + { + return OrbisVideoOutErrorInvalidEventQueue; + } + + lock (_stateGate) + { + var existingIndex = port.VblankEvents.FindIndex(registration => registration.Equeue == equeue); + if (existingIndex >= 0) + { + port.VblankEvents[existingIndex] = new FlipEventRegistration(equeue, userData); + } + else + { + port.VblankEvents.Add(new FlipEventRegistration(equeue, userData)); + } + } + + // A guest that parks its main/render loop on a vblank event needs a + // steady tick to advance; start the emulated vblank cadence on demand. + StartVblankThreadOnce(); + TraceVideoOut($"videoout.add_vblank_event eq=0x{equeue:X16} handle={handle} udata=0x{userData:X16}"); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + + [SysAbiExport( + Nid = "oNOQn3knW6s", + ExportName = "sceVideoOutDeleteVblankEvent", + Target = Generation.Gen4 | Generation.Gen5, + LibraryName = "libSceVideoOut")] + public static int VideoOutDeleteVblankEvent(CpuContext ctx) + { + var equeue = ctx[CpuRegister.Rdi]; + var handle = unchecked((int)ctx[CpuRegister.Rsi]); + if (!TryGetPort(handle, out var port)) + { + return OrbisVideoOutErrorInvalidHandle; + } + + lock (_stateGate) + { + port.VblankEvents.RemoveAll(registration => registration.Equeue == equeue); + } + + TraceVideoOut($"videoout.delete_vblank_event eq=0x{equeue:X16} handle={handle}"); + return (int)OrbisGen2Result.ORBIS_GEN2_OK; + } + [SysAbiExport( Nid = "U46NwOiJpys", ExportName = "sceVideoOutSubmitFlip", @@ -666,8 +609,6 @@ public static class VideoOutExports return SubmitFlip(ctx, handle, bufferIndex, flipMode, flipArg, submitGpuImage: true); } - // Struct layout matches the classic SceVideoOutFlipStatus (40 bytes): - // count, processTime, tsc, flipArg, currentBuffer, flipPendingNum. [SysAbiExport( Nid = "SbU3dwp80lQ", ExportName = "sceVideoOutGetFlipStatus", @@ -682,37 +623,24 @@ public static class VideoOutExports return OrbisVideoOutErrorInvalidAddress; } - VideoOutPortState? port; - lock (_stateGate) - { - _ports.TryGetValue(handle, out port); - } - - if (port is null) + if (!TryGetPort(handle, out var port)) { return OrbisVideoOutErrorInvalidHandle; } ulong count; - long flipArg; uint currentBuffer; lock (_stateGate) { count = port.FlipCount; - flipArg = 0; currentBuffer = unchecked((uint)port.CurrentBuffer); } KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x00, count); KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x08, 0); KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x10, 0); - KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x18, unchecked((ulong)flipArg)); + KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x18, 0); KernelMemoryCompatExports.TryWriteUInt64Compat(ctx, statusAddress + 0x20, currentBuffer); - - if (_logVideoOut) - { - TraceVideoOut($"videoout.get_flip_status handle={handle} count={count} currentBuffer={currentBuffer}"); - } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -752,13 +680,23 @@ public static class VideoOutExports return (int)OrbisGen2Result.ORBIS_GEN2_ERROR_MEMORY_FAULT; } - if (filter != OrbisKernelEventFilterVideoOut || - ident is not (SceVideoOutInternalEventVblank or SceVideoOutInternalEventFlip)) + if (filter != OrbisKernelEventFilterVideoOut) { return OrbisVideoOutErrorInvalidEvent; } - return 0; + // sceVideoOutGetEventId reports the event kind: 0 = flip, 1 = vblank. + if (ident == SceVideoOutInternalEventFlip) + { + return 0; + } + + if (ident == SceVideoOutInternalEventVblank) + { + return 1; + } + + return OrbisVideoOutErrorInvalidEvent; } [SysAbiExport( @@ -783,7 +721,7 @@ public static class VideoOutExports } if (filter != OrbisKernelEventFilterVideoOut || - ident is not (SceVideoOutInternalEventVblank or SceVideoOutInternalEventFlip)) + (ident != SceVideoOutInternalEventFlip && ident != SceVideoOutInternalEventVblank)) { return OrbisVideoOutErrorInvalidEvent; } @@ -1103,58 +1041,6 @@ public static class VideoOutExports return groupIndex < 0 ? groupIndex : setIndex; } - private static void SignalVblank(VideoOutPortState port) - { - // Snapshot the registrations into a pooled rental so the triggers can run outside - // _stateGate without copying the list into a fresh allocation on every edge. - // A per-port reusable buffer would race: the pump thread and a guest thread's - // first-edge signal (AddVblankEvent) can signal the same port concurrently. - FlipEventRegistration[]? vblankEvents = null; - int vblankEventCount; - ulong eventHint; - lock (_stateGate) - { - port.VblankCount++; - eventHint = SceVideoOutInternalEventVblank | - ((port.VblankCount & 0x0000_FFFF_FFFF_FFFFUL) << 16); - vblankEventCount = port.VblankEvents.Count; - if (vblankEventCount != 0) - { - vblankEvents = ArrayPool.Shared.Rent(vblankEventCount); - port.VblankEvents.CopyTo(vblankEvents); - } - } - - var signalCount = Interlocked.Increment(ref _vblankSignalCount); - - if (vblankEvents is not null) - { - try - { - for (var i = 0; i < vblankEventCount; i++) - { - _ = KernelEventQueueCompatExports.TriggerDisplayEvent( - vblankEvents[i].Equeue, - SceVideoOutInternalEventVblank, - OrbisKernelEventFilterVideoOut, - eventHint, - vblankEvents[i].UserData); - } - } - finally - { - ArrayPool.Shared.Return(vblankEvents); - } - } - - if (_logVideoOutSync && (signalCount <= 8 || signalCount % 60 == 0)) - { - Console.Error.WriteLine( - $"[LOADER][SYNC] vblank#{signalCount} handle={port.Handle} count={port.VblankCount} " + - $"queues={vblankEventCount} hint=0x{eventHint:X16}"); - } - } - private static int SubmitFlip( CpuContext ctx, int handle, @@ -1197,6 +1083,9 @@ public static class VideoOutExports } } + PaceFlip(port.FlipRate); + PerfOverlay.RecordSubmit(); + var guestImageSubmitted = false; ulong guestImageAddress = 0; if (submitGpuImage && @@ -1216,8 +1105,13 @@ public static class VideoOutExports _ = TryDumpFrame(ctx, port, bufferIndex, flipMode, flipArg); } - if (flipEvents is not null) + void TriggerFlipEvents() { + if (flipEvents is null) + { + return; + } + try { for (var i = 0; i < flipEventCount; i++) @@ -1233,23 +1127,34 @@ public static class VideoOutExports finally { ArrayPool.Shared.Return(flipEvents); + flipEvents = null; } } - var flipCount = Interlocked.Increment(ref _flipSubmitCount); - if (_logVideoOutSync && (flipCount <= 8 || flipCount % 60 == 0)) + if (submitGpuImage) { - Console.Error.WriteLine( - $"[LOADER][SYNC] flip#{flipCount} handle={handle} buffer={bufferIndex} " + - $"addr=0x{guestImageAddress:X16} submitted={guestImageSubmitted} " + - $"flipQueues={flipEventCount}"); + TriggerFlipEvents(); + } + else if (VulkanVideoPresenter.SubmitOrderedGuestAction( + TriggerFlipEvents, + $"videoout flip complete handle={handle} index={bufferIndex}") == 0) + { + // Headless startup has no render queue to order against. + TriggerFlipEvents(); } - if (_logVideoOut) - { - TraceVideoOut($"videoout.submit_flip handle={handle} index={bufferIndex} mode={flipMode} arg={flipArg} events={flipEventCount}"); - } + TraceVideoOut( + $"videoout.submit_flip handle={handle} index={bufferIndex} mode={flipMode} " + + $"arg={flipArg} addr=0x{guestImageAddress:X16} submitted={guestImageSubmitted} " + + $"events={flipEventCount} ordered_completion={!submitGpuImage}"); ReportFrameRate(presented: false); + var diagnosticFlipNumber = Interlocked.Increment(ref _diagnosticFlipCount); + if (_holdFirstFlipMilliseconds > 0 && diagnosticFlipNumber == _holdFlipNumber) + { + Console.Error.WriteLine( + $"[LOADER][INFO] Holding guest flip #{diagnosticFlipNumber} for {_holdFirstFlipMilliseconds} ms for visual verification."); + Thread.Sleep(_holdFirstFlipMilliseconds); + } return (int)OrbisGen2Result.ORBIS_GEN2_OK; } @@ -1284,11 +1189,140 @@ public static class VideoOutExports var elapsedSeconds = (double)elapsedTicks / Stopwatch.Frequency; var submitted = Interlocked.Exchange(ref _submittedFrameCount, 0); var presentedCount = Interlocked.Exchange(ref _presentedFrameCount, 0); - var missedEdges = Interlocked.Exchange(ref _vblankMissedEdges, 0); + var (draws, drawMs, pipelines, spirvCompiles) = VulkanVideoPresenter.ReadAndResetPerfCounters(); Console.Error.WriteLine( $"[LOADER][PERF] videoout submitted_fps={submitted / elapsedSeconds:F1} " + $"presented_fps={presentedCount / elapsedSeconds:F1} " + - $"vblank_missed={missedEdges}"); + $"draws={draws} draw_ms={drawMs:F0} pipelines={pipelines} spirv={spirvCompiles}"); + } + + private static readonly bool _flipPacingDisabled = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_NO_FLIP_PACING"), + "1", + StringComparison.Ordinal); + private static long _lastFlipPacingTimestamp; + + private static Thread? _vblankThread; + private static readonly object _vblankThreadGate = new(); + + /// + /// Starts the emulated vblank tick once a guest registers interest in vblank + /// events. The tick fires the registered vblank events on their event queues + /// at the display refresh cadence so guests that park their main/render loop + /// on a vblank equeue keep advancing. + /// + private static void StartVblankThreadOnce() + { + if (Volatile.Read(ref _vblankThread) is not null) + { + return; + } + + lock (_vblankThreadGate) + { + if (_vblankThread is not null) + { + return; + } + + var thread = new Thread(VblankTickLoop) + { + IsBackground = true, + Name = "SharpEmu-Vblank", + }; + _vblankThread = thread; + thread.Start(); + } + } + + private static void VblankTickLoop() + { + var pending = new List<(ulong Equeue, ulong DataHint, ulong UserData)>(); + var next = Stopwatch.GetTimestamp(); + while (Volatile.Read(ref _vblankStopRequested) == 0) + { + uint refresh = 60; + pending.Clear(); + lock (_stateGate) + { + foreach (var port in _ports.Values) + { + if (port.VblankEvents.Count == 0) + { + continue; + } + + refresh = port.RefreshRate == 0 ? 60 : port.RefreshRate; + port.VblankCount++; + var dataHint = (port.VblankCount & 0x0000_FFFF_FFFF_FFFFUL) << 16; + foreach (var registration in port.VblankEvents) + { + pending.Add((registration.Equeue, dataHint, registration.UserData)); + } + } + } + + foreach (var (equeue, dataHint, userData) in pending) + { + _ = KernelEventQueueCompatExports.TriggerDisplayEvent( + equeue, + SceVideoOutInternalEventVblank, + OrbisKernelEventFilterVideoOut, + dataHint, + userData); + } + + var interval = Stopwatch.Frequency / Math.Max(1, (long)refresh); + next += interval; + var now = Stopwatch.GetTimestamp(); + if (next < now) + { + next = now; + } + + HostTiming.SleepUntil(next); + } + } + + /// + /// Emulates the display vblank cadence: hardware completes flips at the + /// requested rate, which is what paces the game's main loop. Without this + /// the guest runs as fast as the GPU pipeline drains, so frame delivery + /// is bursty and animation judders. When the emulator runs slower than + /// the target rate the sleep never engages. + /// + private static void PaceFlip(int flipRate) + { + if (_flipPacingDisabled) + { + return; + } + + var refreshRate = flipRate switch + { + 1 => 30, + 2 => 20, + _ => 60, + }; + var intervalTicks = Stopwatch.Frequency / refreshRate; + var now = Stopwatch.GetTimestamp(); + var last = Interlocked.Read(ref _lastFlipPacingTimestamp); + var target = last + intervalTicks; + if (target <= now) + { + Interlocked.CompareExchange(ref _lastFlipPacingTimestamp, now, last); + return; + } + + var waitMilliseconds = (target - now) * 1000 / Stopwatch.Frequency; + if (waitMilliseconds is >= 0 and < 100) + { + // Precise wait: Thread.Sleep alone overshoots by a scheduler + // quantum, which caps the flip rate below the target cadence. + HostTiming.SleepUntil(target); + } + + Interlocked.CompareExchange(ref _lastFlipPacingTimestamp, target, last); } private static int RegisterBufferRange(VideoOutPortState port, int startIndex, ReadOnlySpan addresses, BufferAttribute attribute, int requestedGroupIndex = -1) @@ -1330,11 +1364,10 @@ public static class VideoOutExports slot.AddressRight = 0; } - if (_logVideoOut) - { - TraceVideoOut( - $"videoout.register_buffers handle={port.Handle} group={groupIndex} start={startIndex} count={addresses.Length} fmt=0x{attribute.PixelFormat:X} tile={attribute.TilingMode} {attribute.Width}x{attribute.Height} pitch={attribute.PitchInPixel}"); - } + TraceVideoOut( + $"videoout.register_buffers handle={port.Handle} group={groupIndex} start={startIndex} count={addresses.Length} " + + $"addresses=[{string.Join(',', addresses.ToArray().Select(static address => $"0x{address:X16}"))}] " + + $"fmt=0x{attribute.PixelFormat:X} tile={attribute.TilingMode} {attribute.Width}x{attribute.Height} pitch={attribute.PitchInPixel}"); GuestGpu.Current.EnsureStarted(attribute.Width, attribute.Height); var guestFormat = MapPixelFormatToGuestTextureFormat(attribute.PixelFormat); @@ -1366,9 +1399,9 @@ public static class VideoOutExports private static bool TryReadBufferAttribute(CpuContext ctx, ulong attributeAddress, bool attribute2, out BufferAttribute attribute) { attribute = default; - if (!ctx.TryReadUInt32(attributeAddress + 0x04, out var tilingMode) || - !ctx.TryReadUInt32(attributeAddress + 0x0C, out var width) || - !ctx.TryReadUInt32(attributeAddress + 0x10, out var height)) + if (!TryReadUInt32(ctx, attributeAddress + 0x04, out var tilingMode) || + !TryReadUInt32(ctx, attributeAddress + 0x0C, out var width) || + !TryReadUInt32(ctx, attributeAddress + 0x10, out var height)) { return false; } @@ -1385,10 +1418,10 @@ public static class VideoOutExports return true; } - if (!ctx.TryReadUInt32(attributeAddress + 0x00, out var pixelFormat32) || - !ctx.TryReadUInt32(attributeAddress + 0x08, out var aspectRatio) || - !ctx.TryReadUInt32(attributeAddress + 0x14, out var pitchInPixel) || - !ctx.TryReadUInt32(attributeAddress + 0x18, out var option32)) + if (!TryReadUInt32(ctx, attributeAddress + 0x00, out var pixelFormat32) || + !TryReadUInt32(ctx, attributeAddress + 0x08, out var aspectRatio) || + !TryReadUInt32(ctx, attributeAddress + 0x14, out var pitchInPixel) || + !TryReadUInt32(ctx, attributeAddress + 0x18, out var option32)) { return false; } @@ -1500,7 +1533,7 @@ public static class VideoOutExports var basePath = GetFrameDumpBasePath(frameIndex, port.Handle, bufferIndex); WriteBmp(basePath + ".bmp", attribute.Width, attribute.Height, rgb); WriteFrameMetadata(basePath + ".txt", slot.AddressLeft, attribute, bufferIndex, flipMode, flipArg, "bmp-linear-read", fingerprint); - if (_logVideoOut) + if (_traceVideoOut) { TraceVideoOut($"videoout.dump_frame path={basePath}.bmp addr=0x{slot.AddressLeft:X16} {attribute.Width}x{attribute.Height} fmt=0x{attribute.PixelFormat:X} fingerprint=0x{fingerprint:X16}"); } @@ -1542,7 +1575,7 @@ public static class VideoOutExports var basePath = GetFrameDumpBasePath(frameIndex, handle, bufferIndex); File.WriteAllBytes(basePath + ".raw", bytes); WriteFrameMetadata(basePath + ".txt", address, attribute, bufferIndex, flipMode, flipArg, reason, fingerprint); - if (_logVideoOut) + if (_traceVideoOut) { TraceVideoOut($"videoout.dump_frame path={basePath}.raw addr=0x{address:X16} bytes={byteCount} reason={reason} fingerprint=0x{fingerprint:X16}"); } @@ -1565,14 +1598,35 @@ public static class VideoOutExports private static uint GetBytesPerPixel(ulong pixelFormat) => pixelFormat is SceVideoOutPixelFormatA8R8G8B8Srgb or SceVideoOutPixelFormatA8B8G8R8Srgb or - SceVideoOutPixelFormatB8G8R8A8Unorm or - SceVideoOutPixelFormatR8G8B8A8Unorm or SceVideoOutPixelFormatA2R10G10B10 or SceVideoOutPixelFormatA2R10G10B10Srgb or - SceVideoOutPixelFormatA2R10G10B10Bt2020Pq + SceVideoOutPixelFormatA2R10G10B10Bt2020Pq or + SceVideoOutPixelFormat2R8G8B8A8Srgb or + SceVideoOutPixelFormat2B8G8R8A8Srgb or + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2B10G10R10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2B10G10R10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq or + SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq ? 4u : 0u; + internal static bool IsPacked10BitPixelFormat(ulong pixelFormat) => + IsPacked10BitPixelFormatNormalized(NormalizePixelFormat(pixelFormat)); + + private static bool IsPacked10BitPixelFormatNormalized(ulong pixelFormat) => + pixelFormat is + SceVideoOutPixelFormatA2R10G10B10 or + SceVideoOutPixelFormatA2R10G10B10Srgb or + SceVideoOutPixelFormatA2R10G10B10Bt2020Pq or + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2B10G10R10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2B10G10R10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq or + SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq; + // Maps the PS5 VideoOut pixel format space to the AGC "guest texture format" tags // the backend keys its guest-image registry on (see VulkanVideoPresenter. // GetGuestTextureFormat: format=10 => 56 for 8-bit RGBA variants, format=9 => 9 for 10-bit). @@ -1581,14 +1635,101 @@ public static class VideoOutExports { SceVideoOutPixelFormatA8R8G8B8Srgb or SceVideoOutPixelFormatA8B8G8R8Srgb or - SceVideoOutPixelFormatB8G8R8A8Unorm or - SceVideoOutPixelFormatR8G8B8A8Unorm => 56u, + SceVideoOutPixelFormat2R8G8B8A8Srgb or + SceVideoOutPixelFormat2B8G8R8A8Srgb => 56u, SceVideoOutPixelFormatA2R10G10B10 or SceVideoOutPixelFormatA2R10G10B10Srgb or - SceVideoOutPixelFormatA2R10G10B10Bt2020Pq => 9u, + SceVideoOutPixelFormatA2R10G10B10Bt2020Pq or + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2B10G10R10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2B10G10R10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq or + SceVideoOutPixelFormat2B10G10R10A2Bt2100Pq => 9u, _ => 0u, }; + internal static bool TryPackRgba8Pixel( + ulong pixelFormat, + byte red, + byte green, + byte blue, + byte alpha, + out uint packed) + { + pixelFormat = NormalizePixelFormat(pixelFormat); + if (!IsPacked10BitPixelFormatNormalized(pixelFormat)) + { + packed = 0; + return false; + } + + packed = PackRgba8PixelNormalized(pixelFormat, red, green, blue, alpha); + return true; + } + + private static uint PackRgba8PixelNormalized( + ulong pixelFormat, + byte red, + byte green, + byte blue, + byte alpha) + { + var red10 = ExpandUnorm8To10(red); + var green10 = ExpandUnorm8To10(green); + var blue10 = ExpandUnorm8To10(blue); + var alpha2 = ((uint)alpha * 3u + 127u) / 255u; + return HasRedInLeastSignificantBits(pixelFormat) + ? red10 | (green10 << 10) | (blue10 << 20) | (alpha2 << 30) + : blue10 | (green10 << 10) | (red10 << 20) | (alpha2 << 30); + } + + internal static bool TryConvertPacked10ToRgba8( + uint packed, + ulong pixelFormat, + Span rgba) + { + pixelFormat = NormalizePixelFormat(pixelFormat); + if (rgba.Length < 4 || !IsPacked10BitPixelFormatNormalized(pixelFormat)) + { + return false; + } + + ConvertPacked10ToRgba8Normalized(packed, pixelFormat, rgba); + return true; + } + + private static void ConvertPacked10ToRgba8Normalized( + uint packed, + ulong pixelFormat, + Span rgba) + { + var least = packed & 0x3FFu; + var green = (packed >> 10) & 0x3FFu; + var most = (packed >> 20) & 0x3FFu; + var redIsLeast = HasRedInLeastSignificantBits(pixelFormat); + var red = redIsLeast ? least : most; + var blue = redIsLeast ? most : least; + rgba[0] = ReduceUnorm10To8(red); + rgba[1] = ReduceUnorm10To8(green); + rgba[2] = ReduceUnorm10To8(blue); + rgba[3] = (byte)((((packed >> 30) & 0x3u) * 255u + 1u) / 3u); + } + + private static bool HasRedInLeastSignificantBits(ulong pixelFormat) => + pixelFormat is + SceVideoOutPixelFormat2R10G10B10A2 or + SceVideoOutPixelFormat2R10G10B10A2Srgb or + SceVideoOutPixelFormat2R10G10B10A2Bt2100Pq; + + private static uint ExpandUnorm8To10(byte value) => + ((uint)value * 1023u + 127u) / 255u; + + // Preserve both UNORM endpoints and round to nearest. A plain >> 2 is a + // biased truncation because the 10-bit maximum is 1023, not 1020. + private static byte ReduceUnorm10To8(uint value) => + (byte)((value * 255u + 511u) / 1023u); + private static ulong NormalizePixelFormat(ulong pixelFormat) { if (GetBytesPerPixel(pixelFormat) != 0) @@ -1614,22 +1755,28 @@ public static class VideoOutExports private static void ConvertRowToRgb(ReadOnlySpan source, Span destination, ulong pixelFormat) { + pixelFormat = NormalizePixelFormat(pixelFormat); var dst = 0; + Span rgba = stackalloc byte[4]; + var packed10 = IsPacked10BitPixelFormatNormalized(pixelFormat); for (var src = 0; src + 3 < source.Length; src += 4) { - if (pixelFormat is SceVideoOutPixelFormatA8B8G8R8Srgb or SceVideoOutPixelFormatR8G8B8A8Unorm) + if (packed10) + { + var packed = BinaryPrimitives.ReadUInt32LittleEndian(source[src..(src + 4)]); + ConvertPacked10ToRgba8Normalized(packed, pixelFormat, rgba); + destination[dst++] = rgba[0]; + destination[dst++] = rgba[1]; + destination[dst++] = rgba[2]; + } + else if (pixelFormat is + SceVideoOutPixelFormatA8B8G8R8Srgb or + SceVideoOutPixelFormat2R8G8B8A8Srgb) { destination[dst++] = source[src + 0]; destination[dst++] = source[src + 1]; destination[dst++] = source[src + 2]; } - else if (pixelFormat is SceVideoOutPixelFormatA2R10G10B10 or SceVideoOutPixelFormatA2R10G10B10Srgb or SceVideoOutPixelFormatA2R10G10B10Bt2020Pq) - { - var value = BinaryPrimitives.ReadUInt32LittleEndian(source[src..(src + 4)]); - destination[dst++] = (byte)(((value >> 20) & 0x3FF) >> 2); - destination[dst++] = (byte)(((value >> 10) & 0x3FF) >> 2); - destination[dst++] = (byte)((value & 0x3FF) >> 2); - } else { destination[dst++] = source[src + 2]; @@ -1639,6 +1786,36 @@ public static class VideoOutExports } } + [Conditional("DEBUG")] + private static void RunPixelFormatSelfChecks() + { + Span rgba = stackalloc byte[4]; + Debug.Assert(TryPackRgba8Pixel( + SceVideoOutPixelFormat2R10G10B10A2Srgb, + 255, 0, 0, 255, + out var rFirst)); + Debug.Assert(rFirst == 0xC00003FFu); + Debug.Assert(TryConvertPacked10ToRgba8( + rFirst, + SceVideoOutPixelFormat2R10G10B10A2Srgb, + rgba)); + Debug.Assert(rgba.SequenceEqual(new byte[] { 255, 0, 0, 255 })); + + Debug.Assert(TryPackRgba8Pixel( + SceVideoOutPixelFormat2B10G10R10A2Srgb, + 255, 0, 0, 255, + out var bFirst)); + Debug.Assert(bFirst == 0xFFF00000u); + Debug.Assert(TryConvertPacked10ToRgba8( + bFirst, + SceVideoOutPixelFormat2B10G10R10A2Srgb, + rgba)); + Debug.Assert(rgba.SequenceEqual(new byte[] { 255, 0, 0, 255 })); + Debug.Assert(ReduceUnorm10To8(0) == 0); + Debug.Assert(ReduceUnorm10To8(512) == 128); + Debug.Assert(ReduceUnorm10To8(1023) == 255); + } + private static string GetFrameDumpBasePath(long frameIndex, int handle, int bufferIndex) { var directory = GetLogsDirectory(); @@ -1757,6 +1934,19 @@ public static class VideoOutExports return true; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span buffer = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, buffer)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(buffer); + return true; + } + private static bool TryReadInt16(CpuContext ctx, ulong address, out short value) { Span buffer = stackalloc byte[sizeof(short)]; @@ -1770,8 +1960,22 @@ public static class VideoOutExports return true; } + private static readonly bool _traceVideoOut = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_VIDEOOUT"), + "1", + StringComparison.Ordinal); + private static readonly bool _dumpVideoOut = string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DUMP_VIDEOOUT"), + "1", + StringComparison.Ordinal); + private static void TraceVideoOut(string message) { + if (!_traceVideoOut) + { + return; + } + Console.Error.WriteLine($"[LOADER][TRACE] {message}"); } } diff --git a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs index a766825..29efe6c 100644 --- a/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs +++ b/src/SharpEmu.Libs/VideoOut/VulkanVideoPresenter.cs @@ -6,6 +6,7 @@ using Silk.NET.Core.Native; using Silk.NET.Maths; using SharpEmu.HLE; using SharpEmu.Libs.Agc; +using SharpEmu.Libs.Bink; using Silk.NET.Input; using SharpEmu.Libs.Gpu; using SharpEmu.ShaderCompiler; @@ -48,7 +49,9 @@ internal sealed record VulkanTranslatedGuestDraw( internal sealed record VulkanOffscreenGuestDraw( VulkanTranslatedGuestDraw Draw, IReadOnlyList Targets, - bool PublishTarget); + GuestDepthTarget? DepthTarget, + bool PublishTarget, + ulong ShaderAddress); internal sealed record VulkanComputeGuestDispatch( ulong ShaderAddress, @@ -57,33 +60,205 @@ internal sealed record VulkanComputeGuestDispatch( IReadOnlyList GlobalMemoryBuffers, uint GroupCountX, uint GroupCountY, - uint GroupCountZ); + uint GroupCountZ, + uint BaseGroupX, + uint BaseGroupY, + uint BaseGroupZ, + uint LocalSizeX, + uint LocalSizeY, + uint LocalSizeZ, + bool IsIndirect, + bool WritesGlobalMemory, + uint ThreadCountX = uint.MaxValue, + uint ThreadCountY = uint.MaxValue, + uint ThreadCountZ = uint.MaxValue); + +internal sealed record VulkanOrderedGuestAction( + Action Action, + string DebugName); + +internal sealed record VulkanOrderedGuestFlip( + long Version, + int VideoOutHandle, + int DisplayBufferIndex, + ulong Address, + uint Width, + uint Height, + uint PitchInPixel); + +internal sealed record VulkanOrderedGuestFlipWait( + long Version, + int VideoOutHandle, + int DisplayBufferIndex); + +internal readonly record struct VulkanGuestQueueIdentity( + string Name, + ulong SubmissionId) +{ + public static VulkanGuestQueueIdentity Default { get; } = new("host.default", 0); +} internal static unsafe class VulkanVideoPresenter { private const uint DefaultWindowWidth = 1280; private const uint DefaultWindowHeight = 720; - private const int MaxPendingGuestWork = 16; - // A single guest frame commonly contains 30-50 translated draws. Limiting - // this to 16 split one frame across several 60 Hz window callbacks and - // unnecessarily throttled the producer behind the bounded work queue. - private const int MaxGuestWorkPerRender = 128; + // Vulkan's portable upper bound for minStorageBufferOffsetAlignment is + // 256 bytes. Using that fixed power of two (instead of racing the render + // thread's physical-device query) gives shader translation and descriptor + // creation one stable aliasing contract on every conformant device. + internal const ulong GuestStorageBufferOffsetAlignment = 256; + // Guest draw snapshots churn through a small set of 128 KiB-16 MiB size + // classes thousands of times per second. The process-wide shared pool + // trims and repartitions those large arrays aggressively under GC load, + // causing hundreds of MiB/s of replacement byte[] allocations. Keep a + // bounded, non-shared pool for AGC-to-presenter ownership transfers. + internal static System.Buffers.ArrayPool GuestDataPool { get; } = + System.Buffers.ArrayPool.Create( + maxArrayLength: 16 * 1024 * 1024, + maxArraysPerBucket: 96); + // The pending queue and per-render drain budget bound how much guest GPU + // work can be buffered ahead of the presenter. Draws are batched into + // shared command buffers, so draining a large batch per render tick is + // cheap; small caps here throttle games that issue more than a handful + // of draws per frame to a fraction of the display rate. The pending cap + // stays tighter than the drain budget because queued draws pin their + // pooled guest-data arrays until the render thread uploads them. + private const int MaxPendingGuestWork = 64; + // A captured 4K flip can consume tens of MiB of device-local memory. + // Retain only a short presentation queue while always preserving the + // newest generation; older immutable versions are retired immediately. + private const int MaxPendingGuestFlipVersions = 4; + // A count-only queue bound is not a memory bound: one compute dispatch can + // carry dozens of full-resolution texture snapshots. At 4K, 64 queued + // dispatches retained more than 12 GiB of managed byte arrays before the + // render thread could upload them. Keep the count cap for small work and + // independently apply backpressure to the actual retained payload. + private static readonly ulong _maxPendingGuestWorkBytes = + (ulong.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_PENDING_GUEST_WORK_MB"), + out var pendingGuestWorkMb) && pendingGuestWorkMb > 0 + ? pendingGuestWorkMb + : 256UL) * 1024UL * 1024UL; + private const int MaxGuestWorkPerRender = 256; + // On macOS the whole window loop — including Render() and its guest-work + // drain — runs on the process main thread, so draining a large backlog of + // slow guest work (heavy compute) blocks the Cocoa event pump and marks the + // window "Not Responding" while starving the swapchain present. Cap the + // wall-clock time spent draining per Render() call; leftover work stays + // queued for the next frame. SHARPEMU_RENDER_WORK_BUDGET_MS overrides + // (0 disables the cap); default 12ms keeps the window interactive at ~60Hz. + private static readonly long _renderWorkBudgetTicks = + (long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_RENDER_WORK_BUDGET_MS"), + out var renderBudgetMs) && renderBudgetMs >= 0 + ? renderBudgetMs + : 12L) * System.Diagnostics.Stopwatch.Frequency / 1000L; + // Max time the main-thread Render() will block waiting for a frame slot's + // GPU fence before skipping the frame and returning to the event pump. + // Prevents the window freezing behind a slow-compute GPU backlog. + // SHARPEMU_FRAME_WAIT_BUDGET_MS overrides; default 8ms. + private static readonly ulong _frameSlotWaitBudgetNs = + (ulong.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_FRAME_WAIT_BUDGET_MS"), + out var frameWaitMs) && frameWaitMs > 0 + ? frameWaitMs + : 8UL) * 1_000_000UL; + // Cap the guest-submission fence wait so a GPU submission whose fence never + // signals (a mistranslated compute shader that hangs the Metal queue) cannot + // freeze the render thread forever and starve the swapchain present. + // SHARPEMU_FENCE_WAIT_TIMEOUT_MS overrides; default 3s. + private static readonly ulong _guestFenceWaitTimeoutNs = + ulong.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_FENCE_WAIT_TIMEOUT_MS"), out var fenceMs) && fenceMs > 0 + ? fenceMs * 1_000_000UL + : 3_000_000_000UL; + // When making room in the in-flight submission queue from the macOS MAIN + // thread (Render() -> guest-work drain), block only this long per attempt + // instead of the full fence timeout. If a slow/capped compute submission + // isn't done yet, proceed anyway: the in-flight cap is soft, the fence and + // command-buffer pools are dynamic so a brief overshoot is safe, and the + // queue drains as GPU completions land on later frames. This keeps the + // window responsive (event pump runs) under a heavy compute backlog instead + // of the main thread sitting in vkWaitForFences for up to 3s per chunk. + // SHARPEMU_SUBMISSION_CAPACITY_WAIT_MS overrides; default 100ms; 0 restores + // the full blocking wait. + private static readonly ulong _submissionCapacityWaitNs = + ulong.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_SUBMISSION_CAPACITY_WAIT_MS"), out var capMs) + ? capMs * 1_000_000UL + : 100_000_000UL; + private static readonly HashSet _tracedFenceTimeouts = new(); + private static long _guestQueueBackpressureTraceCount; + // Diagnostic: skip every compute dispatch (mistranslated compute shaders + // run long / GPU-hang and starve the present). Isolates whether the + // geometry+composite path renders on its own. + private static readonly bool _skipAllCompute = + Environment.GetEnvironmentVariable("SHARPEMU_SKIP_ALL_COMPUTE") == "1"; + // Diagnostic: skip compute dispatches whose GroupCountZ is at least this, + // to isolate a specific tall dispatch (e.g. Demon's Souls' 27x15x72 froxel + // shader that hangs the Metal queue) without needing its ASLR-varying + // address. 0 disables. + private static readonly uint _skipTallComputeZ = + uint.TryParse(Environment.GetEnvironmentVariable("SHARPEMU_SKIP_TALL_COMPUTE_Z"), out var z) + ? z + : 0; private const uint GuestPrimitiveRectList = 0x11; - private const uint GuestFormatR32Uint = 0x10004; - private const uint GuestFormatR32Sint = 0x20004; - private const uint GuestFormatR32Sfloat = 0x30004; - private const uint GuestFormatR16G16Uint = 0x10005; - private const uint GuestFormatR16G16Sint = 0x20005; - private const uint GuestFormatR16G16Sfloat = 0x30005; - private const uint GuestFormatR8G8B8A8Uint = 0x1000A; - private const uint GuestFormatR8G8B8A8Sint = 0x2000A; - private const uint GuestFormatR16G16B16A16Uint = 0x1000C; - private const uint GuestFormatR16G16B16A16Sint = 0x2000C; private static readonly object _gate = new(); - private static readonly Queue _pendingGuestWork = new(); + private readonly record struct PendingGuestWork( + object Work, + ulong PayloadBytes, + long Sequence, + long RequiredSequence, + long EnqueuedTicks, + VulkanGuestQueueIdentity Queue); + + // PS5 exposes independent graphics and asynchronous-compute queues. A + // single host FIFO adds dependencies that do not exist in the guest: one + // slow ACB dispatch can delay a graphics clear until the CPU has reused + // that heap. Keep FIFO order within each logical guest queue and schedule + // ready queues round-robin. Explicit WAIT_REG_MEM packets remain the only + // mechanism that orders one logical queue behind another. + private static readonly Dictionary> + _pendingGuestWorkByQueue = new(StringComparer.Ordinal); + private static readonly List _pendingGuestQueueSchedule = []; + private static int _pendingGuestQueueCursor; + private static int _pendingGuestWorkCount; + private static ulong _pendingGuestWorkBytes; + // A flip names an image that was rendered earlier in the command stream. + // Keep a small FIFO of those flips instead of replacing an incomplete one + // with the next frame: the guest can enqueue the next frame before the + // render thread reaches the previous image, which otherwise starves + // presentation indefinitely. + private static readonly Queue _pendingGuestImagePresentations = new(); + private static readonly Dictionary _guestImageWorkSequences = new(); private static readonly Dictionary _availableGuestImages = new(); - private static readonly Dictionary _gpuGuestImages = new(); + private static readonly Dictionary<(int Handle, int BufferIndex), long> + _lastOrderedGuestFlipVersions = new(); + private static long _orderedGuestFlipVersionSequence; + // Storage-image initialization is copied only by the first queued writer. + // Later dispatches targeting the same image must not each retain another + // multi-megabyte guest-memory snapshot while waiting for that first writer + // to reach the presenter. Reference counts let failed/completed work + // retire its reservation without leaving a permanent false cache hit. + private readonly record struct PendingGuestImageUpload(int Count, long OwnerSequence); + private static readonly Dictionary<(ulong Address, uint Format), PendingGuestImageUpload> + _pendingGuestImageUploads = new(); + private static readonly Dictionary _pendingGuestImageInitialData = new(); + private static readonly Dictionary + _guestImageExtents = new(); + private static readonly bool _traceGuestImageEvents = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_DRAWS"), + "1", + StringComparison.Ordinal) || + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_EVENTS"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceGuestWorkCompletion = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_WORK_COMPLETION"), + "1", + StringComparison.Ordinal); private static readonly HashSet<(ulong Address, uint Width, uint Height)> _tracedGuestImageSubmissions = []; private static Thread? _thread; @@ -97,15 +272,66 @@ internal static unsafe class VulkanVideoPresenter private const uint NvidiaVendorId = 0x10DE; private const string PortabilityEnumerationExtensionName = "VK_KHR_portability_enumeration"; private const string PortabilitySubsetExtensionName = "VK_KHR_portability_subset"; + private const int GlfwPlatformHint = 0x00050003; + private const int GlfwPlatformWin32 = 0x00060001; + private const int GlfwPlatformCocoa = 0x00060002; + private const int GlfwPlatformWayland = 0x00060003; + private const int GlfwPlatformX11 = 0x00060004; + private const int GlfwPlatformNull = 0x00060005; private static bool _splashHidden; private static long _enqueuedGuestWorkSequence; + // Largest contiguous completed sequence, retained for compact diagnostics. + // Per-queue scheduling can complete a later global id first, so correctness + // checks use IsGuestWorkCompletedLocked rather than numeric <= comparisons. private static long _completedGuestWorkSequence; + private static readonly HashSet _completedGuestWorkOutOfOrder = []; + private static readonly Dictionary _lastEnqueuedGuestWorkByQueue = + new(StringComparer.Ordinal); + private static long _executingGuestWorkSequence; + [ThreadStatic] + private static VulkanGuestQueueIdentity? _submittingGuestQueue; + [ThreadStatic] + private static bool _enqueueAsImmediateQueueFollowup; + [ThreadStatic] + private static LinkedListNode? _immediateFollowupTail; - private static bool ShouldTracePresentedGuestImageContentsForDiagnostics() + private sealed class GuestQueueScope : IDisposable { - var mode = Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); - return string.Equals(mode, "1", StringComparison.Ordinal) || - string.Equals(mode, "present", StringComparison.OrdinalIgnoreCase); + private readonly VulkanGuestQueueIdentity? _previous; + private bool _disposed; + + public GuestQueueScope(VulkanGuestQueueIdentity queue) + { + _previous = _submittingGuestQueue; + _submittingGuestQueue = queue; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _submittingGuestQueue = _previous; + } + } + + public static IDisposable EnterGuestQueue( + string queueName, + ulong submissionId) => + new GuestQueueScope(new VulkanGuestQueueIdentity( + string.IsNullOrWhiteSpace(queueName) ? "guest.unknown" : queueName, + submissionId)); + + private static long CurrentSubmittingQueueTailLocked() + { + var queue = _submittingGuestQueue; + return queue is { } identity && + _lastEnqueuedGuestWorkByQueue.TryGetValue(identity.Name, out var tail) + ? tail + : 0; } private static bool ShouldTraceGuestImageSubmissionsForDiagnostics() @@ -167,7 +393,7 @@ internal static unsafe class VulkanVideoPresenter 1, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false) : hasSplash ? new Presentation( @@ -177,7 +403,7 @@ internal static unsafe class VulkanVideoPresenter 1, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: true) : new Presentation( null, @@ -186,7 +412,7 @@ internal static unsafe class VulkanVideoPresenter 0, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false); StartPresenterLocked(); } @@ -210,9 +436,8 @@ internal static unsafe class VulkanVideoPresenter sequence, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); Console.Error.WriteLine("[LOADER][INFO] Vulkan VideoOut hid splash"); } } @@ -224,11 +449,6 @@ internal static unsafe class VulkanVideoPresenter return; } - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) - { - Console.Error.WriteLine($"[LOADER][TRACE] vk.submit_call kind=Submit {width}x{height}"); - } - lock (_gate) { if (_closed) @@ -244,9 +464,8 @@ internal static unsafe class VulkanVideoPresenter sequence, GuestDrawKind.None, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: 0, IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); if (_thread is not null) { return; @@ -265,11 +484,6 @@ internal static unsafe class VulkanVideoPresenter return; } - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) - { - Console.Error.WriteLine($"[LOADER][TRACE] vk.submit_call kind=SubmitGuestDraw({drawKind}) {width}x{height}"); - } - lock (_gate) { if (_closed || @@ -289,9 +503,8 @@ internal static unsafe class VulkanVideoPresenter sequence, drawKind, TranslatedDraw: null, - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: CurrentSubmittingQueueTailLocked(), IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); if (_thread is not null) { return; @@ -323,12 +536,6 @@ internal static unsafe class VulkanVideoPresenter return; } - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) - { - Console.Error.WriteLine( - $"[LOADER][TRACE] vk.submit_call kind=SubmitTranslatedDraw {width}x{height} textures={textures.Count}"); - } - lock (_gate) { if (_closed) @@ -355,9 +562,8 @@ internal static unsafe class VulkanVideoPresenter primitiveType, indexBuffer, renderState ?? GuestRenderState.Default), - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + RequiredGuestWorkSequence: CurrentSubmittingQueueTailLocked(), IsSplash: false); - System.Threading.Monitor.PulseAll(_gate); if (_thread is not null) { return; @@ -381,7 +587,9 @@ internal static unsafe class VulkanVideoPresenter uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null) + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0) { SubmitOffscreenTranslatedDraw( pixelSpirv, @@ -395,7 +603,9 @@ internal static unsafe class VulkanVideoPresenter primitiveType, indexBuffer, vertexBuffers, - renderState); + renderState, + depthTarget, + shaderAddress); } public static void SubmitOffscreenTranslatedDraw( @@ -410,7 +620,9 @@ internal static unsafe class VulkanVideoPresenter uint primitiveType = 4, GuestIndexBuffer? indexBuffer = null, IReadOnlyList? vertexBuffers = null, - GuestRenderState? renderState = null) + GuestRenderState? renderState = null, + GuestDepthTarget? depthTarget = null, + ulong shaderAddress = 0) { if (pixelSpirv.Length == 0 || targets.Count == 0 || @@ -447,7 +659,6 @@ internal static unsafe class VulkanVideoPresenter Blends = Enumerable.Repeat(effectiveRenderState.Blends[0], targets.Count).ToArray(), }; } - lock (_gate) { if (_closed) @@ -466,7 +677,7 @@ internal static unsafe class VulkanVideoPresenter } } - EnqueueGuestWorkLocked( + var workSequence = EnqueueGuestWorkLocked( new VulkanOffscreenGuestDraw( new VulkanTranslatedGuestDraw( vertexSpirv ?? [], @@ -481,7 +692,160 @@ internal static unsafe class VulkanVideoPresenter indexBuffer, effectiveRenderState), targets.ToArray(), - PublishTarget: true)); + depthTarget, + PublishTarget: true, + shaderAddress)); + foreach (var target in targets) + { + _guestImageWorkSequences[target.Address] = workSequence; + } + } + } + + public static void SubmitDepthOnlyTranslatedDraw( + byte[] pixelSpirv, + IReadOnlyList textures, + IReadOnlyList globalMemoryBuffers, + uint attributeCount, + GuestDepthTarget depthTarget, + byte[]? vertexSpirv = null, + uint vertexCount = 3, + uint instanceCount = 1, + uint primitiveType = 4, + GuestIndexBuffer? indexBuffer = null, + IReadOnlyList? vertexBuffers = null, + GuestRenderState? renderState = null, + ulong shaderAddress = 0) + { + if (pixelSpirv.Length == 0 || + depthTarget.Address == 0 || + depthTarget.Width == 0 || + depthTarget.Height == 0) + { + return; + } + + lock (_gate) + { + if (_closed) + { + return; + } + + EnqueueGuestWorkLocked( + new VulkanOffscreenGuestDraw( + new VulkanTranslatedGuestDraw( + vertexSpirv ?? [], + pixelSpirv, + textures.ToArray(), + globalMemoryBuffers.ToArray(), + vertexBuffers?.ToArray() ?? [], + attributeCount, + vertexCount, + instanceCount, + primitiveType, + indexBuffer, + renderState ?? GuestRenderState.Default), + [new GuestRenderTarget( + Address: 0, + depthTarget.Width, + depthTarget.Height, + Format: 10, + NumberType: 0)], + depthTarget, + PublishTarget: false, + shaderAddress)); + } + } + + private sealed record VulkanGuestImageWrite( + ulong Address, + byte[]? Pixels, + uint FillValue); + + /// + /// Reports the extent of a live guest image so DMA writes to its backing + /// memory can be mirrored into the Vulkan image (PS5 render targets alias + /// guest memory, so CP DMA fills/copies are visible to later GPU reads). + /// + internal static bool TryGetGuestImageExtent( + ulong address, + out uint width, + out uint height, + out ulong byteCount) + { + lock (_gate) + { + if (_guestImageExtents.TryGetValue(address, out var extent)) + { + (width, height, byteCount) = extent; + return true; + } + } + + width = 0; + height = 0; + byteCount = 0; + return false; + } + + internal static void SubmitGuestImageFill(ulong address, uint fillValue) + { + lock (_gate) + { + if (_closed || !_guestImageExtents.ContainsKey(address)) + { + return; + } + + _guestImageWorkSequences[address] = EnqueueGuestWorkLocked( + new VulkanGuestImageWrite(address, null, fillValue)); + } + } + + internal static void SubmitGuestImageWrite(ulong address, byte[] pixels) + { + lock (_gate) + { + if (_closed || !_guestImageExtents.ContainsKey(address)) + { + return; + } + + _guestImageWorkSequences[address] = EnqueueGuestWorkLocked( + new VulkanGuestImageWrite(address, pixels, 0)); + } + } + + private static long _perfDrawCount; + private static long _perfDrawTicks; + private static long _perfPipelineCreations; + private static long _perfSpirvCompilations; + + internal static (long Draws, double DrawMs, long Pipelines, long SpirvCompilations) + ReadAndResetPerfCounters() + { + var draws = Interlocked.Exchange(ref _perfDrawCount, 0); + var ticks = Interlocked.Exchange(ref _perfDrawTicks, 0); + var pipelines = Interlocked.Exchange(ref _perfPipelineCreations, 0); + var spirv = Interlocked.Exchange(ref _perfSpirvCompilations, 0); + return (draws, ticks * 1000.0 / System.Diagnostics.Stopwatch.Frequency, pipelines, spirv); + } + + internal static void CountSpirvCompilation() => + Interlocked.Increment(ref _perfSpirvCompilations); + + internal static IReadOnlyList<(ulong Address, uint Width, uint Height, ulong ByteCount)> GetGuestImageExtents() + { + lock (_gate) + { + return _guestImageExtents + .Select(entry => ( + entry.Key, + entry.Value.Width, + entry.Value.Height, + entry.Value.ByteCount)) + .ToArray(); } } @@ -491,7 +855,8 @@ internal static unsafe class VulkanVideoPresenter IReadOnlyList globalMemoryBuffers, uint attributeCount, uint width, - uint height) + uint height, + ulong shaderAddress = 0) { if (pixelSpirv.Length == 0 || width == 0 || @@ -528,36 +893,51 @@ internal static unsafe class VulkanVideoPresenter height, Format: 12, NumberType: 7)], - PublishTarget: false)); + DepthTarget: null, + PublishTarget: false, + shaderAddress)); } } - public static void SubmitComputeDispatch( + public static long SubmitComputeDispatch( ulong shaderAddress, byte[] computeSpirv, IReadOnlyList textures, IReadOnlyList globalMemoryBuffers, uint groupCountX, uint groupCountY, - uint groupCountZ) + uint groupCountZ, + uint baseGroupX, + uint baseGroupY, + uint baseGroupZ, + uint localSizeX, + uint localSizeY, + uint localSizeZ, + bool isIndirect, + bool writesGlobalMemory, + uint threadCountX = uint.MaxValue, + uint threadCountY = uint.MaxValue, + uint threadCountZ = uint.MaxValue) { if (computeSpirv.Length == 0 || groupCountX == 0 || groupCountY == 0 || groupCountZ == 0 || - textures.All(texture => !texture.IsStorage)) + textures.All(texture => !texture.IsStorage) && + !writesGlobalMemory) { - return; + return 0; } + long workSequence; lock (_gate) { if (_closed) { - return; + return 0; } - EnqueueGuestWorkLocked( + workSequence = EnqueueGuestWorkLocked( new VulkanComputeGuestDispatch( shaderAddress, computeSpirv, @@ -565,7 +945,127 @@ internal static unsafe class VulkanVideoPresenter globalMemoryBuffers.ToArray(), groupCountX, groupCountY, - groupCountZ)); + groupCountZ, + baseGroupX, + baseGroupY, + baseGroupZ, + localSizeX, + localSizeY, + localSizeZ, + isIndirect, + writesGlobalMemory, + threadCountX, + threadCountY, + threadCountZ)); + foreach (var key in GetStorageImageUploadKeys(textures)) + { + _pendingGuestImageUploads[key] = + _pendingGuestImageUploads.TryGetValue(key, out var pendingUpload) + ? pendingUpload with { Count = checked(pendingUpload.Count + 1) } + : new PendingGuestImageUpload(1, workSequence); + } + + foreach (var texture in textures) + { + if (texture.IsStorage && texture.Address != 0) + { + _guestImageWorkSequences[texture.Address] = workSequence; + } + } + } + + return workSequence; + } + + /// + /// Enqueues a CPU-visible PM4 side effect behind all GPU work submitted + /// before it. The render thread flushes its open batch and waits for the + /// corresponding guest fences before invoking the action. + /// + public static long SubmitOrderedGuestAction(Action action, string debugName) + { + ArgumentNullException.ThrowIfNull(action); + lock (_gate) + { + return _closed || _thread is null + ? 0 + : EnqueueGuestWorkLocked(new VulkanOrderedGuestAction(action, debugName)); + } + } + + /// + /// Sequence currently being executed by the single guest-work consumer. + /// Intended only for address-filtered lifetime diagnostics emitted from a + /// guest-work callback before advances it. + /// + public static long CurrentGuestWorkSequenceForDiagnostics => + Volatile.Read(ref _executingGuestWorkSequence); + + private static bool IsGuestWorkCompletedLocked(long sequence) => + sequence <= 0 || + sequence <= _completedGuestWorkSequence || + _completedGuestWorkOutOfOrder.Contains(sequence); + + public static bool WaitForGuestWork( + long workSequence, + int timeoutMilliseconds = System.Threading.Timeout.Infinite) + { + if (workSequence <= 0) + { + return false; + } + + var waitIndefinitely = timeoutMilliseconds == System.Threading.Timeout.Infinite; + var deadline = waitIndefinitely + ? long.MaxValue + : Environment.TickCount64 + Math.Max(timeoutMilliseconds, 1); + lock (_gate) + { + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_wait_enter sequence={workSequence} " + + $"contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + while (!_closed && !IsGuestWorkCompletedLocked(workSequence)) + { + if (!waitIndefinitely) + { + var remaining = deadline - Environment.TickCount64; + if (remaining <= 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan guest work wait timed out " + + $"sequence={workSequence} contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + return false; + } + + System.Threading.Monitor.Wait( + _gate, + checked((int)Math.Min(remaining, 1_000))); + continue; + } + + // CPU-visible GPU writes are ordering points in the guest + // command stream. First-use shader compilation can take more + // than a minute on MoltenVK; timing out would let the guest + // consume stale zero-filled buffers and permanently corrupt + // the frame. Closing the presenter pulses this monitor, so an + // unbounded correctness wait remains interruptible. + System.Threading.Monitor.Wait(_gate, 1_000); + } + + var completed = IsGuestWorkCompletedLocked(workSequence); + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_wait_exit sequence={workSequence} " + + $"completed={completed} contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + return completed; } } @@ -578,64 +1078,40 @@ internal static unsafe class VulkanVideoPresenter var traceSubmission = false; lock (_gate) { - // VideoOut registration does not imply a rendered Vulkan image. - var known = _gpuGuestImages.ContainsKey(address); - if (ShouldTraceGuestImageSubmissionsForDiagnostics()) + if (_closed || + !_availableGuestImages.ContainsKey(address)) { - Console.Error.WriteLine( - $"[LOADER][TRACE] vk.submit_call kind=TrySubmitGuestImage addr=0x{address:X16} " + - $"{width}x{height} known={known}"); - } - - if (_closed) - { - return false; - } - - // The caller (VideoOutExports.SubmitFlip) reports the flip as successful either - // way, so an unregistered address means the frame is dropped silently; warn once - // per address so that shows up in the log. - if (!known) - { - if (_tracedGuestImageSubmissions.Add((address, width, height))) - { - Console.Error.WriteLine( - $"[LOADER][WARN] vk.submit_guest_image_unknown addr=0x{address:X16} " + - $"{width}x{height} - flip target has no completed GPU render output"); - } - return false; } traceSubmission = _tracedGuestImageSubmissions.Add((address, width, height)); var sequence = (_latestPresentation?.Sequence ?? 0) + 1; - _latestPresentation = new Presentation( + var requiredWorkSequence = _guestImageWorkSequences.TryGetValue( + address, + out var imageWorkSequence) + ? imageWorkSequence + : _completedGuestWorkSequence; + var presentation = new Presentation( null, width, height, sequence, GuestDrawKind.None, TranslatedDraw: null, - // A flip targets the image produced by all work already queued - // for this frame. Do not expose it until those draws finish. - RequiredGuestWorkSequence: _enqueuedGuestWorkSequence, + // Wait only for the work that last wrote this image, not for + // every later command the guest has already queued. Requiring + // the global tail makes a fast guest permanently outrun the + // renderer and turns every flip into a dropped black frame. + RequiredGuestWorkSequence: requiredWorkSequence, IsSplash: false, GuestImageAddress: address); - System.Threading.Monitor.PulseAll(_gate); - if (_thread is not null) + _latestPresentation = presentation; + _pendingGuestImagePresentations.Enqueue(presentation); + while (_pendingGuestImagePresentations.Count > MaxPendingGuestWork) { - return true; + _pendingGuestImagePresentations.Dequeue(); } - - _windowWidth = width; - _windowHeight = height; - _thread = new Thread(Run) - { - IsBackground = true, - Name = "SharpEmu Vulkan VideoOut", - }; - _thread.Start(); } if (traceSubmission) @@ -649,8 +1125,176 @@ internal static unsafe class VulkanVideoPresenter return true; } - // Display buffers registered through sceVideoOutRegisterBuffers are valid flip targets - // even when no AGC render-target write to them was ever observed. + /// + /// Enqueues an AGC flip at its exact position in the logical guest queue. + /// The presenter captures the named image into an immutable Vulkan image + /// before it executes later work from the same queue. Presentation then + /// consumes that captured generation rather than the mutable render target. + /// + public static bool TrySubmitOrderedGuestImageFlip( + int videoOutHandle, + int displayBufferIndex, + ulong address, + uint width, + uint height, + uint pitchInPixel) + { + lock (_gate) + { + if (_closed || + _thread is null || + !_availableGuestImages.ContainsKey(address)) + { + return false; + } + + var version = ++_orderedGuestFlipVersionSequence; + _lastOrderedGuestFlipVersions[(videoOutHandle, displayBufferIndex)] = version; + return EnqueueGuestWorkLocked( + new VulkanOrderedGuestFlip( + version, + videoOutHandle, + displayBufferIndex, + address, + width, + height, + pitchInPixel)) > 0; + } + } + + /// + /// Preserves sceAgcDcbWaitUntilSafeForRendering in queue order. Because an + /// ordered flip first copies the mutable render target into an immutable + /// generation on the same Vulkan queue, reaching this marker proves later + /// rendering cannot change the frame selected by that flip. No CPU wait or + /// event-loop stall is required. + /// + public static long SubmitOrderedGuestFlipWait( + int videoOutHandle, + int displayBufferIndex) + { + lock (_gate) + { + var version = _lastOrderedGuestFlipVersions.TryGetValue( + (videoOutHandle, displayBufferIndex), + out var lastVersion) + ? lastVersion + : 0; + return _closed || _thread is null + ? 0 + : EnqueueGuestWorkLocked( + new VulkanOrderedGuestFlipWait( + version, + videoOutHandle, + displayBufferIndex)); + } + } + + /// + /// On PS5 a render target aliases guest memory, so CPU-prefilled pixels are + /// visible before the first draw. Our Vulkan images start undefined, so the + /// first draw into a new address must seed the image from guest memory. + /// + internal static bool GuestImageWantsInitialData(ulong address) + { + if (address == 0) + { + return false; + } + + lock (_gate) + { + return !_availableGuestImages.ContainsKey(address) && + !_pendingGuestImageInitialData.ContainsKey(address); + } + } + + internal static void ProvideGuestImageInitialData(ulong address, byte[] rgbaPixels) + { + lock (_gate) + { + _pendingGuestImageInitialData[address] = rgbaPixels; + } + } + + private static byte[]? TakeGuestImageInitialData(ulong address) + { + lock (_gate) + { + if (!_pendingGuestImageInitialData.TryGetValue(address, out var data)) + { + return null; + } + + _pendingGuestImageInitialData.Remove(address); + return data; + } + } + + // Mirror of the render thread's texture-cache identities, readable from + // the guest submit thread. The AGC translator used to allocate and copy + // every referenced texture's texels out of guest memory on every draw, + // only for the presenter to discard the bytes on a cache hit — for a + // scene sampling large textures this was by far the dominant CPU cost + // (gigabytes/second of allocation, page faults and GC pressure). + // ConcurrentDictionary keyed set: reads happen per texture per draw on + // the guest submit thread and must not contend with the render thread's + // mutations. + private static readonly System.Collections.Concurrent.ConcurrentDictionary< + TextureContentIdentity, byte> _cachedTextureIdentities = new(); + + internal readonly record struct TextureContentIdentity( + ulong Address, + uint Width, + uint Height, + uint Format, + uint NumberType, + uint DstSelect, + uint TileMode, + uint Pitch, + GuestSampler Sampler); + + // Guest memory handle for render-thread self-healing: when a draw whose + // texel copy was skipped misses the texture cache (eviction, cache + // clear, or any other race), the presenter re-reads the texels itself + // instead of showing a fallback pattern. + private static volatile SharpEmu.HLE.ICpuMemory? _guestMemory; + + internal static void AttachGuestMemory(SharpEmu.HLE.ICpuMemory memory) => + _guestMemory = memory; + + internal static bool IsTextureContentCached(in TextureContentIdentity identity) => + _cachedTextureIdentities.ContainsKey(identity); + + private static void MarkTextureContentCached(in TextureContentIdentity identity) => + _cachedTextureIdentities.TryAdd(identity, 0); + + private static void UnmarkTextureContentCached(in TextureContentIdentity identity) => + _cachedTextureIdentities.TryRemove(identity, out _); + + private static void ClearCachedTextureIdentities() => + _cachedTextureIdentities.Clear(); + + internal static bool IsGuestImageAvailable( + ulong address, + uint format, + uint numberType) + { + var guestFormat = GetGuestTextureFormat(format, numberType); + if (address == 0 || guestFormat == 0) + { + return false; + } + + lock (_gate) + { + return _availableGuestImages.TryGetValue(address, out var availableFormat) && + availableFormat == guestFormat; + } + } + + // Display buffers registered through sceVideoOutRegisterBuffers remain + // valid flip targets even before AGC has rendered into them. internal static void RegisterKnownDisplayBuffer(ulong address, uint guestFormat) { if (address == 0 || guestFormat == 0) @@ -665,6 +1309,19 @@ internal static unsafe class VulkanVideoPresenter } internal static bool IsGpuGuestImageAvailable( + ulong address, + uint format, + uint numberType) => + IsGuestImageAvailable(address, format, numberType); + + /// + /// Returns whether a storage image already exists on the presenter or an + /// earlier queued dispatch owns its one-time guest-memory initialization. + /// This is intentionally separate from : + /// a pending image may skip a duplicate upload but is not yet safe for a + /// flip/presentation lookup. + /// + internal static bool IsGuestImageUploadKnown( ulong address, uint format, uint numberType) @@ -677,8 +1334,9 @@ internal static unsafe class VulkanVideoPresenter lock (_gate) { - return _gpuGuestImages.TryGetValue(address, out var availableFormat) && - availableFormat == guestFormat; + return _availableGuestImages.TryGetValue(address, out var availableFormat) && + availableFormat == guestFormat || + _pendingGuestImageUploads.ContainsKey((address, guestFormat)); } } @@ -687,10 +1345,12 @@ internal static unsafe class VulkanVideoPresenter uint sourceWidth, uint sourceHeight, uint sourceFormat, + uint sourceNumberType, ulong destinationAddress, uint destinationWidth, uint destinationHeight, - uint destinationFormat) + uint destinationFormat, + uint destinationNumberType) { if (sourceAddress == 0 || destinationAddress == 0 || @@ -721,7 +1381,7 @@ internal static unsafe class VulkanVideoPresenter sourceWidth, sourceHeight, sourceFormat, - NumberType: 0, + sourceNumberType, [], IsFallback: false, IsStorage: false), @@ -733,7 +1393,7 @@ internal static unsafe class VulkanVideoPresenter destinationWidth, destinationHeight, destinationFormat, - NumberType: 0)); + destinationNumberType)); return true; } @@ -759,39 +1419,10 @@ internal static unsafe class VulkanVideoPresenter return true; } - private static uint GetGuestTextureFormat(uint format, uint numberType) - { - if (!TryDecodeRenderTargetFormat(format, numberType, out var decoded)) - { - return 0; - } - - return decoded.Format switch - { - Format.R8Unorm => 36, - Format.R8Uint => 49, - Format.R8G8Unorm => 3, - Format.A2R10G10B10UnormPack32 => 9, - Format.B10G11R11UfloatPack32 => 7, - Format.R32Uint => GuestFormatR32Uint, - Format.R32Sint => GuestFormatR32Sint, - Format.R32Sfloat => GuestFormatR32Sfloat, - Format.R16G16Unorm => 5, - Format.R16G16Uint => GuestFormatR16G16Uint, - Format.R16G16Sint => GuestFormatR16G16Sint, - Format.R16G16Sfloat => GuestFormatR16G16Sfloat, - Format.R32G32Sfloat => 75, - Format.R8G8B8A8Unorm => 56, - Format.R8G8B8A8Uint => GuestFormatR8G8B8A8Uint, - Format.R8G8B8A8Sint => GuestFormatR8G8B8A8Sint, - Format.R16G16B16A16Unorm => 12, - Format.R16G16B16A16Uint => GuestFormatR16G16B16A16Uint, - Format.R16G16B16A16Sint => GuestFormatR16G16B16A16Sint, - Format.R16G16B16A16Sfloat => 71, - Format.R32G32B32A32Sfloat => 14, - _ => 0, - }; - } + private static uint GetGuestTextureFormat(uint format, uint numberType) => + IsKnownGuestTextureFormat(format) + ? 0x8000_0000u | ((format & 0x1FFu) << 8) | (numberType & 0xFFu) + : 0; internal static bool TryDecodeRenderTargetFormat( uint dataFormat, @@ -806,23 +1437,19 @@ internal static unsafe class VulkanVideoPresenter (5, 4) => Format.R16G16Uint, (5, 5) => Format.R16G16Sint, (5, 7) => Format.R16G16Sfloat, + (6, 7) or (7, 7) => Format.B10G11R11UfloatPack32, (9, _) => Format.A2R10G10B10UnormPack32, (10, 4) => Format.R8G8B8A8Uint, (10, 5) => Format.R8G8B8A8Sint, + (10, 9) => Format.R8G8B8A8Srgb, (10, _) => Format.R8G8B8A8Unorm, + (11, 7) => Format.R32G32Sfloat, (12, 4) => Format.R16G16B16A16Uint, (12, 5) => Format.R16G16B16A16Sint, (12, 7) => Format.R16G16B16A16Sfloat, - (GuestFormatR32Uint, _) or (20, 0) => Format.R32Uint, - (GuestFormatR32Sint, _) => Format.R32Sint, - (GuestFormatR32Sfloat, _) or (29, 0) or (4, 0) => Format.R32Sfloat, - (GuestFormatR16G16Uint, _) => Format.R16G16Uint, - (GuestFormatR16G16Sint, _) => Format.R16G16Sint, - (GuestFormatR16G16Sfloat, _) => Format.R16G16Sfloat, - (GuestFormatR8G8B8A8Uint, _) => Format.R8G8B8A8Uint, - (GuestFormatR8G8B8A8Sint, _) => Format.R8G8B8A8Sint, - (GuestFormatR16G16B16A16Uint, _) => Format.R16G16B16A16Uint, - (GuestFormatR16G16B16A16Sint, _) => Format.R16G16B16A16Sint, + (13, 7) or (14, 7) => Format.R32G32B32A32Sfloat, + (20, 0) => Format.R32Uint, + (29, 0) or (4, 0) => Format.R32Sfloat, (1, 0) or (36, 0) => Format.R8Unorm, (49, 0) => Format.R8Uint, (3, 0) => Format.R8G8Unorm, @@ -855,7 +1482,7 @@ internal static unsafe class VulkanVideoPresenter } private static bool IsKnownGuestTextureFormat(uint format) => - format is 4 or 5 or 7 or 9 or 13 or 14 or 22 or 29 or 36 or 56 or 62 or 64 or 71; + format is >= 1 and <= 19 or 34 or >= 169 and <= 182; private static byte[] CreateBlackFrame(uint width, uint height) { @@ -879,11 +1506,10 @@ internal static unsafe class VulkanVideoPresenter { if (HostMainThread.IsAvailable) { - // GLFW windowing must run on the process main thread (AppKit on - // macOS, X11's single event queue on Linux), so hand the whole - // window loop to the main-thread pump the CLI parked for us. - // _thread only marks the presenter as running; Run() clears it on - // exit either way. + // AppKit (and therefore GLFW) traps when touched off the process + // main thread on macOS, so hand the whole window loop to the + // main-thread pump the CLI parked for us. _thread only marks the + // presenter as running; Run() clears it on exit either way. _thread = Thread.CurrentThread; HostMainThread.SetShutdownRequestHandler(RequestClose); HostMainThread.Post(Run); @@ -961,25 +1587,6 @@ internal static unsafe class VulkanVideoPresenter } } - // GLFW platform enum (GLFW 3.4): glfwInitHint(GLFW_PLATFORM, ...) selects a - // backend, glfwGetPlatform() reports the one in use. - private const int GlfwPlatformHint = 0x00050003; - private const int GlfwPlatformWin32 = 0x00060001; - private const int GlfwPlatformCocoa = 0x00060002; - private const int GlfwPlatformWayland = 0x00060003; - private const int GlfwPlatformX11 = 0x00060004; - private const int GlfwPlatformNull = 0x00060005; - - /// - /// GLFW's native Wayland backend does not reliably map the Vulkan window - /// with some drivers (notably NVIDIA): the surface presents frames but the - /// window never becomes visible, so the game runs with no picture while - /// audio works. XWayland is dependable, so on a Wayland session that also - /// exposes an X server (DISPLAY set) we force GLFW's X11 backend through - /// its GLFW_PLATFORM init hint before GLFW initializes — the supported way - /// to pick a backend, applied by calling into the same libglfw GLFW loads. - /// Opt back into native Wayland with SHARPEMU_ENABLE_WAYLAND=1. - /// private static unsafe void PreferX11OnLinuxWayland() { if (!OperatingSystem.IsLinux() || @@ -991,10 +1598,6 @@ internal static unsafe class VulkanVideoPresenter return; } - // Only steer on a Wayland session (WAYLAND_DISPLAY set). Forcing X11 - // needs an X server to fall back to (XWayland, DISPLAY set); without - // one, forcing it would make glfwInit fail outright, so leave GLFW - // alone and say why the window may not appear. if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("WAYLAND_DISPLAY"))) { return; @@ -1004,8 +1607,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Wayland session without an X server (DISPLAY unset); " + - "cannot steer GLFW to XWayland. If the window does not appear, install " + - "XWayland, or run natively with SHARPEMU_ENABLE_WAYLAND=1."); + "cannot steer GLFW to XWayland. Set SHARPEMU_ENABLE_WAYLAND=1 to use native Wayland."); return; } @@ -1020,8 +1622,7 @@ internal static unsafe class VulkanVideoPresenter System.Runtime.InteropServices.NativeLibrary.GetExport(glfw, "glfwInitHint"); initHint(GlfwPlatformHint, GlfwPlatformX11); Console.Error.WriteLine( - "[LOADER][INFO] Wayland session detected; requested GLFW X11/XWayland " + - "backend (set SHARPEMU_ENABLE_WAYLAND=1 to force native Wayland)."); + "[LOADER][INFO] Wayland session detected; requested GLFW X11/XWayland backend."); } catch (Exception exception) { @@ -1030,8 +1631,6 @@ internal static unsafe class VulkanVideoPresenter } } - /// Logs the backend GLFW actually selected, so a "no window" - /// report shows Wayland vs X11 at a glance. private static unsafe void LogGlfwPlatformInUse() { if (OperatingSystem.IsWindows() || !TryLoadGlfw(out var glfw)) @@ -1079,8 +1678,8 @@ internal static unsafe class VulkanVideoPresenter height = _windowHeight == 0 ? _latestPresentation?.Height ?? 720 : _windowHeight; } - InitializeMacVulkanLoader(); PreferX11OnLinuxWayland(); + InitializeMacVulkanLoader(); try { @@ -1106,53 +1705,476 @@ internal static unsafe class VulkanVideoPresenter { lock (_gate) { + // Guest flips are retained in submission order. The renderer is + // deliberately allowed to lag a frame or two behind the guest + // while it drains expensive work, so use the first completed flip + // rather than repeatedly asking only for the newest one. + while (_pendingGuestImagePresentations.Count > 0 && + _pendingGuestImagePresentations.Peek().Sequence <= presentedSequence) + { + _pendingGuestImagePresentations.Dequeue(); + } + + if (_pendingGuestImagePresentations.Count > 0) + { + var pending = _pendingGuestImagePresentations.Peek(); + if (IsGuestWorkCompletedLocked(pending.RequiredGuestWorkSequence)) + { + presentation = _pendingGuestImagePresentations.Dequeue(); + TryReplaceWithBinkFrame(ref presentation); + return true; + } + + presentation = default; + return false; + } + if (_latestPresentation is not { } latest || latest.Sequence == presentedSequence || - latest.RequiredGuestWorkSequence > _completedGuestWorkSequence) + !IsGuestWorkCompletedLocked(latest.RequiredGuestWorkSequence)) { + if (_latestPresentation is { } rej && + rej.GuestImageAddress != 0 && + rej.Sequence != presentedSequence && + _tracedGuestImagePresentRejections.Add(rej.Sequence)) + { + var reason = rej.Sequence == presentedSequence + ? "already-presented(seq==presented)" + : !IsGuestWorkCompletedLocked(rej.RequiredGuestWorkSequence) + ? $"work-not-done(req={rej.RequiredGuestWorkSequence}>" + + $"contiguous_done={_completedGuestWorkSequence})" + : "unknown"; + Console.Error.WriteLine( + $"[LOADER][WARN] vk.guest_present_rejected addr=0x{rej.GuestImageAddress:X16} " + + $"seq={rej.Sequence} presentedSeq={presentedSequence} reason={reason}"); + } + presentation = default; return false; } presentation = latest; + TryReplaceWithBinkFrame(ref presentation); return true; } } - private static void EnqueueGuestWorkLocked(object work) + private static void TryReplaceWithBinkFrame(ref Presentation presentation) { - while (!_closed && - _thread is not null && - _pendingGuestWork.Count >= MaxPendingGuestWork) + if (!Bink2MovieBridge.TryDecodeNextFrame(out var pixels, out var width, out var height)) { + return; + } + + presentation = new Presentation( + pixels, + width, + height, + presentation.Sequence, + GuestDrawKind.None, + TranslatedDraw: null, + presentation.RequiredGuestWorkSequence, + IsSplash: false); + } + + private static readonly HashSet _tracedGuestImagePresentRejections = new(); + + private static bool HasPendingGuestPresentation(long presentedSequence) + { + lock (_gate) + { + return _pendingGuestImagePresentations.Count > 0 || + _latestPresentation is { } latest && latest.Sequence > presentedSequence; + } + } + + private static long EnqueueGuestWorkLocked(object work) + { + var payloadBytes = GetGuestWorkPayloadBytes(work); + var backpressureLogged = false; + // Work executed by the render-thread consumer can enqueue an ordered + // same-queue completion marker. Blocking that consumer on the normal + // producer backpressure limit deadlocks a full queue: no other thread + // can drain an item to make room for the marker. The consumer has + // already removed the current item, and each immediate follow-up is + // bounded by that item, so admitting it cannot cause unbounded growth. + while (!_enqueueAsImmediateQueueFollowup && + !_closed && + _thread is not null && + (_pendingGuestWorkCount >= MaxPendingGuestWork || + // Always admit one item when no payload is outstanding, even + // when that single item exceeds the configured budget. This + // avoids an impossible wait while still bounding the normal + // multi-item backlog. + (_pendingGuestWorkBytes != 0 && + payloadBytes > _maxPendingGuestWorkBytes - + Math.Min(_pendingGuestWorkBytes, _maxPendingGuestWorkBytes)))) + { + if (!backpressureLogged) + { + backpressureLogged = true; + var traceCount = Interlocked.Increment( + ref _guestQueueBackpressureTraceCount); + if (traceCount <= 16 || (traceCount & (traceCount - 1)) == 0) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_queue_backpressure " + + $"count={traceCount} " + + $"queued={_pendingGuestWorkCount} " + + $"logical_queues={_pendingGuestWorkByQueue.Count} " + + $"retained_mb={_pendingGuestWorkBytes / (1024 * 1024)} " + + $"incoming_mb={payloadBytes / (1024 * 1024)} " + + $"budget_mb={_maxPendingGuestWorkBytes / (1024 * 1024)} " + + $"work={work.GetType().Name}" + + GetGuestWorkPayloadBreakdown(work)); + } + } + System.Threading.Monitor.Wait(_gate); } if (_closed) + { + return 0; + } + + var queue = _submittingGuestQueue ?? VulkanGuestQueueIdentity.Default; + var sequence = ++_enqueuedGuestWorkSequence; + _lastEnqueuedGuestWorkByQueue[queue.Name] = sequence; + var requiredSequence = GetGuestWorkDependencyLocked(work); + if (!_pendingGuestWorkByQueue.TryGetValue(queue.Name, out var pendingQueue)) + { + pendingQueue = new LinkedList(); + _pendingGuestWorkByQueue.Add(queue.Name, pendingQueue); + _pendingGuestQueueSchedule.Add(queue.Name); + } + + var pending = new PendingGuestWork( + work, + payloadBytes, + sequence, + requiredSequence, + System.Diagnostics.Stopwatch.GetTimestamp(), + queue); + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_enqueue sequence={sequence} " + + $"required={requiredSequence} queue={queue.Name} " + + $"immediate={_enqueueAsImmediateQueueFollowup} " + + $"work={work.GetType().Name}"); + } + if (_enqueueAsImmediateQueueFollowup && + _immediateFollowupTail is { List: not null } tail && + ReferenceEquals(tail.List, pendingQueue)) + { + _immediateFollowupTail = pendingQueue.AddAfter(tail, pending); + } + else if (_enqueueAsImmediateQueueFollowup) + { + _immediateFollowupTail = pendingQueue.AddFirst(pending); + } + else + { + pendingQueue.AddLast(pending); + } + RecordGuestImageWritersLocked(work, sequence); + _pendingGuestWorkCount++; + _pendingGuestWorkBytes = SaturatingAdd(_pendingGuestWorkBytes, payloadBytes); + return sequence; + } + + private static long GetGuestWorkDependencyLocked(object work) + { + IReadOnlyList textures = work switch + { + VulkanOffscreenGuestDraw draw => draw.Draw.Textures, + VulkanComputeGuestDispatch compute => compute.Textures, + _ => Array.Empty(), + }; + var required = 0L; + foreach (var texture in textures) + { + if (!texture.IsStorage || + texture.Address == 0 || + texture.RgbaPixels.Length != 0) + { + continue; + } + + var format = GetGuestTextureFormat(texture.Format, texture.NumberType); + if (_pendingGuestImageUploads.TryGetValue( + (texture.Address, format), + out var pendingUpload)) + { + required = Math.Max(required, pendingUpload.OwnerSequence); + } + } + + return required; + } + + private static void RecordGuestImageWritersLocked(object work, long sequence) + { + static IEnumerable StorageAddresses( + IReadOnlyList textures) => + textures + .Where(static texture => texture.IsStorage && texture.Address != 0) + .Select(static texture => texture.Address); + + IEnumerable addresses = work switch + { + VulkanOffscreenGuestDraw draw => + (draw.PublishTarget + ? draw.Targets + .Where(static target => target.Address != 0) + .Select(static target => target.Address) + : Enumerable.Empty()) + .Concat(StorageAddresses(draw.Draw.Textures)), + VulkanComputeGuestDispatch compute => StorageAddresses(compute.Textures), + VulkanGuestImageWrite imageWrite when imageWrite.Address != 0 => + new[] { imageWrite.Address }, + _ => Array.Empty(), + }; + foreach (var address in addresses.Distinct()) + { + _guestImageWorkSequences[address] = sequence; + } + } + + private static bool TryTakeGuestWork(out PendingGuestWork work) + { + lock (_gate) + { + var queuesToProbe = _pendingGuestQueueSchedule.Count; + while (_pendingGuestQueueSchedule.Count > 0 && queuesToProbe > 0) + { + if (_pendingGuestQueueCursor >= _pendingGuestQueueSchedule.Count) + { + _pendingGuestQueueCursor = 0; + } + + var queueName = _pendingGuestQueueSchedule[_pendingGuestQueueCursor]; + if (!_pendingGuestWorkByQueue.TryGetValue(queueName, out var queue) || + queue.First is not { } first) + { + _pendingGuestWorkByQueue.Remove(queueName); + _pendingGuestQueueSchedule.RemoveAt(_pendingGuestQueueCursor); + queuesToProbe = Math.Min( + queuesToProbe, + _pendingGuestQueueSchedule.Count); + continue; + } + + work = first.Value; + if (!IsGuestWorkCompletedLocked(work.RequiredSequence)) + { + _pendingGuestQueueCursor = + (_pendingGuestQueueCursor + 1) % _pendingGuestQueueSchedule.Count; + queuesToProbe--; + continue; + } + + queue.RemoveFirst(); + _pendingGuestWorkCount--; + if (queue.Count == 0) + { + _pendingGuestWorkByQueue.Remove(queueName); + _pendingGuestQueueSchedule.RemoveAt(_pendingGuestQueueCursor); + } + else + { + _pendingGuestQueueCursor = + (_pendingGuestQueueCursor + 1) % _pendingGuestQueueSchedule.Count; + } + + return true; + } + + work = default; + return false; + } + } + + private static void CompleteGuestWork(in PendingGuestWork pending) + { + SharpEmu.HLE.GuestImageWriteTracker.FlushPendingDiagnostics(); + lock (_gate) + { + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_complete_enter " + + $"sequence={pending.Sequence} work={pending.Work.GetType().Name} " + + $"contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + _pendingGuestWorkBytes = pending.PayloadBytes >= _pendingGuestWorkBytes + ? 0 + : _pendingGuestWorkBytes - pending.PayloadBytes; + ReleasePendingGuestImageUploadsLocked(pending.Work); + if (pending.Sequence == _completedGuestWorkSequence + 1) + { + _completedGuestWorkSequence = pending.Sequence; + while (_completedGuestWorkOutOfOrder.Remove( + _completedGuestWorkSequence + 1)) + { + _completedGuestWorkSequence++; + } + } + else if (pending.Sequence > _completedGuestWorkSequence) + { + // Debug.Assert calls are compiled out of Release builds, so + // never put the state mutation inside its argument. Doing so + // discarded every out-of-order completion in normal runs and + // left the first immediate follow-up as a permanent sequence + // hole, blocking all later CPU-visible GPU waits. + var added = _completedGuestWorkOutOfOrder.Add(pending.Sequence); + System.Diagnostics.Debug.Assert( + added, + "A guest work sequence must complete exactly once."); + } + System.Threading.Monitor.PulseAll(_gate); + if (_traceGuestWorkCompletion) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.guest_work_complete_exit " + + $"sequence={pending.Sequence} contiguous_completed={_completedGuestWorkSequence} " + + $"out_of_order={_completedGuestWorkOutOfOrder.Count}"); + } + } + } + + private static ulong GetGuestWorkPayloadBytes(object work) => work switch + { + VulkanComputeGuestDispatch compute => SaturatingAdd( + GetTexturePayloadBytes(compute.Textures), + GetGlobalBufferPayloadBytes(compute.GlobalMemoryBuffers)), + VulkanOffscreenGuestDraw offscreen => GetDrawPayloadBytes(offscreen.Draw), + VulkanGuestImageWrite { Pixels: { } pixels } => (ulong)pixels.LongLength, + _ => 0, + }; + + private static string GetGuestWorkPayloadBreakdown(object work) + { + static ulong SumTextures(IReadOnlyList textures) => + GetTexturePayloadBytes(textures) / (1024 * 1024); + static ulong SumGlobals(IReadOnlyList buffers) => + GetGlobalBufferPayloadBytes(buffers) / (1024 * 1024); + + return work switch + { + VulkanOffscreenGuestDraw offscreen => + $" textures_mb={SumTextures(offscreen.Draw.Textures)}" + + $" globals_mb={SumGlobals(offscreen.Draw.GlobalMemoryBuffers)}" + + $" vertex_mb={offscreen.Draw.VertexBuffers.Aggregate(0UL, static (sum, buffer) => SaturatingAdd(sum, (ulong)buffer.Data.LongLength)) / (1024 * 1024)}" + + $" index_mb={(ulong)(offscreen.Draw.IndexBuffer?.Data.LongLength ?? 0) / (1024 * 1024)}" + + $" vertex_lengths=[{string.Join(',', offscreen.Draw.VertexBuffers.Select(static buffer => $"{buffer.Length}/{buffer.Data.LongLength}:s{buffer.Stride}:o{buffer.OffsetBytes}"))}]" + + $" global_lengths=[{string.Join(',', offscreen.Draw.GlobalMemoryBuffers.Select(static buffer => buffer.Length))}]", + VulkanComputeGuestDispatch compute => + $" textures_mb={SumTextures(compute.Textures)}" + + $" globals_mb={SumGlobals(compute.GlobalMemoryBuffers)}" + + $" global_lengths=[{string.Join(',', compute.GlobalMemoryBuffers.Select(static buffer => buffer.Length))}]", + _ => string.Empty, + }; + } + + private static ulong GetDrawPayloadBytes(VulkanTranslatedGuestDraw draw) + { + var bytes = GetTexturePayloadBytes(draw.Textures); + bytes = SaturatingAdd(bytes, GetGlobalBufferPayloadBytes(draw.GlobalMemoryBuffers)); + var uniqueVertexData = new HashSet( + System.Collections.Generic.ReferenceEqualityComparer.Instance); + foreach (var vertex in draw.VertexBuffers) + { + if (uniqueVertexData.Add(vertex.Data)) + { + bytes = SaturatingAdd(bytes, (ulong)vertex.Data.LongLength); + } + } + + if (draw.IndexBuffer is { } index) + { + bytes = SaturatingAdd(bytes, (ulong)index.Data.LongLength); + } + + return bytes; + } + + private static ulong GetTexturePayloadBytes( + IReadOnlyList textures) + { + var bytes = 0UL; + foreach (var texture in textures) + { + bytes = SaturatingAdd(bytes, (ulong)texture.RgbaPixels.LongLength); + } + + return bytes; + } + + private static ulong GetGlobalBufferPayloadBytes( + IReadOnlyList buffers) + { + var bytes = 0UL; + foreach (var buffer in buffers) + { + bytes = SaturatingAdd(bytes, (ulong)buffer.Data.LongLength); + } + + return bytes; + } + + private static ulong SaturatingAdd(ulong left, ulong right) => + ulong.MaxValue - left < right ? ulong.MaxValue : left + right; + + private static void ReleasePendingGuestImageUploadsLocked(object work) + { + if (work is not VulkanComputeGuestDispatch compute) { return; } - _pendingGuestWork.Enqueue(work); - _enqueuedGuestWorkSequence++; - System.Threading.Monitor.PulseAll(_gate); - } - - private static bool TryTakeGuestWork(out object work) - { - lock (_gate) + foreach (var key in GetStorageImageUploadKeys(compute.Textures)) { - return _pendingGuestWork.TryDequeue(out work!); + if (!_pendingGuestImageUploads.TryGetValue(key, out var pendingUpload)) + { + continue; + } + + if (pendingUpload.Count <= 1) + { + _pendingGuestImageUploads.Remove(key); + } + else + { + _pendingGuestImageUploads[key] = pendingUpload with + { + Count = pendingUpload.Count - 1, + }; + } } } - private static void CompleteGuestWork() + private static HashSet<(ulong Address, uint Format)> GetStorageImageUploadKeys( + IReadOnlyList textures) { - lock (_gate) + var keys = new HashSet<(ulong Address, uint Format)>(); + foreach (var texture in textures) { - _completedGuestWorkSequence++; - System.Threading.Monitor.PulseAll(_gate); + if (!texture.IsStorage || texture.Address == 0) + { + continue; + } + + var format = GetGuestTextureFormat(texture.Format, texture.NumberType); + if (format != 0) + { + keys.Add((texture.Address, format)); + } } + + return keys; } private readonly record struct Presentation( @@ -1164,7 +2186,8 @@ internal static unsafe class VulkanVideoPresenter VulkanTranslatedGuestDraw? TranslatedDraw, long RequiredGuestWorkSequence, bool IsSplash, - ulong GuestImageAddress = 0); + ulong GuestImageAddress = 0, + long GuestImageVersion = 0); private sealed class Presenter : IDisposable { @@ -1176,13 +2199,6 @@ internal static unsafe class VulkanVideoPresenter private readonly IWindow _window; private const int MaxInFlightGuestSubmissions = 8; - private const double PerformanceHudSampleSeconds = 0.5; - private const uint ThreadQueryLimitedInformation = 0x0800; - private static readonly bool _performanceHudEnabled = - !string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_PERF_HUD"), - "0", - StringComparison.Ordinal); private Vk _vk = null!; private KhrSurface _surfaceApi = null!; private KhrSwapchain _swapchainApi = null!; @@ -1194,11 +2210,21 @@ internal static unsafe class VulkanVideoPresenter private DebugUtilsMessengerEXT _debugMessenger; private ExtDebugUtils? _debugUtils; private PhysicalDevice _physicalDevice; + private uint _maxComputeWorkGroupCountX; + private uint _maxComputeWorkGroupCountY; + private uint _maxComputeWorkGroupCountZ; + private uint _maxComputeWorkGroupSizeX; + private uint _maxComputeWorkGroupSizeY; + private uint _maxComputeWorkGroupSizeZ; + private uint _maxComputeWorkGroupInvocations; + private ulong _minStorageBufferOffsetAlignment = 1; private bool _supportsIndependentBlend; private uint _maxColorAttachments; private Device _device; private PipelineCache _pipelineCache; private string? _pipelineCachePath; + private bool _pipelineCacheDirty; + private long _lastPipelineCacheSaveTick; private Queue _queue; private uint _queueFamilyIndex; private SwapchainKHR _swapchain; @@ -1214,26 +2240,54 @@ internal static unsafe class VulkanVideoPresenter private CommandPool _commandPool; private CommandBuffer _commandBuffer; private CommandBuffer _presentationCommandBuffer; - private VkSemaphore _imageAvailable; - private VkSemaphore _renderFinished; - private Fence _presentationFence; - private bool _presentationInFlight; - private TranslatedDrawResources? _pendingPresentationResources; + // Presentation runs with multiple frames in flight: each frame slot + // owns a command buffer, an acquire semaphore and a fence, so the CPU + // can record frame N+1 while the GPU still executes frame N. The + // per-present vkQueueWaitIdle this replaces made CPU and GPU costs + // strictly additive. Render-finished semaphores are per swapchain + // image because the presentation engine may still wait on them after + // the frame fence has signaled. + private const int MaxFramesInFlight = 2; + private CommandBuffer[] _frameCommandBuffers = []; + private VkSemaphore[] _frameImageAvailable = []; + private VkSemaphore[] _renderFinishedPerImage = []; + private Fence[] _frameFences = []; + private bool[] _frameFencePending = []; + private ulong[] _frameTimelines = []; + private TranslatedDrawResources?[] _frameTranslatedResources = []; + private GuestImageResource?[] _frameGuestImageVersions = []; + private int _currentFrameSlot; + // Monotonic submission/completion counters across every queue submit + // (guest batches, compute chunks and presents). Fences on a single + // queue signal in submission order, so "timeline <= completed" means + // the GPU is done with everything submitted up to that point; this + // lets evicted resources be destroyed without a queue drain. + private ulong _submitTimeline; + private ulong _completedTimeline; + private readonly Queue<(TextureResource Texture, ulong RetireTimeline)> + _deferredTextureDestroys = new(); + private readonly Queue<(TranslatedDrawResources Resources, ulong RetireTimeline)> + _deferredResourceDestroys = new(); + private readonly Queue<(GuestImageResource Image, ulong RetireTimeline)> + _deferredGuestImageVersionDestroys = new(); + private readonly Stack _recycledGuestFences = new(); + private readonly Stack _recycledGuestCommandBuffers = new(); + private readonly List<(VkBuffer Buffer, DeviceMemory Memory)> _batchRetireBuffers = new(); + private const int MaxRecycledGuestFences = 32; + private const int MaxRecycledGuestCommandBuffers = 32; private VkBuffer _stagingBuffer; private DeviceMemory _stagingMemory; private ulong _stagingSize; + // Perf overlay: CPU-rasterized panel copied through per-slot staging + // buffers into one image, then blitted onto the swapchain. + private Image _overlayImage; + private DeviceMemory _overlayImageMemory; + private bool _overlayImageInitialized; + private VkBuffer[] _overlayStagingBuffers = []; + private DeviceMemory[] _overlayStagingMemory = []; + private nint[] _overlayStagingMapped = []; private long _presentedSequence; - private long _performanceHudLastTimestamp; - private TimeSpan _performanceHudLastProcessCpu; - private long _performanceHudPresentedFrames; - private long _performanceHudLastPresentedFrames; - private long _performanceHudLastReadCount; - private long _performanceHudLastReadBytes; - private long _performanceHudLastReadHits; - private long _performanceHudLastReadPvmBytes; - private long _performanceHudLastReadLibcBytes; - private readonly Dictionary _performanceHudThreadCpu = []; - private readonly Dictionary _performanceHudThreadNames = []; + private long _presentNotTakenLoggedSequence = long.MinValue; private bool _vulkanReady; private bool _firstFramePresented; private bool _firstGuestDrawPresented; @@ -1241,25 +2295,62 @@ internal static unsafe class VulkanVideoPresenter private bool _swapchainRecreateDeferred; private bool _tracedPresentedSwapchain; private bool _swapchainReadbackPending; + private long _presentedSwapchainCount; private static int _guestImageDumpSequence; private readonly System.Collections.Concurrent.ConcurrentQueue _pendingAliasImageDumps = new(); private bool _deviceLost; private bool _deviceLostLogged; private int _directPresentationCount; + private readonly Dictionary _presentedGuestImageTraceCounts = new(); private readonly Dictionary _guestImages = new(); + private readonly record struct GuestImageVariantKey( + ulong Address, + uint Width, + uint Height, + uint MipLevels, + uint GuestFormat, + Format Format); + + // A single guest allocation may be rebound through several RT descriptors. + // Keep the inactive Vulkan images instead of destroying their contents each + // time the guest switches size or format at the same address. + private readonly Dictionary + _guestImageVariants = new(); + private readonly Dictionary _guestImageVersions = new(); + private readonly HashSet _capturedGuestFlipVersions = []; + private readonly record struct GuestDepthKey( + ulong Address, + ulong ReadAddress, + uint Width, + uint Height, + uint GuestFormat, + uint SwizzleMode); + + private readonly Dictionary _guestDepthImages = new(); + private readonly Dictionary _depthOnlyColorAddresses = new(); + private ulong _nextDepthOnlyColorAddress = 0xFFFF_FF00_0000_0000UL; private readonly HashSet<(ulong Address, uint Width, uint Height, Format Format)> _tracedTextureCacheHits = new(); - private static readonly bool _dumpTextures = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_DUMP_TEXTURES"), - "1", - StringComparison.Ordinal); + private readonly HashSet<(ulong Address, uint Width, uint Height, uint DstSelect)> _tracedDepthTextureAliases = new(); + private readonly HashSet<(ulong Address, uint Width, uint Height)> _tracedDepthExtentFallbacks = new(); private readonly HashSet<(ulong Address, uint Width, uint Height, Format Format)> _tracedTextureUploads = new(); private readonly HashSet<(ulong Address, uint Width, uint Height, uint Format)> _dumpedTextures = new(); + private readonly HashSet<(ulong Address, uint Width, uint Height, uint Format)> _tracedTextureUploadContents = new(); private readonly HashSet<(ulong Address, int Size)> _tracedGlobalBuffers = new(); + private readonly HashSet<(ulong Address, ulong Size)> _tracedGlobalWritebacks = new(); + private readonly HashSet<(ulong Shader, uint X, uint Y, uint Z, string Reason)> + _rejectedComputeDispatches = new(); + private int _tracedSmallGlobalWritebackEvents; + private int _tracedLargeGlobalWritebackEvents; private readonly HashSet _tracedGuestImageContents = new(); private readonly Dictionary _tracedGuestWriteCounts = new(); + private readonly Dictionary _pixelSpirvWriteCounts = new(); private int _tracedVertexBufferCount; - private readonly Dictionary _computePipelines = - new(ReferenceEqualityComparer.Instance); + private bool _tracedTitleDraw; + // Compute translation can produce an equivalent new byte array on a + // later submit. Reference identity turns that into an expensive new + // MoltenVK pipeline compilation every frame, so key the cache by the + // program content and descriptor-layout shape instead. + private readonly Dictionary _computePipelines = new(); private readonly Dictionary _graphicsPipelines = new(); private readonly Dictionary _samplers = new(); private readonly Dictionary _shaderDigests = @@ -1269,16 +2360,26 @@ internal static unsafe class VulkanVideoPresenter private readonly Dictionary> _hostBufferPool = new(); private readonly Dictionary _hostBufferAllocations = new(); + private readonly List _guestBufferAllocations = []; private readonly Queue _pendingGuestSubmissions = new(); + private readonly Dictionary _lastSubmittedTimelineByGuestQueue = + new(StringComparer.Ordinal); + private readonly Stack _recycledDescriptorPools = new(); + private VulkanGuestQueueIdentity _activeGuestQueue = + VulkanGuestQueueIdentity.Default; + private long _activeGuestWorkSequence; private readonly record struct GraphicsPipelineKey( string VertexShader, string FragmentShader, string RenderTargetLayout, + bool HasDepthAttachment, PrimitiveTopology Topology, string BlendLayout, string ResourceLayout, - string VertexLayout); + string VertexLayout, + GuestRasterState Raster, + GuestDepthState Depth); private readonly record struct HostBufferPoolKey( BufferUsageFlags Usage, @@ -1288,6 +2389,10 @@ internal static unsafe class VulkanVideoPresenter ShaderStageFlags Stages, string Resources); + private readonly record struct ComputePipelineKey( + string ShaderDigest, + string Resources); + private sealed record DescriptorLayoutBundle( DescriptorSetLayout DescriptorSetLayout, PipelineLayout PipelineLayout); @@ -1295,7 +2400,25 @@ internal static unsafe class VulkanVideoPresenter private sealed record HostBufferAllocation( VkBuffer Buffer, DeviceMemory Memory, - HostBufferPoolKey Key); + HostBufferPoolKey Key, + nint Mapped); + + private readonly record struct DirtyGuestBufferRange(ulong Offset, ulong Length); + + private sealed class GuestBufferAllocation + { + public string QueueName = VulkanGuestQueueIdentity.Default.Name; + public ulong BaseAddress; + public ulong Size; + public VkBuffer Buffer; + public DeviceMemory Memory; + public nint Mapped; + public byte[] Shadow = []; + public ulong LastUseTimeline; + public List DirtyRanges { get; } = []; + } + + private const string SharedReadOnlyGuestBufferQueue = "shared.readonly"; private sealed class TranslatedDrawResources { @@ -1317,8 +2440,20 @@ internal static unsafe class VulkanVideoPresenter public uint InstanceCount = 1; public PrimitiveTopology Topology = PrimitiveTopology.TriangleList; public GuestBlendState[] Blends = [GuestBlendState.Default]; + // Vulkan format of this draw's color target. Needed to suppress + // blending on formats Metal cannot blend (integer / 32-bit float), + // which otherwise makes vkCreateGraphicsPipelines fail and can + // trip a Metal validation assertion. + public Format[] TargetFormats = [Format.Undefined]; public GuestRect? Scissor; public GuestViewport? Viewport; + public GuestRasterState Raster = GuestRasterState.Default; + public GuestDepthState Depth = GuestDepthState.Default; + public bool HasDepthAttachment; + // Layout keys are needed twice per draw (pipeline lookup and + // descriptor-layout lookup); cache the built strings. + public string? ResourceLayoutKey; + public string? VertexLayoutKey; public RenderPass TransientRenderPass; public Framebuffer TransientFramebuffer; } @@ -1338,24 +2473,46 @@ internal static unsafe class VulkanVideoPresenter public bool NeedsUpload; public bool OwnsStorage; public bool IsStorage; + public bool Cached; + public ulong CpuContentFingerprint; + public bool UpdatesCpuContent; public GuestSampler SamplerState; public Sampler Sampler; public GuestImageResource? GuestImage; - public ulong CpuContentFingerprint; - public bool UpdatesCpuContent; + public GuestDepthResource? GuestDepth; + // A sampled render-target alias cannot remain bound to the same + // image while that image is a color attachment. The per-draw + // snapshot uses this source to copy the target's pre-draw contents + // before the render pass begins. The snapshot itself is owned by + // this TextureResource and retires with the draw fence. + public GuestImageResource? FeedbackSource; + public GuestDepthResource? DepthFeedbackSource; } private sealed class GlobalBufferResource { + public ulong BaseAddress; + public bool Writable; + public bool WriteBackToGuest; public VkBuffer Buffer; public DeviceMemory Memory; + public nint Mapped; + // DescriptorOffset/Size include the shader-visible byte bias. + public ulong Offset; public ulong Size; + // GuestOffset/Size identify only the original guest resource and + // are used for dirty writeback; descriptor padding must not be + // published over unrelated guest bytes. + public ulong GuestOffset; + public ulong GuestSize; + public GuestBufferAllocation? Allocation; } private sealed class VertexBufferResource { public VkBuffer Buffer; public DeviceMemory Memory; + public bool OwnsBuffer; public ulong Size; public uint Location; public uint ComponentCount; @@ -1365,12 +2522,47 @@ internal static unsafe class VulkanVideoPresenter public uint OffsetBytes; } + private const Format DepthFormat = Format.D32Sfloat; + + private sealed class GuestDepthResource + { + public GuestDepthKey Key; + public ulong Address; + public ulong ReadAddress; + public ulong WriteAddress; + public uint Width; + public uint Height; + public uint GuestFormat; + public uint SwizzleMode; + public Image Image; + public DeviceMemory Memory; + public ImageView View; + public Dictionary SampleViews { get; } = new(); + public bool Initialized; + public ImageLayout Layout = ImageLayout.Undefined; + public float GuestClearDepth = 1f; + public float ClearDepth = 1f; + public string InitializationSource = "none"; + } + + private sealed class DepthFramebufferResource + { + public required GuestDepthResource Depth; + public RenderPass LoadRenderPass; + public RenderPass ColorClearRenderPass; + public RenderPass DepthClearRenderPass; + public RenderPass BothClearRenderPass; + public Framebuffer Framebuffer; + } + private sealed class GuestImageResource { public ulong Address; + public long FlipVersion; public uint Width; public uint Height; public uint MipLevels; + public uint GuestFormat; public Format Format; public Image Image; public DeviceMemory Memory; @@ -1378,7 +2570,9 @@ internal static unsafe class VulkanVideoPresenter public ImageView[] MipViews = []; public Dictionary<(Format Format, uint MipLevel, uint LevelCount, uint DstSelect), ImageView> FormatViews { get; } = new(); public RenderPass RenderPass; + public RenderPass InitialRenderPass; public Framebuffer Framebuffer; + public Dictionary DepthFramebuffers { get; } = new(); public bool Initialized; public bool InitialUploadPending; public bool IsCpuBacked; @@ -1388,9 +2582,13 @@ internal static unsafe class VulkanVideoPresenter private sealed record PendingGuestSubmission( Fence Fence, CommandBuffer CommandBuffer, - TranslatedDrawResources Resources, + IReadOnlyList Resources, IReadOnlyList TraceImages, - string DebugName); + IReadOnlyList<(VkBuffer Buffer, DeviceMemory Memory)> RetireBuffers, + ulong Timeline, + string DebugName, + VulkanGuestQueueIdentity Queue, + long WorkSequence); public Presenter(uint width, uint height) { @@ -1398,21 +2596,21 @@ internal static unsafe class VulkanVideoPresenter options.Size = new Vector2D((int)DefaultWindowWidth, (int)DefaultWindowHeight); options.Title = VideoOutExports.GetWindowTitle(); options.WindowBorder = WindowBorder.Fixed; - // FIFO already provides the presentation clock. Throttling Silk's render loop - // as well can miss alternating vblanks and collapse delivery to 30 FPS or less. - options.VSync = false; - options.FramesPerSecond = 0; - options.UpdatesPerSecond = 0; + options.VSync = true; + options.FramesPerSecond = 60; + options.UpdatesPerSecond = 60; _window = Window.Create(options); _window.Load += Initialize; _window.Render += Render; - _window.Closing += OnWindowClosing; - } - - private void OnWindowClosing() - { - VideoOutExports.NotifyPresentationWindowClosed(); - DisposeVulkan(); + _window.Closing += () => + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan VideoOut window closing; " + + $"requested={Volatile.Read(ref _presenterCloseRequested)} " + + $"deviceLost={_deviceLost}"); + VideoOutExports.NotifyPresentationWindowClosed(); + DisposeVulkan(); + }; } public void Run() => _window.Run(); @@ -1434,12 +2632,6 @@ internal static unsafe class VulkanVideoPresenter private void Initialize() { - if (PngSplashLoader.TryLoadIcon(out var iconPixels, out var iconWidth, out var iconHeight)) - { - var icon = new RawImage((int)iconWidth, (int)iconHeight, iconPixels); - _window.SetWindowIcon(ref icon); - } - LogGlfwPlatformInUse(); if (!OperatingSystem.IsWindows()) { @@ -1870,13 +3062,9 @@ internal static unsafe class VulkanVideoPresenter Check(_vk.EnumeratePhysicalDevices(_instance, &deviceCount, devicePointer), "vkEnumeratePhysicalDevices"); } - // Hybrid laptops enumerate the iGPU first, and AMD's integrated driver - // segfaults compiling some translated shaders, so rank rather than take - // the first hit. SHARPEMU_VK_DEVICE= pins an adapter by name. var deviceOverride = Environment.GetEnvironmentVariable("SHARPEMU_VK_DEVICE"); var bestScore = int.MinValue; var found = false; - foreach (var device in devices) { uint queueCount = 0; @@ -1901,7 +3089,6 @@ internal static unsafe class VulkanVideoPresenter var score = ScorePhysicalDevice(properties, name, deviceOverride); Console.Error.WriteLine( $"[LOADER][INFO] Vulkan candidate: {name} ({properties.DeviceType}) score={score}"); - if (score > bestScore) { bestScore = score; @@ -1919,15 +3106,12 @@ internal static unsafe class VulkanVideoPresenter throw new InvalidOperationException("No Vulkan graphics/present queue was found."); } + LoadComputeDeviceLimits(); _vk.GetPhysicalDeviceProperties(_physicalDevice, out var selected); _maxColorAttachments = selected.Limits.MaxColorAttachments; var selectedName = SilkMarshal.PtrToString((nint)selected.DeviceName) ?? "unknown"; - var apiMajor = (selected.ApiVersion >> 22) & 0x7F; - var apiMinor = (selected.ApiVersion >> 12) & 0x3FF; - var apiPatch = selected.ApiVersion & 0xFFF; Console.Error.WriteLine( - $"[LOADER][INFO] Vulkan device: {selectedName} " + - $"(type={selected.DeviceType}, api={apiMajor}.{apiMinor}.{apiPatch})"); + $"[LOADER][INFO] Vulkan device: {selectedName} ({selected.DeviceType})"); VideoOutExports.SetSelectedGpuName(selectedName); _window.Title = VideoOutExports.GetWindowTitle(); } @@ -1947,7 +3131,6 @@ internal static unsafe class VulkanVideoPresenter PhysicalDeviceType.DiscreteGpu => 300, PhysicalDeviceType.VirtualGpu => 100, PhysicalDeviceType.Cpu => 50, - // Last resort: only picked when nothing else can present. PhysicalDeviceType.IntegratedGpu => -100, _ => 10, }; @@ -1960,6 +3143,59 @@ internal static unsafe class VulkanVideoPresenter return score; } + private void LoadComputeDeviceLimits() + { + _vk.GetPhysicalDeviceProperties(_physicalDevice, out var properties); + var subgroupSizeControl = new PhysicalDeviceSubgroupSizeControlProperties + { + SType = StructureType.PhysicalDeviceSubgroupSizeControlProperties, + }; + var subgroup = new PhysicalDeviceSubgroupProperties + { + SType = StructureType.PhysicalDeviceSubgroupProperties, + PNext = &subgroupSizeControl, + }; + var properties2 = new PhysicalDeviceProperties2 + { + SType = StructureType.PhysicalDeviceProperties2, + PNext = &subgroup, + }; + _vk.GetPhysicalDeviceProperties2(_physicalDevice, &properties2); + _maxComputeWorkGroupCountX = properties.Limits.MaxComputeWorkGroupCount[0]; + _maxComputeWorkGroupCountY = properties.Limits.MaxComputeWorkGroupCount[1]; + _maxComputeWorkGroupCountZ = properties.Limits.MaxComputeWorkGroupCount[2]; + _maxComputeWorkGroupSizeX = properties.Limits.MaxComputeWorkGroupSize[0]; + _maxComputeWorkGroupSizeY = properties.Limits.MaxComputeWorkGroupSize[1]; + _maxComputeWorkGroupSizeZ = properties.Limits.MaxComputeWorkGroupSize[2]; + _maxComputeWorkGroupInvocations = properties.Limits.MaxComputeWorkGroupInvocations; + _minStorageBufferOffsetAlignment = Math.Max( + properties.Limits.MinStorageBufferOffsetAlignment, + 1UL); + if (GuestStorageBufferOffsetAlignment % + _minStorageBufferOffsetAlignment != 0) + { + throw new InvalidOperationException( + $"Vulkan storage-buffer alignment " + + $"{_minStorageBufferOffsetAlignment} is not compatible with " + + $"the portable alias alignment " + + $"{GuestStorageBufferOffsetAlignment}"); + } + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan compute limits groups=" + + $"{_maxComputeWorkGroupCountX}x{_maxComputeWorkGroupCountY}x{_maxComputeWorkGroupCountZ} " + + $"local={_maxComputeWorkGroupSizeX}x{_maxComputeWorkGroupSizeY}x" + + $"{_maxComputeWorkGroupSizeZ} invocations={_maxComputeWorkGroupInvocations} " + + $"storage_alignment={_minStorageBufferOffsetAlignment}"); + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan subgroup default={subgroup.SubgroupSize} " + + $"stages={subgroup.SupportedStages} ops={subgroup.SupportedOperations} " + + $"size_control={subgroupSizeControl.MinSubgroupSize}-" + + $"{subgroupSizeControl.MaxSubgroupSize} " + + $"required_stages={subgroupSizeControl.RequiredSubgroupSizeStages} " + + $"max_compute_subgroups=" + + $"{subgroupSizeControl.MaxComputeWorkgroupSubgroups}"); + } + private void CreateDevice() { var priority = 1.0f; @@ -2126,26 +3362,69 @@ internal static unsafe class VulkanVideoPresenter private void CreatePipelineCache() { - _vk.GetPhysicalDeviceProperties(_physicalDevice, out var properties); - var directory = Path.Combine(AppContext.BaseDirectory, "pipeline-cache"); - _pipelineCachePath = Path.Combine( - directory, - $"vulkan-{properties.VendorID:X4}-{properties.DeviceID:X4}-{properties.DriverVersion:X8}.bin"); - + var cacheMode = Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE"); + // Vulkan cache blobs carry the implementation's compatibility + // header and are rejected/rebuilt below when the device or driver + // changes. MoltenVK compilation of a large translated shader can + // take ten seconds, so discarding a valid cache at every launch is + // much more harmful than using Vulkan's normal persistence path. + // Keep an explicit opt-out for diagnostics and read-only systems. + var persistentCacheEnabled = + !string.Equals(cacheMode, "0", StringComparison.Ordinal); + _pipelineCachePath = persistentCacheEnabled ? GetPipelineCachePath() : null; byte[] initialData = []; try { - if (File.Exists(_pipelineCachePath)) + if (_pipelineCachePath is not null && File.Exists(_pipelineCachePath)) { initialData = File.ReadAllBytes(_pipelineCachePath); } } - catch (IOException exception) + catch (Exception exception) { Console.Error.WriteLine( - $"[LOADER][WARN] Vulkan pipeline cache could not be read: {exception.Message}"); + $"[LOADER][WARN] Vulkan pipeline cache read failed: {exception.Message}"); } + var result = TryCreatePipelineCache(initialData, out _pipelineCache); + if (result != Result.Success && initialData.Length != 0) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache rejected ({result}); rebuilding it."); + result = TryCreatePipelineCache([], out _pipelineCache); + } + + if (result != Result.Success) + { + _pipelineCache = default; + _pipelineCachePath = null; + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache unavailable: {result}"); + return; + } + + SetDebugName( + ObjectType.PipelineCache, + _pipelineCache.Handle, + _pipelineCachePath is null + ? "SharpEmu in-memory pipeline cache" + : "SharpEmu persistent pipeline cache"); + _lastPipelineCacheSaveTick = Environment.TickCount64; + if (_pipelineCachePath is null) + { + Console.Error.WriteLine( + "[LOADER][INFO] Vulkan pipeline cache ready: memory-only " + + "(persistence disabled with SHARPEMU_VK_PIPELINE_CACHE=0)."); + } + else + { + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan pipeline cache ready: path={_pipelineCachePath} initial={initialData.Length} bytes"); + } + } + + private Result TryCreatePipelineCache(byte[] initialData, out PipelineCache pipelineCache) + { fixed (byte* initialDataPointer = initialData) { var createInfo = new PipelineCacheCreateInfo @@ -2154,63 +3433,98 @@ internal static unsafe class VulkanVideoPresenter InitialDataSize = (nuint)initialData.Length, PInitialData = initialData.Length == 0 ? null : initialDataPointer, }; - var result = _vk.CreatePipelineCache( + return _vk.CreatePipelineCache( _device, &createInfo, null, - out _pipelineCache); - if (result != Result.Success && initialData.Length != 0) - { - createInfo.InitialDataSize = 0; - createInfo.PInitialData = null; - Check( - _vk.CreatePipelineCache( - _device, - &createInfo, - null, - out _pipelineCache), - "vkCreatePipelineCache(empty)"); - initialData = []; - } - else - { - Check(result, "vkCreatePipelineCache"); - } + out pipelineCache); } - - Console.Error.WriteLine( - $"[LOADER][INFO] Vulkan pipeline cache: " + - $"{(initialData.Length == 0 ? "new" : $"loaded {initialData.Length} bytes")}"); } - private void SavePipelineCache() + private static string GetPipelineCachePath() + { + var configured = Environment.GetEnvironmentVariable("SHARPEMU_VK_PIPELINE_CACHE_PATH"); + if (!string.IsNullOrWhiteSpace(configured)) + { + return Path.GetFullPath( + Environment.ExpandEnvironmentVariables(configured)); + } + + var root = OperatingSystem.IsMacOS() + ? Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "Library", + "Caches") + : Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + return Path.Combine(root, "SharpEmu", "vulkan-pipeline-cache.bin"); + } + + private void MarkPipelineCacheDirty() + { + if (_pipelineCache.Handle == 0) + { + return; + } + + _pipelineCacheDirty = true; + // Exporting MoltenVK's cache can itself serialize the compiler. + // Gameplay may discover dozens of expensive pipelines in one + // frame, so saving after every slow creation compounds a warm-up + // hitch into a multi-minute stall. Coalesce all creations into one + // periodic snapshot; shutdown still forces a final save. + if (Environment.TickCount64 - _lastPipelineCacheSaveTick >= 30_000) + { + SavePipelineCache(force: false); + } + } + + private void SavePipelineCache(bool force) { if (_pipelineCache.Handle == 0 || string.IsNullOrWhiteSpace(_pipelineCachePath)) { return; } + if (!force && !_pipelineCacheDirty) + { + return; + } + try { nuint size = 0; - Check( - _vk.GetPipelineCacheData(_device, _pipelineCache, &size, null), - "vkGetPipelineCacheData(size)"); - if (size == 0 || size > int.MaxValue) + var result = _vk.GetPipelineCacheData( + _device, + _pipelineCache, + &size, + null); + if (result != Result.Success || size == 0 || size > 256u * 1024u * 1024u) { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache query failed: result={result} size={size}"); return; } - var data = new byte[(int)size]; + var data = new byte[checked((int)size)]; fixed (byte* dataPointer = data) { - Check( - _vk.GetPipelineCacheData( - _device, - _pipelineCache, - &size, - dataPointer), - "vkGetPipelineCacheData"); + result = _vk.GetPipelineCacheData( + _device, + _pipelineCache, + &size, + dataPointer); + } + + if (result != Result.Success) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan pipeline cache export failed: {result}"); + return; + } + + if (size != (nuint)data.Length) + { + Array.Resize(ref data, checked((int)size)); } var directory = Path.GetDirectoryName(_pipelineCachePath); @@ -2218,13 +3532,19 @@ internal static unsafe class VulkanVideoPresenter { Directory.CreateDirectory(directory); } - File.WriteAllBytes(_pipelineCachePath, data.AsSpan(0, (int)size).ToArray()); + + var temporaryPath = _pipelineCachePath + $".{Environment.ProcessId}.tmp"; + File.WriteAllBytes(temporaryPath, data); + File.Move(temporaryPath, _pipelineCachePath, overwrite: true); + _pipelineCacheDirty = false; + _lastPipelineCacheSaveTick = Environment.TickCount64; + Console.Error.WriteLine( + $"[LOADER][INFO] Vulkan pipeline cache saved: path={_pipelineCachePath} bytes={data.Length}"); } - catch (Exception exception) when ( - exception is IOException or UnauthorizedAccessException or InvalidOperationException) + catch (Exception exception) { Console.Error.WriteLine( - $"[LOADER][WARN] Vulkan pipeline cache could not be saved: {exception.Message}"); + $"[LOADER][WARN] Vulkan pipeline cache save failed: {exception.Message}"); } } @@ -2249,30 +3569,7 @@ internal static unsafe class VulkanVideoPresenter var surfaceFormat = ChooseSurfaceFormat(formats); _swapchainFormat = surfaceFormat.Format; _extent = ChooseExtent(capabilities); - uint presentModeCount = 0; - Check( - _surfaceApi.GetPhysicalDeviceSurfacePresentModes( - _physicalDevice, - _surface, - &presentModeCount, - null), - "vkGetPhysicalDeviceSurfacePresentModesKHR"); - var presentModes = new PresentModeKHR[presentModeCount]; - fixed (PresentModeKHR* presentModePointer = presentModes) - { - Check( - _surfaceApi.GetPhysicalDeviceSurfacePresentModes( - _physicalDevice, - _surface, - &presentModeCount, - presentModePointer), - "vkGetPhysicalDeviceSurfacePresentModesKHR"); - } - - var presentMode = presentModes.Contains(PresentModeKHR.MailboxKhr) - ? PresentModeKHR.MailboxKhr - : PresentModeKHR.FifoKhr; - Console.Error.WriteLine($"[LOADER][INFO] Vulkan present mode: {presentMode}"); + var presentMode = ChoosePresentMode(); var imageCount = capabilities.MinImageCount + 1; if (capabilities.MaxImageCount != 0) { @@ -2317,6 +3614,45 @@ internal static unsafe class VulkanVideoPresenter _imageInitialized = new bool[swapchainImageCount]; } + private PresentModeKHR ChoosePresentMode() + { + // MAILBOX never blocks vkQueuePresentKHR on vblank, so a slow + // frame does not quantize the frame rate down to 30/20 fps the + // way FIFO does; the guest side is already paced by PaceFlip. + // FIFO is the only mode guaranteed by the spec and remains the + // fallback (MoltenVK typically exposes FIFO + IMMEDIATE only). + uint modeCount = 0; + if (_surfaceApi.GetPhysicalDeviceSurfacePresentModes( + _physicalDevice, + _surface, + &modeCount, + null) != Result.Success || + modeCount == 0) + { + return PresentModeKHR.FifoKhr; + } + + var modes = stackalloc PresentModeKHR[(int)modeCount]; + if (_surfaceApi.GetPhysicalDeviceSurfacePresentModes( + _physicalDevice, + _surface, + &modeCount, + modes) != Result.Success) + { + return PresentModeKHR.FifoKhr; + } + + for (var index = 0u; index < modeCount; index++) + { + if (modes[index] == PresentModeKHR.MailboxKhr) + { + return PresentModeKHR.MailboxKhr; + } + } + + return PresentModeKHR.FifoKhr; + } + private void CreateCommandResources() { var poolInfo = new CommandPoolCreateInfo @@ -2332,58 +3668,246 @@ internal static unsafe class VulkanVideoPresenter SType = StructureType.CommandBufferAllocateInfo, CommandPool = _commandPool, Level = CommandBufferLevel.Primary, - CommandBufferCount = 1, + CommandBufferCount = MaxFramesInFlight, }; - Check(_vk.AllocateCommandBuffers(_device, &allocateInfo, out _commandBuffer), "vkAllocateCommandBuffers"); - _presentationCommandBuffer = _commandBuffer; + _frameCommandBuffers = new CommandBuffer[MaxFramesInFlight]; + fixed (CommandBuffer* frameCommandBuffers = _frameCommandBuffers) + { + Check( + _vk.AllocateCommandBuffers(_device, &allocateInfo, frameCommandBuffers), + "vkAllocateCommandBuffers"); + } var semaphoreInfo = new SemaphoreCreateInfo { SType = StructureType.SemaphoreCreateInfo, }; - Check(_vk.CreateSemaphore(_device, &semaphoreInfo, null, out _imageAvailable), "vkCreateSemaphore"); - Check(_vk.CreateSemaphore(_device, &semaphoreInfo, null, out _renderFinished), "vkCreateSemaphore"); - var fenceInfo = new FenceCreateInfo { SType = StructureType.FenceCreateInfo, }; - Check( - _vk.CreateFence(_device, &fenceInfo, null, out _presentationFence), - "vkCreateFence(presentation)"); + _frameImageAvailable = new VkSemaphore[MaxFramesInFlight]; + _frameFences = new Fence[MaxFramesInFlight]; + _frameFencePending = new bool[MaxFramesInFlight]; + _frameTimelines = new ulong[MaxFramesInFlight]; + _frameTranslatedResources = new TranslatedDrawResources?[MaxFramesInFlight]; + _frameGuestImageVersions = new GuestImageResource?[MaxFramesInFlight]; + for (var slot = 0; slot < MaxFramesInFlight; slot++) + { + Check( + _vk.CreateSemaphore(_device, &semaphoreInfo, null, out _frameImageAvailable[slot]), + "vkCreateSemaphore"); + Check( + _vk.CreateFence(_device, &fenceInfo, null, out _frameFences[slot]), + "vkCreateFence(frame)"); + } + + _renderFinishedPerImage = new VkSemaphore[_swapchainImages.Length]; + for (var image = 0; image < _renderFinishedPerImage.Length; image++) + { + Check( + _vk.CreateSemaphore(_device, &semaphoreInfo, null, out _renderFinishedPerImage[image]), + "vkCreateSemaphore"); + } + + _currentFrameSlot = 0; + _commandBuffer = _frameCommandBuffers[0]; + _presentationCommandBuffer = _commandBuffer; CreateStagingBuffer((ulong)_extent.Width * _extent.Height * 4); + CreateOverlayResources(); } - private void CompletePendingPresentation(bool wait) + private void CreateOverlayResources() { - if (!_presentationInFlight) + const ulong overlayBytes = PerfOverlay.PanelWidth * PerfOverlay.PanelHeight * 4; + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = Format.B8G8R8A8Unorm, + Extent = new Extent3D(PerfOverlay.PanelWidth, PerfOverlay.PanelHeight, 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.TransferSrcBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check(_vk.CreateImage(_device, &imageInfo, null, out _overlayImage), "vkCreateImage(overlay)"); + _vk.GetImageMemoryRequirements(_device, _overlayImage, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + Check( + _vk.AllocateMemory(_device, &memoryInfo, null, out _overlayImageMemory), + "vkAllocateMemory(overlay)"); + Check( + _vk.BindImageMemory(_device, _overlayImage, _overlayImageMemory, 0), + "vkBindImageMemory(overlay)"); + _overlayImageInitialized = false; + + _overlayStagingBuffers = new VkBuffer[MaxFramesInFlight]; + _overlayStagingMemory = new DeviceMemory[MaxFramesInFlight]; + _overlayStagingMapped = new nint[MaxFramesInFlight]; + for (var slot = 0; slot < MaxFramesInFlight; slot++) + { + _overlayStagingBuffers[slot] = CreateBuffer( + overlayBytes, + BufferUsageFlags.TransferSrcBit, + MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, + out _overlayStagingMemory[slot]); + void* mapped; + Check( + _vk.MapMemory(_device, _overlayStagingMemory[slot], 0, overlayBytes, 0, &mapped), + "vkMapMemory(overlay staging)"); + _overlayStagingMapped[slot] = (nint)mapped; + } + } + + private void RecordOverlayBlit(uint imageIndex, int frameSlot) + { + if (_overlayImage.Handle == 0 || _overlayStagingMapped.Length <= frameSlot) { return; } - var fence = _presentationFence; - var result = wait - ? _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue) - : _vk.GetFenceStatus(_device, fence); - if (!wait && result == Result.NotReady) + int pendingWork; + lock (_gate) { - return; + pendingWork = _pendingGuestWorkCount; } - Check(result, "vkWaitForFences(presentation)"); - _presentationInFlight = false; - if (_pendingPresentationResources is not null) + var pixels = new Span( + (void*)_overlayStagingMapped[frameSlot], + PerfOverlay.PanelWidth * PerfOverlay.PanelHeight * 4); + PerfOverlay.Fill(pixels, pendingWork, _pendingGuestSubmissions.Count); + + var toTransferDst = new ImageMemoryBarrier { - MarkSampledImagesInitialized(_pendingPresentationResources); - MarkStorageImagesInitialized(_pendingPresentationResources); - DestroyTranslatedDrawResources(_pendingPresentationResources); - _pendingPresentationResources = null; - } + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = _overlayImageInitialized ? AccessFlags.TransferReadBit : 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = _overlayImageInitialized + ? ImageLayout.TransferSrcOptimal + : ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _overlayImage, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.TransferBit, + 0, 0, null, 0, null, 1, &toTransferDst); + + var copyRegion = new BufferImageCopy + { + ImageSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1), + ImageExtent = new Extent3D(PerfOverlay.PanelWidth, PerfOverlay.PanelHeight, 1), + }; + _vk.CmdCopyBufferToImage( + _commandBuffer, + _overlayStagingBuffers[frameSlot], + _overlayImage, + ImageLayout.TransferDstOptimal, + 1, + ©Region); + + var toTransferSrc = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _overlayImage, + SubresourceRange = ColorSubresourceRange(), + }; + var swapchainToDst = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.PresentSrcKhr, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _swapchainImages[imageIndex], + SubresourceRange = ColorSubresourceRange(), + }; + var preBlitBarriers = stackalloc ImageMemoryBarrier[2] { toTransferSrc, swapchainToDst }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit | PipelineStageFlags.ColorAttachmentOutputBit, + PipelineStageFlags.TransferBit, + 0, 0, null, 0, null, 2, preBlitBarriers); + + const int margin = 12; + var panelWidth = (int)Math.Min(PerfOverlay.PanelWidth, _extent.Width - margin); + var panelHeight = (int)Math.Min(PerfOverlay.PanelHeight, _extent.Height - margin); + // Source and destination are both B8G8R8A8 and the panel is not + // scaled. MoltenVK has corrupted pixels outside the blit region + // for this transfer-on-swapchain path (horizontal red/yellow + // scanlines across the entire window). An exact image copy has + // the required semantics and avoids the driver's blit conversion + // path altogether. + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1), + DstSubresource = new ImageSubresourceLayers(ImageAspectFlags.ColorBit, 0, 0, 1), + SrcOffset = new Offset3D(0, 0, 0), + DstOffset = new Offset3D(margin, margin, 0), + Extent = new Extent3D((uint)panelWidth, (uint)panelHeight, 1), + }; + _vk.CmdCopyImage( + _commandBuffer, + _overlayImage, + ImageLayout.TransferSrcOptimal, + _swapchainImages[imageIndex], + ImageLayout.TransferDstOptimal, + 1, + ©); + + var swapchainToPresent = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = 0, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.PresentSrcKhr, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = _swapchainImages[imageIndex], + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.BottomOfPipeBit, + 0, 0, null, 0, null, 1, &swapchainToPresent); + _overlayImageInitialized = true; } private CommandBuffer AllocateGuestCommandBuffer() { + // The pool has ResetCommandBufferBit, so vkBeginCommandBuffer + // implicitly resets recycled buffers. + if (_recycledGuestCommandBuffers.TryPop(out var recycled)) + { + return recycled; + } + var allocateInfo = new CommandBufferAllocateInfo { SType = StructureType.CommandBufferAllocateInfo, @@ -2401,11 +3925,14 @@ internal static unsafe class VulkanVideoPresenter return commandBuffer; } - private void SubmitGuestCommandBuffer( - CommandBuffer commandBuffer, - TranslatedDrawResources resources, - IReadOnlyList traceImages) + private Fence AcquireGuestFence() { + // Recycled fences were reset when they were collected. + if (_recycledGuestFences.TryPop(out var recycled)) + { + return recycled; + } + var fenceInfo = new FenceCreateInfo { SType = StructureType.FenceCreateInfo, @@ -2414,6 +3941,163 @@ internal static unsafe class VulkanVideoPresenter Check( _vk.CreateFence(_device, &fenceInfo, null, out fence), "vkCreateFence(guest)"); + return fence; + } + + private void ReleaseGuestCommandBuffer(CommandBuffer commandBuffer) + { + if (_recycledGuestCommandBuffers.Count < MaxRecycledGuestCommandBuffers) + { + _recycledGuestCommandBuffers.Push(commandBuffer); + return; + } + + _vk.FreeCommandBuffers(_device, _commandPool, 1, &commandBuffer); + } + + private void ReleaseGuestFence(Fence fence, bool needsReset) + { + if (_recycledGuestFences.Count < MaxRecycledGuestFences) + { + if (needsReset) + { + Check(_vk.ResetFences(_device, 1, &fence), "vkResetFences(guest)"); + } + + _recycledGuestFences.Push(fence); + return; + } + + _vk.DestroyFence(_device, fence, null); + } + + // Translated draws are recorded into a shared command buffer and + // submitted once per drained work batch: on MoltenVK every + // vkQueueSubmit is a Metal command-buffer commit (~0.8ms), which used + // to be paid per draw and dominated the frame time. + private CommandBuffer _batchCommandBuffer; + private bool _batchOpen; + private int _batchDrawCount; + private readonly List _batchResources = new(); + private readonly List _batchTraceImages = new(); + + // Consecutive draws into the same target stay inside one render pass: + // on MoltenVK every render pass is a Metal render encoder, and one + // encoder per draw was the dominant per-draw fixed cost after submit + // batching. The pass closes when the target changes, when a draw + // needs transfer/storage work outside a pass, or when the batch + // flushes. + private GuestImageResource? _openPassTarget; + + private void CloseOpenTranslatedRenderPass() + { + if (_openPassTarget is not { } target) + { + return; + } + + _openPassTarget = null; + _vk.CmdEndRenderPass(_batchCommandBuffer); + var toShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ColorAttachmentWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.ColorAttachmentOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _batchCommandBuffer, + PipelineStageFlags.ColorAttachmentOutputBit, + PipelineStageFlags.FragmentShaderBit, + 0, + 0, + null, + 0, + null, + 1, + &toShaderRead); + } + + private CommandBuffer BeginBatchedGuestCommands() + { + if (_batchOpen) + { + return _batchCommandBuffer; + } + + _batchCommandBuffer = AllocateGuestCommandBuffer(); + var beginInfo = new CommandBufferBeginInfo + { + SType = StructureType.CommandBufferBeginInfo, + Flags = CommandBufferUsageFlags.OneTimeSubmitBit, + }; + Check( + _vk.BeginCommandBuffer(_batchCommandBuffer, &beginInfo), + "vkBeginCommandBuffer(batch)"); + _batchOpen = true; + _batchDrawCount = 0; + return _batchCommandBuffer; + } + + private void FlushBatchedGuestCommands() + { + if (!_batchOpen) + { + return; + } + + CloseOpenTranslatedRenderPass(); + _batchOpen = false; + try + { + Check(_vk.EndCommandBuffer(_batchCommandBuffer), "vkEndCommandBuffer(batch)"); + SubmitGuestCommandBuffer( + _batchCommandBuffer, + _batchResources.ToArray(), + _batchTraceImages.ToArray(), + _batchRetireBuffers.Count > 0 ? _batchRetireBuffers.ToArray() : []); + } + catch + { + // The batch never reached the queue: release everything it + // owned here so the stale lists cannot ride into the next + // batch's submission. + foreach (var resources in _batchResources) + { + DestroyTranslatedDrawResources(resources); + } + + foreach (var (buffer, memory) in _batchRetireBuffers) + { + _vk.DestroyBuffer(_device, buffer, null); + _vk.FreeMemory(_device, memory, null); + } + + ReleaseGuestCommandBuffer(_batchCommandBuffer); + throw; + } + finally + { + _batchResources.Clear(); + _batchTraceImages.Clear(); + _batchRetireBuffers.Clear(); + _batchCommandBuffer = default; + } + } + + private void SubmitGuestCommandBuffer( + CommandBuffer commandBuffer, + IReadOnlyList resources, + IReadOnlyList traceImages, + IReadOnlyList<(VkBuffer Buffer, DeviceMemory Memory)>? retireBuffers = null, + IReadOnlyList? referencedResources = null) + { + var fence = AcquireGuestFence(); try { var submitInfo = new SubmitInfo @@ -2428,50 +4112,46 @@ internal static unsafe class VulkanVideoPresenter } catch { - _vk.DestroyFence(_device, fence, null); + ReleaseGuestFence(fence, needsReset: false); throw; } + _submitTimeline++; + foreach (var referenced in referencedResources ?? resources) + { + foreach (var globalBuffer in referenced.GlobalMemoryBuffers) + { + if (globalBuffer.Allocation is not { } allocation) + { + continue; + } + + allocation.LastUseTimeline = Math.Max( + allocation.LastUseTimeline, + _submitTimeline); + if (globalBuffer.Writable && globalBuffer.WriteBackToGuest) + { + MarkGuestBufferDirty( + allocation, + globalBuffer.GuestOffset, + globalBuffer.GuestSize); + } + } + } + _pendingGuestSubmissions.Enqueue( new PendingGuestSubmission( fence, commandBuffer, resources, traceImages, - resources.DebugName)); - } - - private void SubmitGuestCommandBufferAndWait(CommandBuffer commandBuffer) - { - var fenceInfo = new FenceCreateInfo - { - SType = StructureType.FenceCreateInfo, - }; - Fence fence; - Check( - _vk.CreateFence(_device, &fenceInfo, null, out fence), - "vkCreateFence(guest chunk)"); - try - { - var submitInfo = new SubmitInfo - { - SType = StructureType.SubmitInfo, - CommandBufferCount = 1, - PCommandBuffers = &commandBuffer, - }; - Check( - _vk.QueueSubmit(_queue, 1, &submitInfo, fence), - "vkQueueSubmit(guest chunk)"); - Check( - _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue), - "vkWaitForFences(guest chunk)"); - } - finally - { - _vk.DestroyFence(_device, fence, null); - } - - _vk.FreeCommandBuffers(_device, _commandPool, 1, &commandBuffer); + retireBuffers ?? [], + _submitTimeline, + resources.Count > 0 ? resources[0].DebugName : "batch", + _activeGuestQueue, + _activeGuestWorkSequence)); + _lastSubmittedTimelineByGuestQueue[_activeGuestQueue.Name] = + _submitTimeline; } private void EnsureGuestSubmissionCapacity() @@ -2479,7 +4159,14 @@ internal static unsafe class VulkanVideoPresenter CollectCompletedGuestSubmissions(waitForOldest: false); if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) { - CollectCompletedGuestSubmissions(waitForOldest: true); + // Bounded wait so the macOS main thread returns to its event + // pump promptly under a slow-compute backlog; if the oldest + // isn't done yet we proceed (soft cap, dynamic pools). + CollectCompletedGuestSubmissions( + waitForOldest: true, + maxWaitNs: _submissionCapacityWaitNs == 0 + ? _guestFenceWaitTimeoutNs + : _submissionCapacityWaitNs); } } @@ -2491,17 +4178,42 @@ internal static unsafe class VulkanVideoPresenter } } - private void CollectCompletedGuestSubmissions(bool waitForOldest) + private void CollectCompletedGuestSubmissions(bool waitForOldest, ulong maxWaitNs = 0) { if (waitForOldest && _pendingGuestSubmissions.TryPeek(out var oldest)) { var fence = oldest.Fence; + // maxWaitNs==0 => the full "is this submission hung" timeout, + // which also emits the one-shot hang warning below. A shorter + // capacity-probe wait (maxWaitNs>0) must NOT report a hang: the + // submission is still tracked and will be collected once the GPU + // finishes it on a later frame. + var isProbeWait = maxWaitNs != 0 && maxWaitNs < _guestFenceWaitTimeoutNs; + var waitNs = maxWaitNs != 0 ? maxWaitNs : _guestFenceWaitTimeoutNs; var result = _vk.WaitForFences( _device, 1, &fence, true, - ulong.MaxValue); + waitNs); + if (result == Result.Timeout) + { + // A GPU submission whose fence never signals (typically a + // mistranslated compute shader that hangs the Metal queue) + // would otherwise block the render thread forever, starving + // the swapchain present (black screen). Log the culprit and + // continue so at least the last good frame can be shown. + if (!isProbeWait && _tracedFenceTimeouts.Add(oldest.DebugName)) + { + Console.Error.WriteLine( + $"[LOADER][WARN] vk.fence_wait_timeout submission='{oldest.DebugName}' " + + $"— GPU work not completing after {_guestFenceWaitTimeoutNs / 1_000_000}ms; " + + "render thread continuing (present not blocked)."); + } + + return; + } + Check(result, $"vkWaitForFences(guest: {oldest.DebugName})"); } @@ -2521,41 +4233,575 @@ internal static unsafe class VulkanVideoPresenter TraceGuestImageContents(image); } - DestroyTranslatedDrawResources(submission.Resources); - var commandBuffer = submission.CommandBuffer; - _vk.FreeCommandBuffers( - _device, - _commandPool, - 1, - &commandBuffer); - _vk.DestroyFence(_device, submission.Fence, null); + foreach (var resources in submission.Resources) + { + DestroyTranslatedDrawResources(resources); + } + + foreach (var (buffer, memory) in submission.RetireBuffers) + { + _vk.DestroyBuffer(_device, buffer, null); + _vk.FreeMemory(_device, memory, null); + } + + ReleaseGuestCommandBuffer(submission.CommandBuffer); + ReleaseGuestFence(submission.Fence, needsReset: true); + if (submission.Timeline > _completedTimeline) + { + _completedTimeline = submission.Timeline; + } } + + ProcessDeferredTextureDestroys(); + } + + private void WaitForAllGuestSubmissionsForCpuVisibility() + { + FlushBatchedGuestCommands(); + while (_pendingGuestSubmissions.TryPeek(out var oldest)) + { + var fence = oldest.Fence; + Check( + _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue), + $"vkWaitForFences(cpu visibility: {oldest.DebugName})"); + CollectCompletedGuestSubmissions(waitForOldest: false); + } + } + + private void WaitForActiveGuestQueueSubmissionsForCpuVisibility() + { + FlushBatchedGuestCommands(); + if (!_lastSubmittedTimelineByGuestQueue.TryGetValue( + _activeGuestQueue.Name, + out var targetTimeline) || + targetTimeline <= _completedTimeline) + { + return; + } + + PendingGuestSubmission? target = null; + foreach (var submission in _pendingGuestSubmissions) + { + if (submission.Timeline == targetTimeline) + { + target = submission; + break; + } + } + + if (target is null) + { + throw new InvalidOperationException( + $"Guest queue '{_activeGuestQueue.Name}' lost pending timeline " + + $"{targetTimeline} (completed={_completedTimeline})."); + } + + var waitStart = System.Diagnostics.Stopwatch.GetTimestamp(); + var fence = target.Fence; + Check( + _vk.WaitForFences(_device, 1, &fence, true, ulong.MaxValue), + $"vkWaitForFences(queue visibility: {_activeGuestQueue.Name})"); + CollectCompletedGuestSubmissions(waitForOldest: false); + var waitedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - waitStart) * + 1000.0 / System.Diagnostics.Stopwatch.Frequency; + TraceVulkanShader( + $"vk.queue_visibility queue={_activeGuestQueue.Name} " + + $"submission={_activeGuestQueue.SubmissionId} " + + $"target_timeline={targetTimeline} completed_timeline={_completedTimeline} " + + $"waited_ms={waitedMs:F3}"); + } + + private void ExecuteOrderedGuestAction(VulkanOrderedGuestAction work) + { + WaitForActiveGuestQueueSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(_activeGuestQueue.Name); + work.Action(); + TraceVulkanShader( + $"vk.ordered_action queue={_activeGuestQueue.Name} " + + $"submission={_activeGuestQueue.SubmissionId} " + + $"work_sequence={_activeGuestWorkSequence} name='{work.DebugName}'"); + } + + private void ExecuteOrderedGuestFlip(VulkanOrderedGuestFlip work) + { + FlushBatchedGuestCommands(); + _guestImages.TryGetValue(work.Address, out var source); + if (_deviceLost || + source is null || + !source.Initialized) + { + Console.Error.WriteLine( + $"[LOADER][WARN] vk.flip_capture_failed version={work.Version} " + + $"queue={_activeGuestQueue.Name} addr=0x{work.Address:X16} " + + $"found={(source is not null)} initialized={(source?.Initialized ?? false)}"); + return; + } + + EnsureGuestSubmissionCapacity(); + var snapshot = CreateGuestFlipSnapshot(source, work.Version); + var commandBuffer = AllocateGuestCommandBuffer(); + var submitted = false; + try + { + var beginInfo = new CommandBufferBeginInfo + { + SType = StructureType.CommandBufferBeginInfo, + Flags = CommandBufferUsageFlags.OneTimeSubmitBit, + }; + Check( + _vk.BeginCommandBuffer(commandBuffer, &beginInfo), + "vkBeginCommandBuffer(flip capture)"); + + var barriers = stackalloc ImageMemoryBarrier[2]; + barriers[0] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ShaderReadBit | + AccessFlags.ShaderWriteBit | + AccessFlags.ColorAttachmentWriteBit | + AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = ImageLayout.ShaderReadOnlyOptimal, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = ColorSubresourceRange(), + }; + barriers[1] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = snapshot.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.AllCommandsBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 2, + barriers); + + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, 0, 0, 1), + DstSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, 0, 0, 1), + Extent = new Extent3D(source.Width, source.Height, 1), + }; + _vk.CmdCopyImage( + commandBuffer, + source.Image, + ImageLayout.TransferSrcOptimal, + snapshot.Image, + ImageLayout.TransferDstOptimal, + 1, + ©); + + barriers[0] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferReadBit, + DstAccessMask = AccessFlags.ShaderReadBit | + AccessFlags.ShaderWriteBit | + AccessFlags.ColorAttachmentWriteBit, + OldLayout = ImageLayout.TransferSrcOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = ColorSubresourceRange(), + }; + barriers[1] = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = snapshot.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.AllCommandsBit, + 0, + 0, + null, + 0, + null, + 2, + barriers); + + Check( + _vk.EndCommandBuffer(commandBuffer), + "vkEndCommandBuffer(flip capture)"); + SubmitGuestCommandBuffer(commandBuffer, [], []); + submitted = true; + snapshot.Initialized = true; + _guestImageVersions.Add(work.Version, snapshot); + _capturedGuestFlipVersions.Add(work.Version); + + lock (_gate) + { + var sequence = (_latestPresentation?.Sequence ?? 0) + 1; + var presentation = new Presentation( + null, + work.Width, + work.Height, + sequence, + GuestDrawKind.None, + TranslatedDraw: null, + RequiredGuestWorkSequence: _activeGuestWorkSequence, + IsSplash: false, + GuestImageAddress: work.Address, + GuestImageVersion: work.Version); + _latestPresentation = presentation; + _pendingGuestImagePresentations.Enqueue(presentation); + while (_pendingGuestImagePresentations.Count > MaxPendingGuestFlipVersions) + { + _pendingGuestImagePresentations.Dequeue(); + } + } + + CollectAbandonedGuestImageVersions(); + + var effectivePitch = work.PitchInPixel == 0 + ? work.Width + : work.PitchInPixel; + TraceVulkanShader( + $"vk.flip_capture version={work.Version} " + + $"queue={_activeGuestQueue.Name} submission={_activeGuestQueue.SubmissionId} " + + $"work_sequence={_activeGuestWorkSequence} addr=0x{work.Address:X16} " + + $"size={work.Width}x{work.Height} pitch={effectivePitch}"); + } + finally + { + if (!submitted) + { + ReleaseGuestCommandBuffer(commandBuffer); + DestroyGuestImage(snapshot); + } + } + } + + private void ExecuteOrderedGuestFlipWait(VulkanOrderedGuestFlipWait work) + { + var captured = work.Version != 0 && + _capturedGuestFlipVersions.Contains(work.Version); + TraceVulkanShader( + $"vk.flip_wait_safe version={work.Version} " + + $"queue={_activeGuestQueue.Name} submission={_activeGuestQueue.SubmissionId} " + + $"handle={work.VideoOutHandle} index={work.DisplayBufferIndex} " + + $"capture_complete={(captured ? 1 : 0)}"); +#if DEBUG + System.Diagnostics.Debug.Assert( + work.Version == 0 || captured, + "An ordered wait-safe marker must execute after its flip capture."); +#endif + } + + private GuestImageResource CreateGuestFlipSnapshot( + GuestImageResource source, + long version) + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = source.Format, + Extent = new Extent3D(source.Width, source.Height, 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = ImageUsageFlags.TransferSrcBit | + ImageUsageFlags.TransferDstBit | + ImageUsageFlags.SampledBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check( + _vk.CreateImage(_device, &imageInfo, null, out var image), + "vkCreateImage(flip snapshot)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var allocationInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + DeviceMemory memory = default; + try + { + Check( + _vk.AllocateMemory(_device, &allocationInfo, null, out memory), + "vkAllocateMemory(flip snapshot)"); + Check( + _vk.BindImageMemory(_device, image, memory, 0), + "vkBindImageMemory(flip snapshot)"); + } + catch + { + if (memory.Handle != 0) + { + _vk.FreeMemory(_device, memory, null); + } + _vk.DestroyImage(_device, image, null); + throw; + } + + SetDebugName( + ObjectType.Image, + image.Handle, + $"guest flip v{version} source 0x{source.Address:X16}"); + return new GuestImageResource + { + Address = source.Address, + FlipVersion = version, + Width = source.Width, + Height = source.Height, + MipLevels = 1, + GuestFormat = source.GuestFormat, + Format = source.Format, + Image = image, + Memory = memory, + }; + } + + private static byte[]? TryReadGuestTexturePixels(GuestDrawTexture texture) + { + var memory = _guestMemory; + if (memory is null || texture.Address == 0) + { + return null; + } + + var width = Math.Max(texture.Width, 1); + var height = Math.Max(texture.Height, 1); + var rowLength = texture.TileMode == 0 + ? Math.Max(texture.Pitch, width) + : width; + var byteCount = GetTextureByteCount(texture.Format, rowLength, height); + if (byteCount == 0 || byteCount > int.MaxValue) + { + return null; + } + + var pixels = new byte[(int)byteCount]; + return memory.TryRead(texture.Address, pixels) ? pixels : null; + } + + /// + /// Returns a skipped draw's pooled data arrays: draws dropped before + /// resource creation would otherwise strand their rented buffers. + /// + private static void ReturnPooledGuestData(VulkanTranslatedGuestDraw draw) + { + foreach (var buffer in draw.GlobalMemoryBuffers) + { + if (buffer.Pooled) + { + GuestDataPool.Return(buffer.Data); + } + } + + foreach (var buffer in draw.VertexBuffers) + { + if (buffer.Pooled) + { + GuestDataPool.Return(buffer.Data); + } + } + + if (draw.IndexBuffer is { Pooled: true } indexBuffer) + { + GuestDataPool.Return(indexBuffer.Data); + } + } + + private void ProcessDeferredTextureDestroys() + { + while (_deferredTextureDestroys.TryPeek(out var entry) && + entry.RetireTimeline <= _completedTimeline) + { + _deferredTextureDestroys.Dequeue(); + DestroyCachedTextureResource(entry.Texture); + } + + while (_deferredResourceDestroys.TryPeek(out var resourceEntry) && + resourceEntry.RetireTimeline <= _completedTimeline) + { + _deferredResourceDestroys.Dequeue(); + DestroyTranslatedDrawResources(resourceEntry.Resources); + } + + while (_deferredGuestImageVersionDestroys.TryPeek(out var imageEntry) && + imageEntry.RetireTimeline <= _completedTimeline) + { + _deferredGuestImageVersionDestroys.Dequeue(); + DestroyGuestImage(imageEntry.Image); + TraceVulkanShader( + $"vk.flip_retired version={imageEntry.Image.FlipVersion} " + + $"timeline={imageEntry.RetireTimeline} reason=presentation-dropped"); + } + } + + private void WaitFrameSlot(int slot) => TryWaitFrameSlot(slot, ulong.MaxValue); + + // Returns false when the slot's fence is still unsignaled after + // timeoutNs (the GPU is behind, e.g. a slow-compute backlog). Callers + // on the macOS main thread must NOT wait forever here or the Cocoa + // event pump stalls and the window goes "Not Responding" (F1 overlay / + // close stop working). A bounded wait lets Render() skip the frame and + // return to the pump; the fence still signals later and the frame is + // retried. + private bool TryWaitFrameSlot(int slot, ulong timeoutNs) + { + if (_frameFencePending.Length <= slot || !_frameFencePending[slot]) + { + if (_frameGuestImageVersions.Length > slot && + _frameGuestImageVersions[slot] is { } unsubmittedVersion) + { + _frameGuestImageVersions[slot] = null; + _capturedGuestFlipVersions.Remove(unsubmittedVersion.FlipVersion); + DestroyGuestImage(unsubmittedVersion); + TraceVulkanShader( + $"vk.flip_retired version={unsubmittedVersion.FlipVersion} " + + $"frame_slot={slot} reason=frame-not-submitted"); + } + return true; + } + + var fence = _frameFences[slot]; + var waitResult = _vk.WaitForFences(_device, 1, &fence, true, timeoutNs); + if (waitResult == Result.Timeout) + { + return false; + } + + Check(waitResult, "vkWaitForFences(frame)"); + Check(_vk.ResetFences(_device, 1, &fence), "vkResetFences(frame)"); + _frameFencePending[slot] = false; + if (_frameTimelines[slot] > _completedTimeline) + { + _completedTimeline = _frameTimelines[slot]; + } + + if (_frameTranslatedResources[slot] is { } translated) + { + _frameTranslatedResources[slot] = null; + DestroyTranslatedDrawResources(translated); + } + + if (_frameGuestImageVersions[slot] is { } guestImageVersion) + { + _frameGuestImageVersions[slot] = null; + _capturedGuestFlipVersions.Remove(guestImageVersion.FlipVersion); + DestroyGuestImage(guestImageVersion); + TraceVulkanShader( + $"vk.flip_retired version={guestImageVersion.FlipVersion} " + + $"frame_slot={slot} timeline={_frameTimelines[slot]}"); + } + + ProcessDeferredTextureDestroys(); + return true; + } + + private void WaitAllFrameSlots() + { + for (var slot = 0; slot < _frameFencePending.Length; slot++) + { + WaitFrameSlot(slot); + } + } + + private void CollectAbandonedGuestImageVersions() + { + if (_guestImageVersions.Count == 0) + { + return; + } + + HashSet referencedVersions; + lock (_gate) + { + referencedVersions = _pendingGuestImagePresentations + .Select(static presentation => presentation.GuestImageVersion) + .Where(static version => version != 0) + .ToHashSet(); + if (_latestPresentation is { GuestImageVersion: not 0 } latest) + { + referencedVersions.Add(latest.GuestImageVersion); + } + } + + foreach (var entry in _guestImageVersions.ToArray()) + { + if (referencedVersions.Contains(entry.Key)) + { + continue; + } + + _guestImageVersions.Remove(entry.Key); + _capturedGuestFlipVersions.Remove(entry.Key); + _deferredGuestImageVersionDestroys.Enqueue( + (entry.Value, _submitTimeline)); + TraceVulkanShader( + $"vk.flip_retire_deferred version={entry.Key} " + + $"timeline={_submitTimeline} reason=presentation-dropped"); + } + } + + // Used on teardown paths that already drained the queue (device + // wait-idle): releases per-slot state without touching fences that + // were never submitted. + private void DrainFrameSlots() + { + WaitAllFrameSlots(); + _completedTimeline = _submitTimeline; + ProcessDeferredTextureDestroys(); } private IReadOnlyList GetTraceImages( TranslatedDrawResources resources, - IReadOnlyList? renderTargets = null) + IReadOnlyList? renderTargets = null, + ulong shaderAddress = 0) { - var images = new HashSet(); + var candidates = new HashSet(); foreach (var renderTarget in renderTargets ?? []) { - if (ShouldTraceGuestImageContents(renderTarget)) - { - images.Add(renderTarget); - } + candidates.Add(renderTarget); } foreach (var texture in resources.Textures) { - if (texture.IsStorage && - texture.GuestImage is { } image && - ShouldTraceGuestImageContents(image)) + if ((texture.IsStorage || _traceGuestImageAddressFilterEnabled) && + texture.GuestImage is { } image) { - images.Add(image); + candidates.Add(image); } } - return images.ToArray(); + return candidates + .Where(image => ShouldTraceGuestImageContents(image, shaderAddress)) + .ToArray(); } private void CreateGuestDrawResources() @@ -2744,6 +4990,7 @@ internal static unsafe class VulkanVideoPresenter null, out _barycentricPipeline), "vkCreateGraphicsPipelines"); + MarkPipelineCacheDirty(); } finally { @@ -2774,15 +5021,71 @@ internal static unsafe class VulkanVideoPresenter VulkanTranslatedGuestDraw draw, RenderPass renderPass, IReadOnlyList renderTargetFormats, - Extent2D extent) + Extent2D extent, + IReadOnlyList? feedbackTargets = null, + bool hasDepthAttachment = false, + GuestDepthResource? feedbackDepth = null) { - var vertexSpirv = draw.VertexSpirv; + var isTitleDraw = draw.VertexBuffers.Any(buffer => + buffer.Location == 0 && + buffer.ComponentCount == 4 && + buffer.DataFormat == 10 && + buffer.NumberFormat == 0 && + buffer.Stride == 16 && + buffer.OffsetBytes == 12 && + buffer.Length == 67568); + var forceFullscreenPipeline = Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_FULLSCREEN_PIPELINE") == "1"; + var forceFullscreenVertex = forceFullscreenPipeline || + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_FULLSCREEN_VERTEX") == "1" || + isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_FULLSCREEN_VERTEX") == "1" || + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_FULLSCREEN_VERTEX_TARGETS", + target.Address)) == true; + var forceRasterState = forceFullscreenPipeline || + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_DEFAULT_RASTER_STATE") == "1" || + isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DEFAULT_RASTER_STATE") == "1" || + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_DEFAULT_RASTER_STATE_TARGETS", + target.Address)) == true; + var forceTitleSolidFragment = + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_SOLID_FRAGMENT") == "1" && + isTitleDraw; + var forceSolidFragment = forceTitleSolidFragment || forceFullscreenPipeline || + Environment.GetEnvironmentVariable("SHARPEMU_FORCE_SOLID_FRAGMENT") == "1" || + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_SOLID_FRAGMENT_TARGETS", + target.Address)) == true; + var forceAttributeFragment = uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_FORCE_ATTRIBUTE_FRAGMENT"), + out var attributeFragmentLocation) && + feedbackTargets?.Any(target => AddressListContains( + "SHARPEMU_FORCE_ATTRIBUTE_FRAGMENT_TARGETS", + target.Address)) == true; + var vertexSpirv = forceFullscreenVertex + ? SpirvFixedShaders.CreateFullscreenVertex(0) + : draw.VertexSpirv; + var fragmentSpirv = forceSolidFragment + ? SpirvFixedShaders.CreateSolidFragment(1f, 0f, 1f, 1f) + : forceAttributeFragment + ? SpirvFixedShaders.CreateAttributeFragment(attributeFragmentLocation) + : draw.PixelSpirv; + var fixedFragmentDumpPath = Environment.GetEnvironmentVariable( + "SHARPEMU_DUMP_FIXED_SOLID_FRAGMENT"); + if (forceSolidFragment && !string.IsNullOrWhiteSpace(fixedFragmentDumpPath)) + { + File.WriteAllBytes(fixedFragmentDumpPath, fragmentSpirv); + } if (draw.RenderState.Blends.Count != renderTargetFormats.Count) { throw new InvalidOperationException( "color attachment formats and blend states must have matching counts"); } - if (vertexSpirv.Length == 0 && !TryCompileFullscreenVertexShader( draw.AttributeCount, @@ -2805,13 +5108,74 @@ internal static unsafe class VulkanVideoPresenter Blends = draw.RenderState.Blends.ToArray(), Scissor = draw.RenderState.Scissor, Viewport = draw.RenderState.Viewport, + Raster = draw.RenderState.Raster, + Depth = draw.RenderState.Depth, + HasDepthAttachment = hasDepthAttachment, + TargetFormats = renderTargetFormats.ToArray(), }; + if (forceFullscreenVertex) + { + resources.VertexCount = 3; + resources.InstanceCount = 1; + resources.Topology = PrimitiveTopology.TriangleList; + } + if (forceRasterState) + { + resources.Blends = Enumerable.Repeat( + GuestBlendState.Default, + renderTargetFormats.Count).ToArray(); + resources.Scissor = null; + resources.Viewport = null; + resources.Raster = GuestRasterState.Default; + resources.Depth = GuestDepthState.Default; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DEFAULT_BLEND") == "1") + { + resources.Blends = Enumerable.Repeat( + GuestBlendState.Default, + renderTargetFormats.Count).ToArray(); + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DEFAULT_VIEWPORT_SCISSOR") == "1") + { + resources.Scissor = null; + resources.Viewport = null; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DISABLE_CULL") == "1") + { + resources.Raster = resources.Raster with + { + CullFront = false, + CullBack = false, + }; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_DISABLE_DEPTH") == "1") + { + resources.Depth = GuestDepthState.Default; + } + if (isTitleDraw && Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_TITLE_STATE") == "1") + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.title_state " + + $"viewport={resources.Viewport} scissor={resources.Scissor} " + + $"raster={resources.Raster} depth={resources.Depth} " + + $"blends=[{string.Join(',', resources.Blends)}]"); + } try { foreach (var texture in draw.Textures) { - if (texture.IsStorage) + // Skip address-0 storage bindings here: the real resolution + // path uses a scratch image for those, but this warm-up pass + // called ResolveStorageGuestImage directly, which throws on + // address 0 and dropped the whole draw (Demon's Souls G-buffer + // normals/IDs passes -> lighting had no input -> black). + if (texture.IsStorage && texture.Address != 0) { _ = ResolveStorageGuestImage(texture); } @@ -2819,28 +5183,70 @@ internal static unsafe class VulkanVideoPresenter for (var index = 0; index < draw.Textures.Count; index++) { - resources.Textures[index] = ResolveTextureResource(draw.Textures[index]); + var texture = draw.Textures[index]; + var resolved = ResolveTextureResource(texture); + var feedbackTarget = !texture.IsStorage + ? feedbackTargets?.FirstOrDefault(target => + ReferenceEquals(resolved.GuestImage, target)) + : null; + // ResolveTextureResource may deliberately decline an + // address alias when the descriptor is incompatible with + // the render-target image. Only snapshot an alias which + // actually resolved to the target; a separately uploaded + // texture has no Vulkan attachment feedback hazard. + resources.Textures[index] = + feedbackDepth is not null && + !texture.IsStorage && + ReferenceEquals(resolved.GuestDepth, feedbackDepth) + ? CreateDepthFeedbackSnapshot(texture, feedbackDepth) + : + feedbackTarget is not null && + !texture.IsStorage && + ReferenceEquals(resolved.GuestImage, feedbackTarget) + ? CreateRenderTargetFeedbackSnapshot(texture, feedbackTarget) + : resolved; } + PrepareGuestBufferAllocations(draw.GlobalMemoryBuffers); for (var index = 0; index < draw.GlobalMemoryBuffers.Count; index++) { resources.GlobalMemoryBuffers[index] = CreateGlobalBufferResource(draw.GlobalMemoryBuffers[index]); } + var sharedVertexResources = new Dictionary< + byte[], VertexBufferResource>( + System.Collections.Generic.ReferenceEqualityComparer.Instance); for (var index = 0; index < draw.VertexBuffers.Count; index++) { - resources.VertexBuffers[index] = - CreateVertexBufferResource(draw.VertexBuffers[index]); + var guestVertex = draw.VertexBuffers[index]; + if (sharedVertexResources.TryGetValue( + guestVertex.Data, + out var sharedVertex)) + { + resources.VertexBuffers[index] = + CreateVertexBufferAlias(sharedVertex, guestVertex); + } + else + { + var vertexResource = + CreateVertexBufferResource(guestVertex); + resources.VertexBuffers[index] = vertexResource; + sharedVertexResources.Add(guestVertex.Data, vertexResource); + } } - if (draw.IndexBuffer is { Data.Length: > 0 } indexBuffer) + if (draw.IndexBuffer is { Length: > 0 } indexBuffer) { resources.IndexBuffer = CreateHostBuffer( - indexBuffer.Data, + indexBuffer.Data.AsSpan(0, indexBuffer.Length), BufferUsageFlags.IndexBufferBit, out resources.IndexMemory); resources.Index32Bit = indexBuffer.Is32Bit; + if (indexBuffer.Pooled) + { + GuestDataPool.Return(indexBuffer.Data); + } } CreateTranslatedDescriptorResources( @@ -2849,7 +5255,7 @@ internal static unsafe class VulkanVideoPresenter CreateTranslatedPipeline( resources, vertexSpirv, - draw.PixelSpirv, + fragmentSpirv, renderPass, renderTargetFormats, extent); @@ -2860,6 +5266,18 @@ internal static unsafe class VulkanVideoPresenter DestroyTranslatedDrawResources(resources); throw; } + finally + { + var returnedVertexData = new HashSet( + System.Collections.Generic.ReferenceEqualityComparer.Instance); + foreach (var vertex in draw.VertexBuffers) + { + if (vertex.Pooled && returnedVertexData.Add(vertex.Data)) + { + GuestDataPool.Return(vertex.Data); + } + } + } } [MethodImpl(MethodImplOptions.NoInlining)] @@ -2887,7 +5305,11 @@ internal static unsafe class VulkanVideoPresenter for (var index = 0; index < dispatch.Textures.Count; index++) { var texture = dispatch.Textures[index]; - if (texture.IsStorage) + // Address-zero storage descriptors are valid scratch bindings. + // ResolveTextureResource creates their transient image below; + // pre-resolving them as guest-backed images throws and drops + // the entire compute dispatch before that path can run. + if (texture.IsStorage && texture.Address != 0) { if (traceResources) { @@ -2895,10 +5317,12 @@ internal static unsafe class VulkanVideoPresenter $"vk.compute_resources storage[{index}] begin " + $"addr=0x{texture.Address:X16} fmt={texture.Format} " + $"size={texture.Width}x{texture.Height} " + - $"mips={texture.MipLevels} level={texture.MipLevel}"); + $"view_mips={texture.BaseMipLevel}+{texture.MipLevels} " + + $"resource_mips={texture.ResourceMipLevels} " + + $"relative_level={texture.MipLevel}"); } - _ = ResolveStorageImageResource(texture); + _ = ResolveStorageGuestImage(texture); if (traceResources) { TraceVulkanShader($"vk.compute_resources storage[{index}] ready"); @@ -2922,6 +5346,7 @@ internal static unsafe class VulkanVideoPresenter TraceVulkanShader("vk.compute_resources resolve ready"); } + PrepareGuestBufferAllocations(dispatch.GlobalMemoryBuffers); for (var index = 0; index < dispatch.GlobalMemoryBuffers.Count; index++) { resources.GlobalMemoryBuffers[index] = @@ -3034,14 +5459,42 @@ internal static unsafe class VulkanVideoPresenter }; } - fixed (DescriptorPoolSize* poolSizePointer = poolSizes) + if (_recycledDescriptorPools.TryPop(out var recycledPool)) { + Check( + _vk.ResetDescriptorPool(_device, recycledPool, 0), + "vkResetDescriptorPool"); + resources.DescriptorPool = recycledPool; + } + else + { + // Generously sized so any draw's set fits, making the pool + // recyclable regardless of the draw's binding mix. AAA titles + // (e.g. Demon's Souls) bind well over 32 textures in a single + // descriptor set, so the sampled-image budget in particular + // must be large enough to avoid a per-draw dynamic fallback. + var genericPoolSizes = stackalloc DescriptorPoolSize[3]; + genericPoolSizes[0] = new DescriptorPoolSize + { + Type = DescriptorType.CombinedImageSampler, + DescriptorCount = 256, + }; + genericPoolSizes[1] = new DescriptorPoolSize + { + Type = DescriptorType.StorageImage, + DescriptorCount = 64, + }; + genericPoolSizes[2] = new DescriptorPoolSize + { + Type = DescriptorType.StorageBuffer, + DescriptorCount = 64, + }; var poolInfo = new DescriptorPoolCreateInfo { SType = StructureType.DescriptorPoolCreateInfo, MaxSets = 1, - PoolSizeCount = (uint)poolSizes.Length, - PPoolSizes = poolSizePointer, + PoolSizeCount = 3, + PPoolSizes = genericPoolSizes, }; DescriptorPool descriptorPool; Check( @@ -3082,7 +5535,7 @@ internal static unsafe class VulkanVideoPresenter bufferInfoPointer[index] = new DescriptorBufferInfo { Buffer = resources.GlobalMemoryBuffers[index].Buffer, - Offset = 0, + Offset = resources.GlobalMemoryBuffers[index].Offset, Range = resources.GlobalMemoryBuffers[index].Size, }; } @@ -3155,13 +5608,16 @@ internal static unsafe class VulkanVideoPresenter GetShaderDigest(vertexSpirv), GetShaderDigest(fragmentSpirv), string.Join(',', renderTargetFormats.Select(format => (uint)format)), + resources.HasDepthAttachment, resources.Topology, string.Join(';', resources.Blends.Select(blend => $"{(blend.Enable ? 1 : 0)}:{blend.ColorSrcFactor}:{blend.ColorDstFactor}:" + $"{blend.ColorFunc}:{blend.AlphaSrcFactor}:{blend.AlphaDstFactor}:" + $"{blend.AlphaFunc}:{(blend.SeparateAlphaBlend ? 1 : 0)}:{blend.WriteMask}")), GetResourceLayoutKey(resources), - GetVertexLayoutKey(resources)); + GetVertexLayoutKey(resources), + resources.Raster, + resources.HasDepthAttachment ? resources.Depth : GuestDepthState.Default); if (_graphicsPipelines.TryGetValue(pipelineKey, out var cachedPipeline)) { resources.Pipeline = cachedPipeline; @@ -3236,6 +5692,14 @@ internal static unsafe class VulkanVideoPresenter { SType = StructureType.PipelineInputAssemblyStateCreateInfo, Topology = resources.Topology, + // Metal always applies primitive restart to strip/fan + // topologies with the max index as the cut value, so + // match that here. Enabling it for lists is what makes + // MoltenVK warn ("Metal does not support disabling + // primitive restart"); lists never carry a restart + // index, so leaving it off for them is both correct + // and warning-free. + PrimitiveRestartEnable = RequiresPrimitiveRestart(resources.Topology), }; var viewport = new Viewport(0, 0, extent.Width, extent.Height, 0, 1); var scissor = new Rect2D(new Offset2D(0, 0), extent); @@ -3247,12 +5711,29 @@ internal static unsafe class VulkanVideoPresenter ScissorCount = 1, PScissors = &scissor, }; + var raster = resources.Raster; + var cullMode = CullModeFlags.None; + if (raster.CullFront) + { + cullMode |= CullModeFlags.FrontBit; + } + + if (raster.CullBack) + { + cullMode |= CullModeFlags.BackBit; + } + var rasterization = new PipelineRasterizationStateCreateInfo { SType = StructureType.PipelineRasterizationStateCreateInfo, + // Wireframe (PolygonMode.Line) needs the fillModeNonSolid + // device feature and is effectively unused by shipping + // titles, so fall back to a solid fill. PolygonMode = PolygonMode.Fill, - CullMode = CullModeFlags.None, - FrontFace = FrontFace.CounterClockwise, + CullMode = cullMode, + FrontFace = raster.FrontFaceClockwise + ? FrontFace.Clockwise + : FrontFace.CounterClockwise, LineWidth = 1, }; var multisample = new PipelineMultisampleStateCreateInfo @@ -3266,7 +5747,8 @@ internal static unsafe class VulkanVideoPresenter var blend = resources.Blends[index]; colorBlendAttachments[index] = new PipelineColorBlendAttachmentState { - BlendEnable = blend.Enable, + BlendEnable = blend.Enable && + IsBlendableFormat(renderTargetFormats[index]), SrcColorBlendFactor = ToVkBlendFactor(blend.ColorSrcFactor), DstColorBlendFactor = ToVkBlendFactor(blend.ColorDstFactor), ColorBlendOp = ToVkBlendOp(blend.ColorFunc), @@ -3297,6 +5779,16 @@ internal static unsafe class VulkanVideoPresenter DynamicStateCount = 2, PDynamicStates = dynamicStateValues, }; + var depth = resources.Depth; + var depthStencil = new PipelineDepthStencilStateCreateInfo + { + SType = StructureType.PipelineDepthStencilStateCreateInfo, + DepthTestEnable = depth.TestEnable, + DepthWriteEnable = depth.WriteEnable, + DepthCompareOp = ToVkCompareOp(depth.CompareOp), + DepthBoundsTestEnable = false, + StencilTestEnable = false, + }; var pipelineInfo = new GraphicsPipelineCreateInfo { SType = StructureType.GraphicsPipelineCreateInfo, @@ -3308,6 +5800,7 @@ internal static unsafe class VulkanVideoPresenter PRasterizationState = &rasterization, PMultisampleState = &multisample, PColorBlendState = &colorBlend, + PDepthStencilState = resources.HasDepthAttachment ? &depthStencil : null, PDynamicState = &dynamicState, Layout = resources.PipelineLayout, RenderPass = renderPass, @@ -3320,12 +5813,14 @@ internal static unsafe class VulkanVideoPresenter _pipelineCache, 1, &pipelineInfo, - null, - out pipeline), - "vkCreateGraphicsPipelines(translated)"); + null, + out pipeline), + "vkCreateGraphicsPipelines(translated)"); + MarkPipelineCacheDirty(); resources.Pipeline = pipeline; resources.PipelineCached = true; _graphicsPipelines.Add(pipelineKey, pipeline); + Interlocked.Increment(ref _perfPipelineCreations); SetDebugName( ObjectType.Pipeline, pipeline.Handle, @@ -3408,6 +5903,18 @@ internal static unsafe class VulkanVideoPresenter pipelineInfo.PSetLayouts = &descriptorSetLayout; } + var computePushConstantRange = new PushConstantRange + { + StageFlags = ShaderStageFlags.ComputeBit, + Offset = 0, + Size = 3 * sizeof(uint), + }; + if ((stageFlags & ShaderStageFlags.ComputeBit) != 0) + { + pipelineInfo.PushConstantRangeCount = 1; + pipelineInfo.PPushConstantRanges = &computePushConstantRange; + } + PipelineLayout pipelineLayout; Check( _vk.CreatePipelineLayout( @@ -3433,7 +5940,13 @@ internal static unsafe class VulkanVideoPresenter return digest; } - private static string GetResourceLayoutKey(TranslatedDrawResources resources) + private static string GetResourceLayoutKey(TranslatedDrawResources resources) => + resources.ResourceLayoutKey ??= BuildResourceLayoutKey(resources); + + private static string GetVertexLayoutKey(TranslatedDrawResources resources) => + resources.VertexLayoutKey ??= BuildVertexLayoutKey(resources); + + private static string BuildResourceLayoutKey(TranslatedDrawResources resources) { var key = new StringBuilder(); key.Append(resources.GlobalMemoryBuffers.Length).Append(':'); @@ -3445,7 +5958,7 @@ internal static unsafe class VulkanVideoPresenter return key.ToString(); } - private static string GetVertexLayoutKey(TranslatedDrawResources resources) + private static string BuildVertexLayoutKey(TranslatedDrawResources resources) { var key = new StringBuilder(); foreach (var buffer in resources.VertexBuffers) @@ -3468,7 +5981,10 @@ internal static unsafe class VulkanVideoPresenter TranslatedDrawResources resources, byte[] computeSpirv) { - if (_computePipelines.TryGetValue(computeSpirv, out var cachedPipeline)) + var pipelineKey = new ComputePipelineKey( + GetShaderDigest(computeSpirv), + GetResourceLayoutKey(resources)); + if (_computePipelines.TryGetValue(pipelineKey, out var cachedPipeline)) { resources.Pipeline = cachedPipeline; resources.PipelineCached = true; @@ -3503,13 +6019,14 @@ internal static unsafe class VulkanVideoPresenter null, out pipeline), "vkCreateComputePipelines(translated)"); + MarkPipelineCacheDirty(); resources.Pipeline = pipeline; resources.PipelineCached = true; SetDebugName( ObjectType.Pipeline, pipeline.Handle, $"SharpEmu compute cs={computeSpirv.Length}b"); - _computePipelines.Add(computeSpirv, pipeline); + _computePipelines.Add(pipelineKey, pipeline); } finally { @@ -3526,16 +6043,20 @@ internal static unsafe class VulkanVideoPresenter return ResolveStorageImageResource(texture); } + if (texture.Address != 0 && + TryResolveGuestDepthTexture(texture, out var depthTexture)) + { + return depthTexture; + } + var vkFormat = GetTextureFormat(texture.Format, texture.NumberType); if (texture.Address != 0 && - _guestImages.TryGetValue(texture.Address, out var guestImage) && - IsCompatibleGuestImageAlias(texture, guestImage) && - IsCompatibleViewFormat(guestImage.Format, vkFormat) && + TryResolveGuestImageAlias(texture, vkFormat, out var guestImage) && TryGetOrCreateGuestImageView( guestImage, vkFormat, - mipLevel: 0, - levelCount: guestImage.MipLevels, + mipLevel: texture.BaseMipLevel, + levelCount: texture.MipLevels, dstSelect: texture.DstSelect, out var view)) { @@ -3570,9 +6091,13 @@ internal static unsafe class VulkanVideoPresenter _pendingAliasImageDumps.Enqueue(guestImage); } - if (TryCreateCpuTextureRefreshResource(texture, guestImage, view, out var refresh)) + if (TryCreateCpuTextureRefreshResource( + texture, + guestImage, + view, + out var refreshResource)) { - return refresh; + return refreshResource; } return new TextureResource @@ -3589,7 +6114,26 @@ internal static unsafe class VulkanVideoPresenter }; } - return CreateTextureResource(texture); + if (ShouldTraceVulkanResources() && texture.Address != 0) + { + if (_guestImages.TryGetValue(texture.Address, out var missImage)) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.alias_miss addr=0x{texture.Address:X16} " + + $"reason={(IsCompatibleGuestImageAlias(texture, missImage) ? "format" : "size")} " + + $"tex={texture.Width}x{texture.Height}/f{texture.Format}/n{texture.NumberType}/vk{vkFormat} " + + $"img={missImage.Width}x{missImage.Height}/imgfmt{missImage.Format} " + + $"init={missImage.Initialized}"); + } + else + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.alias_miss addr=0x{texture.Address:X16} " + + $"reason=absent tex={texture.Width}x{texture.Height}/f{texture.Format}/n{texture.NumberType}"); + } + } + + return GetOrCreateCachedTextureResource(texture); } private bool TryCreateCpuTextureRefreshResource( @@ -3657,6 +6201,318 @@ internal static unsafe class VulkanVideoPresenter return true; } + private bool TryResolveGuestImageAlias( + GuestDrawTexture texture, + Format viewFormat, + out GuestImageResource guestImage) + { + GuestImageResource? best = null; + var bestScore = int.MinValue; + var guestFormat = GetGuestTextureFormat(texture.Format, texture.NumberType); + + void Consider(GuestImageResource candidate, bool isActive) + { + if (!IsUsableGuestImageAlias(texture, viewFormat, candidate)) + { + return; + } + + // Prefer an exact descriptor match. Initialization is important, + // but it must not make a differently sized alias outrank the image + // that the texture descriptor actually names. + var score = 0; + if (candidate.Width == texture.Width && + candidate.Height == texture.Height) + { + score += 32; + } + if (candidate.Format == viewFormat) + { + score += 16; + } + if (candidate.GuestFormat == guestFormat) + { + score += 8; + } + if (candidate.Initialized) + { + score += 4; + } + if (candidate.MipLevels == texture.ResourceMipLevels) + { + score += 2; + } + if (isActive) + { + score += 1; + } + + if (score > bestScore) + { + best = candidate; + bestScore = score; + } + } + + if (_guestImages.TryGetValue(texture.Address, out var active)) + { + Consider(active, isActive: true); + } + + foreach (var (key, candidate) in _guestImageVariants) + { + if (key.Address == texture.Address) + { + Consider(candidate, isActive: false); + } + } + + if (best is not null) + { + guestImage = best; + if (ShouldTraceVulkanResources()) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.texture_variant_hit " + + $"addr=0x{texture.Address:X16} " + + $"tex={texture.Width}x{texture.Height}/{viewFormat} " + + $"image={best.Width}x{best.Height}/{best.Format} " + + $"initialized={best.Initialized}"); + } + return true; + } + + guestImage = null!; + return false; + } + + private static bool IsUsableGuestImageAlias( + GuestDrawTexture texture, + Format viewFormat, + GuestImageResource guestImage) => + texture.BaseMipLevel < guestImage.MipLevels && + IsCompatibleGuestImageAlias(texture, guestImage) && + IsCompatibleViewFormat(guestImage.Format, viewFormat); + + private bool TryResolveGuestDepthTexture( + GuestDrawTexture texture, + out TextureResource resource) + { + foreach (var depth in _guestDepthImages.Values) + { + if (texture.Address != depth.Address && + texture.Address != depth.ReadAddress && + texture.Address != depth.WriteAddress) + { + continue; + } + + if (texture.Width > depth.Width || texture.Height > depth.Height) + { + continue; + } + + if (!depth.SampleViews.TryGetValue(texture.DstSelect, out var view)) + { + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = depth.Image, + ViewType = ImageViewType.Type2D, + Format = DepthFormat, + Components = ToVkComponentMapping(texture.DstSelect), + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1), + }; + Check( + _vk.CreateImageView(_device, &viewInfo, null, out view), + "vkCreateImageView(depth sample)"); + SetDebugName( + ObjectType.ImageView, + view.Handle, + $"SharpEmu guest depth sample 0x{depth.Address:X16} " + + $"dst=0x{texture.DstSelect:X3}"); + depth.SampleViews.Add(texture.DstSelect, view); + } + + if (_tracedDepthTextureAliases.Add( + (texture.Address, texture.Width, texture.Height, texture.DstSelect))) + { + TraceVulkanShader( + $"vk.depth_texture_alias addr=0x{texture.Address:X16} " + + $"depth=0x{depth.Address:X16} " + + $"texture={texture.Width}x{texture.Height} " + + $"surface={depth.Width}x{depth.Height} dst=0x{texture.DstSelect:X3}"); + } + resource = new TextureResource + { + Address = texture.Address, + Image = depth.Image, + View = view, + Width = depth.Width, + Height = depth.Height, + RowLength = depth.Width, + DstSelect = texture.DstSelect, + SamplerState = texture.Sampler, + GuestDepth = depth, + }; + return true; + } + + resource = null!; + return false; + } + + private readonly Dictionary _textureCache = new(); + + /// + /// Guest textures are static assets in the common case, but every draw + /// used to restage and reupload them (a fresh image, device memory and + /// staging buffer per draw). Cache the uploaded resource per descriptor + /// identity and invalidate through the CPU write tracker, so animated + /// or streamed texture memory still refreshes. + /// + private TextureResource GetOrCreateCachedTextureResource(GuestDrawTexture texture) + { + if (texture.Address == 0) + { + return CreateTextureResource(texture); + } + + var key = new TextureContentIdentity( + texture.Address, + texture.Width, + texture.Height, + texture.Format, + texture.NumberType, + texture.DstSelect, + texture.TileMode, + texture.Pitch, + texture.Sampler); + if (_textureCache.TryGetValue(key, out var cached)) + { + return cached; + } + + // Empty pixels mean the submit thread skipped the guest-memory + // copy because this identity was marked cached; a miss here is + // an invalidation race (eviction, cache clear). Self-heal by + // reading the texels directly rather than rendering a fallback. + if (texture.RgbaPixels.Length == 0) + { + var refreshed = TryReadGuestTexturePixels(texture); + if (refreshed is null) + { + return CreateTextureResource(texture); + } + + texture = texture with { RgbaPixels = refreshed }; + } + + var resource = CreateTextureResource(texture); + if (resource.OwnsStorage) + { + resource.Cached = true; + _textureCache[key] = resource; + MarkTextureContentCached(key); + SharpEmu.HLE.GuestImageWriteTracker.Track( + texture.Address, + (ulong)texture.RgbaPixels.Length, + CurrentGuestWorkSequenceForDiagnostics, + "vulkan.texture-cache"); + } + + return resource; + } + + private void EvictDirtyCachedTextures() + { + if (_textureCache.Count == 0) + { + return; + } + + List? evicted = null; + foreach (var entry in _textureCache) + { + if (SharpEmu.HLE.GuestImageWriteTracker.ConsumeDirty(entry.Key.Address)) + { + (evicted ??= []).Add(entry.Key); + } + } + + if (evicted is null && _textureCache.Count <= 2048) + { + return; + } + + // Destruction is deferred until every submission that may still + // reference the texture has completed (fences signal in queue + // order), so eviction never has to drain the GPU. An open batch + // is flushed first so the retire timeline exactly covers every + // recorded reference (nothing may guess which submission lands + // next on the shared queue). + if (_batchOpen) + { + FlushBatchedGuestCommands(); + } + + var retireTimeline = _submitTimeline; + if (_textureCache.Count > 2048) + { + foreach (var entry in _textureCache) + { + _deferredTextureDestroys.Enqueue((entry.Value, retireTimeline)); + } + + _textureCache.Clear(); + ClearCachedTextureIdentities(); + return; + } + + foreach (var key in evicted!) + { + if (_textureCache.Remove(key, out var resource)) + { + UnmarkTextureContentCached(key); + _deferredTextureDestroys.Enqueue((resource, retireTimeline)); + SharpEmu.HLE.GuestImageWriteTracker.Rearm(key.Address); + } + } + } + + private void DestroyCachedTextureResource(TextureResource texture) + { + texture.Cached = false; + if (texture.View.Handle != 0) + { + _vk.DestroyImageView(_device, texture.View, null); + } + + if (texture.Image.Handle != 0 && texture.GuestImage is null) + { + _vk.DestroyImage(_device, texture.Image, null); + if (texture.ImageMemory.Handle != 0) + { + _vk.FreeMemory(_device, texture.ImageMemory, null); + } + } + + if (texture.StagingBuffer.Handle != 0) + { + _vk.DestroyBuffer(_device, texture.StagingBuffer, null); + } + + if (texture.StagingMemory.Handle != 0) + { + _vk.FreeMemory(_device, texture.StagingMemory, null); + } + } + private static bool IsCompatibleGuestImageAlias( GuestDrawTexture texture, GuestImageResource guestImage) @@ -3688,10 +6544,11 @@ internal static unsafe class VulkanVideoPresenter var guestImage = ResolveStorageGuestImage(texture); var vkFormat = GetTextureFormat(texture.Format, texture.NumberType); + var selectedMipLevel = GetStorageMipLevel(texture); var view = GetOrCreateGuestImageView( guestImage, vkFormat, - texture.MipLevel, + selectedMipLevel, levelCount: 1); var resource = new TextureResource { @@ -3718,8 +6575,12 @@ internal static unsafe class VulkanVideoPresenter if ((ulong)texture.RgbaPixels.Length == expectedSize && texture.RgbaPixels.AsSpan().IndexOfAnyExcept((byte)0) >= 0) { + var uploadPixels = texture.Format == 13 + ? ExpandRgb32Pixels(texture.RgbaPixels) + : texture.RgbaPixels; + var uploadSize = (ulong)uploadPixels.Length; resource.StagingBuffer = CreateBuffer( - expectedSize, + uploadSize, BufferUsageFlags.TransferSrcBit, MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, @@ -3731,17 +6592,17 @@ internal static unsafe class VulkanVideoPresenter _device, resource.StagingMemory, 0, - expectedSize, + uploadSize, 0, &mapped), "vkMapMemory(storage texture)"); - fixed (byte* source = texture.RgbaPixels) + fixed (byte* source = uploadPixels) { System.Buffer.MemoryCopy( source, mapped, - texture.RgbaPixels.Length, - texture.RgbaPixels.Length); + uploadPixels.Length, + uploadPixels.Length); } _vk.UnmapMemory(_device, resource.StagingMemory); @@ -3749,7 +6610,8 @@ internal static unsafe class VulkanVideoPresenter guestImage.InitialUploadPending = true; TraceVulkanShader( $"vk.storage_upload addr=0x{texture.Address:X16} " + - $"size={texture.Width}x{texture.Height} bytes={expectedSize}"); + $"size={texture.Width}x{texture.Height} " + + $"logical_bytes={expectedSize} upload_bytes={uploadSize}"); } } @@ -3823,6 +6685,7 @@ internal static unsafe class VulkanVideoPresenter Width = width, Height = height, MipLevels = 1, + GuestFormat = GetGuestTextureFormat(texture.Format, texture.NumberType), Format = vkFormat, Image = image, Memory = memory, @@ -3861,17 +6724,36 @@ internal static unsafe class VulkanVideoPresenter texture.Height, texture.Format, texture.NumberType, - texture.MipLevels), + texture.ResourceMipLevels), format); - if (texture.MipLevel >= guestImage.MipLevels) + var selectedMipLevel = GetStorageMipLevel(texture); + if (selectedMipLevel >= guestImage.MipLevels) { throw new InvalidOperationException( - $"Storage mip {texture.MipLevel} exceeds image mip count {guestImage.MipLevels}."); + $"Storage mip {selectedMipLevel} (base {texture.BaseMipLevel} + relative " + + $"{texture.MipLevel}) exceeds image mip count {guestImage.MipLevels}."); } return guestImage; } + private static uint GetStorageMipLevel(GuestDrawTexture texture) + { + // IMAGE_STORE targets BASE_LEVEL and IMAGE_STORE_MIP's operand is + // expressed in resource-view space, so Vulkan's absolute image + // subresource is descriptor base plus the instruction-relative + // mip. Sampled views achieve the same mapping through their view + // base in ResolveTextureResource. + var selectedMipLevel = (ulong)texture.BaseMipLevel + texture.MipLevel; + if (selectedMipLevel > uint.MaxValue) + { + throw new InvalidOperationException( + $"Storage mip overflow (base {texture.BaseMipLevel} + relative {texture.MipLevel})."); + } + + return (uint)selectedMipLevel; + } + private TextureResource CreateTextureResource(GuestDrawTexture texture) { var width = Math.Max(texture.Width, 1); @@ -3894,7 +6776,16 @@ internal static unsafe class VulkanVideoPresenter var pixels = texture.RgbaPixels.Length == (int)expectedSize ? texture.RgbaPixels : CreateFallbackTexturePixels(texture.Format, rowLength, height, expectedSize); + if (AddressListContains("SHARPEMU_FORCE_WHITE_TEXTURE_TARGETS", texture.Address)) + { + pixels = pixels.ToArray(); + pixels.AsSpan().Fill(0xFF); + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.texture_force_white addr=0x{texture.Address:X16} " + + $"size={width}x{height} bytes={pixels.Length}"); + } DumpTextureUpload(texture, pixels, rowLength, width, height); + TraceTextureUploadContents(texture, pixels, rowLength, width, height, vkFormat); var uploadPixels = texture.Format == 13 ? ExpandRgb32Pixels(pixels) : pixels; @@ -3976,12 +6867,14 @@ internal static unsafe class VulkanVideoPresenter if (texture.Address != 0 && !_guestImages.ContainsKey(texture.Address)) { + var guestFormat = GetGuestTextureFormat(texture.Format, texture.NumberType); var guestImage = new GuestImageResource { Address = texture.Address, Width = width, Height = height, MipLevels = 1, + GuestFormat = guestFormat, Format = vkFormat, Image = image, Memory = imageMemory, @@ -3995,14 +6888,13 @@ internal static unsafe class VulkanVideoPresenter resource.GuestImage = guestImage; lock (_gate) { - var guestFormat = VulkanVideoPresenter.GetGuestTextureFormat( - texture.Format, - texture.NumberType); if (guestFormat != 0) { _availableGuestImages[texture.Address] = guestFormat; - _gpuGuestImages.Remove(texture.Address); } + + _guestImageExtents[texture.Address] = + (width, height, expectedSize); } } @@ -4069,6 +6961,229 @@ internal static unsafe class VulkanVideoPresenter return hash; } + /// + /// Creates a draw-local sampled image containing the render target's + /// value immediately before the draw. RDNA permits a pixel shader to + /// read an attachment while producing its replacement value, whereas + /// core Vulkan does not permit the same subresource to be both a + /// sampled image and a color attachment. Keeping the two images + /// distinct preserves the guest's read-before-write behavior without + /// a queue idle or an undefined Vulkan feedback loop. + /// + private TextureResource CreateRenderTargetFeedbackSnapshot( + GuestDrawTexture texture, + GuestImageResource source) + { + var image = default(Image); + var memory = default(DeviceMemory); + var view = default(ImageView); + try + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + Flags = + ImageCreateFlags.CreateMutableFormatBit | + ImageCreateFlags.CreateExtendedUsageBit, + ImageType = ImageType.Type2D, + Format = source.Format, + Extent = new Extent3D(source.Width, source.Height, 1), + MipLevels = source.MipLevels, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = + ImageUsageFlags.TransferDstBit | + ImageUsageFlags.SampledBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check( + _vk.CreateImage(_device, &imageInfo, null, out image), + "vkCreateImage(render-target feedback snapshot)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + Check( + _vk.AllocateMemory(_device, &memoryInfo, null, out memory), + "vkAllocateMemory(render-target feedback snapshot)"); + Check( + _vk.BindImageMemory(_device, image, memory, 0), + "vkBindImageMemory(render-target feedback snapshot)"); + + var viewFormat = GetTextureFormat(texture.Format, texture.NumberType); + if (!IsCompatibleViewFormat(source.Format, viewFormat)) + { + throw new InvalidOperationException( + $"Feedback view format {viewFormat} is incompatible with " + + $"render-target format {source.Format}."); + } + + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = image, + ViewType = ImageViewType.Type2D, + Format = viewFormat, + Components = ToVkComponentMapping(texture.DstSelect), + SubresourceRange = ColorSubresourceRange(0, source.MipLevels), + }; + Check( + _vk.CreateImageView(_device, &viewInfo, null, out view), + "vkCreateImageView(render-target feedback snapshot)"); + + var debugName = + $"SharpEmu feedback 0x{source.Address:X16} " + + $"{source.Width}x{source.Height} {source.Format}->{viewFormat}"; + SetDebugName(ObjectType.Image, image.Handle, $"{debugName} image"); + SetDebugName(ObjectType.ImageView, view.Handle, $"{debugName} view"); + TraceVulkanShader( + $"vk.feedback_snapshot_create addr=0x{source.Address:X16} " + + $"size={source.Width}x{source.Height} mips={source.MipLevels} " + + $"image_format={source.Format} view_format={viewFormat} " + + $"dst=0x{texture.DstSelect:X3}"); + + return new TextureResource + { + Address = texture.Address, + Image = image, + ImageMemory = memory, + View = view, + Width = source.Width, + Height = source.Height, + RowLength = source.Width, + DstSelect = texture.DstSelect, + OwnsStorage = true, + SamplerState = texture.Sampler, + FeedbackSource = source, + }; + } + catch + { + if (view.Handle != 0) + { + _vk.DestroyImageView(_device, view, null); + } + + if (image.Handle != 0) + { + _vk.DestroyImage(_device, image, null); + } + + if (memory.Handle != 0) + { + _vk.FreeMemory(_device, memory, null); + } + + throw; + } + } + + private TextureResource CreateDepthFeedbackSnapshot( + GuestDrawTexture texture, + GuestDepthResource source) + { + var image = default(Image); + var memory = default(DeviceMemory); + var view = default(ImageView); + try + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = DepthFormat, + Extent = new Extent3D(source.Width, source.Height, 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check( + _vk.CreateImage(_device, &imageInfo, null, out image), + "vkCreateImage(depth feedback snapshot)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType( + requirements.MemoryTypeBits, + MemoryPropertyFlags.DeviceLocalBit), + }; + Check( + _vk.AllocateMemory(_device, &memoryInfo, null, out memory), + "vkAllocateMemory(depth feedback snapshot)"); + Check( + _vk.BindImageMemory(_device, image, memory, 0), + "vkBindImageMemory(depth feedback snapshot)"); + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = image, + ViewType = ImageViewType.Type2D, + Format = DepthFormat, + Components = ToVkComponentMapping(texture.DstSelect), + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1), + }; + Check( + _vk.CreateImageView(_device, &viewInfo, null, out view), + "vkCreateImageView(depth feedback snapshot)"); + SetDebugName( + ObjectType.Image, + image.Handle, + $"SharpEmu depth feedback 0x{source.Address:X16} image"); + SetDebugName( + ObjectType.ImageView, + view.Handle, + $"SharpEmu depth feedback 0x{source.Address:X16} view"); + return new TextureResource + { + Address = texture.Address, + Image = image, + ImageMemory = memory, + View = view, + Width = source.Width, + Height = source.Height, + RowLength = source.Width, + DstSelect = texture.DstSelect, + OwnsStorage = true, + SamplerState = texture.Sampler, + DepthFeedbackSource = source, + }; + } + catch + { + if (view.Handle != 0) + { + _vk.DestroyImageView(_device, view, null); + } + if (image.Handle != 0) + { + _vk.DestroyImage(_device, image, null); + } + if (memory.Handle != 0) + { + _vk.FreeMemory(_device, memory, null); + } + throw; + } + } + private void DumpTextureUpload( GuestDrawTexture texture, byte[] pixels, @@ -4076,7 +7191,10 @@ internal static unsafe class VulkanVideoPresenter uint width, uint height) { - if (!_dumpTextures || + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_DUMP_TEXTURES"), + "1", + StringComparison.Ordinal) || texture.IsFallback || texture.IsStorage || GetTextureBytesPerPixel(texture.Format) != 4 || @@ -4102,6 +7220,63 @@ internal static unsafe class VulkanVideoPresenter WriteRgbaBmp(path, pixels, rowBytes, visibleRowBytes, (int)width, (int)height); } + private void TraceTextureUploadContents( + GuestDrawTexture texture, + byte[] pixels, + uint rowLength, + uint width, + uint height, + Format format) + { + if (!_traceGuestImageAddressFilterEnabled || + !AddressListContains("SHARPEMU_TRACE_GUEST_IMAGE_ADDRS", texture.Address) || + !_tracedTextureUploadContents.Add( + (texture.Address, width, height, texture.Format))) + { + return; + } + + var bytesPerPixel = checked((uint)GetTextureBytesPerPixel(texture.Format)); + var nonzeroBytes = 0L; + ulong hash = 14695981039346656037UL; + foreach (var value in pixels) + { + nonzeroBytes += value == 0 ? 0 : 1; + hash = (hash ^ value) * 1099511628211UL; + } + + var centerOffset = checked( + ((int)(height / 2) * (int)rowLength + (int)(width / 2)) * + (int)bytesPerPixel); + var center = centerOffset + bytesPerPixel <= pixels.Length + ? Convert.ToHexString( + pixels.AsSpan(centerOffset, checked((int)bytesPerPixel))) + : "out-of-range"; + Console.Error.WriteLine( + "[LOADER][TRACE] " + + $"vk.texture_upload_contents addr=0x{texture.Address:X16} " + + $"size={width}x{height} row={rowLength} format={format} " + + $"guest_format={texture.Format} nonzero_bytes={nonzeroBytes}/{pixels.Length} " + + $"nonblack_pixels={CountNonblackPixels(pixels, format, bytesPerPixel)}/{(ulong)rowLength * height} " + + $"center={center} sample_unique={CountSampledUniquePixels(pixels, bytesPerPixel)} " + + $"hash=0x{hash:X16}"); + + var directory = + Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_DUMP_DIR"); + if (string.IsNullOrWhiteSpace(directory)) + { + return; + } + + Directory.CreateDirectory(directory); + var sequence = Interlocked.Increment(ref _guestImageDumpSequence); + var path = Path.Combine( + directory, + $"{sequence:D4}-texture-0x{texture.Address:X16}-{width}x{height}-" + + $"row{rowLength}-fmt{texture.Format}-{format}.rgba"); + File.WriteAllBytes(path, pixels); + } + private static void WriteRgbaBmp( string path, byte[] rgba, @@ -4201,11 +7376,6 @@ internal static unsafe class VulkanVideoPresenter private static ComponentMapping ToVkComponentMapping(uint dstSelect) { - if (dstSelect == 0) - { - dstSelect = 0xFAC; - } - return new ComponentMapping( ToVkComponentSwizzle(dstSelect & 0x7), ToVkComponentSwizzle((dstSelect >> 3) & 0x7), @@ -4242,45 +7412,374 @@ internal static unsafe class VulkanVideoPresenter private GlobalBufferResource CreateGlobalBufferResource( GuestMemoryBuffer guestBuffer) { - var buffer = CreateHostBuffer( - guestBuffer.Data, - BufferUsageFlags.StorageBufferBit, - out var memory); - var size = (ulong)Math.Max(guestBuffer.Data.Length, sizeof(uint)); + if (guestBuffer.BaseAddress == 0) + { + return CreateTransientGlobalBufferResource(guestBuffer); + } + + var size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)); + var endAddress = checked(guestBuffer.BaseAddress + size); + var allocation = _guestBufferAllocations + .Where(candidate => + candidate.BaseAddress <= guestBuffer.BaseAddress && + candidate.BaseAddress + candidate.Size >= endAddress) + .OrderByDescending(candidate => + string.Equals( + candidate.QueueName, + _activeGuestQueue.Name, + StringComparison.Ordinal)) + .ThenByDescending(candidate => + string.Equals( + candidate.QueueName, + SharedReadOnlyGuestBufferQueue, + StringComparison.Ordinal)) + .First(); + var guestOffset = guestBuffer.BaseAddress - allocation.BaseAddress; + var descriptorOffset = guestOffset & + ~(GuestStorageBufferOffsetAlignment - 1); + var byteBias = guestOffset - descriptorOffset; + if (descriptorOffset % _minStorageBufferOffsetAlignment != 0) + { + throw new InvalidOperationException( + $"guest buffer alias offset 0x{descriptorOffset:X} is not aligned to Vulkan's " + + $"minStorageBufferOffsetAlignment={_minStorageBufferOffsetAlignment}"); + } + + var expectedBias = guestBuffer.BaseAddress & + (GuestStorageBufferOffsetAlignment - 1); + if (byteBias != expectedBias) + { + throw new InvalidOperationException( + $"guest buffer allocation base 0x{allocation.BaseAddress:X16} " + + $"does not satisfy alias alignment " + + $"{GuestStorageBufferOffsetAlignment}"); + } + + var source = guestBuffer.Data.AsSpan(0, guestBuffer.Length); + var shadow = allocation.Shadow.AsSpan(checked((int)guestOffset), guestBuffer.Length); + if (!source.SequenceEqual(shadow)) + { + // HOST_COHERENT does not permit racing a mapped CPU write with + // an in-flight shader access. Retire prior users, publish their + // dirty ranges to guest memory, then upload the current guest + // bytes (which may be newer than the parser's captured array). + if (string.Equals( + allocation.QueueName, + SharedReadOnlyGuestBufferQueue, + StringComparison.Ordinal)) + { + // Shared read-only storage can still be referenced by any + // logical guest queue, so replacing its mapped contents + // requires retiring every prior reader. + WaitForAllGuestSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(); + } + else + { + // Writable aliases are private to one logical guest queue. + // Retiring unrelated queues here recreates the global FIFO + // and turns routine buffer refreshes into queue-wide stalls. + WaitForActiveGuestQueueSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(_activeGuestQueue.Name); + } + var live = new byte[guestBuffer.Length]; + if (_guestMemory?.TryRead(guestBuffer.BaseAddress, live) == true) + { + source = live; + } + + source.CopyTo(new Span( + (void*)(allocation.Mapped + checked((nint)guestOffset)), + source.Length)); + source.CopyTo(shadow); + } if (ShouldTraceVulkanResources() && - _tracedGlobalBuffers.Add((guestBuffer.BaseAddress, guestBuffer.Data.Length))) + _tracedGlobalBuffers.Add((guestBuffer.BaseAddress, guestBuffer.Length))) { Console.Error.WriteLine( $"[LOADER][TRACE] vk.global_buffer base=0x{guestBuffer.BaseAddress:X16} " + - $"bytes={guestBuffer.Data.Length}"); + $"bytes={guestBuffer.Length}"); + } + if (_setDebugUtilsObjectName is not null) + { + SetDebugName( + ObjectType.Buffer, + allocation.Buffer.Handle, + $"SharpEmu global 0x{guestBuffer.BaseAddress:X16} {guestBuffer.Length}b"); + } + + if (guestBuffer.Pooled) + { + GuestDataPool.Return(guestBuffer.Data); } - SetDebugName( - ObjectType.Buffer, - buffer.Handle, - $"SharpEmu global 0x{guestBuffer.BaseAddress:X16} {guestBuffer.Data.Length}b"); return new GlobalBufferResource { + BaseAddress = guestBuffer.BaseAddress, + Writable = guestBuffer.Writable, + WriteBackToGuest = guestBuffer.WriteBackToGuest, + Buffer = allocation.Buffer, + Memory = allocation.Memory, + Mapped = allocation.Mapped + checked((nint)guestOffset), + Offset = descriptorOffset, + Size = checked((size + byteBias + 3) & ~3UL), + GuestOffset = guestOffset, + GuestSize = size, + Allocation = allocation, + }; + } + + private GlobalBufferResource CreateTransientGlobalBufferResource( + GuestMemoryBuffer guestBuffer) + { + var buffer = CreateHostBuffer( + guestBuffer.Data.AsSpan(0, guestBuffer.Length), + BufferUsageFlags.StorageBufferBit, + out var memory); + var allocation = _hostBufferAllocations[buffer.Handle]; + if (guestBuffer.Pooled) + { + GuestDataPool.Return(guestBuffer.Data); + } + + return new GlobalBufferResource + { + BaseAddress = 0, + Writable = false, + WriteBackToGuest = false, Buffer = buffer, Memory = memory, - Size = size, + Mapped = allocation.Mapped, + Offset = 0, + Size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)), + GuestOffset = 0, + GuestSize = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)), }; } + private void PrepareGuestBufferAllocations( + IReadOnlyList buffers) + { + if (buffers.Count == 0) + { + return; + } + + var ranges = new List<(ulong Start, ulong End, bool Writable)>(buffers.Count); + foreach (var buffer in buffers) + { + if (buffer.BaseAddress == 0) + { + continue; + } + + var size = (ulong)Math.Max(buffer.Length, sizeof(uint)); + var alignedStart = buffer.BaseAddress & + ~(GuestStorageBufferOffsetAlignment - 1); + var paddedEnd = checked(buffer.BaseAddress + size + 3) & ~3UL; + ranges.Add(( + alignedStart, + paddedEnd, + buffer.Writable && buffer.WriteBackToGuest)); + } + + if (ranges.Count == 0) + { + return; + } + + ranges.Sort(static (left, right) => left.Start.CompareTo(right.Start)); + var merged = new List<(ulong Start, ulong End, bool Writable)>(ranges.Count); + foreach (var range in ranges) + { + if (merged.Count == 0 || range.Start > merged[^1].End) + { + merged.Add(range); + continue; + } + + var previous = merged[^1]; + merged[^1] = ( + previous.Start, + Math.Max(previous.End, range.End), + previous.Writable || range.Writable); + } + + foreach (var range in merged) + { + EnsureGuestBufferAllocation( + range.Start, + range.End, + range.Writable + ? _activeGuestQueue.Name + : SharedReadOnlyGuestBufferQueue); + } + } + + private void EnsureGuestBufferAllocation( + ulong requestedStart, + ulong requestedEnd, + string queueName) + { + var start = requestedStart; + var end = requestedEnd; + List overlaps; + do + { + overlaps = _guestBufferAllocations + .Where(allocation => + string.Equals( + allocation.QueueName, + queueName, + StringComparison.Ordinal) && + allocation.BaseAddress < end && + start < allocation.BaseAddress + allocation.Size) + .ToList(); + var expandedStart = overlaps.Aggregate( + start, + static (value, allocation) => Math.Min(value, allocation.BaseAddress)); + var expandedEnd = overlaps.Aggregate( + end, + static (value, allocation) => + Math.Max(value, allocation.BaseAddress + allocation.Size)); + if (expandedStart == start && expandedEnd == end) + { + break; + } + + start = expandedStart; + end = expandedEnd; + } + while (true); + + if (overlaps.Count == 1 && + overlaps[0].BaseAddress <= requestedStart && + overlaps[0].BaseAddress + overlaps[0].Size >= requestedEnd) + { + return; + } + + if (overlaps.Count > 0) + { + // Growing/merging an aliased allocation is rare. Synchronize + // only this structural transition so no in-flight descriptor + // can observe storage being replaced underneath it. + WaitForAllGuestSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(); + } + + var replacement = CreateGuestBufferAllocation(start, end, queueName); + foreach (var overlap in overlaps) + { + _guestBufferAllocations.Remove(overlap); + DestroyGuestBufferAllocation(overlap); + } + + _guestBufferAllocations.Add(replacement); + _guestBufferAllocations.Sort(static (left, right) => + { + var queueOrder = string.CompareOrdinal(left.QueueName, right.QueueName); + return queueOrder != 0 + ? queueOrder + : left.BaseAddress.CompareTo(right.BaseAddress); + }); + TraceVulkanShader( + $"vk.guest_buffer_allocation queue={queueName} " + + $"base=0x{start:X16} bytes={replacement.Size} " + + $"merged={overlaps.Count}"); + } + + private GuestBufferAllocation CreateGuestBufferAllocation( + ulong start, + ulong end, + string queueName) + { + var size = checked(end - start); + if (size == 0 || size > int.MaxValue) + { + throw new InvalidOperationException( + $"guest buffer allocation is outside the supported host span: " + + $"base=0x{start:X16} bytes={size}"); + } + + var buffer = CreateBuffer( + size, + BufferUsageFlags.StorageBufferBit, + MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, + out var memory); + void* mapped; + Check(_vk.MapMemory(_device, memory, 0, size, 0, &mapped), "vkMapMemory(guest buffer)"); + var shadow = new byte[checked((int)size)]; + _ = _guestMemory?.TryRead(start, shadow); + shadow.CopyTo(new Span(mapped, shadow.Length)); + SetDebugName( + ObjectType.Buffer, + buffer.Handle, + $"SharpEmu guest VA 0x{start:X16}-0x{end:X16}"); + return new GuestBufferAllocation + { + QueueName = queueName, + BaseAddress = start, + Size = size, + Buffer = buffer, + Memory = memory, + Mapped = (nint)mapped, + Shadow = shadow, + }; + } + + private void DestroyGuestBufferAllocation(GuestBufferAllocation allocation) + { + if (allocation.Mapped != 0) + { + _vk.UnmapMemory(_device, allocation.Memory); + } + + _vk.DestroyBuffer(_device, allocation.Buffer, null); + _vk.FreeMemory(_device, allocation.Memory, null); + } + private VertexBufferResource CreateVertexBufferResource( GuestVertexBuffer guestBuffer) { + ReadOnlySpan source = guestBuffer.Data.AsSpan(0, guestBuffer.Length); + byte[]? forcedVertexColors = null; + if (Environment.GetEnvironmentVariable("SHARPEMU_FORCE_TITLE_VERTEX_COLOR_WHITE") == "1" && + guestBuffer.Location == 0 && + guestBuffer.ComponentCount == 4 && + guestBuffer.DataFormat == 10 && + guestBuffer.NumberFormat == 0 && + guestBuffer.Stride == 16 && + guestBuffer.OffsetBytes == 12 && + guestBuffer.Length == 67568) + { + forcedVertexColors = source.ToArray(); + for (var offset = 12; offset + 3 < forcedVertexColors.Length; offset += 16) + { + forcedVertexColors[offset] = 0xFF; + forcedVertexColors[offset + 1] = 0xFF; + forcedVertexColors[offset + 2] = 0xFF; + } + + source = forcedVertexColors; + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.vertex_force_title_color_white " + + $"base=0x{guestBuffer.BaseAddress:X16} bytes={guestBuffer.Length}"); + } var buffer = CreateHostBuffer( - guestBuffer.Data, + source, BufferUsageFlags.VertexBufferBit, out var memory); - var size = (ulong)Math.Max(guestBuffer.Data.Length, sizeof(uint)); - SetDebugName( - ObjectType.Buffer, - buffer.Handle, - $"SharpEmu vertex loc{guestBuffer.Location} " + - $"0x{guestBuffer.BaseAddress:X16} {guestBuffer.Data.Length}b"); + var size = (ulong)Math.Max(guestBuffer.Length, sizeof(uint)); + if (_setDebugUtilsObjectName is not null) + { + SetDebugName( + ObjectType.Buffer, + buffer.Handle, + $"SharpEmu vertex loc{guestBuffer.Location} " + + $"0x{guestBuffer.BaseAddress:X16} {guestBuffer.Length}b"); + } if (_tracedVertexBufferCount++ < 64) { TraceVulkanShader( @@ -4288,13 +7787,29 @@ internal static unsafe class VulkanVideoPresenter $"base=0x{guestBuffer.BaseAddress:X16} stride={guestBuffer.Stride} " + $"offset={guestBuffer.OffsetBytes} comps={guestBuffer.ComponentCount} " + $"fmt={guestBuffer.DataFormat}/num={guestBuffer.NumberFormat} " + - $"bytes={guestBuffer.Data.Length}"); + $"bytes={guestBuffer.Length}"); } + return CreateVertexBufferResource( + buffer, + memory, + size, + guestBuffer, + ownsBuffer: true); + } + + private static VertexBufferResource CreateVertexBufferResource( + VkBuffer buffer, + DeviceMemory memory, + ulong size, + GuestVertexBuffer guestBuffer, + bool ownsBuffer) + { return new VertexBufferResource { Buffer = buffer, Memory = memory, + OwnsBuffer = ownsBuffer, Size = size, Location = guestBuffer.Location, ComponentCount = guestBuffer.ComponentCount, @@ -4305,6 +7820,22 @@ internal static unsafe class VulkanVideoPresenter }; } + private static VertexBufferResource CreateVertexBufferAlias( + VertexBufferResource shared, + GuestVertexBuffer guestBuffer) => new() + { + Buffer = shared.Buffer, + Memory = shared.Memory, + OwnsBuffer = false, + Size = shared.Size, + Location = guestBuffer.Location, + ComponentCount = guestBuffer.ComponentCount, + DataFormat = guestBuffer.DataFormat, + NumberFormat = guestBuffer.NumberFormat, + Stride = guestBuffer.Stride, + OffsetBytes = guestBuffer.OffsetBytes, + }; + private VkBuffer CreateHostBuffer( ReadOnlySpan data, BufferUsageFlags usage, @@ -4331,27 +7862,29 @@ internal static unsafe class VulkanVideoPresenter usage, MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, out var allocatedMemory); - allocation = new HostBufferAllocation(buffer, allocatedMemory, key); + // Persistently mapped: map/unmap per draw was a measurable + // share of the per-draw fixed cost, and HOST_COHERENT memory + // may legally stay mapped for its lifetime. + void* persistentMapping; + Check( + _vk.MapMemory(_device, allocatedMemory, 0, capacity, 0, &persistentMapping), + "vkMapMemory(host persistent)"); + allocation = new HostBufferAllocation( + buffer, + allocatedMemory, + key, + (nint)persistentMapping); _hostBufferAllocations.Add(buffer.Handle, allocation); } memory = allocation.Memory; - void* mapped; - Check(_vk.MapMemory(_device, memory, 0, size, 0, &mapped), "vkMapMemory(host)"); - try + fixed (byte* source = data) { - fixed (byte* source = data) - { - System.Buffer.MemoryCopy( - source, - mapped, - checked((long)size), - data.Length); - } - } - finally - { - _vk.UnmapMemory(_device, memory); + System.Buffer.MemoryCopy( + source, + (void*)allocation.Mapped, + checked((long)allocation.Key.Capacity), + data.Length); } return allocation.Buffer; @@ -4390,6 +7923,13 @@ internal static unsafe class VulkanVideoPresenter _ => PrimitiveTopology.TriangleList, }; + // Strip and fan topologies are the ones for which a restart index + // splits primitives; list topologies never restart. + private static bool RequiresPrimitiveRestart(PrimitiveTopology topology) => + topology is PrimitiveTopology.LineStrip + or PrimitiveTopology.TriangleStrip + or PrimitiveTopology.TriangleFan; + private static Format ToVkVertexFormat( uint dataFormat, uint numberFormat, @@ -4429,12 +7969,15 @@ internal static unsafe class VulkanVideoPresenter (8, 3) => Format.A2B10G10R10SscaledPack32, (8, 4) => Format.A2B10G10R10UintPack32, (8, 5) => Format.A2B10G10R10SintPack32, - (9, 0) => Format.A2R10G10B10UnormPack32, - (9, 1) => Format.A2R10G10B10SNormPack32, - (9, 2) => Format.A2R10G10B10UscaledPack32, - (9, 3) => Format.A2R10G10B10SscaledPack32, - (9, 4) => Format.A2R10G10B10UintPack32, - (9, 5) => Format.A2R10G10B10SintPack32, + // RDNA COLOR_2_10_10_10 stores component 0 (R) in bits + // 0..9 and A in 30..31. Vulkan names that exact bit layout + // A2B10G10R10_PACK32 (the packed name is MSB-to-LSB). + (9, 0) => Format.A2B10G10R10UnormPack32, + (9, 1) => Format.A2B10G10R10SNormPack32, + (9, 2) => Format.A2B10G10R10UscaledPack32, + (9, 3) => Format.A2B10G10R10SscaledPack32, + (9, 4) => Format.A2B10G10R10UintPack32, + (9, 5) => Format.A2B10G10R10SintPack32, (10, 0) => Format.R8G8B8A8Unorm, (10, 1) => Format.R8G8B8A8SNorm, (10, 2) => Format.R8G8B8A8Uscaled, @@ -4702,7 +8245,7 @@ internal static unsafe class VulkanVideoPresenter private static byte[] CreateFallbackTexturePixels(uint format, uint width, uint height, ulong expectedSize) { - if (format is 9 or 10 or 56 or 62 or 64) + if (format is 9 or 10) { return CreateBlackFrame(width, height); } @@ -4726,15 +8269,6 @@ internal static unsafe class VulkanVideoPresenter 12 => 8UL, 13 => 12UL, 14 => 16UL, - 20 => 4UL, - 22 => 8UL, - 29 => 4UL, - 36 => 1UL, - 49 => 1UL, - 56 => 4UL, - 62 => 4UL, - 64 => 4UL, - 71 => 8UL, _ => 4UL, }; @@ -4742,8 +8276,10 @@ internal static unsafe class VulkanVideoPresenter { var blockBytes = format switch { - 169 or 170 => 8UL, - 171 or 172 or 173 or 174 or 175 or 176 or + // BC1 (169/170) and BC4 (175/176) are 8 bytes per 4x4 block; + // BC2/BC3/BC5/BC6H/BC7 are 16 bytes per block. + 169 or 170 or 175 or 176 => 8UL, + 171 or 172 or 173 or 174 or 177 or 178 or 179 or 180 or 181 or 182 => 16UL, _ => 0UL, }; @@ -4761,20 +8297,26 @@ internal static unsafe class VulkanVideoPresenter private static Format GetTextureFormat(uint format, uint numberType) => (format, numberType) switch { - (9, _) => Format.A2R10G10B10UnormPack32, - (GuestFormatR32Uint, _) => Format.R32Uint, - (GuestFormatR32Sint, _) => Format.R32Sint, - (GuestFormatR32Sfloat, _) => Format.R32Sfloat, - (GuestFormatR16G16Uint, _) => Format.R16G16Uint, - (GuestFormatR16G16Sint, _) => Format.R16G16Sint, - (GuestFormatR16G16Sfloat, _) => Format.R16G16Sfloat, - (GuestFormatR8G8B8A8Uint, _) => Format.R8G8B8A8Uint, - (GuestFormatR8G8B8A8Sint, _) => Format.R8G8B8A8Sint, - (GuestFormatR16G16B16A16Uint, _) => Format.R16G16B16A16Uint, - (GuestFormatR16G16B16A16Sint, _) => Format.R16G16B16A16Sint, + (9, _) => Format.A2B10G10R10UnormPack32, (1, 0) => Format.R8Unorm, + (1, 1) => Format.R8SNorm, + (1, 2) => Format.R8Uscaled, + (1, 3) => Format.R8Sscaled, + (1, 4) => Format.R8Uint, + (1, 5) => Format.R8Sint, (2, 7) => Format.R16Sfloat, + (2, 0) => Format.R16Unorm, + (2, 1) => Format.R16SNorm, + (2, 2) => Format.R16Uscaled, + (2, 3) => Format.R16Sscaled, + (2, 4) => Format.R16Uint, + (2, 5) => Format.R16Sint, (3, 0) => Format.R8G8Unorm, + (3, 1) => Format.R8G8SNorm, + (3, 2) => Format.R8G8Uscaled, + (3, 3) => Format.R8G8Sscaled, + (3, 4) => Format.R8G8Uint, + (3, 5) => Format.R8G8Sint, (4, 4) => Format.R32Uint, (4, 5) => Format.R32Sint, (4, 7) => Format.R32Sfloat, @@ -4783,9 +8325,19 @@ internal static unsafe class VulkanVideoPresenter (5, 5) => Format.R16G16Sint, (5, 7) => Format.R16G16Sfloat, (6, 7) => Format.B10G11R11UfloatPack32, + (7, 7) => Format.B10G11R11UfloatPack32, + (8, 0) => Format.A2B10G10R10UnormPack32, + (8, 1) => Format.A2B10G10R10SNormPack32, + (8, 2) => Format.A2B10G10R10UscaledPack32, + (8, 3) => Format.A2B10G10R10SscaledPack32, + (8, 4) => Format.A2B10G10R10UintPack32, + (8, 5) => Format.A2B10G10R10SintPack32, (10, 0) => Format.R8G8B8A8Unorm, (10, 4) => Format.R8G8B8A8Uint, (10, 5) => Format.R8G8B8A8Sint, + (10, 9) => Format.R8G8B8A8Srgb, + (1, 9) => Format.R8Srgb, + (3, 9) => Format.R8G8Srgb, (11, 4) => Format.R32G32Uint, (11, 5) => Format.R32G32Sint, (11, 7) => Format.R32G32Sfloat, @@ -4799,30 +8351,68 @@ internal static unsafe class VulkanVideoPresenter (14, 4) => Format.R32G32B32A32Uint, (14, 5) => Format.R32G32B32A32Sint, (14, 7) => Format.R32G32B32A32Sfloat, - (20, _) => Format.R32Uint, - (4, _) => Format.R32Sfloat, - (5, _) => Format.R16G16Sfloat, - (7, _) => Format.B10G11R11UfloatPack32, - (14, _) => Format.R32G32B32A32Sfloat, - (22, _) => Format.R16G16B16A16Sfloat, - (29, _) => Format.R32Sfloat, - (36, _) => Format.R8Unorm, - (49, _) => Format.R8Uint, - (56, _) => Format.R8G8B8A8Unorm, - (62, _) => Format.R8G8B8A8Unorm, - (64, _) => Format.R8G8B8A8Unorm, - (71, _) => Format.R16G16B16A16Sfloat, - (75, _) => Format.R32G32Sfloat, + (16, 0) => Format.B5G6R5UnormPack16, + (17, 0) => Format.R5G5B5A1UnormPack16, + (19, 0) => Format.R4G4B4A4UnormPack16, + (34, 7) => Format.E5B9G9R9UfloatPack32, (169, _) => Format.BC1RgbaUnormBlock, (170, _) => Format.BC1RgbaSrgbBlock, + (171, _) => Format.BC2UnormBlock, + (172, _) => Format.BC2SrgbBlock, + (173, _) => Format.BC3UnormBlock, + (174, _) => Format.BC3SrgbBlock, + (175, 1) => Format.BC4SNormBlock, + (175, _) => Format.BC4UnormBlock, + (176, _) => Format.BC4SNormBlock, + (177, 1) => Format.BC5SNormBlock, + (177, _) => Format.BC5UnormBlock, + (178, _) => Format.BC5SNormBlock, + (179, _) => Format.BC6HUfloatBlock, + (180, _) => Format.BC6HSfloatBlock, (181, _) => Format.BC7UnormBlock, (182, _) => Format.BC7SrgbBlock, _ => Format.R8G8B8A8Unorm, }; + private static Format GetRenderTargetFormat(uint format, uint numberType) => + (format, numberType) switch + { + (4, 4) => Format.R32Uint, + (4, 5) => Format.R32Sint, + (4, 7) => Format.R32Sfloat, + (5, 4) => Format.R16G16Uint, + (5, 5) => Format.R16G16Sint, + (5, 7) => Format.R16G16Sfloat, + (6, 7) => Format.B10G11R11UfloatPack32, + (7, 7) => Format.B10G11R11UfloatPack32, + (9, _) => Format.A2B10G10R10UnormPack32, + (10, 9) => Format.R8G8B8A8Srgb, + (10, 4) => Format.R8G8B8A8Uint, + (10, 5) => Format.R8G8B8A8Sint, + (10, _) => Format.R8G8B8A8Unorm, + (11, 7) => Format.R32G32Sfloat, + (12, 4) => Format.R16G16B16A16Uint, + (12, 5) => Format.R16G16B16A16Sint, + (12, 7) => Format.R16G16B16A16Sfloat, + (13, 7) => Format.R32G32B32A32Sfloat, + (14, 7) => Format.R32G32B32A32Sfloat, + (_, 0) => GetTextureFormat(format, numberType), + (_, 9) => GetTextureFormat(format, numberType), + _ => Format.Undefined, + }; private static bool IsBlockCompressedFormat(Format format) => format is Format.BC1RgbaUnormBlock or Format.BC1RgbaSrgbBlock or + Format.BC2UnormBlock or + Format.BC2SrgbBlock or + Format.BC3UnormBlock or + Format.BC3SrgbBlock or + Format.BC4UnormBlock or + Format.BC4SNormBlock or + Format.BC5UnormBlock or + Format.BC5SNormBlock or + Format.BC6HUfloatBlock or + Format.BC6HSfloatBlock or Format.BC7UnormBlock or Format.BC7SrgbBlock; @@ -4880,15 +8470,43 @@ internal static unsafe class VulkanVideoPresenter } private const uint MaxComputeZSlicesPerSubmission = 8; + // An indirect guest dispatch above this size is not credible frame work + // (at the minimum 64-thread group used by the captured title this is + // already over one billion invocations). Treat it as poisoned + // indirect-command data and quarantine it instead of feeding a host + // API a multi-billion-workgroup command. This is validation, not a + // clamp: the raw dimensions remain visible in the trace so the + // producer can be fixed without changing the guest value. + private const ulong MaxCredibleGuestWorkgroupsPerDispatch = 16UL * 1024 * 1024; private void ExecuteComputeDispatch(VulkanComputeGuestDispatch work) { + var perfStart = Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _perfDrawCount); + PerfOverlay.RecordDraw(); + try + { + ExecuteComputeDispatchCore(work); + } + finally + { + Interlocked.Add( + ref _perfDrawTicks, + Stopwatch.GetTimestamp() - perfStart); + } + } + + private void ExecuteComputeDispatchCore(VulkanComputeGuestDispatch work) + { + FlushBatchedGuestCommands(); if (_deviceLost) { return; } - if (AddressListContains("SHARPEMU_SKIP_COMPUTE_CS", work.ShaderAddress)) + if (_skipAllCompute || + AddressListContains("SHARPEMU_SKIP_COMPUTE_CS", work.ShaderAddress) || + (_skipTallComputeZ > 0 && work.GroupCountZ >= _skipTallComputeZ)) { TraceVulkanShader( $"vk.compute_skip cs=0x{work.ShaderAddress:X16} " + @@ -4897,9 +8515,16 @@ internal static unsafe class VulkanVideoPresenter return; } + if (!TryValidateComputeDispatch(work, out var validationError)) + { + LogRejectedComputeDispatch(work, validationError); + return; + } + TranslatedDrawResources? resources = null; CommandBuffer commandBuffer = default; var submitted = false; + var chunksSubmitted = 0; try { EnsureGuestSubmissionCapacity(); @@ -4908,6 +8533,12 @@ internal static unsafe class VulkanVideoPresenter var batchCount = Math.Max( 1u, (uint)Math.Ceiling(work.GroupCountZ / (double)MaxComputeZSlicesPerSubmission)); + var threadLimits = stackalloc uint[3] + { + work.ThreadCountX, + work.ThreadCountY, + work.ThreadCountZ, + }; for (var batchIndex = 0u; batchIndex < batchCount; batchIndex++) { @@ -4916,6 +8547,14 @@ internal static unsafe class VulkanVideoPresenter var isFirstBatch = batchIndex == 0; var isLastBatch = batchIndex == batchCount - 1; + if (!isFirstBatch) + { + // Each chunk is its own queue submission; without + // this the in-flight submission cap only applies to + // the first chunk of a tall dispatch. + EnsureGuestSubmissionCapacity(); + } + commandBuffer = AllocateGuestCommandBuffer(); _commandBuffer = commandBuffer; var beginInfo = new CommandBufferBeginInfo @@ -4930,9 +8569,36 @@ internal static unsafe class VulkanVideoPresenter BeginDebugLabel(_commandBuffer, resources.DebugName); if (isFirstBatch) { + RecordGlobalBufferVisibilityBarrier( + _commandBuffer, + resources, + PipelineStageFlags.ComputeShaderBit); RecordTextureUploads(resources, PipelineStageFlags.ComputeShaderBit); RecordStorageImagesForWrite(resources, PipelineStageFlags.ComputeShaderBit); } + else + { + // Chunks are submitted without CPU waits; this + // barrier orders them against the previous chunk's + // shader writes on the same queue. + var chunkBarrier = new MemoryBarrier + { + SType = StructureType.MemoryBarrier, + SrcAccessMask = AccessFlags.ShaderWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.ComputeShaderBit, + PipelineStageFlags.ComputeShaderBit, + 0, + 1, + &chunkBarrier, + 0, + null, + 0, + null); + } _vk.CmdBindPipeline( _commandBuffer, @@ -4952,6 +8618,14 @@ internal static unsafe class VulkanVideoPresenter null); } + _vk.CmdPushConstants( + _commandBuffer, + resources.PipelineLayout, + ShaderStageFlags.ComputeBit, + 0, + 3 * sizeof(uint), + threadLimits); + RecordChunkedComputeDispatch(_commandBuffer, work, zStart, zCount); if (isLastBatch) @@ -4969,22 +8643,37 @@ internal static unsafe class VulkanVideoPresenter { SubmitGuestCommandBuffer( commandBuffer, - resources, - GetTraceImages(resources)); + [resources], + GetTraceImages(resources, shaderAddress: work.ShaderAddress)); submitted = true; } else { - SubmitGuestCommandBufferAndWait(commandBuffer); + SubmitGuestCommandBuffer( + commandBuffer, + [], + [], + referencedResources: [resources]); + chunksSubmitted++; commandBuffer = default; } } MarkSampledImagesInitialized(resources); MarkStorageImagesInitialized(resources, traceContents: false); + if (work.WritesGlobalMemory) + { + // The CPU submit thread may immediately consume an indirect + // argument written by this dispatch. Wait for the specific + // guest fences and publish only dirty ranges; a queue-wide + // idle unnecessarily serialized presentation work too. + WaitForActiveGuestQueueSubmissionsForCpuVisibility(); + WriteBackAllDirtyGuestBuffers(_activeGuestQueue.Name); + } TraceVulkanShader( $"vk.compute_dispatch groups={work.GroupCountX}x" + $"{work.GroupCountY}x{work.GroupCountZ} " + + $"base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ} " + $"textures={work.Textures.Count} cs=0x{work.ShaderAddress:X16} " + $"batches={batchCount}"); } @@ -5013,7 +8702,444 @@ internal static unsafe class VulkanVideoPresenter if (!submitted && resources is not null) { - DestroyTranslatedDrawResources(resources); + if (chunksSubmitted > 0) + { + // Earlier chunks were submitted with empty resource + // lists and may still execute against these + // pipelines/images; destroy only after every + // submission issued so far has completed. + _deferredResourceDestroys.Enqueue((resources, _submitTimeline)); + } + else + { + DestroyTranslatedDrawResources(resources); + } + } + } + } + + private bool TryValidateComputeDispatch( + VulkanComputeGuestDispatch work, + out string error) + { + if (work.LocalSizeX == 0 || work.LocalSizeY == 0 || work.LocalSizeZ == 0) + { + error = "zero-local-size"; + return false; + } + + if (work.LocalSizeX > _maxComputeWorkGroupSizeX || + work.LocalSizeY > _maxComputeWorkGroupSizeY || + work.LocalSizeZ > _maxComputeWorkGroupSizeZ) + { + error = + $"local-size-exceeds-device({work.LocalSizeX}x{work.LocalSizeY}x{work.LocalSizeZ}>" + + $"{_maxComputeWorkGroupSizeX}x{_maxComputeWorkGroupSizeY}x{_maxComputeWorkGroupSizeZ})"; + return false; + } + + var localInvocations = + (ulong)work.LocalSizeX * work.LocalSizeY * work.LocalSizeZ; + if (localInvocations > _maxComputeWorkGroupInvocations) + { + error = + $"local-invocations-exceed-device({localInvocations}>" + + $"{_maxComputeWorkGroupInvocations})"; + return false; + } + + if ((ulong)work.BaseGroupX + work.GroupCountX > _maxComputeWorkGroupCountX || + (ulong)work.BaseGroupY + work.GroupCountY > _maxComputeWorkGroupCountY || + (ulong)work.BaseGroupZ + work.GroupCountZ > _maxComputeWorkGroupCountZ) + { + error = + $"group-range-exceeds-device(base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ}," + + $"count={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ}," + + $"limit={_maxComputeWorkGroupCountX}x{_maxComputeWorkGroupCountY}x" + + $"{_maxComputeWorkGroupCountZ})"; + return false; + } + + if (work.IsIndirect) + { + ulong totalWorkgroups; + try + { + totalWorkgroups = checked( + (ulong)work.GroupCountX * work.GroupCountY * work.GroupCountZ); + } + catch (OverflowException) + { + error = "indirect-workgroup-count-overflow"; + return false; + } + + if (totalWorkgroups > MaxCredibleGuestWorkgroupsPerDispatch) + { + error = + $"poisoned-indirect-workgroup-count({totalWorkgroups}>" + + $"{MaxCredibleGuestWorkgroupsPerDispatch})"; + return false; + } + } + + error = string.Empty; + return true; + } + + private void LogRejectedComputeDispatch( + VulkanComputeGuestDispatch work, + string reason) + { + if (_rejectedComputeDispatches.Count >= 256 || + !_rejectedComputeDispatches.Add( + (work.ShaderAddress, + work.GroupCountX, + work.GroupCountY, + work.GroupCountZ, + reason))) + { + return; + } + + Console.Error.WriteLine( + $"[LOADER][WARN] vk.compute_reject cs=0x{work.ShaderAddress:X16} " + + $"source={(work.IsIndirect ? "indirect" : "direct")} " + + $"groups={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ} " + + $"base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ} " + + $"local={work.LocalSizeX}x{work.LocalSizeY}x{work.LocalSizeZ} " + + $"reason={reason}"); + } + + private void RecordGlobalBufferVisibilityBarrier( + CommandBuffer commandBuffer, + TranslatedDrawResources resources, + PipelineStageFlags destinationStages) + { + if (resources.GlobalMemoryBuffers.Length == 0) + { + return; + } + + // Queue submission order alone is not a shader-memory dependency. + // This makes stores through any aliased guest view available to + // later vertex/fragment/compute reads and writes on the same queue. + var barrier = new MemoryBarrier + { + SType = StructureType.MemoryBarrier, + SrcAccessMask = AccessFlags.ShaderWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit, + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.AllCommandsBit, + destinationStages, + 0, + 1, + &barrier, + 0, + null, + 0, + null); + } + + private static void MarkGuestBufferDirty( + GuestBufferAllocation allocation, + ulong offset, + ulong length) + { + if (length == 0) + { + return; + } + + var start = offset; + var end = checked(offset + length); + for (var index = allocation.DirtyRanges.Count - 1; index >= 0; index--) + { + var existing = allocation.DirtyRanges[index]; + var existingEnd = existing.Offset + existing.Length; + if (end < existing.Offset || existingEnd < start) + { + continue; + } + + start = Math.Min(start, existing.Offset); + end = Math.Max(end, existingEnd); + allocation.DirtyRanges.RemoveAt(index); + } + + allocation.DirtyRanges.Add(new DirtyGuestBufferRange(start, end - start)); + } + + private void WriteBackAllDirtyGuestBuffers(string? queueName = null) + { + var memory = _guestMemory; + if (memory is null) + { + return; + } + + foreach (var allocation in _guestBufferAllocations) + { + if (queueName is not null && + !string.Equals(allocation.QueueName, queueName, StringComparison.Ordinal)) + { + continue; + } + + if (allocation.DirtyRanges.Count != 0 && + allocation.LastUseTimeline > _completedTimeline) + { + // A mapped HOST_COHERENT allocation still cannot be read + // by the CPU while a shader may be writing it. Callers + // normally retire the relevant fences first; keep this + // helper fail-closed if a future path forgets to do so. + TraceVulkanShader( + $"vk.global_writeback_deferred base=0x{allocation.BaseAddress:X16} " + + $"last_use={allocation.LastUseTimeline} completed={_completedTimeline}"); + continue; + } + + for (var index = allocation.DirtyRanges.Count - 1; index >= 0; index--) + { + var range = allocation.DirtyRanges[index]; + if (range.Length == 0 || range.Length > int.MaxValue) + { + continue; + } + + var mappedBytes = new ReadOnlySpan( + (void*)(allocation.Mapped + checked((nint)range.Offset)), + checked((int)range.Length)); + var shadowBytes = allocation.Shadow.AsSpan( + checked((int)range.Offset), + mappedBytes.Length); + var guestAddress = allocation.BaseAddress + range.Offset; + var changedBytes = 0UL; + var changedRuns = 0; + var changedPages = 0; + var writtenRuns = 0; + var writtenPages = 0; + var failedRuns = 0; + var unreadablePages = 0; + var fallbackWrites = 0; + var firstChangedOffset = -1; + allocation.DirtyRanges.RemoveAt(index); + + // A writable descriptor only identifies a potential write + // range. Publishing the entire mapped view would overwrite + // unrelated live CPU data with its old snapshot. Compare + // against the last synchronized image. For each changed + // page, start with current guest bytes and overlay only the + // shader changes before one bounded write. This preserves + // live CPU changes in unchanged bytes without degenerating + // into millions of writes for alternating output patterns. + const int pageSize = 4096; + const int unreadableMergeGap = 16; + var livePageBuffer = GuestDataPool.Rent(pageSize); + var pageRuns = new List<(int Start, int Length)>(64); + try + { + for (var pageStart = 0; + pageStart < mappedBytes.Length; + pageStart += pageSize) + { + var pageEnd = Math.Min(pageStart + pageSize, mappedBytes.Length); + pageRuns.Clear(); + var cursor = pageStart; + while (cursor < pageEnd) + { + while (cursor < pageEnd && + mappedBytes[cursor] == shadowBytes[cursor]) + { + cursor++; + } + + if (cursor == pageEnd) + { + break; + } + + var runStart = cursor; + while (cursor < pageEnd && + mappedBytes[cursor] != shadowBytes[cursor]) + { + cursor++; + } + + var runLength = cursor - runStart; + pageRuns.Add((runStart, runLength)); + changedRuns++; + changedBytes += (ulong)runLength; + if (firstChangedOffset < 0) + { + firstChangedOffset = runStart; + } + } + + if (pageRuns.Count == 0) + { + continue; + } + + changedPages++; + var pageLength = pageEnd - pageStart; + var livePage = livePageBuffer.AsSpan(0, pageLength); + if (memory.TryRead(guestAddress + (ulong)pageStart, livePage)) + { + foreach (var run in pageRuns) + { + mappedBytes.Slice(run.Start, run.Length).CopyTo( + livePage.Slice(run.Start - pageStart, run.Length)); + } + + if (memory.TryWrite(guestAddress + (ulong)pageStart, livePage)) + { + foreach (var run in pageRuns) + { + mappedBytes.Slice(run.Start, run.Length).CopyTo( + shadowBytes.Slice(run.Start, run.Length)); + } + + writtenPages++; + writtenRuns += pageRuns.Count; + continue; + } + + foreach (var run in pageRuns) + { + failedRuns++; + MarkGuestBufferDirty( + allocation, + range.Offset + (ulong)run.Start, + (ulong)run.Length); + } + + continue; + } + + // A partial/unreadable edge cannot be safely + // reconstructed as a page. Fall back to bounded + // changed spans, coalescing only tiny gaps. + unreadablePages++; + for (var runIndex = 0; runIndex < pageRuns.Count; runIndex++) + { + var firstRunIndex = runIndex; + var mergedStart = pageRuns[runIndex].Start; + var mergedEnd = mergedStart + pageRuns[runIndex].Length; + while (runIndex + 1 < pageRuns.Count && + pageRuns[runIndex + 1].Start - mergedEnd <= + unreadableMergeGap) + { + runIndex++; + mergedEnd = pageRuns[runIndex].Start + + pageRuns[runIndex].Length; + } + + var lastRunIndex = runIndex; + var mergedLength = mergedEnd - mergedStart; + var mergedLive = livePageBuffer.AsSpan(0, mergedLength); + if (memory.TryRead( + guestAddress + (ulong)mergedStart, + mergedLive)) + { + for (var overlayIndex = firstRunIndex; + overlayIndex <= lastRunIndex; + overlayIndex++) + { + var run = pageRuns[overlayIndex]; + mappedBytes.Slice(run.Start, run.Length).CopyTo( + mergedLive.Slice( + run.Start - mergedStart, + run.Length)); + } + + fallbackWrites++; + if (memory.TryWrite( + guestAddress + (ulong)mergedStart, + mergedLive)) + { + for (var overlayIndex = firstRunIndex; + overlayIndex <= lastRunIndex; + overlayIndex++) + { + var run = pageRuns[overlayIndex]; + mappedBytes.Slice(run.Start, run.Length).CopyTo( + shadowBytes.Slice(run.Start, run.Length)); + } + + writtenRuns += lastRunIndex - firstRunIndex + 1; + continue; + } + } + + // Even the merged span crosses an unreadable + // edge. Exact changed runs remain safe because + // they never carry stale gap bytes. + for (var exactIndex = firstRunIndex; + exactIndex <= lastRunIndex; + exactIndex++) + { + var run = pageRuns[exactIndex]; + var changed = mappedBytes.Slice(run.Start, run.Length); + fallbackWrites++; + if (memory.TryWrite( + guestAddress + (ulong)run.Start, + changed)) + { + changed.CopyTo(shadowBytes.Slice( + run.Start, + run.Length)); + writtenRuns++; + } + else + { + failedRuns++; + MarkGuestBufferDirty( + allocation, + range.Offset + (ulong)run.Start, + (ulong)run.Length); + } + } + } + } + } + finally + { + GuestDataPool.Return(livePageBuffer); + } + + var probe = mappedBytes[..Math.Min(mappedBytes.Length, 256)]; + var nonzero = 0; + foreach (var value in probe) + { + nonzero += value == 0 ? 0 : 1; + } + + var firstForRange = _tracedGlobalWritebacks.Count < 256 && + _tracedGlobalWritebacks.Add((guestAddress, range.Length)); + var traceSmallMutation = range.Length <= 4096 && + _tracedSmallGlobalWritebackEvents++ < 1024; + var traceLargeMutation = range.Length >= 1024 * 1024 && + _tracedLargeGlobalWritebackEvents++ < 256; + if (firstForRange || traceSmallMutation || traceLargeMutation) + { + var head = firstChangedOffset >= 0 + ? mappedBytes.Slice( + firstChangedOffset, + Math.Min(mappedBytes.Length - firstChangedOffset, 32)) + : ReadOnlySpan.Empty; + TraceVulkanShader( + $"vk.global_writeback base=0x{guestAddress:X16} " + + $"potential_bytes={mappedBytes.Length} changed_bytes={changedBytes} " + + $"changed_runs={changedRuns} changed_pages={changedPages} " + + $"written_pages={writtenPages} written_runs={writtenRuns} " + + $"unreadable_pages={unreadablePages} " + + $"fallback_writes={fallbackWrites} failed_runs={failedRuns} " + + $"probe_nonzero={nonzero}/{probe.Length} " + + $"changed_head={Convert.ToHexString(head)}"); + } } } } @@ -5025,28 +9151,49 @@ internal static unsafe class VulkanVideoPresenter uint zCount) { const uint maxWorkgroupsPerCommand = 4096; - var yChunk = Math.Max( + ulong commandCount = 0; + var maxXChunk = Math.Max( 1u, Math.Min( - work.GroupCountY, - maxWorkgroupsPerCommand / Math.Max(work.GroupCountX, 1u))); - var commandCount = 0u; - - for (var z = zStart; z < zStart + zCount; z++) + work.GroupCountX, + Math.Min(_maxComputeWorkGroupCountX, maxWorkgroupsPerCommand))); + for (var x = 0u; x < work.GroupCountX;) { - for (var y = 0u; y < work.GroupCountY; y += yChunk) + var countX = Math.Min(maxXChunk, work.GroupCountX - x); + var xyBudget = Math.Max(maxWorkgroupsPerCommand / countX, 1u); + var maxYChunk = Math.Max( + 1u, + Math.Min( + work.GroupCountY, + Math.Min(_maxComputeWorkGroupCountY, xyBudget))); + for (var y = 0u; y < work.GroupCountY;) { - var countY = Math.Min(yChunk, work.GroupCountY - y); - _vk.CmdDispatchBase( - commandBuffer, - 0, - y, - z, - work.GroupCountX, - countY, - 1); - commandCount++; + var countY = Math.Min(maxYChunk, work.GroupCountY - y); + var xyzBudget = Math.Max(xyBudget / countY, 1u); + var maxZChunk = Math.Max( + 1u, + Math.Min( + zCount, + Math.Min(_maxComputeWorkGroupCountZ, xyzBudget))); + for (var z = 0u; z < zCount;) + { + var countZ = Math.Min(maxZChunk, zCount - z); + _vk.CmdDispatchBase( + commandBuffer, + checked(work.BaseGroupX + x), + checked(work.BaseGroupY + y), + checked(work.BaseGroupZ + zStart + z), + countX, + countY, + countZ); + commandCount++; + z += countZ; + } + + y += countY; } + + x += countX; } if (commandCount > 1) @@ -5054,7 +9201,11 @@ internal static unsafe class VulkanVideoPresenter TraceVulkanShader( $"vk.compute_chunked cs=0x{work.ShaderAddress:X16} " + $"groups={work.GroupCountX}x{work.GroupCountY}x{work.GroupCountZ} " + - $"z_range={zStart}..{zStart + zCount} commands={commandCount} y_chunk={yChunk}"); + $"base={work.BaseGroupX}x{work.BaseGroupY}x{work.BaseGroupZ} " + + $"z_range={zStart}..{zStart + zCount} commands={commandCount} " + + $"command_budget={maxWorkgroupsPerCommand} " + + $"device_limit={_maxComputeWorkGroupCountX}x" + + $"{_maxComputeWorkGroupCountY}x{_maxComputeWorkGroupCountZ}"); } } @@ -5065,11 +9216,32 @@ internal static unsafe class VulkanVideoPresenter return; } + var perfStart = System.Diagnostics.Stopwatch.GetTimestamp(); + Interlocked.Increment(ref _perfDrawCount); + PerfOverlay.RecordDraw(); + try + { + ExecuteOffscreenDrawCore(work); + } + finally + { + // Single atomic add per draw: staging -start/+end separately + // let the stats window reset land between them and report + // huge negative draw_ms values. + Interlocked.Add( + ref _perfDrawTicks, + System.Diagnostics.Stopwatch.GetTimestamp() - perfStart); + } + } + + private void ExecuteOffscreenDrawCore(VulkanOffscreenGuestDraw work) + { if (work.Targets.Count > _maxColorAttachments) { Console.Error.WriteLine( $"[LOADER][WARN] Vulkan skipped MRT draw requesting {work.Targets.Count} color attachments; " + $"the selected device supports {_maxColorAttachments}."); + ReturnPooledGuestData(work.Draw); return; } @@ -5083,6 +9255,7 @@ internal static unsafe class VulkanVideoPresenter Console.Error.WriteLine( $"[LOADER][WARN] Vulkan skipped MRT draw with unsupported color target " + $"format={target.Format} number_type={target.NumberType}."); + ReturnPooledGuestData(work.Draw); return; } } @@ -5091,6 +9264,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Vulkan skipped MRT draw with mismatched attachment/blend counts."); + ReturnPooledGuestData(work.Draw); return; } @@ -5100,6 +9274,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Vulkan skipped MRT draw with blending enabled for an integer attachment."); + ReturnPooledGuestData(work.Draw); return; } @@ -5108,6 +9283,7 @@ internal static unsafe class VulkanVideoPresenter { Console.Error.WriteLine( "[LOADER][WARN] Vulkan skipped MRT draw requiring unsupported independentBlend."); + ReturnPooledGuestData(work.Draw); return; } @@ -5118,11 +9294,13 @@ internal static unsafe class VulkanVideoPresenter .Select(target => target.Address) .ToHashSet(); if (work.Draw.Textures.Any(texture => - targetAddresses.Contains(texture.Address))) + texture.IsStorage && targetAddresses.Contains(texture.Address))) { Console.Error.WriteLine( - $"[LOADER][WARN] Vulkan skipped render-target feedback loop " + - $"targets={string.Join(',', targetAddresses.Select(address => $"0x{address:X16}"))}"); + $"[LOADER][WARN] Vulkan skipped storage render-target feedback loop " + + $"targets={string.Join(',', targetAddresses.Select(address => $"0x{address:X16}"))}; " + + "sampled aliases use ordered snapshots"); + ReturnPooledGuestData(work.Draw); return; } @@ -5130,7 +9308,19 @@ internal static unsafe class VulkanVideoPresenter EnsureGuestSubmissionCapacity(); for (var index = 0; index < targets.Length; index++) { - targets[index] = GetOrCreateGuestImage(work.Targets[index], formats[index]); + var targetDescriptor = work.Targets[index].Address == 0 && + work.DepthTarget is { } depthOnlyTarget + ? GetDepthOnlyColorTarget(depthOnlyTarget) + : work.Targets[index]; + targets[index] = GetOrCreateGuestImage(targetDescriptor, formats[index]); + if (work.Targets[index].Address != 0 && + TakeGuestImageInitialData(work.Targets[index].Address) is { } initialData && + !targets[index].Initialized && + (ulong)initialData.Length == + (ulong)targets[index].Width * targets[index].Height * 4) + { + UploadGuestImageInitialData(targets[index], initialData); + } } var firstTarget = targets[0]; @@ -5142,24 +9332,111 @@ internal static unsafe class VulkanVideoPresenter try { var extent = new Extent2D(firstTarget.Width, firstTarget.Height); - var renderPass = firstTarget.RenderPass; - var framebuffer = firstTarget.Framebuffer; + var draw = work.Draw; + if (work.DepthTarget?.ReadOnly == true && draw.RenderState.Depth.WriteEnable) + { + draw = draw with + { + RenderState = draw.RenderState with + { + Depth = draw.RenderState.Depth with { WriteEnable = false }, + }, + }; + } + GuestDepthResource? depth = null; + DepthFramebufferResource? depthFramebuffer = null; + if (work.DepthTarget is { } depthTarget && + (draw.RenderState.Depth.TestEnable || + draw.RenderState.Depth.WriteEnable)) + { + var effectiveDepthTarget = depthTarget; + if (depthTarget.Width < firstTarget.Width || depthTarget.Height < firstTarget.Height) + { + var matchingTexture = work.Draw.Textures.FirstOrDefault(texture => + texture.Address == depthTarget.Address && + texture.Width >= firstTarget.Width && + texture.Height >= firstTarget.Height); + if (matchingTexture is not null) + { + effectiveDepthTarget = depthTarget with + { + Width = matchingTexture.Width, + Height = matchingTexture.Height, + }; + } + else if (depthTarget.Width == 1 && depthTarget.Height == 1) + { + // Some Gen5 streams leave DB_DEPTH_SIZE_XY at its + // clear-state value while binding a full-size DB + // surface. The color attachment and viewport still + // define the render extent; treating that stale 1x1 + // value literally discards the entire draw. + effectiveDepthTarget = depthTarget with + { + Width = firstTarget.Width, + Height = firstTarget.Height, + }; + } + + if (effectiveDepthTarget != depthTarget && + _tracedDepthExtentFallbacks.Add( + (depthTarget.Address, + effectiveDepthTarget.Width, + effectiveDepthTarget.Height))) + { + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan repaired stale guest depth extent " + + $"addr=0x{depthTarget.Address:X16} " + + $"{depthTarget.Width}x{depthTarget.Height} -> " + + $"{effectiveDepthTarget.Width}x{effectiveDepthTarget.Height}"); + } + } + + depth = GetOrCreateGuestDepth(effectiveDepthTarget); + PrepareFirstUseDepth(depth, draw.RenderState.Depth); + if (targets.Length == 1) + { + depthFramebuffer = GetOrCreateDepthFramebuffer(firstTarget, depth); + } + } + + var renderPass = depthFramebuffer is null + ? firstTarget.Initialized + ? firstTarget.RenderPass + : firstTarget.InitialRenderPass + : firstTarget.Initialized + ? depth!.Initialized + ? depthFramebuffer.LoadRenderPass + : depthFramebuffer.DepthClearRenderPass + : depth!.Initialized + ? depthFramebuffer.ColorClearRenderPass + : depthFramebuffer.BothClearRenderPass; + var framebuffer = depthFramebuffer?.Framebuffer ?? firstTarget.Framebuffer; if (targets.Length > 1) { (renderPass, framebuffer) = CreateRenderPassAndFramebuffer( formats, - targets.Select(target => target.MipViews[0]).ToArray(), + targets.Select(target => target.MipViews.Length > 0 + ? target.MipViews[0] + : target.View).ToArray(), firstTarget.Width, - firstTarget.Height); + firstTarget.Height, + targets.Select(target => + target.Initialized || target.InitialUploadPending).ToArray(), + depth, + depth?.Initialized == true); transientRenderPass = renderPass; transientFramebuffer = framebuffer; } resources = CreateTranslatedDrawResources( - work.Draw, + draw, renderPass, formats, - extent); + extent, + targets, + hasDepthAttachment: depth is not null, + feedbackDepth: depth); resources.TransientRenderPass = transientRenderPass; resources.TransientFramebuffer = transientFramebuffer; transientRenderPass = default; @@ -5169,18 +9446,39 @@ internal static unsafe class VulkanVideoPresenter $"first=0x{work.Targets[0].Address:X16} " + $"{firstTarget.Width}x{firstTarget.Height}"; - commandBuffer = AllocateGuestCommandBuffer(); + commandBuffer = BeginBatchedGuestCommands(); _commandBuffer = commandBuffer; - var beginInfo = new CommandBufferBeginInfo - { - SType = StructureType.CommandBufferBeginInfo, - Flags = CommandBufferUsageFlags.OneTimeSubmitBit, - }; - Check( - _vk.BeginCommandBuffer(_commandBuffer, &beginInfo), - "vkBeginCommandBuffer(offscreen)"); + + // Lifetime: recorded commands reference these resources, so + // they join the batch before recording and are destroyed only + // after the batch's fence signals. + _batchResources.Add(resources); + submitted = true; BeginDebugLabel(_commandBuffer, resources.DebugName); + var hasStorageImages = false; + foreach (var texture in resources.Textures) + { + if (texture is null) + { + continue; + } + + hasStorageImages |= texture.IsStorage; + } + + CloseOpenTranslatedRenderPass(); + RecordGlobalBufferVisibilityBarrier( + _commandBuffer, + resources, + PipelineStageFlags.VertexShaderBit | + PipelineStageFlags.FragmentShaderBit); + RecordRenderTargetFeedbackSnapshots( + resources, + PipelineStageFlags.FragmentShaderBit); + RecordDepthFeedbackSnapshots( + resources, + PipelineStageFlags.FragmentShaderBit); RecordTextureUploads(resources, PipelineStageFlags.FragmentShaderBit); RecordStorageImagesForWrite(resources, PipelineStageFlags.FragmentShaderBit); @@ -5188,7 +9486,8 @@ internal static unsafe class VulkanVideoPresenter var anyPriorContents = false; for (var index = 0; index < targets.Length; index++) { - var hasPriorContents = targets[index].Initialized || targets[index].InitialUploadPending; + var hasPriorContents = + targets[index].Initialized || targets[index].InitialUploadPending; anyPriorContents |= hasPriorContents; toColorAttachments[index] = new ImageMemoryBarrier { @@ -5219,12 +9518,48 @@ internal static unsafe class VulkanVideoPresenter (uint)targets.Length, toColorAttachments); - RecordTranslatedGraphicsPass( - resources, + if (depth is not null && + depth.Layout == ImageLayout.ShaderReadOnlyOptimal) + { + var toDepthAttachment = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ShaderReadBit, + DstAccessMask = + AccessFlags.DepthStencilAttachmentReadBit | + AccessFlags.DepthStencilAttachmentWriteBit, + OldLayout = ImageLayout.ShaderReadOnlyOptimal, + NewLayout = ImageLayout.DepthStencilAttachmentOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = depth.Image, + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, 0, 1, 0, 1), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.FragmentShaderBit | + PipelineStageFlags.ComputeShaderBit, + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + 0, + 0, + null, + 0, + null, + 1, + &toDepthAttachment); + } + + BeginTranslatedRenderPass( renderPass, framebuffer, - extent); - RecordStorageImagesForRead(resources, PipelineStageFlags.FragmentShaderBit); + extent, + colorAttachmentCount: targets.Length, + hasDepthAttachment: depth is not null, + clearDepth: depth?.ClearDepth ?? 1f); + RecordTranslatedDrawInPass(resources, extent); + _vk.CmdEndRenderPass(_commandBuffer); var toShaderRead = stackalloc ImageMemoryBarrier[targets.Length]; for (var index = 0; index < targets.Length; index++) @@ -5253,17 +9588,34 @@ internal static unsafe class VulkanVideoPresenter null, (uint)targets.Length, toShaderRead); + + if (hasStorageImages) + { + RecordStorageImagesForRead(resources, PipelineStageFlags.FragmentShaderBit); + } + EndDebugLabel(_commandBuffer); - Check(_vk.EndCommandBuffer(_commandBuffer), "vkEndCommandBuffer(offscreen)"); - SubmitGuestCommandBuffer( - commandBuffer, - resources, - GetTraceImages(resources, targets)); - submitted = true; + var traceImages = GetTraceImages(resources, targets, work.ShaderAddress); + _batchTraceImages.AddRange(traceImages); + if (++_batchDrawCount >= 64 || + (_traceGuestImageShaderFilterEnabled && traceImages.Count != 0)) + { + FlushBatchedGuestCommands(); + } + foreach (var target in targets) { target.Initialized = true; + target.InitialUploadPending = false; + } + if (depth is not null) + { + depth.Initialized = true; + if (draw.RenderState.Depth.WriteEnable) + { + depth.InitializationSource = "translated-depth-write"; + } } MarkSampledImagesInitialized(resources); MarkStorageImagesInitialized(resources, traceContents: false); @@ -5283,16 +9635,71 @@ internal static unsafe class VulkanVideoPresenter lock (_gate) { _availableGuestImages[targets[index].Address] = guestTextureFormat; - _gpuGuestImages[targets[index].Address] = guestTextureFormat; } } } + + var guestWritesMode = Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_WRITES"); + var traceAddressWriteOrdinal = 0L; + _ = long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_WRITE_ORDINAL"), + out traceAddressWriteOrdinal); + var traceLargeWriteOrdinal = 0L; + if (guestWritesMode is not null && + guestWritesMode.StartsWith("large@", StringComparison.Ordinal) && + long.TryParse( + guestWritesMode.AsSpan("large@".Length), + out var parsedTraceLargeWriteOrdinal) && + parsedTraceLargeWriteOrdinal > 0) + { + traceLargeWriteOrdinal = parsedTraceLargeWriteOrdinal; + } + + var tracePixelSpirv = false; + if (int.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_PIXEL_SPIRV_BYTES"), + out var tracePixelSpirvBytes) && + tracePixelSpirvBytes == work.Draw.PixelSpirv.Length) + { + var pixelWriteCount = _pixelSpirvWriteCounts.TryGetValue( + tracePixelSpirvBytes, + out var previousPixelWriteCount) + ? previousPixelWriteCount + 1 + : 1; + _pixelSpirvWriteCounts[tracePixelSpirvBytes] = pixelWriteCount; + var tracePixelSpirvOccurrence = 1; + _ = int.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_PIXEL_SPIRV_OCCURRENCE"), + out tracePixelSpirvOccurrence); + tracePixelSpirv = + pixelWriteCount == Math.Max(tracePixelSpirvOccurrence, 1); + } + var traceTitleDraw = + !_tracedTitleDraw && + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_TITLE_DRAW") == "1" && + work.Draw.VertexBuffers.Any(buffer => + buffer.Location == 0 && + buffer.ComponentCount == 4 && + buffer.DataFormat == 10 && + buffer.NumberFormat == 0 && + buffer.Stride == 16 && + buffer.OffsetBytes == 12 && + buffer.Length == 67568); + _tracedTitleDraw |= traceTitleDraw; + foreach (var target in targets) { - var traceSmallWrites = - Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_WRITES") == "small" && + var traceAddressWrite = + ShouldTraceGuestImageWriteForDiagnostics(target.Address); + var traceSmallWrites = guestWritesMode == "small" && target.Width <= 512 && target.Height <= 256; - if (ShouldTraceGuestImageWriteForDiagnostics(target.Address) || traceSmallWrites) + var traceLargeWrites = + (guestWritesMode == "large" || traceLargeWriteOrdinal != 0) && + target.Width >= 2560 && target.Height >= 1440; + if (traceAddressWrite || traceSmallWrites || + traceLargeWrites || tracePixelSpirv || traceTitleDraw) { var writeCount = _tracedGuestWriteCounts.TryGetValue( target.Address, @@ -5300,16 +9707,41 @@ internal static unsafe class VulkanVideoPresenter ? previousCount + 1 : 1; _tracedGuestWriteCounts[target.Address] = writeCount; - if (writeCount <= (traceSmallWrites ? 48 : 3)) + var shouldTraceWrite = tracePixelSpirv || traceTitleDraw + ? true + : traceAddressWrite && traceAddressWriteOrdinal > 0 + ? writeCount == traceAddressWriteOrdinal + : traceLargeWriteOrdinal != 0 + ? writeCount == traceLargeWriteOrdinal + : writeCount <= + (traceLargeWrites ? 2 : traceSmallWrites ? 48 : 3); + if (traceAddressWrite || shouldTraceWrite) { - _commandBuffer = _presentationCommandBuffer; - Check( - _vk.QueueWaitIdle(_queue), - "vkQueueWaitIdle(guest write trace)"); + var sampledTextures = string.Join( + ',', + work.Draw.Textures.Select(texture => + $"0x{texture.Address:X}:{texture.Width}x{texture.Height}:" + + $"f{texture.Format}:n{texture.NumberType}:" + + $"storage={(texture.IsStorage ? 1 : 0)}")); + var pixelDigest = Convert.ToHexString( + SHA256.HashData(work.Draw.PixelSpirv).AsSpan(0, 4)); Console.Error.WriteLine( $"[LOADER][TRACE] vk.guest_write_sample " + $"addr=0x{target.Address:X16} write={writeCount} " + - $"ps_bytes={work.Draw.PixelSpirv.Length}"); + $"vs_bytes={work.Draw.VertexSpirv.Length} " + + $"ps_bytes={work.Draw.PixelSpirv.Length} ps_hash={pixelDigest} " + + $"vertices={work.Draw.VertexCount} instances={work.Draw.InstanceCount} " + + $"primitive=0x{work.Draw.PrimitiveType:X} " + + $"readback={(shouldTraceWrite ? 1 : 0)} textures=[{sampledTextures}]"); + } + + if (shouldTraceWrite) + { + _commandBuffer = _presentationCommandBuffer; + FlushBatchedGuestCommands(); + Check( + _vk.QueueWaitIdle(_queue), + "vkQueueWaitIdle(guest write trace)"); TraceGuestImageContents(target); } } @@ -5334,7 +9766,6 @@ internal static unsafe class VulkanVideoPresenter !failedTarget.Initialized) { _availableGuestImages.Remove(target.Address); - _gpuGuestImages.Remove(target.Address); } } } @@ -5346,15 +9777,10 @@ internal static unsafe class VulkanVideoPresenter finally { _commandBuffer = _presentationCommandBuffer; - if (!submitted && commandBuffer.Handle != 0) - { - _vk.FreeCommandBuffers( - _device, - _commandPool, - 1, - &commandBuffer); - } - + // The command buffer is the shared batch; it is submitted and + // freed by FlushBatchedGuestCommands. Resources joined the + // batch list before recording, so only pre-recording failures + // (submitted still false) own their cleanup here. if (!submitted && resources is not null) { DestroyTranslatedDrawResources(resources); @@ -5373,16 +9799,240 @@ internal static unsafe class VulkanVideoPresenter } [MethodImpl(MethodImplOptions.NoInlining)] + private void ExecuteGuestImageWrite(VulkanGuestImageWrite work) + { + if (_deviceLost || !_guestImages.TryGetValue(work.Address, out var target)) + { + return; + } + + if (work.Pixels is { } pixels) + { + if (pixels.Length > 0) + { + UploadGuestImageInitialData(target, pixels); + } + + return; + } + + // Recorded into the shared batch command buffer: recording order + // preserves queue-order semantics against earlier batched draws, + // and the fill no longer costs a submit + full queue drain. + var commandBuffer = BeginBatchedGuestCommands(); + CloseOpenTranslatedRenderPass(); + var toTransferDst = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = target.Initialized ? AccessFlags.ShaderReadBit : 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = target.Initialized + ? ImageLayout.ShaderReadOnlyOptimal + : ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + target.Initialized + ? PipelineStageFlags.FragmentShaderBit + : PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &toTransferDst); + + var clearValue = new ClearColorValue( + (work.FillValue & 0xFF) / 255f, + ((work.FillValue >> 8) & 0xFF) / 255f, + ((work.FillValue >> 16) & 0xFF) / 255f, + ((work.FillValue >> 24) & 0xFF) / 255f); + var range = ColorSubresourceRange(0, target.MipLevels); + _vk.CmdClearColorImage( + commandBuffer, + target.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &range); + + var toShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.FragmentShaderBit, + 0, + 0, + null, + 0, + null, + 1, + &toShaderRead); + target.Initialized = true; + } + + private void UploadGuestImageInitialData(GuestImageResource target, byte[] pixels) + { + var byteCount = (ulong)pixels.Length; + var staging = CreateBuffer( + byteCount, + BufferUsageFlags.TransferSrcBit, + MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit, + out var stagingMemory); + try + { + void* mapped; + Check( + _vk.MapMemory(_device, stagingMemory, 0, byteCount, 0, &mapped), + "vkMapMemory(guest image init)"); + fixed (byte* source = pixels) + { + System.Buffer.MemoryCopy(source, mapped, pixels.Length, pixels.Length); + } + + _vk.UnmapMemory(_device, stagingMemory); + + // Recorded into the shared batch; the staging buffer joins + // the batch's retire list and is destroyed when the batch + // fence signals, so the upload costs no queue drain. + var commandBuffer = BeginBatchedGuestCommands(); + CloseOpenTranslatedRenderPass(); + + var toTransferDst = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = target.Initialized ? AccessFlags.ShaderReadBit : 0, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = target.Initialized + ? ImageLayout.ShaderReadOnlyOptimal + : ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + target.Initialized + ? PipelineStageFlags.FragmentShaderBit + : PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &toTransferDst); + + var copyRegion = new BufferImageCopy + { + BufferOffset = 0, + BufferRowLength = 0, + BufferImageHeight = 0, + ImageSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, + 0, + 0, + 1), + ImageOffset = default, + ImageExtent = new Extent3D(target.Width, target.Height, 1), + }; + _vk.CmdCopyBufferToImage( + commandBuffer, + staging, + target.Image, + ImageLayout.TransferDstOptimal, + 1, + ©Region); + + var toShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = target.Image, + SubresourceRange = ColorSubresourceRange(0, target.MipLevels), + }; + _vk.CmdPipelineBarrier( + commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.FragmentShaderBit, + 0, + 0, + null, + 0, + null, + 1, + &toShaderRead); + + target.Initialized = true; + _batchRetireBuffers.Add((staging, stagingMemory)); + staging = default; + stagingMemory = default; + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] seeded addr=0x{target.Address:X} " + + $"{target.Width}x{target.Height}"); + } + } + finally + { + if (staging.Handle != 0) + { + _vk.DestroyBuffer(_device, staging, null); + } + + if (stagingMemory.Handle != 0) + { + _vk.FreeMemory(_device, stagingMemory, null); + } + } + } + private GuestImageResource GetOrCreateGuestImage( GuestRenderTarget target, Format format) { var mipLevels = ClampMipLevels(target.Width, target.Height, target.MipLevels); + var guestFormat = GetGuestTextureFormat(target.Format, target.NumberType); + var requestedKey = new GuestImageVariantKey( + target.Address, + target.Width, + target.Height, + mipLevels, + guestFormat, + format); if (_guestImages.TryGetValue(target.Address, out var existing)) { if (existing.Width == target.Width && existing.Height == target.Height && existing.MipLevels == mipLevels && + existing.GuestFormat == guestFormat && existing.Format == format) { existing.IsCpuBacked = false; @@ -5392,30 +10042,90 @@ internal static unsafe class VulkanVideoPresenter var attachmentView = existing.MipViews.Length > 0 ? existing.MipViews[0] : existing.View; - var (promotedRenderPass, promotedFramebuffer) = CreateRenderPassAndFramebuffer( + var promoted = CreateRenderPassAndFramebuffer( existing.Format, attachmentView, existing.Width, existing.Height); - existing.RenderPass = promotedRenderPass; - existing.Framebuffer = promotedFramebuffer; + existing.RenderPass = promoted.RenderPass; + existing.InitialRenderPass = promoted.InitialRenderPass; + existing.Framebuffer = promoted.Framebuffer; + var promotedRenderPass = promoted.RenderPass; + var promotedInitialRenderPass = promoted.InitialRenderPass; + var promotedFramebuffer = promoted.Framebuffer; var promotedName = GuestImageDebugName(target, format); SetDebugName(ObjectType.RenderPass, promotedRenderPass.Handle, $"{promotedName} renderpass"); + SetDebugName(ObjectType.RenderPass, promotedInitialRenderPass.Handle, $"{promotedName} initial-renderpass"); SetDebugName(ObjectType.Framebuffer, promotedFramebuffer.Handle, $"{promotedName} framebuffer"); } return existing; } - CompletePendingPresentation(wait: true); - WaitForAllGuestSubmissions(); - DestroyGuestImage(existing); + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] recreate addr=0x{target.Address:X} " + + $"old={existing.Width}x{existing.Height}/{existing.Format}/m{existing.MipLevels} " + + $"new={target.Width}x{target.Height}/{format}/m{mipLevels} " + + $"initialized={existing.Initialized}"); + } + + _guestImageVariants.Add( + new GuestImageVariantKey( + existing.Address, + existing.Width, + existing.Height, + existing.MipLevels, + existing.GuestFormat, + existing.Format), + existing); _guestImages.Remove(target.Address); lock (_gate) { _availableGuestImages.Remove(target.Address); - _gpuGuestImages.Remove(target.Address); + _guestImageExtents.Remove(target.Address); } + + // Address-filtered readback diagnostics should inspect the + // replacement image too; a resized/reformatted allocation at + // the same guest address is a different piece of evidence. + _tracedGuestImageContents.Remove(target.Address); + + SharpEmu.HLE.GuestImageWriteTracker.Untrack(target.Address); + } + + if (_guestImageVariants.Remove(requestedKey, out var retained)) + { + retained.IsCpuBacked = false; + retained.CpuContentFingerprint = 0; + _guestImages.Add(target.Address, retained); + lock (_gate) + { + _guestImageExtents[target.Address] = ( + target.Width, + target.Height, + GetTextureByteCount(target.Format, target.Width, target.Height)); + } + + if (target.Width <= 1920 && target.Height <= 1080) + { + SharpEmu.HLE.GuestImageWriteTracker.Track( + target.Address, + (ulong)target.Width * target.Height * GetTextureBytesPerPixel(target.Format), + CurrentGuestWorkSequenceForDiagnostics, + "vulkan.render-target"); + } + + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] retained addr=0x{target.Address:X} " + + $"{target.Width}x{target.Height} fmt={format} " + + $"initialized={retained.Initialized}"); + } + + return retained; } var imageInfo = new ImageCreateInfo @@ -5487,11 +10197,12 @@ internal static unsafe class VulkanVideoPresenter mipViews[mipLevel] = mipView; } - var (renderPass, framebuffer) = CreateRenderPassAndFramebuffer( - format, - mipViews[0], - target.Width, - target.Height); + var (renderPass, initialRenderPass, framebuffer) = + CreateRenderPassAndFramebuffer( + format, + mipViews[0], + target.Width, + target.Height); var resource = new GuestImageResource { @@ -5499,12 +10210,14 @@ internal static unsafe class VulkanVideoPresenter Width = target.Width, Height = target.Height, MipLevels = mipLevels, + GuestFormat = guestFormat, Format = format, Image = image, Memory = memory, View = view, MipViews = mipViews, RenderPass = renderPass, + InitialRenderPass = initialRenderPass, Framebuffer = framebuffer, }; var debugName = GuestImageDebugName(target, format); @@ -5518,39 +10231,95 @@ internal static unsafe class VulkanVideoPresenter $"{debugName} mip{mipLevel}"); } SetDebugName(ObjectType.RenderPass, renderPass.Handle, $"{debugName} renderpass"); + SetDebugName(ObjectType.RenderPass, initialRenderPass.Handle, $"{debugName} initial-renderpass"); SetDebugName(ObjectType.Framebuffer, framebuffer.Handle, $"{debugName} framebuffer"); _guestImages.Add(target.Address, resource); + lock (_gate) + { + _guestImageExtents[target.Address] = ( + target.Width, + target.Height, + GetTextureByteCount(target.Format, target.Width, target.Height)); + } + + if (target.Width <= 1920 && target.Height <= 1080) + { + SharpEmu.HLE.GuestImageWriteTracker.Track( + target.Address, + (ulong)target.Width * target.Height * GetTextureBytesPerPixel(target.Format), + CurrentGuestWorkSequenceForDiagnostics, + "vulkan.render-target"); + } + + if (_traceGuestImageEvents) + { + Console.Error.WriteLine( + $"[GIMG] created-as-rt addr=0x{target.Address:X} " + + $"{target.Width}x{target.Height} fmt={format}"); + } + return resource; } - private (RenderPass RenderPass, Framebuffer Framebuffer) CreateRenderPassAndFramebuffer( + private (RenderPass RenderPass, RenderPass InitialRenderPass, Framebuffer Framebuffer) + CreateRenderPassAndFramebuffer( Format format, ImageView attachmentView, uint width, - uint height) => - CreateRenderPassAndFramebuffer([format], [attachmentView], width, height); + uint height) + { + var load = CreateRenderPassAndFramebuffer( + [format], [attachmentView], width, height, [true], null, false); + var initial = CreateRenderPassAndFramebuffer( + [format], [attachmentView], width, height, [false], null, false); + _vk.DestroyFramebuffer(_device, initial.Framebuffer, null); + return (load.RenderPass, initial.RenderPass, load.Framebuffer); + } private (RenderPass RenderPass, Framebuffer Framebuffer) CreateRenderPassAndFramebuffer( IReadOnlyList formats, IReadOnlyList attachmentViews, uint width, - uint height) + uint height) => + CreateRenderPassAndFramebuffer( + formats, + attachmentViews, + width, + height, + Enumerable.Repeat(true, formats.Count).ToArray(), + null, + false); + + private (RenderPass RenderPass, Framebuffer Framebuffer) CreateRenderPassAndFramebuffer( + IReadOnlyList formats, + IReadOnlyList attachmentViews, + uint width, + uint height, + IReadOnlyList initialized, + GuestDepthResource? depth, + bool depthInitialized) { - if (formats.Count == 0 || formats.Count != attachmentViews.Count) + if (formats.Count == 0 || + formats.Count != attachmentViews.Count || + formats.Count != initialized.Count) { - throw new InvalidOperationException("render target formats and views must have matching counts"); + throw new InvalidOperationException( + "render target formats, views, and initialization states must have matching counts"); } - var colorAttachments = stackalloc AttachmentDescription[formats.Count]; + var attachmentCount = formats.Count + (depth is null ? 0 : 1); + var attachments = stackalloc AttachmentDescription[attachmentCount]; var colorReferences = stackalloc AttachmentReference[formats.Count]; - var views = stackalloc ImageView[formats.Count]; + var views = stackalloc ImageView[attachmentCount]; for (var index = 0; index < formats.Count; index++) { - colorAttachments[index] = new AttachmentDescription + attachments[index] = new AttachmentDescription { Format = formats[index], Samples = SampleCountFlags.Count1Bit, - LoadOp = AttachmentLoadOp.Load, + LoadOp = initialized[index] + ? AttachmentLoadOp.Load + : AttachmentLoadOp.Clear, StoreOp = AttachmentStoreOp.Store, StencilLoadOp = AttachmentLoadOp.DontCare, StencilStoreOp = AttachmentStoreOp.DontCare, @@ -5565,17 +10334,44 @@ internal static unsafe class VulkanVideoPresenter views[index] = attachmentViews[index]; } + AttachmentReference depthReference = default; + if (depth is not null) + { + attachments[formats.Count] = new AttachmentDescription + { + Format = DepthFormat, + Samples = SampleCountFlags.Count1Bit, + LoadOp = depthInitialized + ? AttachmentLoadOp.Load + : AttachmentLoadOp.Clear, + StoreOp = AttachmentStoreOp.Store, + StencilLoadOp = AttachmentLoadOp.DontCare, + StencilStoreOp = AttachmentStoreOp.DontCare, + InitialLayout = depthInitialized + ? ImageLayout.DepthStencilAttachmentOptimal + : ImageLayout.Undefined, + FinalLayout = ImageLayout.DepthStencilAttachmentOptimal, + }; + depthReference = new AttachmentReference + { + Attachment = (uint)formats.Count, + Layout = ImageLayout.DepthStencilAttachmentOptimal, + }; + views[formats.Count] = depth.View; + } + var subpass = new SubpassDescription { PipelineBindPoint = PipelineBindPoint.Graphics, ColorAttachmentCount = (uint)formats.Count, PColorAttachments = colorReferences, + PDepthStencilAttachment = depth is null ? null : &depthReference, }; var renderPassInfo = new RenderPassCreateInfo { SType = StructureType.RenderPassCreateInfo, - AttachmentCount = (uint)formats.Count, - PAttachments = colorAttachments, + AttachmentCount = (uint)attachmentCount, + PAttachments = attachments, SubpassCount = 1, PSubpasses = &subpass, }; @@ -5587,7 +10383,7 @@ internal static unsafe class VulkanVideoPresenter { SType = StructureType.FramebufferCreateInfo, RenderPass = renderPass, - AttachmentCount = (uint)formats.Count, + AttachmentCount = (uint)attachmentCount, PAttachments = views, Width = width, Height = height, @@ -5600,6 +10396,333 @@ internal static unsafe class VulkanVideoPresenter return (renderPass, framebuffer); } + private (Image Image, DeviceMemory Memory, ImageView View) CreateDepthAttachment( + uint width, + uint height) + { + var imageInfo = new ImageCreateInfo + { + SType = StructureType.ImageCreateInfo, + ImageType = ImageType.Type2D, + Format = DepthFormat, + Extent = new Extent3D(Math.Max(width, 1), Math.Max(height, 1), 1), + MipLevels = 1, + ArrayLayers = 1, + Samples = SampleCountFlags.Count1Bit, + Tiling = ImageTiling.Optimal, + Usage = + ImageUsageFlags.DepthStencilAttachmentBit | + ImageUsageFlags.SampledBit | + ImageUsageFlags.TransferSrcBit | + ImageUsageFlags.TransferDstBit, + SharingMode = SharingMode.Exclusive, + InitialLayout = ImageLayout.Undefined, + }; + Check(_vk.CreateImage(_device, &imageInfo, null, out var image), "vkCreateImage(depth)"); + _vk.GetImageMemoryRequirements(_device, image, out var requirements); + var memoryInfo = new MemoryAllocateInfo + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = requirements.Size, + MemoryTypeIndex = FindMemoryType(requirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit), + }; + Check(_vk.AllocateMemory(_device, &memoryInfo, null, out var memory), "vkAllocateMemory(depth)"); + Check(_vk.BindImageMemory(_device, image, memory, 0), "vkBindImageMemory(depth)"); + + var viewInfo = new ImageViewCreateInfo + { + SType = StructureType.ImageViewCreateInfo, + Image = image, + ViewType = ImageViewType.Type2D, + Format = DepthFormat, + SubresourceRange = new ImageSubresourceRange(ImageAspectFlags.DepthBit, 0, 1, 0, 1), + }; + Check(_vk.CreateImageView(_device, &viewInfo, null, out var view), "vkCreateImageView(depth)"); + return (image, memory, view); + } + + private GuestDepthResource GetOrCreateGuestDepth(GuestDepthTarget target) + { + var key = new GuestDepthKey( + target.Address, + target.ReadAddress, + target.Width, + target.Height, + target.GuestFormat, + target.SwizzleMode); + if (_guestDepthImages.TryGetValue(key, out var existing)) + { + existing.GuestClearDepth = target.ClearDepth; + if (!existing.Initialized && existing.InitializationSource == "none") + { + existing.ClearDepth = target.ClearDepth; + } + return existing; + } + + var (image, memory, view) = CreateDepthAttachment(target.Width, target.Height); + var resource = new GuestDepthResource + { + Key = key, + Address = target.Address, + ReadAddress = target.ReadAddress, + WriteAddress = target.WriteAddress, + Width = target.Width, + Height = target.Height, + GuestFormat = target.GuestFormat, + SwizzleMode = target.SwizzleMode, + Image = image, + Memory = memory, + View = view, + GuestClearDepth = target.ClearDepth, + ClearDepth = target.ClearDepth, + }; + SetDebugName( + ObjectType.Image, + image.Handle, + $"SharpEmu guest depth 0x{target.Address:X16} {target.Width}x{target.Height}"); + SetDebugName( + ObjectType.ImageView, + view.Handle, + $"SharpEmu guest depth view 0x{target.Address:X16}"); + _guestDepthImages.Add(key, resource); + if (_traceGuestImageEvents || _traceVulkanShaderEnabled) + { + Console.Error.WriteLine( + $"[GIMG] created-depth addr=0x{target.Address:X} " + + $"read=0x{target.ReadAddress:X} write=0x{target.WriteAddress:X} " + + $"{target.Width}x{target.Height} zfmt={target.GuestFormat} " + + $"sw={target.SwizzleMode} clear={target.ClearDepth:0.######}"); + } + + return resource; + } + + private static void PrepareFirstUseDepth( + GuestDepthResource depth, + GuestDepthState state) + { + if (depth.Initialized || depth.InitializationSource != "none") + { + return; + } + + var effectiveClear = depth.GuestClearDepth; + var source = "guest-clear"; + if (state.TestEnable && !state.WriteEnable) + { + switch (state.CompareOp) + { + case 1: // Less + case 3: // LessOrEqual + effectiveClear = 1f; + source = "neutral-first-use"; + break; + case 4: // Greater + case 6: // GreaterOrEqual + effectiveClear = 0f; + source = "neutral-first-use"; + break; + case 7: // Always + break; + default: // Never, Equal, NotEqual + source = "guest-clear-ambiguous"; + Console.Error.WriteLine( + $"[LOADER][WARN] Vulkan has no neutral first-use depth clear " + + $"addr=0x{depth.Address:X16} compare={state.CompareOp}; " + + $"using guest clear {depth.GuestClearDepth:0.######}."); + break; + } + } + + depth.ClearDepth = effectiveClear; + depth.InitializationSource = source; + if (_traceDepthInitialization) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.depth_init " + + $"addr=0x{depth.Address:X16} size={depth.Width}x{depth.Height} " + + $"source={source} guest_clear={depth.GuestClearDepth:0.######} " + + $"effective_clear={effectiveClear:0.######} compare={state.CompareOp} " + + $"test={(state.TestEnable ? 1 : 0)} write={(state.WriteEnable ? 1 : 0)} " + + $"initialized=0"); + } + } + + private GuestRenderTarget GetDepthOnlyColorTarget(GuestDepthTarget depth) + { + var key = new GuestDepthKey( + depth.Address, + depth.ReadAddress, + depth.Width, + depth.Height, + depth.GuestFormat, + depth.SwizzleMode); + if (!_depthOnlyColorAddresses.TryGetValue(key, out var address)) + { + address = _nextDepthOnlyColorAddress; + _nextDepthOnlyColorAddress = checked(_nextDepthOnlyColorAddress + 0x1000_0000UL); + _depthOnlyColorAddresses.Add(key, address); + } + + // The translated fragment module still declares a color output, + // even for a guest depth-only pass. A private, never-published + // color attachment keeps that output legal while the persistent + // guest DB surface remains the only observable result. + return new GuestRenderTarget( + address, + depth.Width, + depth.Height, + Format: 10, + NumberType: 0); + } + + private DepthFramebufferResource GetOrCreateDepthFramebuffer( + GuestImageResource color, + GuestDepthResource depth) + { + if (color.DepthFramebuffers.TryGetValue(depth.Key, out var existing)) + { + return existing; + } + + if (depth.Width < color.Width || depth.Height < color.Height) + { + throw new InvalidOperationException( + $"guest depth 0x{depth.Address:X16} extent {depth.Width}x{depth.Height} " + + $"is smaller than color target 0x{color.Address:X16} " + + $"{color.Width}x{color.Height}"); + } + + var attachmentView = color.MipViews.Length > 0 ? color.MipViews[0] : color.View; + var loadRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: false, + clearDepth: false); + var colorClearRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: true, + clearDepth: false); + var depthClearRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: false, + clearDepth: true); + var bothClearRenderPass = CreateDepthRenderPass( + color.Format, + clearColor: true, + clearDepth: true); + var attachments = stackalloc ImageView[2]; + attachments[0] = attachmentView; + attachments[1] = depth.View; + var framebufferInfo = new FramebufferCreateInfo + { + SType = StructureType.FramebufferCreateInfo, + RenderPass = loadRenderPass, + AttachmentCount = 2, + PAttachments = attachments, + Width = color.Width, + Height = color.Height, + Layers = 1, + }; + Check( + _vk.CreateFramebuffer(_device, &framebufferInfo, null, out var framebuffer), + "vkCreateFramebuffer(offscreen depth)"); + var resource = new DepthFramebufferResource + { + Depth = depth, + LoadRenderPass = loadRenderPass, + ColorClearRenderPass = colorClearRenderPass, + DepthClearRenderPass = depthClearRenderPass, + BothClearRenderPass = bothClearRenderPass, + Framebuffer = framebuffer, + }; + var name = $"SharpEmu color 0x{color.Address:X16} depth 0x{depth.Address:X16}"; + SetDebugName(ObjectType.RenderPass, loadRenderPass.Handle, $"{name} load"); + SetDebugName(ObjectType.RenderPass, colorClearRenderPass.Handle, $"{name} color-clear"); + SetDebugName(ObjectType.RenderPass, depthClearRenderPass.Handle, $"{name} depth-clear"); + SetDebugName(ObjectType.RenderPass, bothClearRenderPass.Handle, $"{name} both-clear"); + SetDebugName(ObjectType.Framebuffer, framebuffer.Handle, $"{name} framebuffer"); + color.DepthFramebuffers.Add(depth.Key, resource); + return resource; + } + + private RenderPass CreateDepthRenderPass( + Format colorFormat, + bool clearColor, + bool clearDepth) + { + var attachments = stackalloc AttachmentDescription[2]; + attachments[0] = new AttachmentDescription + { + Format = colorFormat, + Samples = SampleCountFlags.Count1Bit, + LoadOp = clearColor ? AttachmentLoadOp.Clear : AttachmentLoadOp.Load, + StoreOp = AttachmentStoreOp.Store, + StencilLoadOp = AttachmentLoadOp.DontCare, + StencilStoreOp = AttachmentStoreOp.DontCare, + InitialLayout = ImageLayout.ColorAttachmentOptimal, + FinalLayout = ImageLayout.ColorAttachmentOptimal, + }; + attachments[1] = new AttachmentDescription + { + Format = DepthFormat, + Samples = SampleCountFlags.Count1Bit, + LoadOp = clearDepth ? AttachmentLoadOp.Clear : AttachmentLoadOp.Load, + StoreOp = AttachmentStoreOp.Store, + StencilLoadOp = AttachmentLoadOp.DontCare, + StencilStoreOp = AttachmentStoreOp.DontCare, + InitialLayout = clearDepth + ? ImageLayout.Undefined + : ImageLayout.DepthStencilAttachmentOptimal, + FinalLayout = ImageLayout.DepthStencilAttachmentOptimal, + }; + var colorReference = new AttachmentReference + { + Attachment = 0, + Layout = ImageLayout.ColorAttachmentOptimal, + }; + var depthReference = new AttachmentReference + { + Attachment = 1, + Layout = ImageLayout.DepthStencilAttachmentOptimal, + }; + var subpass = new SubpassDescription + { + PipelineBindPoint = PipelineBindPoint.Graphics, + ColorAttachmentCount = 1, + PColorAttachments = &colorReference, + PDepthStencilAttachment = &depthReference, + }; + var dependency = new SubpassDependency + { + SrcSubpass = Vk.SubpassExternal, + DstSubpass = 0, + SrcStageMask = PipelineStageFlags.LateFragmentTestsBit, + DstStageMask = + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + SrcAccessMask = AccessFlags.DepthStencilAttachmentWriteBit, + DstAccessMask = + AccessFlags.DepthStencilAttachmentReadBit | + AccessFlags.DepthStencilAttachmentWriteBit, + DependencyFlags = DependencyFlags.ByRegionBit, + }; + var createInfo = new RenderPassCreateInfo + { + SType = StructureType.RenderPassCreateInfo, + AttachmentCount = 2, + PAttachments = attachments, + SubpassCount = 1, + PSubpasses = &subpass, + DependencyCount = 1, + PDependencies = &dependency, + }; + Check( + _vk.CreateRenderPass(_device, &createInfo, null, out var renderPass), + "vkCreateRenderPass(offscreen depth)"); + return renderPass; + } + private static uint ClampMipLevels(uint width, uint height, uint requestedMipLevels) { var largestDimension = Math.Max(width, height); @@ -5615,6 +10738,12 @@ internal static unsafe class VulkanVideoPresenter private void DestroyGuestImage(GuestImageResource resource) { + foreach (var depthFramebuffer in resource.DepthFramebuffers.Values) + { + DestroyDepthFramebuffer(depthFramebuffer); + } + resource.DepthFramebuffers.Clear(); + foreach (var view in resource.FormatViews.Values) { if (view.Handle != 0) @@ -5634,6 +10763,11 @@ internal static unsafe class VulkanVideoPresenter _vk.DestroyRenderPass(_device, resource.RenderPass, null); } + if (resource.InitialRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.InitialRenderPass, null); + } + if (resource.View.Handle != 0) { _vk.DestroyImageView(_device, resource.View, null); @@ -5656,28 +10790,58 @@ internal static unsafe class VulkanVideoPresenter { _vk.FreeMemory(_device, resource.Memory, null); } + } - private static uint GetGuestTextureFormat(Format format) => - format switch + private void DestroyDepthFramebuffer(DepthFramebufferResource resource) + { + if (resource.Framebuffer.Handle != 0) { - Format.A2R10G10B10UnormPack32 => 9, - Format.R8G8B8A8Unorm => 56, - Format.R16G16Unorm => 5, - Format.R16G16B16A16Unorm => 12, - Format.R32Uint => GuestFormatR32Uint, - Format.R32Sint => GuestFormatR32Sint, - Format.R32Sfloat => GuestFormatR32Sfloat, - Format.R16G16Uint => GuestFormatR16G16Uint, - Format.R16G16Sint => GuestFormatR16G16Sint, - Format.R16G16Sfloat => GuestFormatR16G16Sfloat, - Format.R8G8B8A8Uint => GuestFormatR8G8B8A8Uint, - Format.R8G8B8A8Sint => GuestFormatR8G8B8A8Sint, - Format.R16G16B16A16Uint => GuestFormatR16G16B16A16Uint, - Format.R16G16B16A16Sint => GuestFormatR16G16B16A16Sint, - Format.R16G16B16A16Sfloat => 71, - _ => 0, - }; + _vk.DestroyFramebuffer(_device, resource.Framebuffer, null); + } + + if (resource.LoadRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.LoadRenderPass, null); + } + if (resource.ColorClearRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.ColorClearRenderPass, null); + } + if (resource.DepthClearRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.DepthClearRenderPass, null); + } + if (resource.BothClearRenderPass.Handle != 0) + { + _vk.DestroyRenderPass(_device, resource.BothClearRenderPass, null); + } + } + + private void DestroyGuestDepth(GuestDepthResource resource) + { + foreach (var sampleView in resource.SampleViews.Values) + { + if (sampleView.Handle != 0) + { + _vk.DestroyImageView(_device, sampleView, null); + } + } + resource.SampleViews.Clear(); + + if (resource.View.Handle != 0) + { + _vk.DestroyImageView(_device, resource.View, null); + } + if (resource.Image.Handle != 0) + { + _vk.DestroyImage(_device, resource.Image, null); + } + if (resource.Memory.Handle != 0) + { + _vk.FreeMemory(_device, resource.Memory, null); + } + } private bool TryGetOrCreateGuestImageView( GuestImageResource resource, @@ -5793,9 +10957,11 @@ internal static unsafe class VulkanVideoPresenter Format.R16G16Sint or Format.R16G16Sfloat or Format.R8G8B8A8Unorm or + Format.R8G8B8A8Srgb or Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or Format.A2R10G10B10UnormPack32 or + Format.A2B10G10R10UnormPack32 or Format.B10G11R11UfloatPack32 => 32, Format.R32G32Uint or Format.R32G32Sint or @@ -5811,214 +10977,14 @@ internal static unsafe class VulkanVideoPresenter _ => 0, }; - private void UpdatePerformanceHud() - { - if (!_performanceHudEnabled) - { - return; - } - - var now = Stopwatch.GetTimestamp(); - if (_performanceHudLastTimestamp != 0 && - Stopwatch.GetElapsedTime(_performanceHudLastTimestamp, now).TotalSeconds < - PerformanceHudSampleSeconds) - { - return; - } - - try - { - using var process = Process.GetCurrentProcess(); - var processCpu = process.TotalProcessorTime; - var currentThreadCpu = new Dictionary(); - var currentThreadIds = new HashSet(); - var hottestThreadId = 0; - var hottestThreadCpuSeconds = 0.0; - - // Per-thread CPU times and thread names come from Windows-only - // APIs; on POSIX the HUD reports process totals with an "idle" - // hottest-thread slot. - if (OperatingSystem.IsWindows()) - { - foreach (ProcessThread thread in process.Threads) - { - using (thread) - { - try - { - var threadId = thread.Id; - var cpu = thread.TotalProcessorTime; - currentThreadIds.Add(threadId); - currentThreadCpu[threadId] = cpu; - if (_performanceHudThreadCpu.TryGetValue(threadId, out var previousCpu)) - { - var deltaSeconds = Math.Max(0.0, (cpu - previousCpu).TotalSeconds); - if (deltaSeconds > hottestThreadCpuSeconds) - { - hottestThreadCpuSeconds = deltaSeconds; - hottestThreadId = threadId; - } - } - } - catch (InvalidOperationException) - { - } - } - } - } - - if (_performanceHudLastTimestamp != 0) - { - var elapsedSeconds = Math.Max( - Stopwatch.GetElapsedTime(_performanceHudLastTimestamp, now).TotalSeconds, - 0.001); - var processCpuPercent = Math.Max( - 0.0, - (processCpu - _performanceHudLastProcessCpu).TotalSeconds / - elapsedSeconds / - Math.Max(Environment.ProcessorCount, 1) * - 100.0); - var hottestThreadPercent = hottestThreadCpuSeconds / elapsedSeconds * 100.0; - var presentedFrames = _performanceHudPresentedFrames; - var fps = (presentedFrames - _performanceHudLastPresentedFrames) / elapsedSeconds; - var hotName = hottestThreadId == 0 - ? "idle" - : GetPerformanceThreadName(hottestThreadId); - long guestBacklog; - int queuedGuestWork; - lock (_gate) - { - guestBacklog = Math.Max( - 0, - _enqueuedGuestWorkSequence - _completedGuestWorkSequence); - queuedGuestWork = _pendingGuestWork.Count; - } - - var gpuInFlight = _pendingGuestSubmissions.Count + - (_presentationInFlight ? 1 : 0); - var readCount = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadCount); - var readBytes = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadBytes); - var readHits = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadCacheHits); - var readPvmBytes = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadPvmBytes); - var readLibcBytes = Interlocked.Read( - ref Gen5ShaderScalarEvaluator.GlobalMemoryReadLibcBytes); - var readsPerSecond = - (readCount - _performanceHudLastReadCount) / elapsedSeconds; - var readMbPerSecond = - (readBytes - _performanceHudLastReadBytes) / - elapsedSeconds / - (1024.0 * 1024.0); - var readHitsPerSecond = - (readHits - _performanceHudLastReadHits) / elapsedSeconds; - var readPvmMbPerSecond = - (readPvmBytes - _performanceHudLastReadPvmBytes) / - elapsedSeconds / - (1024.0 * 1024.0); - var readLibcMbPerSecond = - (readLibcBytes - _performanceHudLastReadLibcBytes) / - elapsedSeconds / - (1024.0 * 1024.0); - _performanceHudLastReadCount = readCount; - _performanceHudLastReadBytes = readBytes; - _performanceHudLastReadHits = readHits; - _performanceHudLastReadPvmBytes = readPvmBytes; - _performanceHudLastReadLibcBytes = readLibcBytes; - _window.Title = - $"FPS {fps:0.0} CPU {processCpuPercent:0}% | " + - $"HOT {hotName}#{hottestThreadId} {hottestThreadPercent:0}% | " + - $"WORK {guestBacklog} (q{queuedGuestWork}/gpu{gpuInFlight}) | " + - $"RD {readsPerSecond:0}/s {readMbPerSecond:0}MB/s h{readHitsPerSecond:0}/s " + - $"P{readPvmMbPerSecond:0} L{readLibcMbPerSecond:0} | " + - VideoOutExports.GetWindowTitle(); - _performanceHudLastPresentedFrames = presentedFrames; - } - - _performanceHudThreadCpu.Clear(); - foreach (var (threadId, cpu) in currentThreadCpu) - { - _performanceHudThreadCpu[threadId] = cpu; - } - - foreach (var staleThreadId in _performanceHudThreadNames.Keys - .Where(threadId => !currentThreadIds.Contains(threadId)) - .ToArray()) - { - _performanceHudThreadNames.Remove(staleThreadId); - } - - _performanceHudLastProcessCpu = processCpu; - _performanceHudLastTimestamp = now; - } - catch (Exception exception) when ( - exception is InvalidOperationException or System.ComponentModel.Win32Exception) - { - _performanceHudLastTimestamp = now; - } - } - - private string GetPerformanceThreadName(int threadId) - { - if (_performanceHudThreadNames.TryGetValue(threadId, out var cached)) - { - return cached; - } - - var name = "tid"; - var handle = OpenThread(ThreadQueryLimitedInformation, false, (uint)threadId); - if (handle != 0) - { - try - { - if (GetThreadDescription(handle, out var description) >= 0 && description != 0) - { - try - { - var described = Marshal.PtrToStringUni(description); - if (!string.IsNullOrWhiteSpace(described)) - { - name = described.Length <= 28 ? described : described[..28]; - } - } - finally - { - LocalFree(description); - } - } - } - finally - { - CloseHandle(handle); - } - } - - _performanceHudThreadNames[threadId] = name; - return name; - } - - [DllImport("kernel32.dll", SetLastError = true)] - private static extern nint OpenThread(uint desiredAccess, bool inheritHandle, uint threadId); - - [DllImport("kernel32.dll")] - private static extern int GetThreadDescription(nint thread, out nint description); - - [DllImport("kernel32.dll")] - private static extern nint LocalFree(nint memory); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(System.Runtime.InteropServices.UnmanagedType.Bool)] - private static extern bool CloseHandle(nint handle); - private void WaitForRenderWork() { - var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || _presentationInFlight; + var gpuWorkInFlight = _pendingGuestSubmissions.Count > 0 || + Array.Exists(_frameFencePending, static pending => pending); lock (_gate) { if (_closed || - _pendingGuestWork.Count > 0 || + _pendingGuestWorkCount > 0 || (_latestPresentation is { } latest && latest.Sequence != _presentedSequence && latest.RequiredGuestWorkSequence <= _completedGuestWorkSequence)) @@ -6034,6 +11000,7 @@ internal static unsafe class VulkanVideoPresenter { if (Volatile.Read(ref _presenterCloseRequested)) { + Console.Error.WriteLine("[LOADER][WARN] Vulkan VideoOut closing on host shutdown request."); _window.Close(); return; } @@ -6043,19 +11010,84 @@ internal static unsafe class VulkanVideoPresenter return; } - WaitForRenderWork(); - UpdatePerformanceHud(); + // Reuse of a frame slot waits only on that slot's fence, keeping + // up to MaxFramesInFlight frames pipelined between CPU and GPU. + var frameSlot = _currentFrameSlot; + if (!TryWaitFrameSlot(frameSlot, _frameSlotWaitBudgetNs)) + { + // The GPU is still finishing this slot's previous frame (slow + // compute backlog). Don't block the macOS main thread — return + // to the Cocoa event pump so the window keeps handling input + // (F1 overlay, drag, close) and redrawing. The frame is retried + // next Render(); the fence signals once the GPU catches up. + return; + } + _presentationCommandBuffer = _frameCommandBuffers[frameSlot]; _commandBuffer = _presentationCommandBuffer; if (!_deviceLost) { CollectCompletedGuestSubmissions(waitForOldest: false); } + EvictDirtyCachedTextures(); var completedWork = 0; - while (completedWork < MaxGuestWorkPerRender && - TryTakeGuestWork(out var work)) + var renderWorkDeadline = _renderWorkBudgetTicks > 0 + ? System.Diagnostics.Stopwatch.GetTimestamp() + _renderWorkBudgetTicks + : long.MaxValue; + while (completedWork < MaxGuestWorkPerRender) { + // Never block the macOS main thread waiting for in-flight GPU + // work to drain. If submission is at capacity (a slow-compute + // backlog), stop processing and let the event pump run; the + // remaining queued work is picked up on later frames as the GPU + // completions free up capacity (collected non-blockingly here). + CollectCompletedGuestSubmissions(waitForOldest: false); + if (_pendingGuestSubmissions.Count >= MaxInFlightGuestSubmissions) + { + break; + } + + if (!TryTakeGuestWork(out var pendingGuestWork)) + { + break; + } + + if (!string.Equals( + _activeGuestQueue.Name, + pendingGuestWork.Queue.Name, + StringComparison.Ordinal)) + { + // A host command buffer must never contain commands from + // two independent guest queues: an ordered action fences + // only its own queue's predecessor submissions. + FlushBatchedGuestCommands(); + } + + _activeGuestQueue = pendingGuestWork.Queue; + _activeGuestWorkSequence = pendingGuestWork.Sequence; + Volatile.Write( + ref _executingGuestWorkSequence, + pendingGuestWork.Sequence); + using var guestQueueScope = EnterGuestQueue( + pendingGuestWork.Queue.Name, + pendingGuestWork.Queue.SubmissionId); + _enqueueAsImmediateQueueFollowup = true; + _immediateFollowupTail = null; + var work = pendingGuestWork.Work; + + var traceWork = ShouldTracePresentedGuestImageContentsForDiagnostics(); + var workStart = traceWork ? System.Diagnostics.Stopwatch.GetTimestamp() : 0L; + if (traceWork && work is VulkanComputeGuestDispatch or VulkanOffscreenGuestDraw) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.render_work_enter #{completedWork} " + + $"sequence={pendingGuestWork.Sequence} " + + $"queue={pendingGuestWork.Queue.Name} " + + $"submission={pendingGuestWork.Queue.SubmissionId} " + + $"queued_ms={(System.Diagnostics.Stopwatch.GetTimestamp() - pendingGuestWork.EnqueuedTicks) * 1000.0 / System.Diagnostics.Stopwatch.Frequency:F3} " + + work.GetType().Name); + } try { switch (work) @@ -6066,21 +11098,92 @@ internal static unsafe class VulkanVideoPresenter case VulkanComputeGuestDispatch computeDispatch: ExecuteComputeDispatch(computeDispatch); break; + case VulkanGuestImageWrite guestImageWrite: + ExecuteGuestImageWrite(guestImageWrite); + break; + case VulkanOrderedGuestAction orderedAction: + ExecuteOrderedGuestAction(orderedAction); + break; + case VulkanOrderedGuestFlip orderedFlip: + ExecuteOrderedGuestFlip(orderedFlip); + break; + case VulkanOrderedGuestFlipWait flipWait: + ExecuteOrderedGuestFlipWait(flipWait); + break; } } finally { - CompleteGuestWork(); + CompleteGuestWork(pendingGuestWork); + _enqueueAsImmediateQueueFollowup = false; + _immediateFollowupTail = null; + Volatile.Write(ref _executingGuestWorkSequence, 0); + } + + if (workStart != 0) + { + var elapsedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - workStart) + * 1000.0 / System.Diagnostics.Stopwatch.Frequency; + if (elapsedMs > 250.0) + { + var desc = work switch + { + VulkanComputeGuestDispatch c => $"compute cs=0x{c.ShaderAddress:X16} groups={c.GroupCountX}x{c.GroupCountY}x{c.GroupCountZ}", + VulkanOffscreenGuestDraw d => + $"draw mrt={d.Targets.Count} " + + $"rt=0x{d.Targets[0].Address:X16} " + + $"{d.Targets[0].Width}x{d.Targets[0].Height}", + _ => work.GetType().Name, + }; + Console.Error.WriteLine( + $"[LOADER][WARN] vk.slow_render_work {elapsedMs:F0}ms " + + $"queue={pendingGuestWork.Queue.Name} " + + $"submission={pendingGuestWork.Queue.SubmissionId} " + + $"sequence={pendingGuestWork.Sequence}: {desc}"); + } } completedWork++; + + // Return to the main-thread event pump + present once the + // per-frame budget is spent; remaining guest work is drained + // on subsequent Render() calls. Without this a compute-heavy + // backlog freezes the window (macOS "Not Responding"). + if (System.Diagnostics.Stopwatch.GetTimestamp() >= renderWorkDeadline) + { + break; + } } + FlushBatchedGuestCommands(); + CollectAbandonedGuestImageVersions(); + if (!TryTakePresentation(_presentedSequence, out var presentation)) { + // A render-loop tick with no newer flip is normal. Warn only when + // an actual queued presentation is waiting on unfinished guest work. + if (ShouldTracePresentedGuestImageContentsForDiagnostics() && + HasPendingGuestPresentation(_presentedSequence) && + _presentNotTakenLoggedSequence != _presentedSequence) + { + _presentNotTakenLoggedSequence = _presentedSequence; + Console.Error.WriteLine( + $"[LOADER][WARN] vk.present_not_taken seq={_presentedSequence} " + + "— presentation submitted but its required guest work isn't complete; nothing shown."); + } + return; } + if (ShouldTracePresentedGuestImageContentsForDiagnostics()) + { + Console.Error.WriteLine( + $"[LOADER][TRACE] vk.present_taken addr=0x{presentation.GuestImageAddress:X16} " + + $"version={presentation.GuestImageVersion} " + + $"drawKind={presentation.DrawKind} hasPixels={presentation.Pixels is not null} " + + $"hasTranslatedDraw={presentation.TranslatedDraw is not null}"); + } + if (presentation.Pixels is null && presentation.DrawKind != GuestDrawKind.FullscreenBarycentric && presentation.TranslatedDraw is null && @@ -6089,8 +11192,6 @@ internal static unsafe class VulkanVideoPresenter return; } - CompletePendingPresentation(wait: true); - byte[]? pixels = null; if (presentation.Pixels is { } sourcePixels) { @@ -6110,22 +11211,57 @@ internal static unsafe class VulkanVideoPresenter TranslatedDrawResources? translatedResources = null; GuestImageResource? presentedGuestImage = null; - var tracePresentedGuestImage = false; - if (presentation.GuestImageAddress != 0 && - (!_guestImages.TryGetValue( - presentation.GuestImageAddress, - out presentedGuestImage) || - !presentedGuestImage.Initialized)) + var ownsPresentedGuestImageVersion = false; + if (presentation.GuestImageVersion != 0) { + ownsPresentedGuestImageVersion = _guestImageVersions.Remove( + presentation.GuestImageVersion, + out presentedGuestImage); + } + else if (presentation.GuestImageAddress != 0) + { + _guestImages.TryGetValue( + presentation.GuestImageAddress, + out presentedGuestImage); + } + + if (presentation.GuestImageAddress != 0 && + (presentedGuestImage is null || !presentedGuestImage.Initialized)) + { + if (ShouldTracePresentedGuestImageContentsForDiagnostics()) + { + Console.Error.WriteLine( + $"[LOADER][WARN] vk.present_dropped addr=0x{presentation.GuestImageAddress:X16} " + + $"version={presentation.GuestImageVersion} " + + $"found={(presentedGuestImage is not null)} " + + $"initialized={(presentedGuestImage?.Initialized ?? false)} " + + $"— no swapchain present this frame (black)."); + } + + if (ownsPresentedGuestImageVersion && presentedGuestImage is not null) + { + DestroyGuestImage(presentedGuestImage); + } + return; } + if (ownsPresentedGuestImageVersion) + { + System.Diagnostics.Debug.Assert( + _frameGuestImageVersions[frameSlot] is null, + "A reusable frame slot cannot still own a flip version."); + _frameGuestImageVersions[frameSlot] = presentedGuestImage; + } if (presentedGuestImage is not null) { _directPresentationCount++; - if (ShouldSamplePresentedGuestImageForDiagnostics( - _directPresentationCount)) + var traceAddressedPresentation = + ShouldTraceAddressedPresentedGuestImage(presentedGuestImage); + if (traceAddressedPresentation || + ShouldTracePresentedGuestImageContentsForDiagnostics() && + (_directPresentationCount is 1 or 30 or 120 || + _directPresentationCount % 600 == 0)) { - tracePresentedGuestImage = true; Console.Error.WriteLine( $"[LOADER][TRACE] vk.present_sample frame={_directPresentationCount} " + $"addr=0x{presentedGuestImage.Address:X16}"); @@ -6142,19 +11278,14 @@ internal static unsafe class VulkanVideoPresenter [_swapchainFormat], _extent); if (ShouldTracePresentedGuestImageContentsForDiagnostics() && - !_firstGuestDrawPresented) + !_firstGuestDrawPresented && + translatedResources.Textures is + [ + { GuestImage: { } guestImage }, + ] && + _tracedGuestImageContents.Add(guestImage.Address)) { - Console.Error.WriteLine( - $"[LOADER][TRACE] vk.translated_draw kind={presentation.DrawKind} " + - $"textures={translatedResources.Textures.Length}"); - foreach (var boundTexture in translatedResources.Textures) - { - if (boundTexture.GuestImage is { } guestImage && - _tracedGuestImageContents.Add(guestImage.Address)) - { - TraceGuestImageContents(guestImage); - } - } + TraceGuestImageContents(guestImage); } } catch (Exception exception) @@ -6171,7 +11302,7 @@ internal static unsafe class VulkanVideoPresenter _device, _swapchain, ulong.MaxValue, - _imageAvailable, + _frameImageAvailable[frameSlot], default, &imageIndex); if (acquireResult == Result.ErrorOutOfDateKhr) @@ -6181,6 +11312,19 @@ internal static unsafe class VulkanVideoPresenter { DestroyTranslatedDrawResources(translatedResources); } + if (ownsPresentedGuestImageVersion && presentedGuestImage is not null) + { + if (_frameGuestImageVersions.Length > frameSlot && + ReferenceEquals( + _frameGuestImageVersions[frameSlot], + presentedGuestImage)) + { + _frameGuestImageVersions[frameSlot] = null; + _capturedGuestFlipVersions.Remove( + presentedGuestImage.FlipVersion); + DestroyGuestImage(presentedGuestImage); + } + } return; } @@ -6190,6 +11334,10 @@ internal static unsafe class VulkanVideoPresenter if (pixels is not null) { + // The staging buffer is shared across frame slots; a CPU + // pixel upload (splash / host frames) degrades to serial + // presentation rather than corrupting an in-flight copy. + WaitAllFrameSlots(); void* mapped; Check( _vk.MapMemory(_device, _stagingMemory, 0, (ulong)pixels.Length, 0, &mapped), @@ -6255,11 +11403,16 @@ internal static unsafe class VulkanVideoPresenter $"Unsupported translated guest draw: {presentation.DrawKind}."); } + if (PerfOverlay.Enabled) + { + RecordOverlayBlit(imageIndex, frameSlot); + } + Check(_vk.EndCommandBuffer(_commandBuffer), "vkEndCommandBuffer"); - var imageAvailable = _imageAvailable; + var imageAvailable = _frameImageAvailable[frameSlot]; var commandBuffer = _commandBuffer; - var renderFinished = _renderFinished; + var renderFinished = _renderFinishedPerImage[imageIndex]; var submitInfo = new SubmitInfo { SType = StructureType.SubmitInfo, @@ -6271,16 +11424,21 @@ internal static unsafe class VulkanVideoPresenter SignalSemaphoreCount = 1, PSignalSemaphores = &renderFinished, }; - var presentationFence = _presentationFence; Check( - _vk.ResetFences(_device, 1, &presentationFence), - "vkResetFences(presentation)"); - Check( - _vk.QueueSubmit(_queue, 1, &submitInfo, presentationFence), + _vk.QueueSubmit(_queue, 1, &submitInfo, _frameFences[frameSlot]), "vkQueueSubmit"); - _presentationInFlight = true; - _pendingPresentationResources = translatedResources; - translatedResources = null; + _submitTimeline++; + _frameTimelines[frameSlot] = _submitTimeline; + _frameFencePending[frameSlot] = true; + _frameTranslatedResources[frameSlot] = translatedResources; + if (translatedResources is not null) + { + // CPU-side layout bookkeeping only; later command buffers are + // recorded after this submission, so queue order makes the + // flags valid before any dependent GPU work runs. + MarkSampledImagesInitialized(translatedResources); + MarkStorageImagesInitialized(translatedResources); + } var swapchain = _swapchain; var presentInfo = new PresentInfoKHR @@ -6295,14 +11453,8 @@ internal static unsafe class VulkanVideoPresenter var presentResult = _swapchainApi.QueuePresent(_queue, &presentInfo); if (presentResult == Result.ErrorOutOfDateKhr) { - Check(_vk.QueueWaitIdle(_queue), "vkQueueWaitIdle"); - CompletePendingPresentation(wait: false); - CollectCompletedGuestSubmissions(waitForOldest: false); - if (translatedResources is not null) - { - DestroyTranslatedDrawResources(translatedResources); - } - + // The submitted frame still executes; RecreateSwapchainResources + // drains it (and every frame slot) before destroying anything. RecreateSwapchainResources("vkQueuePresentKHR", presentResult); return; } @@ -6310,28 +11462,25 @@ internal static unsafe class VulkanVideoPresenter CheckSwapchainResult(presentResult, "vkQueuePresentKHR"); recreateAfterPresent |= presentResult == Result.SuboptimalKhr; VideoOutExports.ReportPresentedFrame(); - _performanceHudPresentedFrames++; - if (_swapchainReadbackPending) + PerfOverlay.RecordPresent(); + if (_swapchainReadbackPending || !_pendingAliasImageDumps.IsEmpty) { - CompletePendingPresentation(wait: true); - TraceSwapchainReadback(); - } - // Report the actual presented pixels before starting the larger - // source-image readback. If a guest draw wedges the GPU, the - // source probe can block in vkQueueWaitIdle; doing it first used - // to hide whether the swapchain itself was black and made the - // diagnostic run stop immediately after vk.present_sample. - if (tracePresentedGuestImage && presentedGuestImage is not null) - { - TraceGuestImageContents(presentedGuestImage); - } - while (_pendingAliasImageDumps.TryDequeue(out var aliasImage)) - { - TraceGuestImageContents(aliasImage); - } - CollectCompletedGuestSubmissions(waitForOldest: false); + // Diagnostics read back GPU memory and need this frame done. + WaitFrameSlot(frameSlot); + if (_swapchainReadbackPending) + { + TraceSwapchainReadback(); + } + while (_pendingAliasImageDumps.TryDequeue(out var aliasImage)) + { + TraceGuestImageContents(aliasImage); + } + } + + CollectCompletedGuestSubmissions(waitForOldest: false); _imageInitialized[imageIndex] = true; + _currentFrameSlot = (frameSlot + 1) % MaxFramesInFlight; _presentedSequence = presentation.Sequence; if (presentation.IsSplash && !_splashPresented) { @@ -6490,6 +11639,34 @@ internal static unsafe class VulkanVideoPresenter try { var bytes = new ReadOnlySpan(mapped, checked((int)byteCount)); + if (GuestImageTraceInterval() is not null && bytesPerPixel == 4) + { + long r = 0, g = 0, b = 0, a = 0, samples = 0; + for (var offset = 0; offset + 4 <= bytes.Length; offset += 4 * 251) + { + r += bytes[offset]; + g += bytes[offset + 1]; + b += bytes[offset + 2]; + a += bytes[offset + 3]; + samples++; + } + + if (samples > 0) + { + Console.Error.WriteLine( + $"[RB] addr=0x{image.Address:X} " + + $"mean={r / samples},{g / samples},{b / samples},A{a / samples} " + + $"sample_unique={CountSampledUniquePixels(bytes, bytesPerPixel)}"); + } + + if (++_intervalReadbackCount % 25 == 0) + { + DumpGuestImageBytes(image, bytes); + } + + return; + } + var nonzeroBytes = 0L; ulong hash = 14695981039346656037UL; foreach (var value in bytes) @@ -6514,7 +11691,8 @@ internal static unsafe class VulkanVideoPresenter $"size={image.Width}x{image.Height} format={image.Format} " + $"nonzero_bytes={nonzeroBytes}/{byteCount} " + $"nonblack_pixels={nonblackPixels}/{(ulong)image.Width * image.Height} " + - $"center={center} hash=0x{hash:X16}"); + $"center={center} sample_unique={CountSampledUniquePixels(bytes, bytesPerPixel)} " + + $"hash=0x{hash:X16}"); DumpGuestImageBytes(image, bytes); } finally @@ -6529,6 +11707,37 @@ internal static unsafe class VulkanVideoPresenter } } + private static int CountSampledUniquePixels( + ReadOnlySpan bytes, + uint bytesPerPixel) + { + if (bytesPerPixel == 0) + { + return 0; + } + + var unique = new HashSet(); + var stride = checked((int)bytesPerPixel * 251); + for (var offset = 0; + offset + bytesPerPixel <= bytes.Length; + offset += stride) + { + ulong hash = 14695981039346656037UL; + for (var index = 0; index < bytesPerPixel; index++) + { + hash = (hash ^ bytes[offset + index]) * 1099511628211UL; + } + + unique.Add(hash); + if (unique.Count > 256) + { + return 257; + } + } + + return unique.Count; + } + private static void DumpGuestImageBytes( GuestImageResource image, ReadOnlySpan bytes) @@ -6548,22 +11757,49 @@ internal static unsafe class VulkanVideoPresenter File.WriteAllBytes(path, bytes.ToArray()); } + // Metal cannot blend into integer render targets or 32-bit-per-channel + // float targets (unsupported on Apple-family GPUs). Enabling blend on + // one makes vkCreateGraphicsPipelines fail with ErrorInitializationFailed + // (and trips a Metal "not blendable" validation assertion), so the draw + // is silently dropped. Force blend off for those; blending on an integer + // target is meaningless on real hardware anyway. + private static bool IsBlendableFormat(Format format) => + format switch + { + Format.R8Uint or Format.R8Sint or + Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or + Format.R16G16Uint or Format.R16G16Sint or + Format.R16G16B16A16Uint or Format.R16G16B16A16Sint or + Format.R32Uint or Format.R32Sint or + Format.R32G32Uint or Format.R32G32Sint or + Format.R32G32B32A32Uint or Format.R32G32B32A32Sint or + Format.R32Sfloat or + Format.R32G32Sfloat or + Format.R32G32B32A32Sfloat => false, + _ => true, + }; + private static uint GetReadbackBytesPerPixel(Format format) => format switch { Format.R8Unorm or Format.R8Uint or Format.R8Sint => 1, + Format.R8G8Unorm or + Format.R8G8Uint or + Format.R8G8Sint => 2, Format.R32Uint or Format.R32Sint or Format.R32Sfloat or + Format.B10G11R11UfloatPack32 or Format.R16G16Uint or Format.R16G16Sint or Format.R16G16Sfloat or Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or Format.R8G8B8A8Unorm or - Format.A2R10G10B10UnormPack32 => 4, + Format.A2R10G10B10UnormPack32 or + Format.A2B10G10R10UnormPack32 => 4, Format.R16G16B16A16Uint or Format.R16G16B16A16Sint or Format.R16G16B16A16Sfloat => 8, @@ -6587,7 +11823,8 @@ internal static unsafe class VulkanVideoPresenter var pixel = bytes.Slice(offset, (int)bytesPerPixel); var hasColor = format switch { - Format.A2R10G10B10UnormPack32 => + Format.A2R10G10B10UnormPack32 or + Format.A2B10G10R10UnormPack32 => (BitConverter.ToUInt32(pixel) & 0x3FFFFFFFu) != 0, Format.R8G8B8A8Uint or Format.R8G8B8A8Sint or @@ -6608,6 +11845,11 @@ internal static unsafe class VulkanVideoPresenter private void RecordTranslatedDraw(uint imageIndex, TranslatedDrawResources resources) { BeginDebugLabel(_commandBuffer, "SharpEmu swapchain draw"); + RecordGlobalBufferVisibilityBarrier( + _commandBuffer, + resources, + PipelineStageFlags.VertexShaderBit | + PipelineStageFlags.FragmentShaderBit); RecordTextureUploads(resources, PipelineStageFlags.FragmentShaderBit); RecordStorageImagesForWrite(resources, PipelineStageFlags.FragmentShaderBit); RecordTranslatedGraphicsPass( @@ -6625,29 +11867,21 @@ internal static unsafe class VulkanVideoPresenter { foreach (var texture in resources.Textures) { + if (texture.GuestDepth is { } depth) + { + RecordGuestDepthForSampling(depth, shaderStage); + } + if (!texture.NeedsUpload) { continue; } - // InitialUploadPending means this upload still has to perform - // the image's first layout transition. Treating it as prior - // contents records ShaderReadOnlyOptimal as oldLayout even - // though a freshly created image is still Undefined. Linux - // validation reports VUID-vkCmdDraw-None-09600 and NVIDIA - // samples the uninitialized (black) image in that case. - var hasPriorContents = - texture.GuestImage is { Initialized: true }; var toTransfer = new ImageMemoryBarrier { SType = StructureType.ImageMemoryBarrier, - SrcAccessMask = hasPriorContents - ? AccessFlags.ShaderReadBit - : 0, DstAccessMask = AccessFlags.TransferWriteBit, - OldLayout = hasPriorContents - ? ImageLayout.ShaderReadOnlyOptimal - : ImageLayout.Undefined, + OldLayout = ImageLayout.Undefined, NewLayout = ImageLayout.TransferDstOptimal, SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, DstQueueFamilyIndex = Vk.QueueFamilyIgnored, @@ -6656,9 +11890,7 @@ internal static unsafe class VulkanVideoPresenter }; _vk.CmdPipelineBarrier( _commandBuffer, - hasPriorContents - ? PipelineStageFlags.AllCommandsBit - : PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TopOfPipeBit, PipelineStageFlags.TransferBit, 0, 0, @@ -6711,6 +11943,426 @@ internal static unsafe class VulkanVideoPresenter null, 1, &toShaderRead); + if (texture.Cached) + { + // The queue executes command buffers in submission order, + // so once this upload is recorded every later draw can + // reuse the image without restaging it. + texture.NeedsUpload = false; + } + } + } + + private void RecordGuestDepthForSampling( + GuestDepthResource depth, + PipelineStageFlags shaderStage) + { + if (depth.Layout == ImageLayout.ShaderReadOnlyOptimal) + { + return; + } + + if (!depth.Initialized) + { + var depthRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1); + var toTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = depth.Layout, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = depth.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &toTransfer); + var clearValue = new ClearDepthStencilValue(depth.ClearDepth, 0); + _vk.CmdClearDepthStencilImage( + _commandBuffer, + depth.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &depthRange); + depth.Initialized = true; + if (depth.InitializationSource == "none") + { + depth.InitializationSource = "sample-clear"; + } + depth.Layout = ImageLayout.TransferDstOptimal; + } + + var barrier = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = depth.Layout == ImageLayout.TransferDstOptimal + ? AccessFlags.TransferWriteBit + : AccessFlags.DepthStencilAttachmentWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = depth.Layout, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = depth.Image, + SubresourceRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + depth.Layout == ImageLayout.TransferDstOptimal + ? PipelineStageFlags.TransferBit + : PipelineStageFlags.LateFragmentTestsBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &barrier); + depth.Layout = ImageLayout.ShaderReadOnlyOptimal; + } + + private void RecordRenderTargetFeedbackSnapshots( + TranslatedDrawResources resources, + PipelineStageFlags shaderStage) + { + foreach (var texture in resources.Textures) + { + if (texture.FeedbackSource is not { } source) + { + continue; + } + + // Initialize every destination mip to a deterministic zero. + // Render-target writes currently populate mip 0; leaving the + // remaining sampled mips undefined turns guest LOD selection + // into driver-dependent colored garbage. + var destinationToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = ColorSubresourceRange(0, source.MipLevels), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &destinationToTransfer); + + var clearValue = new ClearColorValue(0f, 0f, 0f, 0f); + // Avoid overlapping a clear and copy on mip 0: without an + // intervening dependency two transfer writes to one + // subresource are not ordered merely because they were + // recorded in that order. Initialized sources overwrite mip + // 0 directly and clear only the otherwise undefined tail. + var clearBaseMip = source.Initialized ? 1u : 0u; + if (clearBaseMip < source.MipLevels) + { + var destinationRange = ColorSubresourceRange( + clearBaseMip, + source.MipLevels - clearBaseMip); + _vk.CmdClearColorImage( + _commandBuffer, + texture.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &destinationRange); + } + + if (source.Initialized) + { + var sourceToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.ShaderReadBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = ImageLayout.ShaderReadOnlyOptimal, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + // Only mip 0 has defined render-target contents. + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + shaderStage | + PipelineStageFlags.ColorAttachmentOutputBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &sourceToTransfer); + + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, + 0, + 0, + 1), + DstSubresource = new ImageSubresourceLayers( + ImageAspectFlags.ColorBit, + 0, + 0, + 1), + Extent = new Extent3D(source.Width, source.Height, 1), + }; + _vk.CmdCopyImage( + _commandBuffer, + source.Image, + ImageLayout.TransferSrcOptimal, + texture.Image, + ImageLayout.TransferDstOptimal, + 1, + ©); + + var sourceToShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferReadBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferSrcOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = ColorSubresourceRange(), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &sourceToShaderRead); + } + + var destinationToShaderRead = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = ColorSubresourceRange(0, source.MipLevels), + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &destinationToShaderRead); + + TraceVulkanShader( + $"vk.feedback_snapshot_copy addr=0x{source.Address:X16} " + + $"size={source.Width}x{source.Height} initialized={source.Initialized}"); + } + } + + private void RecordDepthFeedbackSnapshots( + TranslatedDrawResources resources, + PipelineStageFlags shaderStage) + { + foreach (var texture in resources.Textures) + { + if (texture.DepthFeedbackSource is not { } source) + { + continue; + } + + var depthRange = new ImageSubresourceRange( + ImageAspectFlags.DepthBit, + 0, + 1, + 0, + 1); + var destinationToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + DstAccessMask = AccessFlags.TransferWriteBit, + OldLayout = ImageLayout.Undefined, + NewLayout = ImageLayout.TransferDstOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TopOfPipeBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &destinationToTransfer); + + if (source.Initialized) + { + var sourceToTransfer = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = source.Layout == ImageLayout.ShaderReadOnlyOptimal + ? AccessFlags.ShaderReadBit + : AccessFlags.DepthStencilAttachmentWriteBit, + DstAccessMask = AccessFlags.TransferReadBit, + OldLayout = source.Layout, + NewLayout = ImageLayout.TransferSrcOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + shaderStage | + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + PipelineStageFlags.TransferBit, + 0, + 0, + null, + 0, + null, + 1, + &sourceToTransfer); + var copy = new ImageCopy + { + SrcSubresource = new ImageSubresourceLayers( + ImageAspectFlags.DepthBit, + 0, + 0, + 1), + DstSubresource = new ImageSubresourceLayers( + ImageAspectFlags.DepthBit, + 0, + 0, + 1), + Extent = new Extent3D(source.Width, source.Height, 1), + }; + _vk.CmdCopyImage( + _commandBuffer, + source.Image, + ImageLayout.TransferSrcOptimal, + texture.Image, + ImageLayout.TransferDstOptimal, + 1, + ©); + + var sourceToAttachment = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferReadBit, + DstAccessMask = + AccessFlags.DepthStencilAttachmentReadBit | + AccessFlags.DepthStencilAttachmentWriteBit, + OldLayout = ImageLayout.TransferSrcOptimal, + NewLayout = ImageLayout.DepthStencilAttachmentOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = source.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + PipelineStageFlags.EarlyFragmentTestsBit | + PipelineStageFlags.LateFragmentTestsBit, + 0, + 0, + null, + 0, + null, + 1, + &sourceToAttachment); + source.Layout = ImageLayout.DepthStencilAttachmentOptimal; + } + else + { + var clearValue = new ClearDepthStencilValue(source.ClearDepth, 0); + _vk.CmdClearDepthStencilImage( + _commandBuffer, + texture.Image, + ImageLayout.TransferDstOptimal, + &clearValue, + 1, + &depthRange); + } + + var destinationToShader = new ImageMemoryBarrier + { + SType = StructureType.ImageMemoryBarrier, + SrcAccessMask = AccessFlags.TransferWriteBit, + DstAccessMask = AccessFlags.ShaderReadBit, + OldLayout = ImageLayout.TransferDstOptimal, + NewLayout = ImageLayout.ShaderReadOnlyOptimal, + SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, + DstQueueFamilyIndex = Vk.QueueFamilyIgnored, + Image = texture.Image, + SubresourceRange = depthRange, + }; + _vk.CmdPipelineBarrier( + _commandBuffer, + PipelineStageFlags.TransferBit, + shaderStage, + 0, + 0, + null, + 0, + null, + 1, + &destinationToShader); } } @@ -6824,11 +12476,9 @@ internal static unsafe class VulkanVideoPresenter guestImage.Initialized = true; guestImage.InitialUploadPending = false; - var format = GetGuestTextureFormat(guestImage.Format); - if (format != 0) + if (guestImage.GuestFormat != 0) { - _availableGuestImages[texture.Address] = format; - _gpuGuestImages[texture.Address] = format; + _availableGuestImages[texture.Address] = guestImage.GuestFormat; } if (traceContents && @@ -6876,27 +12526,200 @@ internal static unsafe class VulkanVideoPresenter } } - private bool ShouldTraceGuestImageContents(GuestImageResource image) + private bool ShouldTraceGuestImageContents( + GuestImageResource image, + ulong shaderAddress = 0) { if (image.Address == 0) { return false; } + if (_traceGuestImageShaderFilterEnabled && + !AddressListContains( + "SHARPEMU_TRACE_GUEST_IMAGE_SHADER_ADDRS", + shaderAddress)) + { + return false; + } + + if ((_traceGuestImageWidth > 0 && image.Width != _traceGuestImageWidth) || + (_traceGuestImageHeight > 0 && image.Height != _traceGuestImageHeight)) + { + return false; + } + + if (!string.IsNullOrWhiteSpace(_traceGuestImageFormat) && + (!Enum.TryParse( + _traceGuestImageFormat, + ignoreCase: true, + out var expectedFormat) || + image.Format != expectedFormat)) + { + return false; + } + var addressMatched = ShouldTraceGuestImageAddressForDiagnostics(image.Address); + if (addressMatched && _traceGuestImageOccurrence > 0) + { + var count = _guestImageTraceCounts.TryGetValue(image.Address, out var previous) + ? previous + 1 + : 1; + _guestImageTraceCounts[image.Address] = count; + return count == _traceGuestImageOccurrence; + } + var broadTrace = ShouldTraceGuestImageContentsForDiagnostics() && image.Width >= 1280 && image.Height >= 720; + if (GuestImageTraceInterval() is { } interval) + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_IMAGE_ADDRS"); + if (!string.IsNullOrWhiteSpace(addressFilter) && !addressMatched) + { + return false; + } + + if (image.Width < 1280 || image.Height < 720) + { + return false; + } + + _globalGuestImageDrawCount++; + if (_globalGuestImageDrawCount < GuestImageTraceStartAfter() || + _intervalReadbackCount > 3000) + { + return false; + } + + var count = _guestImageTraceCounts.TryGetValue(image.Address, out var previous) + ? previous + 1 + : 1; + _guestImageTraceCounts[image.Address] = count; + return count % interval == 0; + } + return (addressMatched || broadTrace) && _tracedGuestImageContents.Add(image.Address); } - private static bool ShouldTraceGuestImageContentsForDiagnostics() => + private readonly Dictionary _guestImageTraceCounts = new(); + private long _globalGuestImageDrawCount; + private long _intervalReadbackCount; + + private static long? _cachedGuestImageTraceInterval = long.MinValue; + private static long _cachedGuestImageTraceStartAfter; + + // SHARPEMU_TRACE_GUEST_IMAGES=every:N[@M] — read back 1280x720+ guest + // images every Nth draw into each, starting after M total such draws. + private static long? GuestImageTraceInterval() + { + if (_cachedGuestImageTraceInterval == long.MinValue) + { + var mode = Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); + long? interval = null; + if (mode is not null && mode.StartsWith("every:", StringComparison.Ordinal)) + { + var spec = mode["every:".Length..]; + var at = spec.IndexOf('@'); + var intervalText = at < 0 ? spec : spec[..at]; + if (long.TryParse(intervalText, out var parsed) && parsed > 0) + { + interval = parsed; + } + + if (at >= 0 && long.TryParse(spec[(at + 1)..], out var after) && after > 0) + { + _cachedGuestImageTraceStartAfter = after; + } + } + + _cachedGuestImageTraceInterval = interval; + } + + return _cachedGuestImageTraceInterval; + } + + private static long GuestImageTraceStartAfter() + { + _ = GuestImageTraceInterval(); + return _cachedGuestImageTraceStartAfter; + } + + // Diagnostics toggles are read once: these run per draw / per cached + // texture hit, and env lookups plus string parsing are far too + // expensive there (and non-trivially so under Rosetta 2). + private static readonly string? _traceGuestImagesMode = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"); + private static readonly bool _traceGuestImagesEnabled = + string.Equals(_traceGuestImagesMode, "1", StringComparison.Ordinal); + private static readonly bool _tracePresentedGuestImagesEnabled = + _traceGuestImagesEnabled || + string.Equals(_traceGuestImagesMode, "present", StringComparison.OrdinalIgnoreCase); + private static readonly bool _traceVulkanResourcesEnabled = string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGES"), + Environment.GetEnvironmentVariable("SHARPEMU_LOG_VK_RESOURCES"), "1", StringComparison.Ordinal); + private static readonly bool _traceVulkanShaderEnabled = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), + "1", + StringComparison.Ordinal) || + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC_SHADER"), + "1", + StringComparison.Ordinal); + private static readonly bool _traceDepthInitialization = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_DEPTH_INIT"), + "1", + StringComparison.Ordinal); + private static readonly long _traceGuestImageOccurrence = + long.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_OCCURRENCE"), + out var traceGuestImageOccurrence) && + traceGuestImageOccurrence > 0 + ? traceGuestImageOccurrence + : 0; + private static readonly uint _traceGuestImageWidth = + uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_WIDTH"), + out var traceGuestImageWidth) + ? traceGuestImageWidth + : 0; + private static readonly uint _traceGuestImageHeight = + uint.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_HEIGHT"), + out var traceGuestImageHeight) + ? traceGuestImageHeight + : 0; + private static readonly string? _traceGuestImageFormat = + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_GUEST_IMAGE_FORMAT"); + private static readonly long _tracePresentedGuestImageOccurrence = + long.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_PRESENTED_GUEST_IMAGE_OCCURRENCE"), + out var tracePresentedGuestImageOccurrence) && + tracePresentedGuestImageOccurrence > 0 + ? tracePresentedGuestImageOccurrence + : 0; + private static readonly bool _traceGuestImageShaderFilterEnabled = + !string.IsNullOrWhiteSpace( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_IMAGE_SHADER_ADDRS")); + private static readonly bool _traceGuestImageAddressFilterEnabled = + !string.IsNullOrWhiteSpace( + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_GUEST_IMAGE_ADDRS")); + private static readonly System.Collections.Concurrent.ConcurrentDictionary< + string, + (bool Wildcard, ulong[] Addresses)> _cachedAddressLists = new(); + + private static bool ShouldTraceGuestImageContentsForDiagnostics() => + _traceGuestImagesEnabled; private static bool ShouldTraceGuestImageAddressForDiagnostics(ulong address) { @@ -6916,19 +12739,27 @@ internal static unsafe class VulkanVideoPresenter string environmentVariable, ulong address) { - var addresses = Environment.GetEnvironmentVariable(environmentVariable); + var (wildcard, addresses) = _cachedAddressLists.GetOrAdd( + environmentVariable, + static name => ParseAddressList(Environment.GetEnvironmentVariable(name))); + return wildcard || Array.IndexOf(addresses, address) >= 0; + } + + private static (bool Wildcard, ulong[] Addresses) ParseAddressList(string? addresses) + { if (string.IsNullOrWhiteSpace(addresses)) { - return false; + return (false, []); } + var parsedAddresses = new List(); foreach (var token in addresses.Split( [',', ';', ' ', '\t'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { if (token == "*") { - return true; + return (true, []); } var span = token.AsSpan(); @@ -6941,21 +12772,40 @@ internal static unsafe class VulkanVideoPresenter span, System.Globalization.NumberStyles.HexNumber, System.Globalization.CultureInfo.InvariantCulture, - out var parsed) && - parsed == address) + out var parsed)) { - return true; + parsedAddresses.Add(parsed); } } - return false; + return (false, parsedAddresses.ToArray()); + } + + private static bool ShouldTracePresentedGuestImageContentsForDiagnostics() => + _tracePresentedGuestImagesEnabled; + + private bool ShouldTraceAddressedPresentedGuestImage(GuestImageResource image) + { + if (!AddressListContains( + "SHARPEMU_TRACE_PRESENTED_GUEST_IMAGE_ADDRS", + image.Address)) + { + return false; + } + + var count = _presentedGuestImageTraceCounts.TryGetValue( + image.Address, + out var previous) + ? previous + 1 + : 1; + _presentedGuestImageTraceCounts[image.Address] = count; + return _tracePresentedGuestImageOccurrence == 0 + ? count == 1 + : count == _tracePresentedGuestImageOccurrence; } private static bool ShouldTraceVulkanResources() => - string.Equals( - Environment.GetEnvironmentVariable("SHARPEMU_LOG_VK_RESOURCES"), - "1", - StringComparison.Ordinal); + _traceVulkanResourcesEnabled; private void RecordTranslatedGraphicsPass( TranslatedDrawResources resources, @@ -6963,20 +12813,54 @@ internal static unsafe class VulkanVideoPresenter Framebuffer framebuffer, Extent2D extent) { - var clearValue = default(ClearValue); + BeginTranslatedRenderPass(renderPass, framebuffer, extent); + RecordTranslatedDrawInPass(resources, extent); + _vk.CmdEndRenderPass(_commandBuffer); + } + + private void BeginTranslatedRenderPass( + RenderPass renderPass, + Framebuffer framebuffer, + Extent2D extent, + int colorAttachmentCount = 1, + bool hasDepthAttachment = false, + float clearDepth = 1f) + { + colorAttachmentCount = Math.Max(colorAttachmentCount, 1); + var clearValueCount = colorAttachmentCount + (hasDepthAttachment ? 1 : 0); + var clearValues = stackalloc ClearValue[clearValueCount]; + for (var index = 0; index < colorAttachmentCount; index++) + { + clearValues[index] = default; + } + // Reverse-Z is not assumed; clear depth to 1.0 (far) so a standard + // LessOrEqual/Less test keeps the nearest fragment. + if (hasDepthAttachment) + { + clearValues[colorAttachmentCount] = new ClearValue + { + DepthStencil = new ClearDepthStencilValue(clearDepth, 0), + }; + } var renderPassInfo = new RenderPassBeginInfo { SType = StructureType.RenderPassBeginInfo, RenderPass = renderPass, Framebuffer = framebuffer, RenderArea = new Rect2D(new Offset2D(0, 0), extent), - ClearValueCount = 1, - PClearValues = &clearValue, + ClearValueCount = (uint)clearValueCount, + PClearValues = clearValues, }; _vk.CmdBeginRenderPass( _commandBuffer, &renderPassInfo, SubpassContents.Inline); + } + + private void RecordTranslatedDrawInPass( + TranslatedDrawResources resources, + Extent2D extent) + { _vk.CmdBindPipeline( _commandBuffer, PipelineBindPoint.Graphics, @@ -6998,7 +12882,6 @@ internal static unsafe class VulkanVideoPresenter var drawScissor = ClampScissor(resources.Scissor, extent); if (drawScissor.Width == 0 || drawScissor.Height == 0) { - _vk.CmdEndRenderPass(_commandBuffer); return; } @@ -7027,36 +12910,68 @@ internal static unsafe class VulkanVideoPresenter offsets); } - var scissor = new Rect2D( - new Offset2D(drawScissor.X, drawScissor.Y), - new Extent2D(drawScissor.Width, drawScissor.Height)); - _vk.CmdSetScissor(_commandBuffer, 0, 1, &scissor); + // Replaying a full-screen primitive once per 512x512 scissor tile + // multiplies an ordinary 4K composite into 32 complete draws. On + // MoltenVK this starves the render thread and makes the guest fall + // behind its own flip queue. Vulkan clips a normal fullscreen draw + // efficiently; keep tiling only as an explicit driver diagnostic. + var maxPixelsPerDraw = Environment.GetEnvironmentVariable( + "SHARPEMU_ENABLE_CHUNKED_DRAWS") == "1" + ? 512u * 512u + : uint.MaxValue; + var rowsPerDraw = Math.Max( + 1u, + Math.Min(drawScissor.Height, maxPixelsPerDraw / Math.Max(drawScissor.Width, 1u))); + var drawCount = 0u; + for (var y = 0u; y < drawScissor.Height; y += rowsPerDraw) + { + var scissor = new Rect2D( + new Offset2D( + drawScissor.X, + checked(drawScissor.Y + (int)y)), + new Extent2D( + drawScissor.Width, + Math.Min(rowsPerDraw, drawScissor.Height - y))); + _vk.CmdSetScissor(_commandBuffer, 0, 1, &scissor); - if (resources.IndexBuffer.Handle != 0) - { - _vk.CmdBindIndexBuffer( - _commandBuffer, - resources.IndexBuffer, - 0, - resources.Index32Bit ? IndexType.Uint32 : IndexType.Uint16); - _vk.CmdDrawIndexed( - _commandBuffer, - resources.VertexCount, - resources.InstanceCount, - 0, - 0, - 0); + if (resources.IndexBuffer.Handle != 0) + { + _vk.CmdBindIndexBuffer( + _commandBuffer, + resources.IndexBuffer, + 0, + resources.Index32Bit ? IndexType.Uint32 : IndexType.Uint16); + _vk.CmdDrawIndexed( + _commandBuffer, + resources.VertexCount, + resources.InstanceCount, + 0, + 0, + 0); + } + else + { + _vk.CmdDraw( + _commandBuffer, + resources.VertexCount, + resources.InstanceCount, + 0, + 0); + } + + drawCount++; } - else + + if (drawCount > 1) { - _vk.CmdDraw( - _commandBuffer, - resources.VertexCount, - resources.InstanceCount, - 0, - 0); + TraceVulkanShader( + $"vk.graphics_chunked target={extent.Width}x{extent.Height} " + + $"draws={drawCount} rows={rowsPerDraw} " + + $"scissor={drawScissor.X},{drawScissor.Y},{drawScissor.Width}x{drawScissor.Height} " + + $"viewport={drawViewport.X:0.###},{drawViewport.Y:0.###}," + + $"{drawViewport.Width:0.###}x{drawViewport.Height:0.###} " + + $"name={resources.DebugName}"); } - _vk.CmdEndRenderPass(_commandBuffer); } private void DestroyTranslatedDrawResources(TranslatedDrawResources resources) @@ -7073,7 +12988,7 @@ internal static unsafe class VulkanVideoPresenter foreach (var texture in resources.Textures) { - if (texture is null) + if (texture is null || texture.Cached) { continue; } @@ -7112,7 +13027,7 @@ internal static unsafe class VulkanVideoPresenter foreach (var globalBuffer in resources.GlobalMemoryBuffers) { - if (globalBuffer is null) + if (globalBuffer is null || globalBuffer.Allocation is not null) { continue; } @@ -7122,7 +13037,7 @@ internal static unsafe class VulkanVideoPresenter foreach (var vertexBuffer in resources.VertexBuffers) { - if (vertexBuffer is null) + if (vertexBuffer is null || !vertexBuffer.OwnsBuffer) { continue; } @@ -7139,7 +13054,14 @@ internal static unsafe class VulkanVideoPresenter if (resources.DescriptorPool.Handle != 0) { - _vk.DestroyDescriptorPool(_device, resources.DescriptorPool, null); + if (_recycledDescriptorPools.Count < 256) + { + _recycledDescriptorPools.Push(resources.DescriptorPool); + } + else + { + _vk.DestroyDescriptorPool(_device, resources.DescriptorPool, null); + } } if (!resources.DescriptorLayoutCached && @@ -7232,9 +13154,12 @@ internal static unsafe class VulkanVideoPresenter uint imageIndex, GuestImageResource source) { + var presentedCount = Interlocked.Increment(ref _presentedSwapchainCount); + var periodicDumpInterval = SwapchainDumpInterval(); var traceDestination = ShouldTracePresentedGuestImageContentsForDiagnostics() && - !_tracedPresentedSwapchain; + (!_tracedPresentedSwapchain || + periodicDumpInterval > 0 && presentedCount % periodicDumpInterval == 0); _tracedPresentedSwapchain |= traceDestination; BeginDebugLabel( _commandBuffer, @@ -7465,6 +13390,27 @@ internal static unsafe class VulkanVideoPresenter $"format={_swapchainFormat} nonzero_bytes={nonzeroBytes}/{byteCount} " + $"nonblack_pixels={nonblackPixels}/{(ulong)_extent.Width * _extent.Height} " + $"hash=0x{hash:X16}"); + + var dumpDir = Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_DUMP_DIR"); + if (!string.IsNullOrWhiteSpace(dumpDir)) + { + Directory.CreateDirectory(dumpDir); + var seq = Interlocked.Increment(ref _guestImageDumpSequence); + var path = Path.Combine( + dumpDir, + $"present-{seq:D4}-{_extent.Width}x{_extent.Height}-{_swapchainFormat}.bgra"); + File.WriteAllBytes(path, bytes.ToArray()); + Console.Error.WriteLine($"[LOADER][TRACE] vk.swapchain_dump path={path}"); + // Continuous readback is intentionally opt-in: each 1080p frame + // is several megabytes and synchronously waits for the GPU. + if (string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_GUEST_IMAGE_DUMP_CONTINUOUS"), + "1", + StringComparison.Ordinal)) + { + _tracedPresentedSwapchain = false; + } + } } finally { @@ -7472,6 +13418,16 @@ internal static unsafe class VulkanVideoPresenter } } + private static readonly long _swapchainDumpInterval = ParseSwapchainDumpInterval(); + + private static long SwapchainDumpInterval() => _swapchainDumpInterval; + + private static long ParseSwapchainDumpInterval() + { + var raw = Environment.GetEnvironmentVariable("SHARPEMU_SWAPCHAIN_DUMP_EVERY"); + return long.TryParse(raw, out var interval) && interval > 0 ? interval : 0; + } + private Extent2D ChooseExtent(SurfaceCapabilitiesKHR capabilities) { if (capabilities.CurrentExtent.Width != uint.MaxValue) @@ -7598,9 +13554,10 @@ internal static unsafe class VulkanVideoPresenter } _vulkanReady = false; _vk.DeviceWaitIdle(_device); - CompletePendingPresentation(wait: false); + SavePipelineCache(force: true); + DrainFrameSlots(); CollectCompletedGuestSubmissions(waitForOldest: false); - SavePipelineCache(); + ClearCachedTextureIdentities(); foreach (var pipeline in _computePipelines.Values) { _vk.DestroyPipeline(_device, pipeline, null); @@ -7611,11 +13568,6 @@ internal static unsafe class VulkanVideoPresenter _vk.DestroyPipeline(_device, pipeline, null); } _graphicsPipelines.Clear(); - if (_pipelineCache.Handle != 0) - { - _vk.DestroyPipelineCache(_device, _pipelineCache, null); - _pipelineCache = default; - } foreach (var layout in _descriptorLayouts.Values) { _vk.DestroyPipelineLayout(_device, layout.PipelineLayout, null); @@ -7634,6 +13586,12 @@ internal static unsafe class VulkanVideoPresenter } _samplers.Clear(); _shaderDigests.Clear(); + WriteBackAllDirtyGuestBuffers(); + foreach (var allocation in _guestBufferAllocations) + { + DestroyGuestBufferAllocation(allocation); + } + _guestBufferAllocations.Clear(); foreach (var allocation in _hostBufferAllocations.Values) { _vk.DestroyBuffer(_device, allocation.Buffer, null); @@ -7646,14 +13604,39 @@ internal static unsafe class VulkanVideoPresenter DestroyGuestImage(guestImage); } _guestImages.Clear(); + foreach (var guestImageVariant in _guestImageVariants.Values) + { + DestroyGuestImage(guestImageVariant); + } + _guestImageVariants.Clear(); + foreach (var guestImageVersion in _guestImageVersions.Values) + { + DestroyGuestImage(guestImageVersion); + } + _guestImageVersions.Clear(); + _capturedGuestFlipVersions.Clear(); + while (_deferredGuestImageVersionDestroys.TryDequeue(out var deferredVersion)) + { + DestroyGuestImage(deferredVersion.Image); + } + foreach (var guestDepth in _guestDepthImages.Values) + { + DestroyGuestDepth(guestDepth); + } + _guestDepthImages.Clear(); lock (_gate) { _availableGuestImages.Clear(); - _gpuGuestImages.Clear(); + _lastOrderedGuestFlipVersions.Clear(); } DestroySwapchainResources(); if (_device.Handle != 0) { + if (_pipelineCache.Handle != 0) + { + _vk.DestroyPipelineCache(_device, _pipelineCache, null); + _pipelineCache = default; + } _vk.DestroyDevice(_device, null); _device = default; } @@ -7707,7 +13690,7 @@ internal static unsafe class VulkanVideoPresenter Console.Error.WriteLine( $"[LOADER][INFO] Vulkan VideoOut recreating swapchain after {operation}: {result}"); _vk.DeviceWaitIdle(_device); - CompletePendingPresentation(wait: false); + DrainFrameSlots(); CollectCompletedGuestSubmissions(waitForOldest: false); DestroySwapchainResources(); CreateSwapchain(); @@ -7731,20 +13714,62 @@ internal static unsafe class VulkanVideoPresenter _stagingMemory = default; _stagingSize = 0; } - if (_imageAvailable.Handle != 0) + foreach (var semaphore in _frameImageAvailable) { - _vk.DestroySemaphore(_device, _imageAvailable, null); - _imageAvailable = default; + if (semaphore.Handle != 0) + { + _vk.DestroySemaphore(_device, semaphore, null); + } } - if (_renderFinished.Handle != 0) + _frameImageAvailable = []; + foreach (var semaphore in _renderFinishedPerImage) { - _vk.DestroySemaphore(_device, _renderFinished, null); - _renderFinished = default; + if (semaphore.Handle != 0) + { + _vk.DestroySemaphore(_device, semaphore, null); + } } - if (_presentationFence.Handle != 0) + _renderFinishedPerImage = []; + if (_overlayImage.Handle != 0) { - _vk.DestroyFence(_device, _presentationFence, null); - _presentationFence = default; + _vk.DestroyImage(_device, _overlayImage, null); + _overlayImage = default; + } + if (_overlayImageMemory.Handle != 0) + { + _vk.FreeMemory(_device, _overlayImageMemory, null); + _overlayImageMemory = default; + } + for (var slot = 0; slot < _overlayStagingBuffers.Length; slot++) + { + if (_overlayStagingBuffers[slot].Handle != 0) + { + _vk.DestroyBuffer(_device, _overlayStagingBuffers[slot], null); + } + if (_overlayStagingMemory[slot].Handle != 0) + { + _vk.FreeMemory(_device, _overlayStagingMemory[slot], null); + } + } + _overlayStagingBuffers = []; + _overlayStagingMemory = []; + _overlayStagingMapped = []; + _overlayImageInitialized = false; + foreach (var fence in _frameFences) + { + if (fence.Handle != 0) + { + _vk.DestroyFence(_device, fence, null); + } + } + _frameFences = []; + _frameFencePending = []; + _frameTimelines = []; + _frameTranslatedResources = []; + _frameGuestImageVersions = []; + while (_recycledGuestFences.TryPop(out var recycledFence)) + { + _vk.DestroyFence(_device, recycledFence, null); } if (_barycentricPipeline.Handle != 0) { @@ -7777,6 +13802,10 @@ internal static unsafe class VulkanVideoPresenter } if (_commandPool.Handle != 0) { + // Destroying the pool frees every command buffer allocated + // from it, including recycled and per-frame ones. + _recycledGuestCommandBuffers.Clear(); + _frameCommandBuffers = []; _vk.DestroyCommandPool(_device, _commandPool, null); _commandPool = default; _commandBuffer = default; @@ -7833,8 +13862,7 @@ internal static unsafe class VulkanVideoPresenter private static void TraceVulkanShader(string message) { - if (!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC"), "1", StringComparison.Ordinal) && - !string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_LOG_AGC_SHADER"), "1", StringComparison.Ordinal)) + if (!_traceVulkanShaderEnabled) { return; } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs index f5a1d52..5b28d7a 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.Alu.cs @@ -19,11 +19,77 @@ public static partial class Gen5SpirvTranslator return true; } + if (instruction.Control is Gen5SdwaControl sdwa && + (sdwa.Source0Select == 7 || + sdwa.Source1Select == 7 || + sdwa.DestinationSelect == 7 || + sdwa.DestinationUnused == 3)) + { + error = $"reserved SDWA selector/modifier in {instruction.Opcode}"; + return false; + } + + if (instruction.Control is Gen5DppControl dppControl && + !IsSupportedDppControl(dppControl.Control)) + { + error = $"unsupported DPP16 control 0x{dppControl.Control:X3}"; + return false; + } + if (instruction.Opcode.StartsWith("VCmp", StringComparison.Ordinal)) { return TryEmitVectorCompare(instruction, out error); } + if (instruction.Opcode == "VReadfirstlaneB32") + { + if (instruction.Destinations.Count == 0 || + instruction.Destinations[0].Kind != Gen5OperandKind.ScalarRegister || + instruction.Sources.Count == 0) + { + error = "invalid read-first-lane operands"; + return false; + } + + var value = GetRawSource(instruction, 0); + if (_subgroupInvocationIdInput != 0) + { + if (_emulateWave64) + { + value = BroadcastFirstWave64Active(value); + } + else + { + // SPIR-V's BroadcastFirst uses the first host-active + // invocation. Guest EXEC is modeled as data, so obtain the + // guest-active mask explicitly and broadcast from its first + // set lane instead. This also updates the private SGPR copy + // for lanes that are currently disabled and may be restored + // by a later saveexec sequence. + var activeLanes = _module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + _uvec4Type, + UInt(3), + Load(_boolType, _exec)); + var activeLow = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + activeLanes, + 0); + var firstActiveLane = Ext(73, _uintType, activeLow); + value = _module.AddInstruction( + SpirvOp.GroupNonUniformBroadcast, + _uintType, + UInt(3), + value, + firstActiveLane); + } + } + + StoreS(instruction.Destinations[0].Value, value); + return true; + } + if (instruction.Opcode == "VReadlaneB32") { return TryEmitReadlane(instruction, out error); @@ -39,7 +105,6 @@ public static partial class Gen5SpirvTranslator switch (instruction.Opcode) { case "VMovB32": - case "VReadfirstlaneB32": result = GetRawSource(instruction, 0); break; case "VWritelaneB32": @@ -288,8 +353,8 @@ public static partial class Gen5SpirvTranslator case "VFmaF32": case "VMadMkF32": case "VMadAkF32": - case "VFmamkF32": - case "VFmaakF32": + case "VFmaMkF32": + case "VFmaAkF32": result = EmitFloatResult( instruction, Ext( @@ -328,6 +393,12 @@ public static partial class Gen5SpirvTranslator case "VXorB32": result = EmitIntegerBinary(instruction, SpirvOp.BitwiseXor); break; + case "VXnorB32": + { + var xor = EmitIntegerBinary(instruction, SpirvOp.BitwiseXor); + result = _module.AddInstruction(SpirvOp.Not, _uintType, xor); + break; + } case "VNotB32": result = _module.AddInstruction( SpirvOp.Not, @@ -353,6 +424,7 @@ public static partial class Gen5SpirvTranslator result = EmitIntegerBinary(instruction, SpirvOp.IAdd); break; case "VAddcU32": + case "VAddCoCiU32": result = EmitAddWithCarry(instruction); break; case "VSubI32": @@ -421,12 +493,16 @@ public static partial class Gen5SpirvTranslator _longType, wideLeft, wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, + result = Bitcast( _uintType, - ShiftRightLogical64( - Bitcast(_ulongType, product), - _module.Constant64(_ulongType, 32))); + _module.AddInstruction( + SpirvOp.SConvert, + _intType, + _module.AddInstruction( + SpirvOp.ShiftRightArithmetic, + _longType, + product, + _module.Constant64(_longType, 32)))); break; } case "VBcntU32B32": @@ -437,6 +513,72 @@ public static partial class Gen5SpirvTranslator GetRawSource(instruction, 0)), GetRawSource(instruction, 1)); break; + case "VMbcntHiU32B32": + { + var guestLane = GuestWaveLane(); + var lane = BitwiseAnd(guestLane, UInt(31)); + var partialMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), lane), + UInt(1)); + var countedBits = _module.AddInstruction( + SpirvOp.BitCount, + _uintType, + BitwiseAnd(GetRawSource(instruction, 0), partialMask)); + var isUpperHalf = _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + guestLane, + UInt(32)); + result = IAdd( + GetRawSource(instruction, 1), + _module.AddInstruction( + SpirvOp.Select, + _uintType, + isUpperHalf, + countedBits, + UInt(0))); + break; + } + case "VMbcntLoU32B32": + { + var guestLane = GuestWaveLane(); + var lane = BitwiseAnd(guestLane, UInt(31)); + var partialMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), lane), + UInt(1)); + var lowBitsMask = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + guestLane, + UInt(32)), + UInt(uint.MaxValue), + partialMask); + result = IAdd( + GetRawSource(instruction, 1), + _module.AddInstruction( + SpirvOp.BitCount, + _uintType, + BitwiseAnd(GetRawSource(instruction, 0), lowBitsMask))); + break; + } + case "VBfmB32": + { + var width = BitwiseAnd(GetRawSource(instruction, 0), UInt(31)); + var lowMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), width), + UInt(1)); + result = ShiftLeftLogical(lowMask, GetRawSource(instruction, 1)); + break; + } case "VMadU32U24": { var left = BitwiseAnd( @@ -634,18 +776,6 @@ public static partial class Gen5SpirvTranslator Ext(38, _uintType, high, right)); break; } - case "VSadU32": - { - var left = GetRawSource(instruction, 0); - var right = GetRawSource(instruction, 1); - var difference = _module.AddInstruction( - SpirvOp.ISub, - _uintType, - Ext(41, _uintType, left, right), - Ext(38, _uintType, left, right)); - result = IAdd(difference, GetRawSource(instruction, 2)); - break; - } case "VMed3I32": { var left = Bitcast(_intType, GetRawSource(instruction, 0)); @@ -718,6 +848,50 @@ public static partial class Gen5SpirvTranslator StoreCarryOut(instruction, borrow); break; } + case "VMadU64U32": + { + // V_MAD_U64_U32 writes a 64-bit VGPR pair. The first two + // sources are 32-bit factors; the third is a 64-bit addend + // held in a VGPR or SGPR pair. Its SDST receives the carry + // mask for the unsigned 64-bit addition. + var wideLeft = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + GetRawSource(instruction, 0)); + var wideRight = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + GetRawSource(instruction, 1)); + var product = _module.AddInstruction( + SpirvOp.IMul, + _ulongType, + wideLeft, + wideRight); + var addend = GetRawSource64(instruction, 2); + var wideResult = _module.AddInstruction( + SpirvOp.IAdd, + _ulongType, + product, + addend); + var carry = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + wideResult, + addend); + result = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + wideResult); + var high = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + ShiftRightLogical64( + wideResult, + _module.Constant64(_ulongType, 32))); + StoreV(destination + 1, high); + StoreCarryOut(instruction, carry); + break; + } case "VBfeU32": { var width = BitwiseAnd(GetRawSource(instruction, 2), UInt(31)); @@ -753,21 +927,51 @@ public static partial class Gen5SpirvTranslator first, second); result = Ext(58, _uintType, vector); + StorePackedHalf( + destination, + vector); break; } - case "VCvtPkU16U32": - case "VCvtPkI16I32": - result = BitwiseOr( - BitwiseAnd(GetRawSource(instruction, 0), UInt(0xFFFF)), - ShiftLeftLogical( - BitwiseAnd(GetRawSource(instruction, 1), UInt(0xFFFF)), - UInt(16))); + case "VCvtPknormI16F32": + case "VCvtPknormU16F32": + { + var vector = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + GetFloatSource(instruction, 0), + GetFloatSource(instruction, 1)); + // GLSL.std.450 PackSnorm2x16 / PackUnorm2x16 match the + // RDNA saturated normalized conversion and packing rules. + result = Ext( + instruction.Opcode == "VCvtPknormI16F32" ? 56u : 57u, + _uintType, + vector); break; + } default: error = $"unsupported vector opcode {instruction.Opcode}"; return false; } + if (instruction.Control is Gen5DppControl dpp) + { + result = _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsDppWriteEnabled(dpp), + result, + LoadV(destination)); + } + + if (instruction.Control is Gen5SdwaControl destinationControl && + destinationControl.ScalarDestination is null) + { + result = ApplySdwaDestination( + destinationControl, + result, + LoadV(destination)); + } + StoreV(destination, result); return true; } @@ -894,21 +1098,17 @@ public static partial class Gen5SpirvTranslator { condition = _module.ConstantBool(true); } - else if (opcode is "VCmpOF32" or "VCmpxOF32" or "VCmpUF32" or "VCmpxUF32") + else if (opcode is + "VCmpOF32" or "VCmpxOF32" or + "VCmpUF32" or "VCmpxUF32") { - // The ordered/unordered predicates only test whether either - // operand is NaN. SPIR-V's OpOrdered/OpUnordered are Kernel-only, - // so build the same result from OpIsNan, which needs no extra - // capability: unordered = isnan(a) || isnan(b), ordered = !that. var left = GetFloatSource(instruction, 0); var right = GetFloatSource(instruction, 1); - var nanLeft = _module.AddInstruction(SpirvOp.IsNan, _boolType, left); - var nanRight = _module.AddInstruction(SpirvOp.IsNan, _boolType, right); var unordered = _module.AddInstruction( SpirvOp.LogicalOr, _boolType, - nanLeft, - nanRight); + _module.AddInstruction(SpirvOp.IsNan, _boolType, left), + _module.AddInstruction(SpirvOp.IsNan, _boolType, right)); condition = opcode is "VCmpUF32" or "VCmpxUF32" ? unordered : _module.AddInstruction(SpirvOp.LogicalNot, _boolType, unordered); @@ -927,11 +1127,11 @@ public static partial class Gen5SpirvTranslator "VCmpLgF32" or "VCmpxLgF32" => SpirvOp.FOrdNotEqual, "VCmpGeF32" or "VCmpxGeF32" => SpirvOp.FOrdGreaterThanEqual, "VCmpNeqF32" or "VCmpxNeqF32" => SpirvOp.FUnordNotEqual, - "VCmpNlgF32" or "VCmpxNlgF32" => SpirvOp.FUnordEqual, "VCmpNltF32" or "VCmpxNltF32" => SpirvOp.FUnordGreaterThanEqual, "VCmpNleF32" or "VCmpxNleF32" => SpirvOp.FUnordGreaterThan, "VCmpNgtF32" or "VCmpxNgtF32" => SpirvOp.FUnordLessThanEqual, "VCmpNgeF32" or "VCmpxNgeF32" => SpirvOp.FUnordLessThan, + "VCmpNlgF32" or "VCmpxNlgF32" => SpirvOp.FUnordEqual, _ => SpirvOp.Nop, }; if (operation == SpirvOp.Nop) @@ -978,20 +1178,52 @@ public static partial class Gen5SpirvTranslator condition = _module.AddInstruction(operation, _boolType, left, right); } - // On gfx10, VCmpx writes EXEC only and preserves VCC; the sdst - // operand was removed from the cmpx encodings on this generation. + if (_state.Program.Address == 0x0000000500781200ul && + ((instruction.Pc == 0x4D4 && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_COMPARE_4D4") == "1") || + (instruction.Pc == 0x540 && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_COMPARE_540") == "1"))) + { + condition = _module.ConstantBool(true); + } + + // Vector compares fully overwrite the destination mask, but only + // lanes enabled by EXEC can pass the test: VCC = EXEC & condition. + // Balloting the raw condition leaks results from disabled lanes + // into later saveexec/branch sequences. + var activeCondition = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + Load(_boolType, _exec), + condition); + if (instruction.Control is Gen5DppControl compareDpp) + { + activeCondition = _module.AddInstruction( + SpirvOp.Select, + _boolType, + IsDppWriteEnabled(compareDpp), + activeCondition, + Load(_boolType, _vcc)); + } + if (opcode.StartsWith("VCmpx", StringComparison.Ordinal)) { - var active = _module.AddInstruction( - SpirvOp.LogicalAnd, - _boolType, - Load(_boolType, _exec), - condition); - StoreWaveMask(126, active); + // GFX10 VCMPX is EXEC-only. The SDWA bits that older + // generations exposed as an explicit scalar destination must + // not overwrite VCC/an SGPR pair here. Guest shaders rely on + // those registers retaining a saved EXEC mask for later + // reconvergence. + StoreWaveMask(126, activeCondition); } else { - StoreWaveMask(106, condition); + var compareDestination = instruction.Control is Gen5SdwaControl + { ScalarDestination: { } scalarDestination } + ? scalarDestination + : 106u; + StoreWaveMask(compareDestination, activeCondition); } return true; @@ -1062,6 +1294,56 @@ public static partial class Gen5SpirvTranslator } var left = GetRawSource(instruction, 0); + if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) + { + var oldExec64 = BooleanToWaveMask(Load(_boolType, _exec)); + var oldExec = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + oldExec64); + var notLeft = _module.AddInstruction(SpirvOp.Not, _uintType, left); + var notOldExec = _module.AddInstruction(SpirvOp.Not, _uintType, oldExec); + var newExec = instruction.Opcode switch + { + "SAndSaveexecB32" => BitwiseAnd(oldExec, left), + "SOrSaveexecB32" => BitwiseOr(oldExec, left), + "SXorSaveexecB32" => _module.AddInstruction( + SpirvOp.BitwiseXor, _uintType, oldExec, left), + "SAndn1SaveexecB32" => BitwiseAnd(notLeft, oldExec), + "SAndn2SaveexecB32" => BitwiseAnd(left, notOldExec), + "SOrn1SaveexecB32" => BitwiseOr(notLeft, oldExec), + "SOrn2SaveexecB32" => BitwiseOr(left, notOldExec), + "SNandSaveexecB32" => _module.AddInstruction( + SpirvOp.Not, _uintType, BitwiseAnd(left, oldExec)), + "SNorSaveexecB32" => _module.AddInstruction( + SpirvOp.Not, _uintType, BitwiseOr(left, oldExec)), + "SXnorSaveexecB32" => _module.AddInstruction( + SpirvOp.Not, + _uintType, + _module.AddInstruction( + SpirvOp.BitwiseXor, + _uintType, + left, + oldExec)), + _ => 0u, + }; + if (newExec == 0) + { + error = $"unsupported scalar 32-bit saveexec opcode {instruction.Opcode}"; + return false; + } + + StoreS(destination, oldExec); + // B32 saveexec is the Wave32 form; EXEC_HI is zero. + var newExec64 = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + newExec); + StoreS64(126, newExec64); + Store(_scc, IsNotZero(newExec)); + return true; + } + uint result; switch (instruction.Opcode) { @@ -1224,19 +1506,11 @@ public static partial class Gen5SpirvTranslator break; case "SMulHiU32": { - var wideLeft = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - left); - var wideRight = _module.AddInstruction( - SpirvOp.UConvert, - _ulongType, - right); var product = _module.AddInstruction( SpirvOp.IMul, _ulongType, - wideLeft, - wideRight); + _module.AddInstruction(SpirvOp.UConvert, _ulongType, left), + _module.AddInstruction(SpirvOp.UConvert, _ulongType, right)); result = _module.AddInstruction( SpirvOp.UConvert, _uintType, @@ -1245,29 +1519,6 @@ public static partial class Gen5SpirvTranslator _module.Constant64(_ulongType, 32))); break; } - case "SMulHiI32": - { - var wideLeft = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, left)); - var wideRight = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, right)); - var product = _module.AddInstruction( - SpirvOp.IMul, - _longType, - wideLeft, - wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - ShiftRightLogical64( - Bitcast(_ulongType, product), - _module.Constant64(_ulongType, 32))); - break; - } case "SAndB32": result = BitwiseAnd(left, right); Store(_scc, IsNotZero(result)); @@ -1389,43 +1640,6 @@ public static partial class Gen5SpirvTranslator Store(_scc, IsNotZero(result)); break; } - case "SAbsdiffI32": - { - var wideLeft = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, left)); - var wideRight = _module.AddInstruction( - SpirvOp.SConvert, - _longType, - Bitcast(_intType, right)); - var difference = _module.AddInstruction( - SpirvOp.ISub, - _longType, - wideLeft, - wideRight); - result = _module.AddInstruction( - SpirvOp.UConvert, - _uintType, - Ext(5, _longType, difference)); - Store(_scc, IsNotZero(result)); - break; - } - case "SPackLlB32B16": - result = BitwiseOr( - BitwiseAnd(left, UInt(0xFFFF)), - ShiftLeftLogical(right, UInt(16))); - break; - case "SPackLhB32B16": - result = BitwiseOr( - BitwiseAnd(left, UInt(0xFFFF)), - BitwiseAnd(right, UInt(0xFFFF0000))); - break; - case "SPackHhB32B16": - result = BitwiseOr( - ShiftRightLogical(left, UInt(16)), - BitwiseAnd(right, UInt(0xFFFF0000))); - break; case "SCselectB32": result = _module.AddInstruction( SpirvOp.Select, @@ -1489,6 +1703,21 @@ public static partial class Gen5SpirvTranslator right); break; } + case "SPackLlB32B16": + result = BitwiseOr( + BitwiseAnd(left, UInt(0xFFFF)), + ShiftLeftLogical(right, UInt(16))); + break; + case "SPackLhB32B16": + result = BitwiseOr( + BitwiseAnd(left, UInt(0xFFFF)), + BitwiseAnd(right, UInt(0xFFFF0000))); + break; + case "SPackHhB32B16": + result = BitwiseOr( + ShiftRightLogical(left, UInt(16)), + BitwiseAnd(right, UInt(0xFFFF0000))); + break; default: error = $"unsupported scalar opcode {instruction.Opcode}"; return false; @@ -1610,7 +1839,7 @@ public static partial class Gen5SpirvTranslator var left = GetRawSource64(instruction, 0); if (instruction.Opcode.EndsWith("SaveexecB64", StringComparison.Ordinal)) { - var oldExec = BooleanToLaneMask(Load(_boolType, _exec)); + var oldExec = BooleanToWaveMask(Load(_boolType, _exec)); var notLeft = _module.AddInstruction(SpirvOp.Not, _ulongType, left); var newExec = instruction.Opcode switch { @@ -1798,11 +2027,70 @@ public static partial class Gen5SpirvTranslator return true; } + if (instruction.Opcode == "SBfmB64") + { + if (instruction.Sources.Count < 2) + { + error = "missing scalar 64-bit bitfield-mask source"; + return false; + } + + var width = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + BitwiseAnd(GetRawSource(instruction, 0), UInt(63))); + var offset = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + BitwiseAnd(GetRawSource(instruction, 1), UInt(63))); + // Width is masked to 0..63, so (1 << width) never invokes an + // undefined 64-bit shift. This naturally yields zero for a + // zero-width mask and avoids OpBitFieldInsert, which the + // MoltenVK/SPIRV-Cross path rejects for 64-bit integers. + var lowMask = _module.AddInstruction( + SpirvOp.ISub, + _ulongType, + ShiftLeftLogical64( + _module.Constant64(_ulongType, 1), + width), + _module.Constant64(_ulongType, 1)); + var maskValue = ShiftLeftLogical64(lowMask, offset); + StoreS64(destination, maskValue); + Store(_scc, IsNotZero64(maskValue)); + return true; + } + uint value; - if (instruction.Opcode is "SMovB64" or "SWqmB64") + if (instruction.Opcode == "SMovB64") { value = left; } + else if (instruction.Opcode == "SWqmB64") + { + var quadAny = _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + left, + _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + ShiftRightLogical64(left, _module.Constant64(_ulongType, 1)), + _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + ShiftRightLogical64(left, _module.Constant64(_ulongType, 2)), + ShiftRightLogical64(left, _module.Constant64(_ulongType, 3))))); + quadAny = _module.AddInstruction( + SpirvOp.BitwiseAnd, + _ulongType, + quadAny, + _module.Constant64(_ulongType, 0x1111_1111_1111_1111UL)); + value = _module.AddInstruction( + SpirvOp.IMul, + _ulongType, + quadAny, + _module.Constant64(_ulongType, 0xFUL)); + } else if (instruction.Opcode == "SNotB64") { value = _module.AddInstruction(SpirvOp.Not, _ulongType, left); @@ -1896,7 +2184,8 @@ public static partial class Gen5SpirvTranslator private uint GetRawSource( Gen5ShaderInstruction instruction, - int sourceIndex) + int sourceIndex, + bool applySdwaIntegerModifiers = true) { if ((uint)sourceIndex >= instruction.Sources.Count) { @@ -1909,12 +2198,48 @@ public static partial class Gen5SpirvTranslator Gen5OperandKind.VectorRegister => LoadV(operand.Value), Gen5OperandKind.ScalarRegister => LoadS(operand.Value), Gen5OperandKind.LiteralConstant => UInt(operand.Value), + Gen5OperandKind.EncodedConstant when operand.Value == 251 => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + LogicalNot(SubgroupAny(Load(_boolType, _vcc))), + UInt(1), + UInt(0)), + Gen5OperandKind.EncodedConstant when operand.Value == 252 => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + LogicalNot(SubgroupAny(Load(_boolType, _exec))), + UInt(1), + UInt(0)), + Gen5OperandKind.EncodedConstant when operand.Value == 253 => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _scc), + UInt(1), + UInt(0)), Gen5OperandKind.EncodedConstant when TryDecodeInlineConstant( operand.Value, out var inline) => UInt(inline), _ => throw new InvalidOperationException($"unsupported source {operand}"), }; + // DPP16 remaps src0 across lanes before the VALU operation. The IR + // has always preserved this control, but treating it as an ordinary + // local VGPR read breaks every wave reduction used by the XPR + // renderer (min/max/OR scans become value-with-self operations). + if (sourceIndex == 0 && + instruction.Control is Gen5DppControl dpp) + { + value = ApplyDppSource(dpp, value); + } + else if (sourceIndex == 0 && + instruction.Control is Gen5Dpp8Control dpp8) + { + value = ApplyDpp8Source(dpp8, value); + } + if (instruction.Control is Gen5SdwaControl sdwa) { var selector = sourceIndex switch @@ -1933,11 +2258,326 @@ public static partial class Gen5SpirvTranslator 5 => BitwiseAnd(ShiftRightLogical(value, UInt(16)), UInt(0xFFFF)), _ => value, }; + var signExtend = sourceIndex switch + { + 0 => sdwa.Source0SignExtend, + 1 => sdwa.Source1SignExtend, + _ => false, + }; + if (signExtend && selector != 6) + { + var width = selector <= 3 ? 8u : 16u; + value = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, value), + UInt(0), + UInt(width))); + } + + if (applySdwaIntegerModifiers) + { + if ((sdwa.AbsoluteMask & (1u << sourceIndex)) != 0) + { + value = Bitcast( + _uintType, + Ext(5, _intType, Bitcast(_intType, value))); + } + + if ((sdwa.NegateMask & (1u << sourceIndex)) != 0) + { + value = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + UInt(0), + value); + } + } } return value; } + private uint ApplyDpp8Source(Gen5Dpp8Control control, uint value) + { + var lane = GuestWaveLane(); + var laneInGroup = BitwiseAnd(lane, UInt(7)); + var selector = UInt(control.LaneSelectors & 7); + for (var index = 1u; index < 8; index++) + { + selector = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + laneInGroup, + UInt(index)), + UInt((control.LaneSelectors >> checked((int)(index * 3))) & 7), + selector); + } + + var targetLane = IAdd(BitwiseAnd(lane, UInt(0xFFFF_FFF8)), selector); + targetLane = BitwiseAnd(targetLane, UInt(31)); + var shuffled = _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + targetLane); + if (control.FetchInactive) + { + return shuffled; + } + + var activeWord = _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _exec), + UInt(1), + UInt(0)); + var sourceActive = IsNotZero( + _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + activeWord, + targetLane)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + sourceActive, + shuffled, + UInt(0)); + } + + private uint ApplyDppSource(Gen5DppControl control, uint value) + { + GetDppSourceLane(control, out var targetLane, out var inRange); + var lane = GuestWaveLane(); + var safeTarget = _module.AddInstruction( + SpirvOp.Select, + _uintType, + inRange, + targetLane, + lane); + safeTarget = BitwiseAnd(safeTarget, UInt(31)); + var shuffled = _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + value, + safeTarget); + + var sourceAvailable = inRange; + if (!control.FetchInactive) + { + var activeWord = _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _exec), + UInt(1), + UInt(0)); + var shuffledActive = _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + activeWord, + safeTarget); + sourceAvailable = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + sourceAvailable, + IsNotZero(shuffledActive)); + } + + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + sourceAvailable, + shuffled, + UInt(0)); + } + + private uint ApplySdwaDestination( + Gen5SdwaControl control, + uint value, + uint previous) + { + var (shift, width) = control.DestinationSelect switch + { + 0 => (0u, 8u), + 1 => (8u, 8u), + 2 => (16u, 8u), + 3 => (24u, 8u), + 4 => (0u, 16u), + 5 => (16u, 16u), + _ => (0u, 32u), + }; + if (width == 32) + { + return value; + } + + var lowMask = width == 8 ? 0xFFu : 0xFFFFu; + var fieldMask = lowMask << checked((int)shift); + var upperStart = shift + width; + var upperMask = upperStart == 32 + ? 0u + : uint.MaxValue << checked((int)upperStart); + var positioned = ShiftLeftLogical( + BitwiseAnd(value, UInt(lowMask)), + UInt(shift)); + return control.DestinationUnused switch + { + 0 => positioned, + 1 => BitwiseOr( + positioned, + _module.AddInstruction( + SpirvOp.Select, + _uintType, + IsNotZero(BitwiseAnd(positioned, UInt(1u << checked((int)(shift + width - 1))))), + UInt(upperMask), + UInt(0))), + 2 => BitwiseOr( + BitwiseAnd(previous, UInt(~fieldMask)), + positioned), + _ => throw new InvalidOperationException("reserved SDWA destination-unused mode"), + }; + } + + private static bool IsSupportedDppControl(uint control) => + control <= 0xFF || + control is >= 0x101 and <= 0x10F or + >= 0x111 and <= 0x11F or + >= 0x121 and <= 0x12F or + 0x140 or 0x141 or + >= 0x150 and <= 0x15F or + >= 0x160 and <= 0x16F; + + private void GetDppSourceLane( + Gen5DppControl control, + out uint targetLane, + out uint inRange) + { + var lane = GuestWaveLane(); + var rowBase = BitwiseAnd(lane, UInt(0xFFFF_FFF0)); + var rowLane = BitwiseAnd(lane, UInt(15)); + var dpp = control.Control; + inRange = _module.ConstantBool(true); + + if (dpp <= 0xFF) + { + var quadLane = BitwiseAnd(lane, UInt(3)); + var selected = UInt(dpp & 3); + for (var index = 1u; index < 4; index++) + { + selected = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + quadLane, + UInt(index)), + UInt((dpp >> checked((int)(index * 2))) & 3), + selected); + } + + targetLane = IAdd(BitwiseAnd(lane, UInt(0xFFFF_FFFC)), selected); + return; + } + + if (dpp is >= 0x101 and <= 0x10F) + { + var shift = UInt(dpp & 15); + var shifted = IAdd(rowLane, shift); + inRange = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + shifted, + UInt(16)); + targetLane = IAdd(rowBase, BitwiseAnd(shifted, UInt(15))); + return; + } + + if (dpp is >= 0x111 and <= 0x11F) + { + var shift = UInt(dpp & 15); + inRange = _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + rowLane, + shift); + targetLane = IAdd( + rowBase, + BitwiseAnd( + _module.AddInstruction(SpirvOp.ISub, _uintType, rowLane, shift), + UInt(15))); + return; + } + + if (dpp is >= 0x121 and <= 0x12F) + { + targetLane = IAdd( + rowBase, + BitwiseAnd( + _module.AddInstruction( + SpirvOp.ISub, + _uintType, + rowLane, + UInt(dpp & 15)), + UInt(15))); + return; + } + + targetLane = dpp switch + { + 0x140 => IAdd(rowBase, _module.AddInstruction( + SpirvOp.ISub, _uintType, UInt(15), rowLane)), + 0x141 => IAdd( + BitwiseAnd(lane, UInt(0xFFFF_FFF8)), + _module.AddInstruction( + SpirvOp.ISub, + _uintType, + UInt(7), + BitwiseAnd(lane, UInt(7)))), + >= 0x150 and <= 0x15F => IAdd(rowBase, UInt(dpp & 15)), + >= 0x160 and <= 0x16F => IAdd( + rowBase, + BitwiseXor(rowLane, UInt(dpp & 15))), + _ => lane, + }; + } + + private uint IsDppWriteEnabled(Gen5DppControl control) + { + GetDppSourceLane(control, out _, out var inRange); + var lane = GuestWaveLane(); + var row = ShiftRightLogical(lane, UInt(4)); + var bank = BitwiseAnd(lane, UInt(3)); + var rowEnabled = IsNotZero(BitwiseAnd( + UInt(control.RowMask), + ShiftLeftLogical(UInt(1), row))); + var bankEnabled = IsNotZero(BitwiseAnd( + UInt(control.BankMask), + ShiftLeftLogical(UInt(1), bank))); + var sourceAllowsWrite = control.BoundControl + ? _module.ConstantBool(true) + : inRange; + return _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + rowEnabled, + _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + bankEnabled, + sourceAllowsWrite)); + } + private uint GetFloatSource( Gen5ShaderInstruction instruction, int sourceIndex) @@ -1956,7 +2596,12 @@ public static partial class Gen5SpirvTranslator } else { - value = Bitcast(_floatType, GetRawSource(instruction, sourceIndex)); + value = Bitcast( + _floatType, + GetRawSource( + instruction, + sourceIndex, + applySdwaIntegerModifiers: false)); } uint absoluteMask = 0; @@ -2000,6 +2645,35 @@ public static partial class Gen5SpirvTranslator return LoadS64(operand.Value); } + if (operand.Kind == Gen5OperandKind.VectorRegister) + { + var vectorLow = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + LoadV(operand.Value)); + var high = _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + LoadV(operand.Value + 1)); + high = ShiftLeftLogical64( + high, + _module.Constant64(_ulongType, 32)); + return _module.AddInstruction( + SpirvOp.BitwiseOr, + _ulongType, + vectorLow, + high); + } + + // Scalar inline negative constants are signed immediates. B64 + // consumers sign-extend them, so -1 denotes a full 64-bit mask. + if (operand.Kind == Gen5OperandKind.EncodedConstant && + operand.Value is >= 193 and <= 208) + { + var signed = -(long)(operand.Value - 192); + return _module.Constant64(_ulongType, unchecked((ulong)signed)); + } + var low = GetRawSource(instruction, sourceIndex); return _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); } @@ -2296,10 +2970,13 @@ public static partial class Gen5SpirvTranslator { var left = GetRawSource(instruction, 0); var right = GetRawSource(instruction, 1); + var carryMask = instruction.Sources.Count > 2 + ? IsCurrentLaneSet(GetRawSource64(instruction, 2)) + : Load(_boolType, _vcc); var carryIn = _module.AddInstruction( SpirvOp.Select, _uintType, - Load(_boolType, _vcc), + carryMask, UInt(1), UInt(0)); var partial = IAdd(left, right); @@ -2309,7 +2986,7 @@ public static partial class Gen5SpirvTranslator _boolType, _module.AddInstruction(SpirvOp.ULessThan, _boolType, partial, left), _module.AddInstruction(SpirvOp.ULessThan, _boolType, result, partial)); - StoreWaveMask(106, carry); + StoreCarryOut(instruction, carry); return result; } @@ -2319,10 +2996,13 @@ public static partial class Gen5SpirvTranslator { var left = GetRawSource(instruction, reverse ? 1 : 0); var right = GetRawSource(instruction, reverse ? 0 : 1); + var borrowMask = instruction.Sources.Count > 2 + ? IsCurrentLaneSet(GetRawSource64(instruction, 2)) + : Load(_boolType, _vcc); var borrowIn = _module.AddInstruction( SpirvOp.Select, _uintType, - Load(_boolType, _vcc), + borrowMask, UInt(1), UInt(0)); var partial = _module.AddInstruction(SpirvOp.ISub, _uintType, left, right); @@ -2340,7 +3020,67 @@ public static partial class Gen5SpirvTranslator _boolType, partial, borrowIn)); - StoreWaveMask(106, borrow); + StoreCarryOut(instruction, borrow); + return result; + } + + private uint BroadcastFirstWave64Active(uint value) + { + var lane = GuestWaveLane(); + EmitConditional( + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + lane, + UInt(0)), + () => Store(WaveBroadcastScratchPointer(), UInt(0))); + EmitWave64Barrier(); + + var activeMask = BooleanToWaveMask(Load(_boolType, _exec)); + var lowMask = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + activeMask); + var highMask = _module.AddInstruction( + SpirvOp.UConvert, + _uintType, + ShiftRightLogical64( + activeMask, + _module.Constant64(_ulongType, 32))); + var hasLow = IsNotZero(lowMask); + var hasHigh = IsNotZero(highMask); + var firstLow = Ext(73, _uintType, lowMask); + var firstHigh = IAdd(UInt(32), Ext(73, _uintType, highMask)); + var firstLane = _module.AddInstruction( + SpirvOp.Select, + _uintType, + hasLow, + firstLow, + _module.AddInstruction( + SpirvOp.Select, + _uintType, + hasHigh, + firstHigh, + UInt(0))); + var isFirst = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + lane, + firstLane); + EmitConditional( + _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + isFirst, + _module.AddInstruction( + SpirvOp.LogicalOr, + _boolType, + hasLow, + hasHigh)), + () => Store(WaveBroadcastScratchPointer(), value)); + EmitWave64Barrier(); + var result = Load(_uintType, WaveBroadcastScratchPointer()); + EmitWave64Barrier(); return result; } @@ -2348,25 +3088,18 @@ public static partial class Gen5SpirvTranslator Gen5ShaderInstruction instruction, uint carry) { + var activeCarry = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + Load(_boolType, _exec), + carry); if (instruction.Control is Gen5Vop3Control { ScalarDestination: { } register }) { - StoreS( - register, - _module.AddInstruction( - SpirvOp.Select, - _uintType, - carry, - UInt(1), - UInt(0))); - if (register == 106) - { - StoreWaveMask(106, carry); - } - + StoreWaveMask(register, activeCarry); return; } - StoreWaveMask(106, carry); + StoreWaveMask(106, activeCarry); } private bool TryEmitReadlane( @@ -2409,10 +3142,21 @@ public static partial class Gen5SpirvTranslator Gen5ShaderInstruction instruction, bool exchangeRows) { + if (instruction.Control is not Gen5Vop3Control control || + (control.OperandSelect & ~3u) != 0 || + control.AbsoluteMask != 0 || + control.NegateMask != 0 || + control.OutputModifier != 0 || + control.Clamp) + { + throw new InvalidOperationException( + $"invalid permlane modifiers for {instruction.Opcode}"); + } + var value = GetRawSource(instruction, 0); var selectorLow = GetRawSource(instruction, 1); var selectorHigh = GetRawSource(instruction, 2); - var lane = Load(_uintType, _subgroupInvocationIdInput); + var lane = GuestWaveLane(); var localLane = BitwiseAnd(lane, UInt(15)); var lowHalf = _module.AddInstruction( SpirvOp.ULessThan, @@ -2445,12 +3189,38 @@ public static partial class Gen5SpirvTranslator } var targetLane = IAdd(rowBase, selector); - return _module.AddInstruction( + targetLane = BitwiseAnd(targetLane, UInt(31)); + var shuffled = _module.AddInstruction( SpirvOp.GroupNonUniformShuffle, _uintType, UInt(3), value, targetLane); + var fetchInactive = (control.OperandSelect & 1) != 0; + if (fetchInactive) + { + return shuffled; + } + + var activeWord = _module.AddInstruction( + SpirvOp.Select, + _uintType, + Load(_boolType, _exec), + UInt(1), + UInt(0)); + var sourceActive = IsNotZero( + _module.AddInstruction( + SpirvOp.GroupNonUniformShuffle, + _uintType, + UInt(3), + activeWord, + targetLane)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + sourceActive, + shuffled, + UInt(0)); } private uint EmitFloatResult( diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs index c2cbebe..60722e2 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/Gen5SpirvTranslator.cs @@ -10,6 +10,12 @@ public static partial class Gen5SpirvTranslator private const uint ScalarRegisterCount = 256; private const uint VectorRegisterCount = 512; private const uint LdsDwordCount = 8192; + // Graphics stages model LDS as a per-invocation Private array rather than + // real workgroup-shared memory. A full 32 KB Private array per vertex/pixel + // invocation is wasteful and risks Metal compile limits, and per-invocation + // write-then-read correctness only needs deterministic address→slot masking, + // so a smaller array is safe. + private const uint PrivateLdsDwordCount = 2048; private const uint RdnaWaveLaneCount = 32; public static bool TryCompilePixelShader( @@ -21,17 +27,24 @@ public static partial class Gen5SpirvTranslator int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) => + int initialScalarBufferIndex = -1, + int pixelRenderTargetSlot = 0, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1) => TryCompilePixelShader( state, evaluation, - [new Gen5PixelOutputBinding(0, 0, outputKind)], + [new Gen5PixelOutputBinding((uint)pixelRenderTargetSlot, 0, outputKind)], out shader, out error, globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex); + initialScalarBufferIndex, + pixelInputEnable, + pixelInputAddress, + storageBufferOffsetAlignment); public static bool TryCompilePixelShader( Gen5ShaderState state, @@ -42,7 +55,10 @@ public static partial class Gen5SpirvTranslator int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int initialScalarBufferIndex = -1, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + ulong storageBufferOffsetAlignment = 1) { if (outputs.Count > 8 || outputs.Any(output => output.GuestSlot > 7)) { @@ -80,7 +96,10 @@ public static partial class Gen5SpirvTranslator globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex); + initialScalarBufferIndex, + pixelInputEnable: pixelInputEnable, + pixelInputAddress: pixelInputAddress, + storageBufferOffsetAlignment: storageBufferOffsetAlignment); return context.TryCompile(out shader, out error); } @@ -92,7 +111,9 @@ public static partial class Gen5SpirvTranslator int globalBufferBase = 0, int totalGlobalBufferCount = -1, int imageBindingBase = 0, - int scalarRegisterBufferIndex = -1) + int initialScalarBufferIndex = -1, + int requiredVertexOutputCount = 0, + ulong storageBufferOffsetAlignment = 1) { var context = new CompilationContext( Gen5SpirvStage.Vertex, @@ -105,7 +126,9 @@ public static partial class Gen5SpirvTranslator globalBufferBase, totalGlobalBufferCount, imageBindingBase, - scalarRegisterBufferIndex); + initialScalarBufferIndex, + requiredVertexOutputCount: requiredVertexOutputCount, + storageBufferOffsetAlignment: storageBufferOffsetAlignment); return context.TryCompile(out shader, out error); } @@ -116,7 +139,11 @@ public static partial class Gen5SpirvTranslator uint localSizeY, uint localSizeZ, out Gen5SpirvShader shader, - out string error) + out string error, + int totalGlobalBufferCount = -1, + int initialScalarBufferIndex = -1, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1) { var context = new CompilationContext( Gen5SpirvStage.Compute, @@ -127,26 +154,79 @@ public static partial class Gen5SpirvTranslator Math.Max(localSizeY, 1), Math.Max(localSizeZ, 1), 0, - -1, + totalGlobalBufferCount, 0, - -1); + initialScalarBufferIndex, + waveLaneCount: waveLaneCount, + storageBufferOffsetAlignment: storageBufferOffsetAlignment); return context.TryCompile(out shader, out error); } private sealed partial class CompilationContext { + private const uint ImageDescriptorDwords = 8; + private const uint SamplerDescriptorDwords = 4; + private const int ScalarRegisterCount = 128; + private const long InitialScalarDefinition = -1; + private const long ConflictingScalarDefinition = -2; + private const long UnreachableScalarDefinition = -3; + private readonly SpirvModuleBuilder _module = new(); private readonly Gen5SpirvStage _stage; private readonly Gen5ShaderState _state; private readonly Gen5ShaderEvaluation _evaluation; private readonly IReadOnlyList _pixelOutputBindings; + private readonly uint _waveLaneCount; + private readonly bool _emulateWave64; + + // Safety valve for the PC-dispatcher loop. Each iteration executes one + // GCN basic block; a correctly-translated shader always reaches its + // terminal block (pc out of range -> default -> exit) well within any + // real control flow. A mistranslated shader whose loop-exit condition is + // wrong would otherwise spin the dispatcher forever, hanging the single + // Metal queue and freezing every later submission (black screen, no + // recovery). Bounding the iteration count guarantees the invocation + // terminates instead: the effect may be wrong for that shader, but the + // GPU never wedges. 0 disables the guard (original unbounded behaviour). + private static readonly int _maxDispatcherSteps = + int.TryParse( + Environment.GetEnvironmentVariable("SHARPEMU_SHADER_MAX_STEPS"), + out var maxSteps) && maxSteps >= 0 + ? maxSteps + : 100_000; + + // Diagnostic coverage probe. When enabled, every selected MRT export + // writes opaque magenta while preserving the shader's control flow, + // EXEC mask, geometry and raster state. This separates missing + // rasterization from valid fragments whose translated values are zero. + private static readonly bool _forcePixelMagenta = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_FORCE_PIXEL_MAGENTA"), + "1", + StringComparison.Ordinal); + + // Which pixel-shader MRT export target (EXP_MRT0..7 == render-target + // slot) is routed to the single fragment output. The offscreen draw + // path renders one bound color target per pass, so a multi-render-target + // (deferred G-buffer) draw compiles one pixel variant per slot, each + // selecting that slot's export here. + // Vertex stage only: the fragment shader paired with this vertex shader + // declares interpolated inputs for locations 0..(this-1). Metal requires + // every fragment input location to be written by the vertex shader, so + // the vertex stage must export at least this many param outputs (any it + // does not naturally export are zero-filled) or pipeline creation fails + // with "Fragment input(s) `user(locnN)` ... not written by vertex shader". + private readonly int _requiredVertexOutputCount; private readonly uint _localSizeX; private readonly uint _localSizeY; private readonly uint _localSizeZ; private readonly int _globalBufferBase; private readonly int _totalGlobalBufferCount; private readonly int _imageBindingBase; - private readonly int _scalarRegisterBufferIndex; + private readonly int _initialScalarBufferIndex; + private readonly uint _pixelInputEnable; + private readonly uint _pixelInputAddress; + private readonly ulong _storageBufferOffsetAlignment; private readonly List _interfaces = []; private readonly Dictionary _pixelInputs = []; private readonly Dictionary _pixelOutputs = []; @@ -155,6 +235,7 @@ public static partial class Gen5SpirvTranslator private readonly List _imageResources = []; private readonly Dictionary _imageBindingByPc = []; private readonly Dictionary _bufferBindingByPc = []; + private readonly Dictionary _scalarDefinitionsBeforePc = []; private uint _voidType; private uint _boolType; private uint _uintType; @@ -169,25 +250,40 @@ public static partial class Gen5SpirvTranslator private uint _uvec3Type; private uint _uvec4Type; private uint _privateUintPointer; + private uint _privateVec2Pointer; private uint _privateBoolPointer; + private uint _runtimeBufferBiases; private uint _scalarRegisters; private uint _vectorRegisters; + private uint _packedHalfRegisters; private uint _scc; private uint _vcc; private uint _exec; + private uint _reachedPixelExport; private uint _programCounter; private uint _programActive; + private uint _iterationGuard; private uint _globalBuffers; + private uint _gfx10BufferFormatTable; + private uint _storageBlockPointer; private uint _storageUintPointer; private uint _lds; - private uint _workgroupUintPointer; + private uint _ldsElementPointer; + private uint _ldsDwordMask; private uint _positionOutput; private uint _vertexIndexInput; private uint _instanceIndexInput; private uint _fragCoordInput; private uint _localInvocationIdInput; + private uint _localInvocationIndexInput; private uint _workGroupIdInput; + private uint _computeDispatchLimit; + private uint _pushConstantUintPointer; private uint _subgroupInvocationIdInput; + private uint _waveMaskScratch; + private uint _waveMaskScratchElementPointer; + private uint _waveBroadcastScratch; + private bool _waveScratchInLds; private uint _glsl; private enum ImageComponentKind @@ -227,12 +323,23 @@ public static partial class Gen5SpirvTranslator int globalBufferBase, int totalGlobalBufferCount, int imageBindingBase, - int scalarRegisterBufferIndex) + int initialScalarBufferIndex, + uint pixelInputEnable = 0, + uint pixelInputAddress = 0, + int requiredVertexOutputCount = 0, + uint waveLaneCount = 32, + ulong storageBufferOffsetAlignment = 1) { _stage = stage; + _requiredVertexOutputCount = requiredVertexOutputCount; _state = state; _evaluation = evaluation; _pixelOutputBindings = pixelOutputBindings; + _waveLaneCount = waveLaneCount == 64 ? 64u : 32u; + _emulateWave64 = + stage == Gen5SpirvStage.Compute && + _waveLaneCount == 64 && + (ulong)localSizeX * localSizeY * localSizeZ == 64; _localSizeX = localSizeX; _localSizeY = localSizeY; _localSizeZ = localSizeZ; @@ -241,7 +348,20 @@ public static partial class Gen5SpirvTranslator ? evaluation.GlobalMemoryBindings.Count : totalGlobalBufferCount; _imageBindingBase = imageBindingBase; - _scalarRegisterBufferIndex = scalarRegisterBufferIndex; + _initialScalarBufferIndex = initialScalarBufferIndex; + _pixelInputEnable = pixelInputEnable; + _pixelInputAddress = pixelInputAddress; + if (storageBufferOffsetAlignment == 0 || + (storageBufferOffsetAlignment & (storageBufferOffsetAlignment - 1)) != 0 || + storageBufferOffsetAlignment > uint.MaxValue) + { + throw new ArgumentOutOfRangeException( + nameof(storageBufferOffsetAlignment), + storageBufferOffsetAlignment, + "storage-buffer offset alignment must be a uint-sized power of two"); + } + + _storageBufferOffsetAlignment = storageBufferOffsetAlignment; } public bool TryCompile(out Gen5SpirvShader shader, out string error) @@ -250,6 +370,75 @@ public static partial class Gen5SpirvTranslator error = string.Empty; try { + if (Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_TITLE_INTERFACE") == "1" && + _state.Program.Address is 0x0000000500780000ul or + 0x0000000500781200ul) + { + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] stage={_stage} " + + $"address=0x{_state.Program.Address:X16} " + + $"required_vertex_outputs={_requiredVertexOutputCount} " + + $"ps_ena=0x{_pixelInputEnable:X8} ps_addr=0x{_pixelInputAddress:X8}"); + foreach (var instruction in _state.Program.Instructions) + { + if (instruction.Control is Gen5ExportControl export) + { + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " + + $"export_target={export.Target} mask=0x{export.EnableMask:X} " + + $"compressed={export.Compressed} src=[" + + string.Join(',', instruction.Sources) + "]"); + } + else if (instruction.Control is Gen5InterpolationControl interpolation) + { + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " + + $"attribute={interpolation.Attribute} " + + $"channel={interpolation.Channel} dst=[" + + string.Join(',', instruction.Destinations) + "]"); + } + else if (instruction.Control is Gen5ScalarMemoryControl scalarMemory) + { + var bindingIndex = -1; + for (var index = 0; + index < _evaluation.GlobalMemoryBindings.Count; + index++) + { + if (_evaluation.GlobalMemoryBindings[index] + .InstructionPcs.Contains(instruction.Pc)) + { + bindingIndex = index; + break; + } + } + + var binding = bindingIndex >= 0 + ? _evaluation.GlobalMemoryBindings[bindingIndex] + : null; + var byteOffset = scalarMemory.ImmediateOffsetBytes; + var sample = binding is not null && + scalarMemory.DynamicOffsetRegister is null && + byteOffset >= 0 && + byteOffset < binding.DataLength + ? Convert.ToHexString( + binding.Data.AsSpan( + byteOffset, + Math.Min( + checked((int)scalarMemory.DestinationCount * 4), + binding.DataLength - byteOffset))) + : "dynamic-or-unavailable"; + Console.Error.WriteLine( + $"[AGC][TITLE-INTERFACE] pc=0x{instruction.Pc:X4} " + + $"scalar_load={instruction.Opcode} binding={bindingIndex} " + + $"scalar=s{binding?.ScalarAddress} " + + $"base=0x{binding?.BaseAddress:X16} length={binding?.DataLength} " + + $"offset={byteOffset} dynamic={scalarMemory.DynamicOffsetRegister} " + + $"bytes={sample}"); + } + } + } + DeclareModule(); var blocks = BuildBasicBlocks(_state.Program.Instructions); if (blocks.Count == 0) @@ -258,10 +447,39 @@ public static partial class Gen5SpirvTranslator return false; } + BuildScalarDefinitionInfo(blocks, _state.Program.Instructions); + var functionType = _module.TypeFunction(_voidType); var main = _module.BeginFunction(_voidType, functionType); _module.AddName(main, "main"); _module.AddLabel(); + if (_stage == Gen5SpirvStage.Pixel && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_EARLY_COLOR") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + var earlyOutput = _pixelOutputs + .OrderBy(static pair => pair.Key) + .Select(static pair => pair.Value) + .First(); + var earlyColor = earlyOutput.Kind switch + { + Gen5PixelOutputKind.Float => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + earlyOutput.Type, + Float(1f), + Float(0f), + Float(1f), + Float(1f)), + Gen5PixelOutputKind.Sint => + _module.ConstantNull(earlyOutput.Type), + _ => _module.ConstantNull(earlyOutput.Type), + }; + Store(earlyOutput.Variable, earlyColor); + _module.AddStatement(SpirvOp.Return); + _module.AddLabel(); + } EmitInitialState(); var loopHeader = _module.AllocateId(); @@ -314,12 +532,91 @@ public static partial class Gen5SpirvTranslator _module.AddStatement(SpirvOp.Branch, loopContinue); _module.AddLabel(loopContinue); var active = Load(_boolType, _programActive); + if (_maxDispatcherSteps > 0) + { + var steps = IAdd(Load(_uintType, _iterationGuard), UInt(1)); + Store(_iterationGuard, steps); + var withinLimit = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + steps, + UInt((uint)_maxDispatcherSteps)); + active = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + active, + withinLimit); + } + _module.AddStatement( SpirvOp.BranchConditional, active, loopHeader, loopMerge); _module.AddLabel(loopMerge); + if (_stage == Gen5SpirvStage.Pixel && + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_TITLE_SHADER_STATE") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + var stateOutput = _pixelOutputs + .OrderBy(static pair => pair.Key) + .Select(static pair => pair.Value) + .FirstOrDefault(static output => + output.Kind == Gen5PixelOutputKind.Float); + if (stateOutput.Variable != 0) + { + uint EncodeBool(uint condition) => + _module.AddInstruction( + SpirvOp.Select, + _floatType, + condition, + Float(1f), + Float(0f)); + + Store( + stateOutput.Variable, + _module.AddInstruction( + SpirvOp.CompositeConstruct, + stateOutput.Type, + EncodeBool(Load(_boolType, _exec)), + EncodeBool(IsWaveMaskActive(LoadS64(52))), + EncodeBool(Load(_boolType, _reachedPixelExport)), + Float(1f))); + } + + StoreS64( + 126, + _module.Constant64(_ulongType, 1)); + } + if (_stage == Gen5SpirvStage.Pixel) + { + // A fragment lane removed from EXEC is not a request to + // write the output variable's zero initializer. It is a + // killed fragment and must not participate in color, + // depth, or blend operations. Keep EXEC masking during + // translation, then terminate lanes that remain inactive + // when the guest pixel shader exits. + var returnLabel = _module.AllocateId(); + var killLabel = _module.AllocateId(); + // Materialize the condition before SelectionMerge: SPIR-V + // requires the merge instruction to be immediately followed + // by its structured branch terminator. + var laneActive = Load(_boolType, _exec); + _module.AddStatement( + SpirvOp.SelectionMerge, + returnLabel, + 0); + _module.AddStatement( + SpirvOp.BranchConditional, + laneActive, + returnLabel, + killLabel); + _module.AddLabel(killLabel); + _module.AddStatement(SpirvOp.Kill); + _module.AddLabel(returnLabel); + } + _module.AddStatement(SpirvOp.Return); _module.EndFunction(); @@ -394,6 +691,11 @@ public static partial class Gen5SpirvTranslator { _module.AddCapability(SpirvCapability.GroupNonUniformVote); } + + if (UsesSubgroupBroadcast() || UsesWaveControl()) + { + _module.AddCapability(SpirvCapability.GroupNonUniformBallot); + } } _glsl = _module.ImportExtInst("GLSL.std.450"); @@ -412,15 +714,20 @@ public static partial class Gen5SpirvTranslator _uvec4Type = _module.TypeVector(_uintType, 4); _privateUintPointer = _module.TypePointer(SpirvStorageClass.Private, _uintType); + _privateVec2Pointer = + _module.TypePointer(SpirvStorageClass.Private, _vec2Type); _privateBoolPointer = _module.TypePointer(SpirvStorageClass.Private, _boolType); var scalarArrayType = _module.TypeArray(_uintType, ScalarRegisterCount); var vectorArrayType = _module.TypeArray(_uintType, VectorRegisterCount); + var packedHalfArrayType = _module.TypeArray(_vec2Type, VectorRegisterCount); var privateScalarArrayPointer = _module.TypePointer(SpirvStorageClass.Private, scalarArrayType); var privateVectorArrayPointer = _module.TypePointer(SpirvStorageClass.Private, vectorArrayType); + var privatePackedHalfArrayPointer = + _module.TypePointer(SpirvStorageClass.Private, packedHalfArrayType); _scalarRegisters = _module.AddGlobalVariable( privateScalarArrayPointer, SpirvStorageClass.Private, @@ -429,6 +736,10 @@ public static partial class Gen5SpirvTranslator privateVectorArrayPointer, SpirvStorageClass.Private, _module.ConstantNull(vectorArrayType)); + _packedHalfRegisters = _module.AddGlobalVariable( + privatePackedHalfArrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(packedHalfArrayType)); _scc = _module.AddGlobalVariable( _privateBoolPointer, SpirvStorageClass.Private, @@ -441,6 +752,10 @@ public static partial class Gen5SpirvTranslator _privateBoolPointer, SpirvStorageClass.Private, _module.ConstantBool(true)); + _reachedPixelExport = _module.AddGlobalVariable( + _privateBoolPointer, + SpirvStorageClass.Private, + _module.ConstantBool(false)); _programCounter = _module.AddGlobalVariable( _privateUintPointer, SpirvStorageClass.Private, @@ -449,37 +764,155 @@ public static partial class Gen5SpirvTranslator _privateBoolPointer, SpirvStorageClass.Private, _module.ConstantBool(true)); + if (_maxDispatcherSteps > 0) + { + _iterationGuard = _module.AddGlobalVariable( + _privateUintPointer, + SpirvStorageClass.Private, + _module.Constant(_uintType, 0)); + _interfaces.Add(_iterationGuard); + _module.AddName(_iterationGuard, "pcGuard"); + } + _interfaces.Add(_scalarRegisters); _interfaces.Add(_vectorRegisters); + _interfaces.Add(_packedHalfRegisters); _interfaces.Add(_scc); _interfaces.Add(_vcc); _interfaces.Add(_exec); + _interfaces.Add(_reachedPixelExport); _interfaces.Add(_programCounter); _interfaces.Add(_programActive); _module.AddName(_scalarRegisters, "sgpr"); _module.AddName(_vectorRegisters, "vgpr"); + _module.AddName(_packedHalfRegisters, "vgprPackedHalf"); + + var runtimeBufferBiasCount = + _globalBufferBase + _evaluation.GlobalMemoryBindings.Count; + if (_initialScalarBufferIndex >= 0 && runtimeBufferBiasCount > 0) + { + var biasArrayType = _module.TypeArray( + _uintType, + (uint)runtimeBufferBiasCount); + var privateBiasArrayPointer = _module.TypePointer( + SpirvStorageClass.Private, + biasArrayType); + _runtimeBufferBiases = _module.AddGlobalVariable( + privateBiasArrayPointer, + SpirvStorageClass.Private, + _module.ConstantNull(biasArrayType)); + _module.AddName(_runtimeBufferBiases, "guestBufferByteBias"); + _interfaces.Add(_runtimeBufferBiases); + } DeclareBuffers(); DeclareImages(); DeclareLds(); + DeclareWave64Scratch(); DeclareStageInterface(); + DeclareComputeDispatchLimit(); } - private void DeclareLds() + private void DeclareComputeDispatchLimit() { - if (_stage != Gen5SpirvStage.Compute || !UsesLds()) + if (_stage != Gen5SpirvStage.Compute) { return; } - var ldsArrayType = _module.TypeArray(_uintType, LdsDwordCount); - var ldsPointer = - _module.TypePointer(SpirvStorageClass.Workgroup, ldsArrayType); - _workgroupUintPointer = + // RDNA DISPATCH_* can express exact thread dimensions, including + // a final partially populated workgroup. Vulkan dispatches whole + // workgroups, so the command path supplies the exact exclusive + // thread bounds through a small push-constant block and excess + // invocations are disabled before any guest instruction executes. + var block = _module.TypeStruct(_uvec3Type); + _module.AddDecoration(block, SpirvDecoration.Block); + _module.AddMemberDecoration(block, 0, SpirvDecoration.Offset, 0); + var blockPointer = + _module.TypePointer(SpirvStorageClass.PushConstant, block); + _pushConstantUintPointer = + _module.TypePointer(SpirvStorageClass.PushConstant, _uintType); + _computeDispatchLimit = _module.AddGlobalVariable( + blockPointer, + SpirvStorageClass.PushConstant); + _module.AddName(_computeDispatchLimit, "dispatchThreadLimit"); + _interfaces.Add(_computeDispatchLimit); + } + + private void DeclareWave64Scratch() + { + if (!_emulateWave64 || !UsesSubgroupOperations()) + { + return; + } + + // Metal exposes 32 KiB of threadgroup memory on the Apple GPUs we + // target. Some PS5 compute shaders legitimately request all of it, + // so allocating another workgroup variable for the wave64 bridge + // makes pipeline creation fail. Reuse the final three dwords of the + // existing LDS allocation in that case. The translator already + // bounds guest LDS accesses to this fixed allocation; keeping the + // bridge inside it preserves the host limit and still provides the + // cross-subgroup rendezvous needed to model one 64-lane guest wave. + if (_lds != 0) + { + _waveScratchInLds = true; + _waveMaskScratchElementPointer = _ldsElementPointer; + return; + } + + var maskArrayType = _module.TypeArray(_uintType, 2); + var maskArrayPointer = + _module.TypePointer(SpirvStorageClass.Workgroup, maskArrayType); + _waveMaskScratchElementPointer = _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); - _lds = _module.AddGlobalVariable( - ldsPointer, + _waveMaskScratch = _module.AddGlobalVariable( + maskArrayPointer, SpirvStorageClass.Workgroup); + _module.AddName(_waveMaskScratch, "wave64MaskScratch"); + _interfaces.Add(_waveMaskScratch); + + var uintPointer = + _module.TypePointer(SpirvStorageClass.Workgroup, _uintType); + _waveBroadcastScratch = _module.AddGlobalVariable( + uintPointer, + SpirvStorageClass.Workgroup); + _module.AddName(_waveBroadcastScratch, "wave64BroadcastScratch"); + _interfaces.Add(_waveBroadcastScratch); + } + + private void DeclareLds() + { + if (!UsesLds()) + { + return; + } + + // Compute shaders get genuine workgroup-shared LDS. Graphics stages + // (NGG export/vertex, pixel) cannot use the Workgroup storage class + // in SPIR-V, but they still emit ds_write/ds_read — typically as + // per-invocation scratch/spill or as NGG staging whose cross-lane + // reads don't feed this stage's exports. Model those as a + // per-invocation Private array so the shader is valid SPIR-V and its + // draw stops being dropped. Index masking in LdsPointer keeps the + // arbitrary computed addresses inside the array. + var storageClass = _stage == Gen5SpirvStage.Compute + ? SpirvStorageClass.Workgroup + : SpirvStorageClass.Private; + var dwordCount = _stage == Gen5SpirvStage.Compute + ? LdsDwordCount + : PrivateLdsDwordCount; + _ldsDwordMask = dwordCount - 1; + + var ldsArrayType = _module.TypeArray(_uintType, dwordCount); + var ldsPointer = _module.TypePointer(storageClass, ldsArrayType); + _ldsElementPointer = _module.TypePointer(storageClass, _uintType); + _lds = storageClass == SpirvStorageClass.Workgroup + ? _module.AddGlobalVariable(ldsPointer, storageClass) + : _module.AddGlobalVariable( + ldsPointer, + storageClass, + _module.ConstantNull(ldsArrayType)); _module.AddName(_lds, "lds"); _interfaces.Add(_lds); } @@ -509,6 +942,8 @@ public static partial class Gen5SpirvTranslator (uint)_totalGlobalBufferCount); var descriptorsPointer = _module.TypePointer(SpirvStorageClass.StorageBuffer, descriptors); + _storageBlockPointer = + _module.TypePointer(SpirvStorageClass.StorageBuffer, block); _storageUintPointer = _module.TypePointer(SpirvStorageClass.StorageBuffer, _uintType); _globalBuffers = _module.AddGlobalVariable( @@ -600,8 +1035,15 @@ public static partial class Gen5SpirvTranslator return (SpirvImageFormat.Unknown, ImageComponentKind.Float); } - var dataFormat = (descriptor[1] >> 20) & 0x1FFu; - var numberType = (descriptor[1] >> 26) & 0xFu; + var unifiedFormat = (descriptor[1] >> 20) & 0x1FFu; + if (!Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out var dataFormat, + out var numberType)) + { + return (SpirvImageFormat.Unknown, ImageComponentKind.Float); + } + return (dataFormat, numberType) switch { (1, _) => (SpirvImageFormat.R8, ImageComponentKind.Float), @@ -668,6 +1110,18 @@ public static partial class Gen5SpirvTranslator SpirvDecoration.BuiltIn, (uint)SpirvBuiltIn.SubgroupLocalInvocationId); _interfaces.Add(_subgroupInvocationIdInput); + + if (_waveLaneCount == 64) + { + _localInvocationIndexInput = _module.AddGlobalVariable( + subgroupPointer, + SpirvStorageClass.Input); + _module.AddDecoration( + _localInvocationIndexInput, + SpirvDecoration.BuiltIn, + (uint)SpirvBuiltIn.LocalInvocationIndex); + _interfaces.Add(_localInvocationIndexInput); + } } if (_stage == Gen5SpirvStage.Vertex) @@ -710,6 +1164,13 @@ public static partial class Gen5SpirvTranslator .OfType() .Where(export => export.Target is >= 32 and < 64) .Select(export => export.Target - 32) + // Cover every location the paired fragment shader reads, even + // ones this vertex program never exports, so Metal's exact + // vertex-out/fragment-in interface match succeeds. Extras are + // zero-filled in EmitInitialState. + .Concat(Enumerable + .Range(0, Math.Max(_requiredVertexOutputCount, 0)) + .Select(location => (uint)location)) .Distinct() .Order() .ToArray(); @@ -753,7 +1214,13 @@ public static partial class Gen5SpirvTranslator (uint)SpirvBuiltIn.FragCoord); _interfaces.Add(_fragCoordInput); - foreach (var binding in _pixelOutputBindings) + var declaredPixelOutputs = + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_SINGLE_MRT") == "1" && + _state.Program.Address == 0x0000000500781200ul + ? _pixelOutputBindings.Take(1) + : _pixelOutputBindings; + foreach (var binding in declaredPixelOutputs) { var outputType = GetPixelOutputType(binding.Kind); var outputPointer = @@ -833,11 +1300,37 @@ public static partial class Gen5SpirvTranslator private void EmitInitialState() { - if (_scalarRegisterBufferIndex >= 0) + if (_initialScalarBufferIndex >= 0) { - for (uint index = 0; index < ScalarRegisterCount; index++) + // Initial scalar registers arrive in a per-draw buffer instead + // of being baked as constants, so animated user data (colors, + // scroll offsets) reuses one translation and pipeline. Only + // registers the program can observe need loading. + var consumed = Gen5ShaderTranslator.ComputeConsumedScalarMask(_state.Program); + for (uint index = 0; + index < _evaluation.InitialScalarRegisters.Count && + index < ScalarRegisterCount; + index++) { - StoreS(index, LoadBufferWord(_scalarRegisterBufferIndex, UInt(index))); + if (Gen5ShaderTranslator.IsScalarConsumed(consumed, index)) + { + StoreS( + index, + LoadBufferWord(_initialScalarBufferIndex, UInt(index))); + } + } + + var runtimeBufferBiasCount = + _globalBufferBase + _evaluation.GlobalMemoryBindings.Count; + for (var binding = 0; + binding < runtimeBufferBiasCount; + binding++) + { + Store( + RuntimeBufferBiasPointer(binding), + LoadBufferWord( + _initialScalarBufferIndex, + UInt(checked(256u + (uint)binding)))); } } else @@ -856,6 +1349,7 @@ public static partial class Gen5SpirvTranslator } Store(_scc, _module.ConstantBool(false)); + Store(_reachedPixelExport, _module.ConstantBool(false)); if (_subgroupInvocationIdInput != 0) { StoreWaveMask(106, _module.ConstantBool(false)); @@ -863,8 +1357,14 @@ public static partial class Gen5SpirvTranslator } else { - Store(_vcc, _module.ConstantBool(false)); - Store(_exec, _module.ConstantBool(true)); + // Graphics stages emulate one logical wave lane. Keep the + // guest-visible VCC/EXEC scalar pairs synchronized with the + // internal booleans: shaders commonly save EXEC from s126:s127 + // and restore it after divergent work. Initializing only _exec + // left those registers at zero, so the first restore disabled + // every fragment before its color export. + StoreS64(106, _module.Constant64(_ulongType, 0)); + StoreS64(126, _module.Constant64(_ulongType, 1)); } Store(_programCounter, UInt(0)); Store(_programActive, _module.ConstantBool(true)); @@ -873,22 +1373,21 @@ public static partial class Gen5SpirvTranslator { StoreV(5, Load(_uintType, _vertexIndexInput), guardWithExec: false); StoreV(8, Load(_uintType, _instanceIndexInput), guardWithExec: false); + + // Give every declared param output a defined starting value. + // Outputs the program actually exports overwrite this; the + // extras that only exist to satisfy the fragment interface stay + // zero. The explicit store also keeps SPIRV-Cross from pruning + // an unexported output (which would re-break the interface). + foreach (var output in _vertexOutputs.Values) + { + Store(output, _module.ConstantNull(_vec4Type)); + } } else if (_stage == Gen5SpirvStage.Pixel) { var fragCoord = Load(_vec4Type, _fragCoordInput); - var x = _module.AddInstruction( - SpirvOp.CompositeExtract, - _floatType, - fragCoord, - 0); - var y = _module.AddInstruction( - SpirvOp.CompositeExtract, - _floatType, - fragCoord, - 1); - StoreV(2, Bitcast(_uintType, x), guardWithExec: false); - StoreV(3, Bitcast(_uintType, y), guardWithExec: false); + EmitPixelInputState(fragCoord); foreach (var output in _pixelOutputs.Values) { Store(output.Variable, _module.ConstantNull(output.Type)); @@ -897,19 +1396,57 @@ public static partial class Gen5SpirvTranslator else { var localId = Load(_uvec3Type, _localInvocationIdInput); + var workGroupId = Load(_uvec3Type, _workGroupIdInput); + var invocationInBounds = _module.ConstantBool(true); for (uint component = 0; component < 3; component++) { - var value = _module.AddInstruction( + var localComponent = _module.AddInstruction( SpirvOp.CompositeExtract, _uintType, localId, component); - StoreV(component, value, guardWithExec: false); + StoreV(component, localComponent, guardWithExec: false); + + var groupComponent = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + workGroupId, + component); + var localSize = component switch + { + 0 => _localSizeX, + 1 => _localSizeY, + _ => _localSizeZ, + }; + var globalComponent = IAdd( + _module.AddInstruction( + SpirvOp.IMul, + _uintType, + groupComponent, + UInt(localSize)), + localComponent); + var limitPointer = _module.AddInstruction( + SpirvOp.AccessChain, + _pushConstantUintPointer, + _computeDispatchLimit, + UInt(0), + UInt(component)); + var componentInBounds = _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + globalComponent, + Load(_uintType, limitPointer)); + invocationInBounds = _module.AddInstruction( + SpirvOp.LogicalAnd, + _boolType, + invocationInBounds, + componentInBounds); } + Store(_programActive, invocationInBounds); + if (_state.ComputeSystemRegisters is { } registers) { - var workGroupId = Load(_uvec3Type, _workGroupIdInput); StoreComputeSystemRegister( registers.WorkGroupXRegister, workGroupId, @@ -932,6 +1469,70 @@ public static partial class Gen5SpirvTranslator } } + private void EmitPixelInputState(uint fragCoord) + { + uint vgpr = 0; + + // Pixel input VGPRs are compacted in SPI_PS_INPUT_ADDR order. The + // interpolation inputs occupy register slots even though V_INTERP + // is lowered directly from SPIR-V interpolants; position inputs + // following them must still land in the hardware-selected VGPRs. + AdvancePixelInput(0, 2, ref vgpr); // PERSP_SAMPLE + AdvancePixelInput(1, 2, ref vgpr); // PERSP_CENTER + AdvancePixelInput(2, 2, ref vgpr); // PERSP_CENTROID + AdvancePixelInput(3, 3, ref vgpr); // PERSP_PULL_MODEL + AdvancePixelInput(4, 2, ref vgpr); // LINEAR_SAMPLE + AdvancePixelInput(5, 2, ref vgpr); // LINEAR_CENTER + AdvancePixelInput(6, 2, ref vgpr); // LINEAR_CENTROID + AdvancePixelInput(7, 1, ref vgpr); // LINE_STIPPLE + + EmitPixelPositionInput(8, 0, fragCoord, ref vgpr); // POS_X_FLOAT + EmitPixelPositionInput(9, 1, fragCoord, ref vgpr); // POS_Y_FLOAT + EmitPixelPositionInput(10, 2, fragCoord, ref vgpr); // POS_Z_FLOAT + EmitPixelPositionInput(11, 3, fragCoord, ref vgpr); // POS_W_FLOAT + + // FRONT_FACE, ANCILLARY, SAMPLE_COVERAGE and POS_FIXED_PT follow + // position inputs. Reserve their compact slots until their SPIR-V + // builtins are needed by a guest shader. + AdvancePixelInput(12, 1, ref vgpr); + AdvancePixelInput(13, 1, ref vgpr); + AdvancePixelInput(14, 1, ref vgpr); + AdvancePixelInput(15, 1, ref vgpr); + } + + private void AdvancePixelInput(int bit, uint dwordCount, ref uint vgpr) + { + if ((_pixelInputAddress & (1u << bit)) != 0) + { + vgpr += dwordCount; + } + } + + private void EmitPixelPositionInput( + int bit, + uint component, + uint fragCoord, + ref uint vgpr) + { + var mask = 1u << bit; + if ((_pixelInputAddress & mask) == 0) + { + return; + } + + if ((_pixelInputEnable & mask) != 0) + { + var value = _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + fragCoord, + component); + StoreV(vgpr, Bitcast(_uintType, value), guardWithExec: false); + } + + vgpr++; + } + private void StoreComputeSystemRegister( uint? register, uint workGroupId, @@ -970,6 +1571,11 @@ public static partial class Gen5SpirvTranslator error = $"pc=0x{instruction.Pc:X} {instruction.Opcode}: {error}"; return false; } + + CapturePixelVgprs(instruction); + CapturePixelVgprPoints(instruction); + MarkPixelPath(instruction); + CapturePixelExec(instruction); } var terminator = _state.Program.Instructions[block.EndIndex - 1]; @@ -1180,9 +1786,8 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (_stage != Gen5SpirvStage.Compute || - _lds == 0 || - _workgroupUintPointer == 0 || + if (_lds == 0 || + _ldsElementPointer == 0 || instruction.Control is not Gen5DataShareControl control) { error = "invalid LDS instruction"; @@ -1197,6 +1802,24 @@ public static partial class Gen5SpirvTranslator switch (instruction.Opcode) { + case "DsAddU32": + { + if (instruction.Sources.Count < 2) + { + error = "missing LDS atomic-add source"; + return false; + } + + var address = GetRawSource(instruction, 0); + _module.AddInstruction( + SpirvOp.AtomicIAdd, + _uintType, + LdsPointer(address, control.Offset0), + UInt(2), // Workgroup scope. + UInt(0x108), // AcquireRelease | WorkgroupMemory. + GetRawSource(instruction, 1)); + return true; + } case "DsWriteB32": { if (instruction.Sources.Count < 2) @@ -1207,7 +1830,7 @@ public static partial class Gen5SpirvTranslator var address = GetRawSource(instruction, 0); StoreLds( - LdsPointer(address, EffectiveDsOffsetBytes(control.Offset0)), + LdsPointer(address, control.Offset0), GetRawSource(instruction, 1)); return true; } @@ -1220,13 +1843,36 @@ public static partial class Gen5SpirvTranslator } var address = GetRawSource(instruction, 0); - var offset = EffectiveDsOffsetBytes(control.Offset0); + var offset = control.Offset0; StoreLds(LdsPointer(address, offset), GetRawSource(instruction, 1)); StoreLds( LdsPointer(address, offset + sizeof(uint)), GetRawSource(instruction, 2)); return true; } + case "DsWriteB96": + case "DsWriteB128": + { + // ds_write_b96 stores 3 consecutive dwords, ds_write_b128 + // stores 4, from data0..data0+N at the address's offset. + var dwordCount = instruction.Opcode == "DsWriteB128" ? 4 : 3; + if (instruction.Sources.Count < 1 + dwordCount) + { + error = "missing LDS write128 source"; + return false; + } + + var address = GetRawSource(instruction, 0); + var offset = control.Offset0; + for (var dword = 0; dword < dwordCount; dword++) + { + StoreLds( + LdsPointer(address, offset + (uint)(dword * sizeof(uint))), + GetRawSource(instruction, 1 + dword)); + } + + return true; + } case "DsWrite2B32": case "DsWrite2St64B32": { @@ -1241,12 +1887,12 @@ public static partial class Gen5SpirvTranslator StoreLds( LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset0, st64)), + EffectiveDsPairOffsetBytes(control.Offset0, st64)), GetRawSource(instruction, 1)); StoreLds( LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset1, st64)), + EffectiveDsPairOffsetBytes(control.Offset1, st64)), GetRawSource(instruction, 2)); return true; } @@ -1262,10 +1908,35 @@ public static partial class Gen5SpirvTranslator var address = GetRawSource(instruction, 0); var value = Load( _uintType, - LdsPointer(address, EffectiveDsOffsetBytes(control.Offset0))); + LdsPointer(address, control.Offset0)); StoreV(instruction.Destinations[0].Value, value); return true; } + case "DsReadB96": + case "DsReadB128": + { + // ds_read_b96 loads 3 consecutive dwords, ds_read_b128 loads + // 4, into dest..dest+N from the address's offset. + var dwordCount = instruction.Opcode == "DsReadB128" ? 4 : 3; + if (instruction.Destinations.Count < dwordCount || + instruction.Sources.Count < 1) + { + error = "missing LDS read128 operand"; + return false; + } + + var address = GetRawSource(instruction, 0); + var offset = control.Offset0; + for (var dword = 0; dword < dwordCount; dword++) + { + var value = Load( + _uintType, + LdsPointer(address, offset + (uint)(dword * sizeof(uint)))); + StoreV(instruction.Destinations[dword].Value, value); + } + + return true; + } case "DsRead2B32": case "DsRead2St64B32": { @@ -1282,12 +1953,12 @@ public static partial class Gen5SpirvTranslator _uintType, LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset0, st64))); + EffectiveDsPairOffsetBytes(control.Offset0, st64))); var second = Load( _uintType, LdsPointer( address, - EffectiveDsOffsetBytes(control.Offset1, st64))); + EffectiveDsPairOffsetBytes(control.Offset1, st64))); StoreV(instruction.Destinations[0].Value, first); StoreV(instruction.Destinations[1].Value, second); return true; @@ -1298,7 +1969,7 @@ public static partial class Gen5SpirvTranslator } } - private static uint EffectiveDsOffsetBytes(uint offset, bool st64 = false) => + private static uint EffectiveDsPairOffsetBytes(uint offset, bool st64 = false) => offset * (st64 ? 256u : sizeof(uint)); private uint LdsPointer(uint address, uint offsetBytes) @@ -1306,10 +1977,16 @@ public static partial class Gen5SpirvTranslator var addressWithOffset = offsetBytes == 0 ? address : IAdd(address, UInt(offsetBytes)); - var index = ShiftRightLogical(addressWithOffset, UInt(2)); + // Mask the dword index into the array bounds. LDS is a power-of-two + // dword count, so this is a no-op for in-range compute addresses but + // prevents out-of-bounds access when a graphics-stage scratch write + // uses an arbitrary computed byte address. + var index = BitwiseAnd( + ShiftRightLogical(addressWithOffset, UInt(2)), + UInt(_ldsDwordMask)); return _module.AddInstruction( SpirvOp.AccessChain, - _workgroupUintPointer, + _ldsElementPointer, _lds, index); } @@ -1357,7 +2034,17 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (!_bufferBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex)) + var scalarAddress = instruction.Sources.Count != 0 && + instruction.Sources[0].Kind == Gen5OperandKind.ScalarRegister + ? instruction.Sources[0].Value + : uint.MaxValue; + if (!TryResolveDominatingBufferBinding( + instruction.Pc, + scalarAddress, + registerCount: instruction.Opcode.StartsWith( + "SBufferLoad", + StringComparison.Ordinal) ? 4u : 2u, + out var bindingIndex)) { foreach (var destination in instruction.Destinations) { @@ -1376,6 +2063,7 @@ public static partial class Gen5SpirvTranslator var byteAddress = IAdd( dynamicOffset, UInt(unchecked((uint)control.ImmediateOffsetBytes))); + byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress); var dwordAddress = ShiftRightLogical(byteAddress, UInt(2)); for (var index = 0; index < instruction.Destinations.Count; index++) { @@ -1401,7 +2089,11 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (!_bufferBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex)) + if (!TryResolveDominatingBufferBinding( + instruction.Pc, + control.ScalarAddress, + registerCount: 2, + out var bindingIndex)) { error = "missing global-memory binding"; return false; @@ -1410,15 +2102,95 @@ public static partial class Gen5SpirvTranslator var byteAddress = IAdd( LoadV(control.VectorAddress), UInt(unchecked((uint)control.OffsetBytes))); + byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress); var dwordAddress = ShiftRightLogical(byteAddress, UInt(2)); + + if (instruction.Opcode is "GlobalAtomicAdd" or "GlobalAtomicUMax") + { + EmitExecConditional(() => + { + EmitConditional(IsBufferWordInRange(bindingIndex, dwordAddress), () => + { + var original = _module.AddInstruction( + instruction.Opcode == "GlobalAtomicAdd" + ? SpirvOp.AtomicIAdd + : SpirvOp.AtomicUMax, + _uintType, + BufferWordPointer(bindingIndex, dwordAddress), + UInt(1), + UInt(0x48), + LoadV(control.VectorData)); + if (control.Glc) + { + StoreV(control.VectorData, original); + } + }); + }); + return true; + } + + if (instruction.Opcode.StartsWith("GlobalStore", StringComparison.Ordinal)) + { + EmitExecConditional(() => + { + if (TryGetSubdwordStoreInfo( + instruction.Opcode, + out var byteCount, + out var sourceShift)) + { + StoreBufferBytes( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + byteCount, + sourceShift); + return; + } + + for (uint index = 0; index < control.DwordCount; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); + StoreBufferBytes( + bindingIndex, + address, + LoadV(control.VectorData + index), + sizeof(uint), + 0); + } + }); + return true; + } + + if (TryGetSubdwordLoadInfo( + instruction.Opcode, + out var loadByteCount, + out var signExtend, + out var d16, + out var d16High)) + { + StoreV( + control.VectorData, + LoadSubdwordBufferValue( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + loadByteCount, + signExtend, + d16, + d16High)); + return true; + } + for (uint index = 0; index < control.DwordCount; index++) { var address = index == 0 - ? dwordAddress - : IAdd(dwordAddress, UInt(index)); + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); StoreV( control.VectorData + index, - LoadBufferWord(bindingIndex, address)); + LoadUnalignedBufferWord(bindingIndex, address)); } return true; @@ -1436,14 +2208,11 @@ public static partial class Gen5SpirvTranslator return TryEmitVertexInputFetch(control, vertexInput, out error); } - if (_stage == Gen5SpirvStage.Vertex && - IsFormatBufferLoad(instruction.Opcode)) - { - error = $"missing vertex input for {instruction.Opcode} pc=0x{instruction.Pc:X}"; - return false; - } - - if (!_bufferBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex)) + if (!TryResolveDominatingBufferBinding( + instruction.Pc, + control.ScalarResource, + registerCount: 4, + out var bindingIndex)) { error = "missing buffer-memory binding"; return false; @@ -1467,44 +2236,90 @@ public static partial class Gen5SpirvTranslator byteAddress = IAdd( byteAddress, _module.AddInstruction(SpirvOp.IMul, _uintType, vectorIndex, stride)); + byteAddress = ApplyGuestBufferByteBias(bindingIndex, byteAddress); var dwordAddress = ShiftRightLogical(byteAddress, UInt(2)); - if (instruction.Opcode == "BufferAtomicAdd") + if (instruction.Opcode is "BufferAtomicAdd" or "BufferAtomicUMax") { EmitExecConditional(() => { - var original = _module.AddInstruction( - SpirvOp.AtomicIAdd, - _uintType, - BufferWordPointer(bindingIndex, dwordAddress), - UInt(1), - UInt(0x48), - LoadV(control.VectorData)); - if (control.Glc) + var inRange = IsBufferWordInRange(bindingIndex, dwordAddress); + EmitConditional(inRange, () => { - StoreV(control.VectorData, original); + var original = _module.AddInstruction( + instruction.Opcode == "BufferAtomicAdd" + ? SpirvOp.AtomicIAdd + : SpirvOp.AtomicUMax, + _uintType, + BufferWordPointer(bindingIndex, dwordAddress), + UInt(1), + UInt(0x48), + LoadV(control.VectorData)); + if (control.Glc) + { + StoreV(control.VectorData, original); + } + }); + }); + + return true; + } + + if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("BufferStoreFormat", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("BufferStoreByte", StringComparison.Ordinal) || + instruction.Opcode.StartsWith("BufferStoreShort", StringComparison.Ordinal)) + { + EmitExecConditional(() => + { + if (TryGetSubdwordStoreInfo( + instruction.Opcode, + out var byteCount, + out var sourceShift)) + { + StoreBufferBytes( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + byteCount, + sourceShift); + return; + } + + for (uint index = 0; index < control.DwordCount; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); + StoreBufferBytes( + bindingIndex, + address, + LoadV(control.VectorData + index), + sizeof(uint), + 0); } }); return true; } - if (instruction.Opcode.StartsWith("BufferStoreDword", StringComparison.Ordinal)) + if (TryGetSubdwordLoadInfo( + instruction.Opcode, + out var loadByteCount, + out var signExtend, + out var d16, + out var d16High)) { - EmitExecConditional(() => - { - for (uint index = 0; index < control.DwordCount; index++) - { - var address = index == 0 - ? dwordAddress - : IAdd(dwordAddress, UInt(index)); - StoreBufferWord( - bindingIndex, - address, - LoadV(control.VectorData + index)); - } - }); - + StoreV( + control.VectorData, + LoadSubdwordBufferValue( + bindingIndex, + byteAddress, + LoadV(control.VectorData), + loadByteCount, + signExtend, + d16, + d16High)); return true; } @@ -1515,23 +2330,715 @@ public static partial class Gen5SpirvTranslator return false; } + // MUBUF format loads take their element format and destination + // swizzle from the GFX10 buffer descriptor. Keep raw dword loads + // on the byte-address >> 2 path below: unlike typed loads they do + // not perform component conversion or dst_sel processing. + // Vertex shaders normally expose indexed format loads as Vulkan + // attributes. Loads with an additional per-lane byte offset cannot + // be represented by a fixed attribute description, so the scalar + // evaluator captures their descriptor as storage instead. Preserve + // typed conversion for both MUBUF and MTBUF in that fallback path. + if (IsFormatBufferLoad(instruction.Opcode)) + { + EmitBufferFormatLoad( + bindingIndex, + byteAddress, + control.ScalarResource, + control.VectorData, + control.DwordCount); + return true; + } + for (uint index = 0; index < control.DwordCount; index++) { var address = index == 0 - ? dwordAddress - : IAdd(dwordAddress, UInt(index)); + ? byteAddress + : IAdd(byteAddress, UInt(index * sizeof(uint))); StoreV( control.VectorData + index, - LoadBufferWord(bindingIndex, address)); + LoadUnalignedBufferWord(bindingIndex, address)); } return true; } + private void EmitBufferFormatLoad( + int bindingIndex, + uint byteAddress, + uint scalarResource, + uint vectorData, + uint componentCount) + { + var descriptorWord3 = LoadS(scalarResource + 3); + var unifiedFormat = BitwiseAnd( + ShiftRightLogical(descriptorWord3, UInt(12)), + UInt(0x7F)); + var (dataFormat, numberFormat) = DecodeGfx10BufferFormat(unifiedFormat); + + var canonical = new uint[4]; + for (var component = 0; component < canonical.Length; component++) + { + canonical[component] = LoadGfx10BufferFormatComponent( + bindingIndex, + byteAddress, + dataFormat, + numberFormat, + component); + } + + var one = Gfx10FormatOne(numberFormat); + for (uint destination = 0; destination < componentCount; destination++) + { + var selector = BitwiseAnd( + ShiftRightLogical(descriptorWord3, UInt(destination * 3)), + UInt(7)); + var value = UInt(0); + value = SelectUInt(selector, 1, one, value); + value = SelectUInt(selector, 4, canonical[0], value); + value = SelectUInt(selector, 5, canonical[1], value); + value = SelectUInt(selector, 6, canonical[2], value); + value = SelectUInt(selector, 7, canonical[3], value); + StoreV(vectorData + destination, value); + } + } + + private (uint DataFormat, uint NumberFormat) DecodeGfx10BufferFormat( + uint unifiedFormat) + { + // The descriptor is loaded at execution time, so format decoding + // must remain dynamic too. Generate one module-level lookup table + // from the same authoritative decoder used by descriptor + // evaluation rather than specializing the shader to the SRD seen + // at compile time (compiled compute shaders may be reused with new + // SRDs). A table also avoids emitting 77 compares at every format + // load site, which matters in buffer-heavy compute kernels. + if (_gfx10BufferFormatTable == 0) + { + const uint formatCount = 128; + var entries = new uint[formatCount]; + for (uint format = 0; format < formatCount; format++) + { + Gfx10UnifiedFormat.TryDecode( + format, + out var decodedDataFormat, + out var decodedNumberFormat); + entries[format] = UInt( + decodedDataFormat | (decodedNumberFormat << 8)); + } + + var tableType = _module.TypeArray(_uintType, formatCount); + var tablePointer = _module.TypePointer( + SpirvStorageClass.Private, + tableType); + _gfx10BufferFormatTable = _module.AddGlobalVariable( + tablePointer, + SpirvStorageClass.Private, + _module.ConstantComposite(tableType, entries)); + _module.AddName(_gfx10BufferFormatTable, "gfx10BufferFormats"); + _interfaces.Add(_gfx10BufferFormatTable); + } + + var entryPointer = _module.AddInstruction( + SpirvOp.AccessChain, + _privateUintPointer, + _gfx10BufferFormatTable, + unifiedFormat); + var entry = Load(_uintType, entryPointer); + return ( + BitwiseAnd(entry, UInt(0xFF)), + BitwiseAnd( + ShiftRightLogical(entry, UInt(8)), + UInt(0xFF))); + } + + private uint LoadGfx10BufferFormatComponent( + int bindingIndex, + uint elementAddress, + uint dataFormat, + uint numberFormat, + int component) + { + var byteOffset = UInt(0); + var bitOffset = UInt(0); + var bitCount = UInt(0); + + void SetLayout(uint format, uint bytes, uint bits, uint count) + { + var matches = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + dataFormat, + UInt(format)); + byteOffset = _module.AddInstruction( + SpirvOp.Select, + _uintType, + matches, + UInt(bytes), + byteOffset); + bitOffset = _module.AddInstruction( + SpirvOp.Select, + _uintType, + matches, + UInt(bits), + bitOffset); + bitCount = _module.AddInstruction( + SpirvOp.Select, + _uintType, + matches, + UInt(count), + bitCount); + } + + // Legacy DATA_FORMAT layouts selected by the GFX10 unified format. + // Packed formats keep their bit offset in the first dword; byte + // offsets are used for naturally aligned vector components. + switch (component) + { + case 0: + SetLayout(1, 0, 0, 8); // 8 + SetLayout(2, 0, 0, 16); // 16 + SetLayout(3, 0, 0, 8); // 8_8 + SetLayout(4, 0, 0, 32); // 32 + SetLayout(5, 0, 0, 16); // 16_16 + SetLayout(6, 0, 0, 10); // 10_11_11 + SetLayout(7, 0, 0, 11); // 11_11_10 + SetLayout(8, 0, 0, 10); // 10_10_10_2 + SetLayout(9, 0, 0, 2); // 2_10_10_10 + SetLayout(10, 0, 0, 8); // 8_8_8_8 + SetLayout(11, 0, 0, 32); // 32_32 + SetLayout(12, 0, 0, 16); // 16_16_16_16 + SetLayout(13, 0, 0, 32); // 32_32_32 + SetLayout(14, 0, 0, 32); // 32_32_32_32 + break; + case 1: + SetLayout(3, 1, 0, 8); + SetLayout(5, 2, 0, 16); + SetLayout(6, 0, 10, 11); + SetLayout(7, 0, 11, 11); + SetLayout(8, 0, 10, 10); + SetLayout(9, 0, 2, 10); + SetLayout(10, 1, 0, 8); + SetLayout(11, 4, 0, 32); + SetLayout(12, 2, 0, 16); + SetLayout(13, 4, 0, 32); + SetLayout(14, 4, 0, 32); + break; + case 2: + SetLayout(6, 0, 21, 11); + SetLayout(7, 0, 22, 10); + SetLayout(8, 0, 20, 10); + SetLayout(9, 0, 12, 10); + SetLayout(10, 2, 0, 8); + SetLayout(12, 4, 0, 16); + SetLayout(13, 8, 0, 32); + SetLayout(14, 8, 0, 32); + break; + case 3: + SetLayout(8, 0, 30, 2); + SetLayout(9, 0, 22, 10); + SetLayout(10, 3, 0, 8); + SetLayout(12, 6, 0, 16); + SetLayout(14, 12, 0, 32); + break; + } + + var packed = LoadUnalignedBufferWord( + bindingIndex, + IAdd(elementAddress, byteOffset)); + var raw = _module.AddInstruction( + SpirvOp.BitFieldUExtract, + _uintType, + packed, + bitOffset, + bitCount); + var converted = ConvertGfx10BufferComponent( + raw, + bitCount, + numberFormat, + dataFormat); + var valid = _module.AddInstruction( + SpirvOp.INotEqual, + _boolType, + bitCount, + UInt(0)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + valid, + converted, + component == 3 ? Gfx10FormatOne(numberFormat) : UInt(0)); + } + + private uint ConvertGfx10BufferComponent( + uint raw, + uint bitCount, + uint numberFormat, + uint dataFormat) + { + var widthIs32 = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + bitCount, + UInt(32)); + var lowMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), bitCount), + UInt(1)); + lowMask = _module.AddInstruction( + SpirvOp.Select, + _uintType, + widthIs32, + UInt(uint.MaxValue), + lowMask); + + var signedRaw = _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, raw), + UInt(0), + bitCount); + var signedBits = Bitcast(_uintType, signedRaw); + var unsignedFloat = _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + raw); + var signedFloat = _module.AddInstruction( + SpirvOp.ConvertSToF, + _floatType, + signedRaw); + + var unorm = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.FDiv, + _floatType, + unsignedFloat, + _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + lowMask))); + var signedMaximum = ShiftRightLogical(lowMask, UInt(1)); + var snormFloat = _module.AddInstruction( + SpirvOp.FDiv, + _floatType, + signedFloat, + _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + signedMaximum)); + snormFloat = _module.AddInstruction( + SpirvOp.Select, + _floatType, + _module.AddInstruction( + SpirvOp.FOrdLessThan, + _boolType, + snormFloat, + Float(-1f)), + Float(-1f), + snormFloat); + var snorm = Bitcast(_uintType, snormFloat); + var uscaled = Bitcast(_uintType, unsignedFloat); + var sscaled = Bitcast(_uintType, signedFloat); + + var unpackedHalf = Ext(62, _vec2Type, BitwiseAnd(raw, UInt(0xFFFF))); + var half = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedHalf, + 0)); + var floating = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + bitCount, + UInt(16)), + half, + raw); + + // DATA_FORMAT 10_11_11 and 11_11_10 use unsigned mini-floats + // when NUM_FORMAT is FLOAT, not ordinary integer bit patterns. + var isPackedFloat = _module.AddInstruction( + SpirvOp.LogicalOr, + _boolType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + dataFormat, + UInt(6)), + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + dataFormat, + UInt(7))); + floating = _module.AddInstruction( + SpirvOp.Select, + _uintType, + isPackedFloat, + DecodeUnsignedMiniFloat(raw, bitCount), + floating); + + var result = raw; + result = SelectUInt(numberFormat, 0, unorm, result); + result = SelectUInt(numberFormat, 1, snorm, result); + result = SelectUInt(numberFormat, 2, uscaled, result); + result = SelectUInt(numberFormat, 3, sscaled, result); + result = SelectUInt(numberFormat, 4, raw, result); + result = SelectUInt(numberFormat, 5, signedBits, result); + result = SelectUInt(numberFormat, 7, floating, result); + return result; + } + + private uint DecodeUnsignedMiniFloat(uint raw, uint bitCount) + { + var mantissaBits = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + bitCount, + UInt(5)); + var mantissaMask = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + ShiftLeftLogical(UInt(1), mantissaBits), + UInt(1)); + var mantissa = BitwiseAnd(raw, mantissaMask); + var exponent = BitwiseAnd( + ShiftRightLogical(raw, mantissaBits), + UInt(0x1F)); + var mantissaShift = _module.AddInstruction( + SpirvOp.ISub, + _uintType, + UInt(23), + mantissaBits); + var normalBits = BitwiseOr( + ShiftLeftLogical(IAdd(exponent, UInt(112)), UInt(23)), + ShiftLeftLogical(mantissa, mantissaShift)); + var subnormal = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.FMul, + _floatType, + _module.AddInstruction( + SpirvOp.ConvertUToF, + _floatType, + mantissa), + _module.AddInstruction( + SpirvOp.Select, + _floatType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + mantissaBits, + UInt(6)), + Float(1f / 1_048_576f), // 2^-20 for 11-bit UFLOAT + Float(1f / 524_288f)))); // 2^-19 for 10-bit UFLOAT + var special = BitwiseOr( + UInt(0x7F800000), + ShiftLeftLogical(mantissa, mantissaShift)); + var result = _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + exponent, + UInt(0)), + subnormal, + normalBits); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + exponent, + UInt(31)), + special, + result); + } + + private uint Gfx10FormatOne(uint numberFormat) + { + var isUint = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + numberFormat, + UInt(4)); + var isSint = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + numberFormat, + UInt(5)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.LogicalOr, + _boolType, + isUint, + isSint), + UInt(1), + UInt(0x3F800000)); + } + + private uint SelectUInt( + uint selector, + uint expected, + uint whenTrue, + uint whenFalse) => + _module.AddInstruction( + SpirvOp.Select, + _uintType, + _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + selector, + UInt(expected)), + whenTrue, + whenFalse); + + private uint LoadUnalignedBufferWord(int bindingIndex, uint byteAddress) + { + var result = UInt(0); + for (uint index = 0; index < 4; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index)); + var dwordAddress = ShiftRightLogical(address, UInt(2)); + var bitOffset = ShiftLeftLogical(BitwiseAnd(address, UInt(3)), UInt(3)); + var value = BitwiseAnd( + ShiftRightLogical(LoadBufferWord(bindingIndex, dwordAddress), bitOffset), + UInt(0xFF)); + result = BitwiseOr(result, ShiftLeftLogical(value, UInt(index * 8))); + } + + return result; + } + + private uint LoadSubdwordBufferValue( + int bindingIndex, + uint byteAddress, + uint previous, + uint byteCount, + bool signExtend, + bool d16, + bool d16High) + { + var width = byteCount * 8; + var raw = BitwiseAnd( + LoadUnalignedBufferWord(bindingIndex, byteAddress), + UInt(byteCount == 1 ? 0xFFu : 0xFFFFu)); + if (signExtend) + { + raw = Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, raw), + UInt(0), + UInt(width))); + } + + if (!d16) + { + return raw; + } + + var half = BitwiseAnd(raw, UInt(0xFFFF)); + return d16High + ? BitwiseOr( + BitwiseAnd(previous, UInt(0x0000_FFFF)), + ShiftLeftLogical(half, UInt(16))) + : BitwiseOr( + BitwiseAnd(previous, UInt(0xFFFF_0000)), + half); + } + + private void StoreBufferBytes( + int bindingIndex, + uint byteAddress, + uint value, + uint byteCount, + uint sourceShift) + { + value = ShiftRightLogical(value, UInt(sourceShift)); + for (uint index = 0; index < byteCount; index++) + { + var address = index == 0 + ? byteAddress + : IAdd(byteAddress, UInt(index)); + var dwordAddress = ShiftRightLogical(address, UInt(2)); + var shift = ShiftLeftLogical(BitwiseAnd(address, UInt(3)), UInt(3)); + var oldValue = LoadBufferWord(bindingIndex, dwordAddress); + var byteMask = ShiftLeftLogical(UInt(0xFF), shift); + var sourceByte = BitwiseAnd( + ShiftRightLogical(value, UInt(index * 8)), + UInt(0xFF)); + var updated = BitwiseOr( + BitwiseAnd( + oldValue, + _module.AddInstruction(SpirvOp.Not, _uintType, byteMask)), + ShiftLeftLogical(sourceByte, shift)); + StoreBufferWord(bindingIndex, dwordAddress, updated); + } + } + + private static bool TryGetSubdwordLoadInfo( + string opcode, + out uint byteCount, + out bool signExtend, + out bool d16, + out bool d16High) + { + byteCount = opcode.Contains("byte", StringComparison.OrdinalIgnoreCase) ? 1u : 2u; + signExtend = opcode.Contains("Sbyte", StringComparison.Ordinal) || + opcode.Contains("Sshort", StringComparison.Ordinal); + d16 = opcode.Contains("D16", StringComparison.Ordinal); + d16High = opcode.EndsWith("D16Hi", StringComparison.Ordinal); + return opcode.Contains("LoadUbyte", StringComparison.Ordinal) || + opcode.Contains("LoadSbyte", StringComparison.Ordinal) || + opcode.Contains("LoadUshort", StringComparison.Ordinal) || + opcode.Contains("LoadSshort", StringComparison.Ordinal) || + opcode.Contains("LoadShortD16", StringComparison.Ordinal); + } + + private static bool TryGetSubdwordStoreInfo( + string opcode, + out uint byteCount, + out uint sourceShift) + { + byteCount = opcode.Contains("StoreByte", StringComparison.Ordinal) ? 1u : 2u; + sourceShift = opcode.EndsWith("D16Hi", StringComparison.Ordinal) ? 16u : 0u; + return opcode.Contains("StoreByte", StringComparison.Ordinal) || + opcode.Contains("StoreShort", StringComparison.Ordinal); + } + private static bool IsFormatBufferLoad(string opcode) => opcode.StartsWith("BufferLoadFormat", StringComparison.Ordinal) || opcode.StartsWith("TBufferLoadFormat", StringComparison.Ordinal); + private static bool UsesSampler(string opcode) => + opcode.StartsWith("ImageSample", StringComparison.Ordinal) || + opcode.StartsWith("ImageGather", StringComparison.Ordinal); + + private bool TryResolveDominatingBufferBinding( + uint pc, + uint scalarRegister, + uint registerCount, + out int bindingIndex) + { + if (_bufferBindingByPc.TryGetValue(pc, out bindingIndex)) + { + return true; + } + + for (var index = 0; index < _evaluation.GlobalMemoryBindings.Count; index++) + { + var binding = _evaluation.GlobalMemoryBindings[index]; + if (binding.ScalarAddress != scalarRegister) + { + continue; + } + + foreach (var candidatePc in binding.InstructionPcs) + { + if (!HasSameScalarDefinitions( + candidatePc, + pc, + scalarRegister, + registerCount)) + { + continue; + } + + bindingIndex = _globalBufferBase + index; + _bufferBindingByPc.Add(pc, bindingIndex); + return true; + } + } + + bindingIndex = -1; + return false; + } + + private bool TryResolveDominatingImageBinding( + Gen5ShaderInstruction instruction, + Gen5ImageControl control, + out int bindingIndex) + { + if (_imageBindingByPc.TryGetValue(instruction.Pc, out bindingIndex) && + bindingIndex < _imageResources.Count) + { + return true; + } + + var storage = Gen5ShaderTranslator.IsStorageImageOperation(instruction.Opcode); + for (var index = 0; index < _evaluation.ImageBindings.Count; index++) + { + var candidate = _evaluation.ImageBindings[index]; + if (candidate.Control.ScalarResource != control.ScalarResource || + candidate.Control.ScalarSampler != control.ScalarSampler || + Gen5ShaderTranslator.IsStorageImageOperation(candidate.Opcode) != storage || + !HasSameScalarDefinitions( + candidate.Pc, + instruction.Pc, + control.ScalarResource, + ImageDescriptorDwords) || + UsesSampler(instruction.Opcode) && + !HasSameScalarDefinitions( + candidate.Pc, + instruction.Pc, + control.ScalarSampler, + SamplerDescriptorDwords)) + { + continue; + } + + bindingIndex = index; + _imageBindingByPc.Add(instruction.Pc, index); + return true; + } + + bindingIndex = -1; + return false; + } + + private bool HasSameScalarDefinitions( + uint candidatePc, + uint targetPc, + uint firstRegister, + uint registerCount) + { + if (firstRegister + registerCount > ScalarRegisterCount || + !_scalarDefinitionsBeforePc.TryGetValue(candidatePc, out var candidate) || + !_scalarDefinitionsBeforePc.TryGetValue(targetPc, out var target)) + { + return false; + } + + for (var register = firstRegister; + register < firstRegister + registerCount; + register++) + { + var definition = candidate[register]; + if (definition is ConflictingScalarDefinition or + UnreachableScalarDefinition || + target[register] != definition) + { + return false; + } + } + + return true; + } + private bool TryEmitVertexInputFetch( Gen5BufferMemoryControl control, SpirvVertexInput input, @@ -1569,10 +3076,21 @@ public static partial class Gen5SpirvTranslator out string error) { error = string.Empty; - if (!_imageBindingByPc.TryGetValue(instruction.Pc, out var bindingIndex) || - bindingIndex >= _imageResources.Count) + if (!TryResolveDominatingImageBinding(instruction, image, out var bindingIndex)) { - error = "unresolved image binding"; + var candidates = _evaluation.ImageBindings + .Where(binding => + binding.Control.ScalarResource == image.ScalarResource && + binding.Control.ScalarSampler == image.ScalarSampler) + .Take(16) + .Select(binding => + $"{binding.Opcode}@0x{binding.Pc:X}" + + $"/r={HasSameScalarDefinitions(binding.Pc, instruction.Pc, image.ScalarResource, ImageDescriptorDwords)}" + + $"/s={!UsesSampler(instruction.Opcode) || HasSameScalarDefinitions(binding.Pc, instruction.Pc, image.ScalarSampler, SamplerDescriptorDwords)}"); + error = + $"unresolved image binding t=s{image.ScalarResource} " + + $"s=s{image.ScalarSampler} " + + $"candidates=[{string.Join(',', candidates)}]"; return false; } @@ -1638,7 +3156,10 @@ public static partial class Gen5SpirvTranslator { if ((image.Dmask & (1u << component)) != 0) { - var raw = LoadV(image.VectorData + sourceIndex++); + var raw = LoadImageStoreComponent( + image, + resource, + sourceIndex++); components[component] = resource.ComponentKind switch { ImageComponentKind.Sint => Bitcast(_intType, raw), @@ -1662,32 +3183,21 @@ public static partial class Gen5SpirvTranslator SpirvOp.CompositeConstruct, resource.VectorType, components); - if (TryGetImageBounds( - _evaluation.ImageBindings[bindingIndex].ResourceDescriptor, - out var width, - out var height)) - { - EmitBoundsCheckedImageWrite( - coordinates, - width, - height, - imageObject, - texel); - } - else - { - EmitExecConditional( - () => _module.AddStatement( - SpirvOp.ImageWrite, - imageObject, - coordinates, - texel)); - } + var imageSize = _module.AddInstruction( + SpirvOp.ImageQuerySize, + _module.TypeVector(_intType, 2), + imageObject); + EmitBoundsCheckedImageWrite( + coordinates, + imageSize, + imageObject, + texel); return true; } - if (resource.IsStorage) + if (resource.IsStorage && + instruction.Opcode is not ("ImageLoad" or "ImageLoadMip")) { error = $"unsupported storage image opcode {instruction.Opcode}"; return false; @@ -1697,24 +3207,46 @@ public static partial class Gen5SpirvTranslator var writeAllComponents = false; if (instruction.Opcode is "ImageLoad" or "ImageLoadMip") { - var coordinates = TryGetImageBounds( - _evaluation.ImageBindings[bindingIndex].ResourceDescriptor, - out var width, - out var height) - ? BuildClampedIntegerCoordinates(image, 0, width, height) - : BuildIntegerCoordinates(image, 0); - var mipLevel = _evaluation.ImageBindings[bindingIndex].MipLevel ?? 0; - var fetchedImage = _module.AddInstruction( - SpirvOp.Image, - resource.ImageType, - imageObject); - sampled = _module.AddInstruction( - SpirvOp.ImageFetch, - resource.VectorType, - fetchedImage, - coordinates, - 2, - UInt(mipLevel)); + if (resource.IsStorage) + { + var imageSize = _module.AddInstruction( + SpirvOp.ImageQuerySize, + _module.TypeVector(_intType, 2), + imageObject); + var coordinates = BuildClampedIntegerCoordinates( + image, + 0, + imageSize); + sampled = _module.AddInstruction( + SpirvOp.ImageRead, + resource.VectorType, + imageObject, + coordinates); + } + else + { + var mipLevel = _evaluation.ImageBindings[bindingIndex].MipLevel ?? 0; + var fetchedImage = _module.AddInstruction( + SpirvOp.Image, + resource.ImageType, + imageObject); + var imageSize = _module.AddInstruction( + SpirvOp.ImageQuerySizeLod, + _module.TypeVector(_intType, 2), + fetchedImage, + UInt(mipLevel)); + var coordinates = BuildClampedIntegerCoordinates( + image, + 0, + imageSize); + sampled = _module.AddInstruction( + SpirvOp.ImageFetch, + resource.VectorType, + fetchedImage, + coordinates, + 2, + UInt(mipLevel)); + } } else if (instruction.Opcode.StartsWith( "ImageSample", @@ -1724,22 +3256,86 @@ public static partial class Gen5SpirvTranslator instruction.Opcode.EndsWith("O", StringComparison.Ordinal); var hasCompare = instruction.Opcode.Contains("SampleC", StringComparison.Ordinal); - var start = (hasOffset ? 1 : 0) + (hasCompare ? 1 : 0); - var coordinates = BuildFloatCoordinates(image, start); - var explicitLod = - instruction.Opcode.Contains("Lz", StringComparison.Ordinal) || + var hasGradients = + instruction.Opcode.Contains("SampleD", StringComparison.Ordinal); + var hasZeroLod = + instruction.Opcode.Contains("Lz", StringComparison.Ordinal); + var hasLod = !hasZeroLod && instruction.Opcode.Contains("SampleL", StringComparison.Ordinal); - var lod = instruction.Opcode.Contains("Lz", StringComparison.Ordinal) - ? Float(0) - : Bitcast( - _floatType, - LoadV(image.GetAddressRegister(start + 2))); - var offset = hasOffset ? BuildImageOffset(image, 0) : 0u; - var imageOperands = - (explicitLod ? 2u : 0u) | (hasOffset ? 0x10u : 0u); - var reference = hasCompare - ? Bitcast(_floatType, LoadV(image.GetAddressRegister(hasOffset ? 1 : 0))) + var hasBias = + instruction.Opcode.Contains("SampleB", StringComparison.Ordinal); + + // RDNA MIMG address operands are ordered as + // {offset}{bias/lod}{z-compare}{derivatives}{body}. The old + // lowering treated SAMPLE_D as body-first and consequently + // sampled gradients as coordinates in every captured + // derivative operation. + var addressCursor = 0; + var offset = 0u; + if (hasOffset) + { + addressCursor = AlignFullImageAddress(image, addressCursor); + offset = BuildImageOffset(image, addressCursor); + addressCursor += ImageFullAddressSlots(image); + } + + // SAMPLE_B prefixes the body with a bias. SAMPLE_L instead + // carries LOD as the final body component (x, y, lod for 2D), + // per the RDNA image-address table. + var lodOrBias = hasBias + ? LoadImageFloatAddress(image, addressCursor++) : 0u; + var reference = 0u; + if (hasCompare) + { + // PCF references remain full-width even when A16 packs the + // ordinary address components two per VGPR. + addressCursor = AlignFullImageAddress(image, addressCursor); + reference = Bitcast( + _floatType, + LoadV(image.GetAddressRegister( + ImageAddressRegister(image, addressCursor)))); + addressCursor += ImageFullAddressSlots(image); + } + + var gradientX = hasGradients + ? BuildFloatCoordinates(image, addressCursor) + : 0u; + var gradientY = hasGradients + ? BuildFloatCoordinates(image, addressCursor + 2) + : 0u; + if (hasGradients) + { + addressCursor += 4; + } + + var coordinates = BuildFloatCoordinates(image, addressCursor); + var explicitLod = hasGradients || hasZeroLod || hasLod; + var lod = hasZeroLod + ? Float(0) + : hasLod + ? LoadImageFloatAddress(image, addressCursor + 2) + : lodOrBias; + if (hasOffset) + { + // Vulkan before maintenance8 forbids the dynamic Offset + // image operand on non-gather sampling operations. RDNA + // offsets are per-lane VGPR values, so ConstOffset is not + // equivalent. Fold the texel offset into normalized sample + // coordinates using the queried mip extent instead. + var offsetLod = explicitLod && !hasGradients + ? lod + : Float(0); + coordinates = ApplyDynamicSampleOffset( + resource, + imageObject, + coordinates, + offset, + offsetLod); + } + + var imageOperands = + hasGradients ? 4u : explicitLod ? 2u : hasBias ? 1u : 0u; var operands = new List { imageObject, @@ -1749,15 +3345,20 @@ public static partial class Gen5SpirvTranslator if (imageOperands != 0) { operands.Add(imageOperands); - if (explicitLod) + if (hasGradients) + { + operands.Add(gradientX); + operands.Add(gradientY); + } + else if (explicitLod) { operands.Add(lod); } - - if (hasOffset) + else if (hasBias) { - operands.Add(offset); + operands.Add(lodOrBias); } + } sampled = _module.AddInstruction( @@ -1779,12 +3380,26 @@ public static partial class Gen5SpirvTranslator instruction.Opcode.EndsWith("O", StringComparison.Ordinal); var hasCompare = instruction.Opcode.Contains("Gather4C", StringComparison.Ordinal); - var start = (hasOffset ? 1 : 0) + (hasCompare ? 1 : 0); - var coordinates = BuildFloatCoordinates(image, start); - var offset = hasOffset ? BuildImageOffset(image, 0) : 0u; - var reference = hasCompare - ? Bitcast(_floatType, LoadV(image.GetAddressRegister(hasOffset ? 1 : 0))) - : 0u; + var addressCursor = 0; + var offset = 0u; + if (hasOffset) + { + offset = BuildImageOffset(image, addressCursor); + addressCursor += ImageFullAddressSlots(image); + } + + var reference = 0u; + if (hasCompare) + { + addressCursor = AlignFullImageAddress(image, addressCursor); + reference = Bitcast( + _floatType, + LoadV(image.GetAddressRegister( + ImageAddressRegister(image, addressCursor)))); + addressCursor += ImageFullAddressSlots(image); + } + + var coordinates = BuildFloatCoordinates(image, addressCursor); var operands = new List { imageObject, @@ -1843,7 +3458,7 @@ public static partial class Gen5SpirvTranslator return false; } - uint output = 0; + var outputValues = new List(4); for (uint component = 0; component < 4; component++) { if (!writeAllComponents && @@ -1862,7 +3477,55 @@ public static partial class Gen5SpirvTranslator ImageComponentKind.Uint => value, _ => Bitcast(_uintType, value), }; - StoreV(image.VectorData + output++, raw); + outputValues.Add(raw); + } + + if (_stage == Gen5SpirvStage.Pixel && + PixelImageCaptureAddressMatches() && + uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_IMAGE_PC"), + out var captureImagePc) && + instruction.Pc == captureImagePc) + { + var captureBase = 248u; + if (uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_IMAGE_VGPR_BASE"), + out var requestedCaptureBase)) + { + captureBase = requestedCaptureBase; + } + captureBase = captureBase <= 252 ? captureBase : 248u; + for (var component = 0; component < 4; component++) + { + StoreV( + captureBase + (uint)component, + component < outputValues.Count + ? outputValues[component] + : Bitcast(_uintType, Float(1))); + } + } + + if (image.D16) + { + for (var index = 0; index < outputValues.Count; index += 2) + { + var low = outputValues[index]; + var high = index + 1 < outputValues.Count + ? outputValues[index + 1] + : UInt(0); + StoreV( + image.VectorData + (uint)(index / 2), + PackImageD16(resource, low, high)); + } + } + else + { + for (var index = 0; index < outputValues.Count; index++) + { + StoreV(image.VectorData + (uint)index, outputValues[index]); + } } return true; @@ -1931,12 +3594,8 @@ public static partial class Gen5SpirvTranslator private uint BuildFloatCoordinates(Gen5ImageControl image, int start) { - var x = Bitcast( - _floatType, - LoadV(image.GetAddressRegister(start))); - var y = Bitcast( - _floatType, - LoadV(image.GetAddressRegister(start + 1))); + var x = LoadImageFloatAddress(image, start); + var y = LoadImageFloatAddress(image, start + 1); return _module.AddInstruction( SpirvOp.CompositeConstruct, _vec2Type, @@ -1944,15 +3603,113 @@ public static partial class Gen5SpirvTranslator y); } + private static int ImageAddressRegister( + Gen5ImageControl image, + int component) => image.A16 ? component / 2 : component; + + private static int ImageFullAddressSlots(Gen5ImageControl image) => + image.A16 ? 2 : 1; + + private static int AlignFullImageAddress( + Gen5ImageControl image, + int component) => image.A16 ? (component + 1) & ~1 : component; + + private uint LoadImageFloatAddress(Gen5ImageControl image, int component) + { + var raw = LoadV(image.GetAddressRegister( + ImageAddressRegister(image, component))); + if (!image.A16) + { + return Bitcast(_floatType, raw); + } + + var unpacked = Ext(62, _vec2Type, raw); + return _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpacked, + (uint)(component & 1)); + } + + private uint LoadImageIntegerAddress(Gen5ImageControl image, int component) + { + var raw = LoadV(image.GetAddressRegister( + ImageAddressRegister(image, component))); + if (!image.A16) + { + return raw; + } + + return BitwiseAnd( + ShiftRightLogical(raw, UInt((uint)((component & 1) * 16))), + UInt(0xFFFF)); + } + + private uint LoadImageStoreComponent( + Gen5ImageControl image, + SpirvImageResource resource, + uint component) + { + if (!image.D16) + { + return LoadV(image.VectorData + component); + } + + var packed = LoadV(image.VectorData + component / 2); + if (resource.ComponentKind == ImageComponentKind.Float) + { + var unpacked = Ext(62, _vec2Type, packed); + return Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpacked, + component & 1)); + } + + var shifted = ShiftRightLogical(packed, UInt((component & 1) * 16)); + var low = BitwiseAnd(shifted, UInt(0xFFFF)); + if (resource.ComponentKind != ImageComponentKind.Sint) + { + return low; + } + + return Bitcast( + _uintType, + _module.AddInstruction( + SpirvOp.BitFieldSExtract, + _intType, + Bitcast(_intType, low), + UInt(0), + UInt(16))); + } + + private uint PackImageD16( + SpirvImageResource resource, + uint low, + uint high) + { + if (resource.ComponentKind == ImageComponentKind.Float) + { + var pair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Bitcast(_floatType, low), + Bitcast(_floatType, high)); + return Ext(58, _uintType, pair); + } + + return BitwiseOr( + BitwiseAnd(low, UInt(0xFFFF)), + ShiftLeftLogical(BitwiseAnd(high, UInt(0xFFFF)), UInt(16))); + } + private uint BuildIntegerCoordinates(Gen5ImageControl image, int start) { var ivec2 = _module.TypeVector(_intType, 2); - var x = Bitcast( - _intType, - LoadV(image.GetAddressRegister(start))); - var y = Bitcast( - _intType, - LoadV(image.GetAddressRegister(start + 1))); + var x = Bitcast(_intType, LoadImageIntegerAddress(image, start)); + var y = Bitcast(_intType, LoadImageIntegerAddress(image, start + 1)); return _module.AddInstruction( SpirvOp.CompositeConstruct, ivec2, @@ -1963,20 +3720,27 @@ public static partial class Gen5SpirvTranslator private uint BuildClampedIntegerCoordinates( Gen5ImageControl image, int start, - uint width, - uint height) + uint imageSize) { var ivec2 = _module.TypeVector(_intType, 2); var x = ClampSignedCoordinate( Bitcast( _intType, - LoadV(image.GetAddressRegister(start))), - width); + LoadImageIntegerAddress(image, start)), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 0)); var y = ClampSignedCoordinate( Bitcast( _intType, - LoadV(image.GetAddressRegister(start + 1))), - height); + LoadImageIntegerAddress(image, start + 1)), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 1)); return _module.AddInstruction( SpirvOp.CompositeConstruct, ivec2, @@ -1987,7 +3751,11 @@ public static partial class Gen5SpirvTranslator private uint ClampSignedCoordinate(uint value, uint extent) { var zero = _module.Constant(_intType, 0); - var max = _module.Constant(_intType, Math.Max(extent, 1) - 1); + var max = _module.AddInstruction( + SpirvOp.ISub, + _intType, + extent, + _module.Constant(_intType, 1)); var belowZero = _module.AddInstruction( SpirvOp.SLessThan, _boolType, @@ -2014,8 +3782,7 @@ public static partial class Gen5SpirvTranslator private void EmitBoundsCheckedImageWrite( uint coordinates, - uint width, - uint height, + uint imageSize, uint imageObject, uint texel) { @@ -2029,6 +3796,16 @@ public static partial class Gen5SpirvTranslator _intType, coordinates, 1); + var width = _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 0); + var height = _module.AddInstruction( + SpirvOp.CompositeExtract, + _intType, + imageSize, + 1); var zero = _module.Constant(_intType, 0); var xNonNegative = _module.AddInstruction( SpirvOp.SGreaterThanEqual, @@ -2044,12 +3821,12 @@ public static partial class Gen5SpirvTranslator SpirvOp.SLessThan, _boolType, x, - _module.Constant(_intType, width)); + width); var yInRange = _module.AddInstruction( SpirvOp.SLessThan, _boolType, y, - _module.Constant(_intType, height)); + height); var lowerInRange = _module.AddInstruction( SpirvOp.LogicalAnd, _boolType, @@ -2088,30 +3865,13 @@ public static partial class Gen5SpirvTranslator _module.AddLabel(mergeLabel); } - private static bool TryGetImageBounds( - IReadOnlyList descriptor, - out uint width, - out uint height) - { - width = 0; - height = 0; - if (descriptor.Count < 3) - { - return false; - } - - width = (((descriptor[1] >> 30) & 0x3u) | - ((descriptor[2] & 0xFFFu) << 2)) + 1; - height = ((descriptor[2] >> 14) & 0x3FFFu) + 1; - return width != 0 && height != 0 && width <= 16384 && height <= 16384; - } - private uint BuildImageOffset(Gen5ImageControl image, int component) { var ivec2 = _module.TypeVector(_intType, 2); var packed = Bitcast( _intType, - LoadV(image.GetAddressRegister(component))); + LoadV(image.GetAddressRegister( + ImageAddressRegister(image, component)))); var x = _module.AddInstruction( SpirvOp.BitFieldSExtract, _intType, @@ -2131,6 +3891,58 @@ public static partial class Gen5SpirvTranslator y); } + private uint ApplyDynamicSampleOffset( + SpirvImageResource resource, + uint sampledImage, + uint coordinates, + uint texelOffset, + uint lod) + { + var ivec2 = _module.TypeVector(_intType, 2); + var image = _module.AddInstruction( + SpirvOp.Image, + resource.ImageType, + sampledImage); + var signedLod = _module.AddInstruction( + SpirvOp.ConvertFToS, + _intType, + lod); + var lodIsNegative = _module.AddInstruction( + SpirvOp.SLessThan, + _boolType, + signedLod, + _module.Constant(_intType, 0)); + var clampedLod = _module.AddInstruction( + SpirvOp.Select, + _intType, + lodIsNegative, + _module.Constant(_intType, 0), + signedLod); + var size = _module.AddInstruction( + SpirvOp.ImageQuerySizeLod, + ivec2, + image, + clampedLod); + var sizeFloat = _module.AddInstruction( + SpirvOp.ConvertSToF, + _vec2Type, + size); + var offsetFloat = _module.AddInstruction( + SpirvOp.ConvertSToF, + _vec2Type, + texelOffset); + var normalizedOffset = _module.AddInstruction( + SpirvOp.FDiv, + _vec2Type, + offsetFloat, + sizeFloat); + return _module.AddInstruction( + SpirvOp.FAdd, + _vec2Type, + coordinates, + normalizedOffset); + } + private bool TryEmitExport( Gen5ShaderInstruction instruction, Gen5ExportControl export, @@ -2150,6 +3962,8 @@ public static partial class Gen5SpirvTranslator return true; } + Store(_reachedPixelExport, _module.ConstantBool(true)); + var values = new uint[4]; for (var component = 0; component < 4; component++) { @@ -2202,6 +4016,105 @@ public static partial class Gen5SpirvTranslator SpirvOp.CompositeConstruct, output.Type, values); + if (output.Kind == Gen5PixelOutputKind.Float && + PixelExportVgprAddressMatches() && + uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_VGPR_BASE"), + out var debugVgprBase)) + { + var registerBase = debugVgprBase + export.Target * 4; + vector = _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + Bitcast(_floatType, LoadV(registerBase)), + Bitcast(_floatType, LoadV(registerBase + 1)), + Bitcast(_floatType, LoadV(registerBase + 2)), + Bitcast(_floatType, LoadV(registerBase + 3))); + } + if (output.Kind == Gen5PixelOutputKind.Float && + PixelExportVgprAddressMatches() && + uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_PACK_VGPR_BASE"), + out var debugPackVgprBase)) + { + var registerBase = debugPackVgprBase + export.Target * 4; + var lowPair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase))), + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 1)))); + var highPair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 2))), + TruncateFloat32ForPack(Bitcast(_floatType, LoadV(registerBase + 3)))); + var unpackedLow = Ext(62, _vec2Type, Ext(58, _uintType, lowPair)); + var unpackedHigh = Ext(62, _vec2Type, Ext(58, _uintType, highPair)); + vector = _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedLow, + 0), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedLow, + 1), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedHigh, + 0), + _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + unpackedHigh, + 1)); + } + if (_forcePixelMagenta && PixelExportDebugAddressMatches()) + { + vector = output.Kind switch + { + Gen5PixelOutputKind.Float => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + Float(1f), + Float(0f), + Float(1f), + Float(1f)), + Gen5PixelOutputKind.Sint => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + Bitcast(_intType, UInt(1)), + Bitcast(_intType, UInt(0)), + Bitcast(_intType, UInt(1)), + Bitcast(_intType, UInt(1))), + _ => + _module.AddInstruction( + SpirvOp.CompositeConstruct, + output.Type, + UInt(1), + UInt(0), + UInt(1), + UInt(1)), + }; + } + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_EXPORT_EXEC") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + Store(_exec, _module.ConstantBool(true)); + StoreS64( + 126, + _module.Constant64(_ulongType, 1)); + } vector = _module.AddInstruction( SpirvOp.Select, output.Type, @@ -2253,6 +4166,19 @@ public static partial class Gen5SpirvTranslator SpirvOp.CompositeConstruct, _vec4Type, components); + if (_state.Program.Address == 0x0000000500780000ul && + export.Target is >= 32 and < 36 && + Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_TITLE_VERTEX_OUTPUTS_ONE") == "1") + { + outputValue = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec4Type, + Float(1f), + Float(1f), + Float(1f), + Float(1f)); + } outputValue = _module.AddInstruction( SpirvOp.Select, _vec4Type, @@ -2263,10 +4189,288 @@ public static partial class Gen5SpirvTranslator return true; } + private bool PixelExportDebugAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return true; + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + + private bool PixelImageCaptureAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_IMAGE_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return false; + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + + private void CapturePixelVgprs(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches() || + !uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_PC"), + out var capturePc) || + instruction.Pc != capturePc) + { + return; + } + + var sourceText = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_SOURCES"); + if (string.IsNullOrWhiteSpace(sourceText)) + { + return; + } + + var destinationBase = 248u; + if (uint.TryParse( + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_DEST_BASE"), + out var requestedDestinationBase)) + { + destinationBase = requestedDestinationBase; + } + + var sources = sourceText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries); + if (sources.Length is 0 or > 4 || + destinationBase > 252 || + destinationBase + (uint)sources.Length > 256) + { + return; + } + + for (var index = 0; index < sources.Length; index++) + { + if (!uint.TryParse(sources[index], out var source) || + source >= 256) + { + return; + } + } + + for (var index = 0; index < sources.Length; index++) + { + _ = uint.TryParse(sources[index], out var source); + StoreV( + destinationBase + (uint)index, + LoadV(source), + guardWithExec: + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_IGNORE_EXEC") != "1"); + } + } + + private bool PixelVgprCaptureAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return false; + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + + private void CapturePixelVgprPoints(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches()) + { + return; + } + + var captureText = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_POINTS"); + if (string.IsNullOrWhiteSpace(captureText)) + { + return; + } + + foreach (var capture in captureText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries)) + { + var fields = capture.Split(':'); + if (fields.Length != 3 || + !uint.TryParse(fields[0], out var pc) || + !uint.TryParse(fields[1], out var source) || + !uint.TryParse(fields[2], out var destination) || + pc != instruction.Pc || source >= 256 || destination >= 256) + { + continue; + } + + StoreV( + destination, + LoadV(source), + guardWithExec: + Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_VGPR_IGNORE_EXEC") != "1"); + } + } + + private void MarkPixelPath(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches()) + { + return; + } + + var markerText = Environment.GetEnvironmentVariable( + "SHARPEMU_MARK_PIXEL_PCS"); + if (string.IsNullOrWhiteSpace(markerText)) + { + return; + } + + foreach (var marker in markerText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries)) + { + var separator = marker.IndexOf(':'); + if (separator <= 0 || separator == marker.Length - 1 || + !uint.TryParse(marker.AsSpan(0, separator), out var pc) || + !uint.TryParse(marker.AsSpan(separator + 1), out var register) || + pc != instruction.Pc || register >= 256) + { + continue; + } + + StoreV( + register, + Bitcast(_uintType, Float(1)), + guardWithExec: false); + } + } + + private void CapturePixelExec(Gen5ShaderInstruction instruction) + { + if (_stage != Gen5SpirvStage.Pixel || + !PixelVgprCaptureAddressMatches()) + { + return; + } + + var captureText = Environment.GetEnvironmentVariable( + "SHARPEMU_CAPTURE_PIXEL_EXEC_PCS"); + if (string.IsNullOrWhiteSpace(captureText)) + { + return; + } + + foreach (var capture in captureText.Split( + ',', + StringSplitOptions.RemoveEmptyEntries | + StringSplitOptions.TrimEntries)) + { + var separator = capture.IndexOf(':'); + if (separator <= 0 || separator == capture.Length - 1 || + !uint.TryParse(capture.AsSpan(0, separator), out var pc) || + !uint.TryParse(capture.AsSpan(separator + 1), out var register) || + pc != instruction.Pc || register >= 256) + { + continue; + } + + var value = _module.AddInstruction( + SpirvOp.Select, + _floatType, + Load(_boolType, _exec), + Float(1), + Float(0)); + StoreV( + register, + Bitcast(_uintType, value), + guardWithExec: false); + } + } + + private bool PixelExportVgprAddressMatches() + { + var addressFilter = Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PIXEL_EXPORT_VGPR_ADDRESS"); + if (string.IsNullOrWhiteSpace(addressFilter)) + { + return PixelExportDebugAddressMatches(); + } + + var span = addressFilter.AsSpan(); + if (span.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + { + span = span[2..]; + } + + return ulong.TryParse( + span, + System.Globalization.NumberStyles.HexNumber, + System.Globalization.CultureInfo.InvariantCulture, + out var address) && + _state.Program.Address == address; + } + private uint LoadCompressedExportComponent( Gen5ShaderInstruction instruction, int component) { + if (TryLoadPackedHalfExportComponent( + instruction, + component, + out var shadowValue)) + { + return shadowValue; + } + var packed = LoadV(instruction.Sources[component >> 1].Value); var unpacked = Ext(62, _vec2Type, packed); return _module.AddInstruction( @@ -2276,6 +4480,132 @@ public static partial class Gen5SpirvTranslator (uint)(component & 1)); } + private bool TryLoadPackedHalfExportComponent( + Gen5ShaderInstruction exportInstruction, + int component, + out uint value) + { + value = 0; + var packedSource = exportInstruction.Sources[component >> 1]; + var tracePackedExport = + Environment.GetEnvironmentVariable( + "SHARPEMU_TRACE_PACKED_EXPORT") == "1" && + _state.Program.Address == 0x0000000500781200ul; + if (tracePackedExport) + { + Console.Error.WriteLine( + $"[AGC][PACKED-EXPORT] exp_pc=0x{exportInstruction.Pc:X} " + + $"component={component} source={packedSource.Kind}:" + + $"{packedSource.Value}"); + if (component == 0 && exportInstruction.Pc == 0x630) + { + foreach (var decoded in _state.Program.Instructions.Where( + static decoded => decoded.Pc <= 0x640)) + { + Console.Error.WriteLine( + $"[AGC][TITLE-IR] 0x{decoded.Pc:X4} " + + $"{decoded.Opcode} dst=[" + + string.Join(',', decoded.Destinations) + + "] src=[" + + string.Join(',', decoded.Sources) + "] words=[" + + string.Join(',', decoded.Words.Select(static word => $"{word:X8}")) + + "] ctrl=" + decoded.Control); + } + } + } + if (packedSource.Kind != Gen5OperandKind.VectorRegister) + { + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] rejected: source is not a VGPR"); + } + return false; + } + + for (var index = _state.Program.Instructions.Count - 1; index >= 0; index--) + { + var candidate = _state.Program.Instructions[index]; + if (candidate.Pc >= exportInstruction.Pc) + { + continue; + } + + if (exportInstruction.Pc - candidate.Pc > 128) + { + break; + } + + if (!candidate.Destinations.Any(destination => + destination.Kind == Gen5OperandKind.VectorRegister && + destination.Value == packedSource.Value)) + { + continue; + } + + if (tracePackedExport) + { + Console.Error.WriteLine( + $"[AGC][PACKED-EXPORT] nearest_pc=0x{candidate.Pc:X} " + + $"opcode={candidate.Opcode} distance=" + + $"{exportInstruction.Pc - candidate.Pc}"); + } + + if (candidate.Opcode != "VCvtPkrtzF16F32" || + candidate.Sources.Count < 2) + { + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] rejected: nearest writer is " + + candidate.Opcode); + } + return false; + } + + var packedPointer = PackedHalfPointer(packedSource.Value); + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PACKED_EXPORT_STORE_ONE") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + Store( + packedPointer, + _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Float(1f), + Float(1f))); + } + + var packedPair = Load( + _vec2Type, + packedPointer); + value = _module.AddInstruction( + SpirvOp.CompositeExtract, + _floatType, + packedPair, + (uint)(component & 1)); + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PACKED_EXPORT_ONE") == "1") + { + value = Float(1f); + } + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] selected shadow pair"); + } + return true; + } + + if (tracePackedExport) + { + Console.Error.WriteLine( + "[AGC][PACKED-EXPORT] rejected: no nearby writer"); + } + return false; + } + private uint GetPixelOutputType(Gen5PixelOutputKind kind) => kind switch { @@ -2286,14 +4616,87 @@ public static partial class Gen5SpirvTranslator private uint LoadBufferWord(int binding, uint dwordAddress) { - var pointer = BufferWordPointer(binding, dwordAddress); - return Load(_uintType, pointer); + var inRange = IsBufferWordInRange(binding, dwordAddress); + var safeAddress = _module.AddInstruction( + SpirvOp.Select, + _uintType, + inRange, + dwordAddress, + UInt(0)); + var value = Load(_uintType, BufferWordPointer(binding, safeAddress)); + return _module.AddInstruction( + SpirvOp.Select, + _uintType, + inRange, + value, + UInt(0)); + } + + private uint ApplyGuestBufferByteBias(int binding, uint byteAddress) + { + var evaluationBinding = binding - _globalBufferBase; + if ((uint)evaluationBinding >= + (uint)_evaluation.GlobalMemoryBindings.Count) + { + // Runtime SGPR blocks and other synthetic descriptors do not + // alias guest virtual memory and are always bound at offset 0. + return byteAddress; + } + + if (_initialScalarBufferIndex >= 0) + { + // Descriptor offsets must satisfy Vulkan's storage-buffer + // alignment. The presenter therefore rounds the shared guest + // allocation offset down and packs the discarded low address + // bits after the 256 initial SGPRs in the per-dispatch runtime + // block. Keeping this value runtime-stable prevents rotating + // guest allocations from producing a new multi-megabyte SPIR-V + // module and Metal pipeline while preserving exact byte access. + var runtimeByteBias = Load( + _uintType, + RuntimeBufferBiasPointer(binding)); + return IAdd(byteAddress, runtimeByteBias); + } + + // The presenter binds the shared allocation at the largest aligned + // offset not greater than this guest resource's offset. Because the + // allocation base is aligned to the same power of two, the bytes + // discarded from the descriptor offset are exactly the low address + // bits below. Adding them here keeps scalar, MUBUF and GLOBAL paths + // byte-exact, including atomics and resources that overlap another + // descriptor at an unaligned guest address. + var byteBias = + _evaluation.GlobalMemoryBindings[evaluationBinding].BaseAddress & + (_storageBufferOffsetAlignment - 1); + return byteBias == 0 + ? byteAddress + : IAdd(byteAddress, UInt(checked((uint)byteBias))); } private void StoreBufferWord(int binding, uint dwordAddress, uint value) { - var pointer = BufferWordPointer(binding, dwordAddress); - Store(pointer, value); + EmitConditional( + IsBufferWordInRange(binding, dwordAddress), + () => Store(BufferWordPointer(binding, dwordAddress), value)); + } + + private uint IsBufferWordInRange(int binding, uint dwordAddress) + { + var buffer = _module.AddInstruction( + SpirvOp.AccessChain, + _storageBlockPointer, + _globalBuffers, + UInt((uint)binding)); + var length = _module.AddInstruction( + SpirvOp.ArrayLength, + _uintType, + buffer, + 0); + return _module.AddInstruction( + SpirvOp.ULessThan, + _boolType, + dwordAddress, + length); } private uint BufferWordPointer(int binding, uint dwordAddress) => @@ -2312,6 +4715,13 @@ public static partial class Gen5SpirvTranslator _scalarRegisters, UInt(register)); + private uint RuntimeBufferBiasPointer(int binding) => + _module.AddInstruction( + SpirvOp.AccessChain, + _privateUintPointer, + _runtimeBufferBiases, + UInt(checked((uint)binding))); + private uint VectorPointer(uint register) => _module.AddInstruction( SpirvOp.AccessChain, @@ -2319,6 +4729,13 @@ public static partial class Gen5SpirvTranslator _vectorRegisters, UInt(register)); + private uint PackedHalfPointer(uint register) => + _module.AddInstruction( + SpirvOp.AccessChain, + _privateVec2Pointer, + _packedHalfRegisters, + UInt(register)); + private uint LoadS(uint register) => Load(_uintType, ScalarPointer(register)); private uint LoadV(uint register) => Load(_uintType, VectorPointer(register)); @@ -2353,8 +4770,52 @@ public static partial class Gen5SpirvTranslator Store(VectorPointer(register), value); } - private uint Load(uint type, uint pointer) => - _module.AddInstruction(SpirvOp.Load, type, pointer); + private void StorePackedHalf(uint register, uint value) + { + var active = Load(_boolType, _exec); + if (Environment.GetEnvironmentVariable( + "SHARPEMU_FORCE_PACKED_STORE_EXEC_VALUES") == "1" && + _state.Program.Address == 0x0000000500781200ul) + { + var activePair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Float(1f), + Float(1f)); + var inactivePair = _module.AddInstruction( + SpirvOp.CompositeConstruct, + _vec2Type, + Float(0.5f), + Float(0.5f)); + value = _module.AddInstruction( + SpirvOp.Select, + _vec2Type, + active, + activePair, + inactivePair); + Store(PackedHalfPointer(register), value); + return; + } + + value = _module.AddInstruction( + SpirvOp.Select, + _vec2Type, + active, + value, + Load(_vec2Type, PackedHalfPointer(register))); + Store(PackedHalfPointer(register), value); + } + + private uint Load(uint type, uint pointer) + { + if (pointer == 0) + { + throw new InvalidOperationException( + "SPIR-V generator attempted OpLoad from id 0."); + } + + return _module.AddInstruction(SpirvOp.Load, type, pointer); + } private void Store(uint pointer, uint value) => _module.AddStatement(SpirvOp.Store, pointer, value); @@ -2412,6 +4873,9 @@ public static partial class Gen5SpirvTranslator private uint BitwiseAnd64(uint left, uint right) => _module.AddInstruction(SpirvOp.BitwiseAnd, _ulongType, left, right); + private uint BitwiseOr64(uint left, uint right) => + _module.AddInstruction(SpirvOp.BitwiseOr, _ulongType, left, right); + private uint BitwiseOr(uint left, uint right) => _module.AddInstruction(SpirvOp.BitwiseOr, _uintType, left, right); @@ -2424,12 +4888,35 @@ public static partial class Gen5SpirvTranslator private uint SubgroupAny(uint condition) => _subgroupInvocationIdInput == 0 ? condition + : _emulateWave64 + ? IsNotZero64(BooleanToWaveMask(condition)) : _module.AddInstruction( SpirvOp.GroupNonUniformAny, _boolType, UInt(3), condition); + private uint GuestWaveLane() + { + if (_waveLaneCount == 64 && _localInvocationIndexInput != 0) + { + return BitwiseAnd( + Load(_uintType, _localInvocationIndexInput), + UInt(63)); + } + + if (_subgroupInvocationIdInput != 0) + { + return BitwiseAnd( + Load(_uintType, _subgroupInvocationIdInput), + UInt(31)); + } + + // Graphics stages without subgroup support have one logical lane; + // they must not emit OpLoad for absent SPIR-V input ID zero. + return UInt(0); + } + private uint CurrentLaneBit() { if (_subgroupInvocationIdInput == 0) @@ -2437,8 +4924,7 @@ public static partial class Gen5SpirvTranslator return _module.Constant64(_ulongType, 1); } - var lane = Load(_uintType, _subgroupInvocationIdInput); - var maskedLane = BitwiseAnd(lane, UInt(RdnaWaveLaneCount - 1)); + var maskedLane = GuestWaveLane(); var shifted = ShiftLeftLogical64( _module.Constant64(_ulongType, 1), _module.AddInstruction( @@ -2458,7 +4944,7 @@ public static partial class Gen5SpirvTranslator SpirvOp.ULessThan, _boolType, Load(_uintType, _subgroupInvocationIdInput), - UInt(RdnaWaveLaneCount)); + UInt(32)); private uint BooleanToLaneMask(uint condition) => _module.AddInstruction( @@ -2468,6 +4954,105 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit(), _module.Constant64(_ulongType, 0)); + private uint BooleanToWaveMask(uint condition) + { + if (_subgroupInvocationIdInput == 0) + { + return BooleanToLaneMask(condition); + } + + var ballot = _module.AddInstruction( + SpirvOp.GroupNonUniformBallot, + _uvec4Type, + UInt(3), + condition); + var low = _module.AddInstruction( + SpirvOp.CompositeExtract, + _uintType, + ballot, + 0); + if (_emulateWave64) + { + var subgroupLane = + Load(_uintType, _subgroupInvocationIdInput); + var firstLane = _module.AddInstruction( + SpirvOp.IEqual, + _boolType, + subgroupLane, + UInt(0)); + var half = ShiftRightLogical(GuestWaveLane(), UInt(5)); + EmitConditional(firstLane, () => + { + Store(WaveMaskScratchPointer(half), low); + }); + EmitWave64Barrier(); + var lowMask = Load( + _uintType, + WaveMaskScratchPointer(UInt(0))); + var highMask = Load( + _uintType, + WaveMaskScratchPointer(UInt(1))); + var combined = BitwiseOr64( + _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + lowMask), + ShiftLeftLogical64( + _module.AddInstruction( + SpirvOp.UConvert, + _ulongType, + highMask), + _module.Constant64(_ulongType, 32))); + EmitWave64Barrier(); + return combined; + } + + var widened = _module.AddInstruction(SpirvOp.UConvert, _ulongType, low); + if (_waveLaneCount != 64) + { + return widened; + } + + return _module.AddInstruction( + SpirvOp.Select, + _ulongType, + _module.AddInstruction( + SpirvOp.UGreaterThanEqual, + _boolType, + GuestWaveLane(), + UInt(32)), + ShiftLeftLogical64( + widened, + _module.Constant64(_ulongType, 32)), + widened); + } + + private uint WaveMaskScratchPointer(uint index) => + _module.AddInstruction( + SpirvOp.AccessChain, + _waveMaskScratchElementPointer, + _waveScratchInLds ? _lds : _waveMaskScratch, + _waveScratchInLds ? IAdd(UInt(LdsDwordCount - 3), index) : index); + + private uint WaveBroadcastScratchPointer() => + _waveScratchInLds + ? _module.AddInstruction( + SpirvOp.AccessChain, + _ldsElementPointer, + _lds, + UInt(LdsDwordCount - 1)) + : _waveBroadcastScratch; + + private void EmitWave64Barrier() + { + var workgroup = UInt(2); + _module.AddStatement( + SpirvOp.ControlBarrier, + workgroup, + workgroup, + UInt(0x108)); + } + private uint IsWaveMaskActive(uint mask) => _subgroupInvocationIdInput == 0 ? IsNotZero64(mask) @@ -2482,17 +5067,22 @@ public static partial class Gen5SpirvTranslator CurrentLaneBit())); private void StoreWaveMask(uint register, uint condition) => - StoreS64(register, BooleanToLaneMask(condition)); + StoreS64(register, BooleanToWaveMask(condition)); private void EmitExecConditional(Action emit) + { + var active = Load(_boolType, _exec); + EmitConditional(active, emit); + } + + private void EmitConditional(uint condition, Action emit) { var activeLabel = _module.AllocateId(); var mergeLabel = _module.AllocateId(); - var active = Load(_boolType, _exec); _module.AddStatement(SpirvOp.SelectionMerge, mergeLabel, 0); _module.AddStatement( SpirvOp.BranchConditional, - active, + condition, activeLabel, mergeLabel); _module.AddLabel(activeLabel); @@ -2507,8 +5097,13 @@ public static partial class Gen5SpirvTranslator private bool UsesSubgroupShuffle() => _state.Program.Instructions.Any(instruction => + instruction.Control is Gen5DppControl or Gen5Dpp8Control || instruction.Opcode is "VPermlane16B32" or "VPermlanex16B32" or "VReadlaneB32"); + private bool UsesSubgroupBroadcast() => + _state.Program.Instructions.Any(instruction => + instruction.Opcode == "VReadfirstlaneB32"); + private bool UsesWaveControl() => _state.Program.Instructions.Any(instruction => instruction.Opcode.Contains("Saveexec", StringComparison.Ordinal) || @@ -2520,7 +5115,11 @@ public static partial class Gen5SpirvTranslator private bool UsesSubgroupOperations() => _stage == Gen5SpirvStage.Compute && - (UsesSubgroupShuffle() || UsesWaveControl()); + (UsesSubgroupShuffle() || + UsesSubgroupBroadcast() || + UsesWaveControl() || + _state.Program.Instructions.Any(static instruction => + instruction.Opcode is "VMbcntLoU32B32" or "VMbcntHiU32B32")); private static bool IsWaveMaskOperand(Gen5Operand operand) => operand.Kind == Gen5OperandKind.ScalarRegister && @@ -2613,6 +5212,198 @@ public static partial class Gen5SpirvTranslator return blocks; } + private void BuildScalarDefinitionInfo( + IReadOnlyList blocks, + IReadOnlyList instructions) + { + var predecessors = new HashSet[blocks.Count]; + for (var index = 0; index < blocks.Count; index++) + { + predecessors[index] = []; + } + + void AddEdge(int source, int destination) + { + if (destination < 0 || destination >= blocks.Count) + { + return; + } + + predecessors[destination].Add(source); + } + + for (var blockIndex = 0; blockIndex < blocks.Count; blockIndex++) + { + var block = blocks[blockIndex]; + var terminator = instructions[block.EndIndex - 1]; + var hasFallthrough = blockIndex + 1 < blocks.Count; + if (terminator.Opcode == "SEndpgm") + { + continue; + } + + if (terminator.Opcode == "SBranch") + { + if (TryGetBranchTargetPc(terminator, out var targetPc) && + TryFindBlock(blocks, targetPc, out var targetBlock)) + { + AddEdge(blockIndex, targetBlock); + } + + continue; + } + + if (terminator.Opcode.StartsWith("SCbranch", StringComparison.Ordinal)) + { + if (TryGetBranchTargetPc(terminator, out var targetPc) && + TryFindBlock(blocks, targetPc, out var targetBlock)) + { + AddEdge(blockIndex, targetBlock); + } + + if (hasFallthrough) + { + AddEdge(blockIndex, blockIndex + 1); + } + + continue; + } + + if (hasFallthrough) + { + AddEdge(blockIndex, blockIndex + 1); + } + } + + var blockInputs = new long[blocks.Count][]; + var blockOutputs = new long[blocks.Count][]; + var hasOutput = new bool[blocks.Count]; + var initialDefinitions = Enumerable.Repeat( + InitialScalarDefinition, + ScalarRegisterCount).ToArray(); + + static void MergeDefinitions( + long[] destination, + long[] source, + ref bool hasInput) + { + if (!hasInput) + { + Array.Copy(source, destination, ScalarRegisterCount); + hasInput = true; + return; + } + + for (var register = 0; register < ScalarRegisterCount; register++) + { + if (destination[register] != source[register]) + { + destination[register] = ConflictingScalarDefinition; + } + } + } + + static void ApplyScalarDefinitions( + long[] definitions, + ShaderBlock block, + IReadOnlyList blockInstructions) + { + for (var instructionIndex = block.StartIndex; + instructionIndex < block.EndIndex; + instructionIndex++) + { + var instruction = blockInstructions[instructionIndex]; + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value < ScalarRegisterCount) + { + definitions[destination.Value] = instruction.Pc + 1L; + } + } + } + } + + var changed = true; + while (changed) + { + changed = false; + for (var blockIndex = 0; blockIndex < blocks.Count; blockIndex++) + { + var input = Enumerable.Repeat( + UnreachableScalarDefinition, + ScalarRegisterCount).ToArray(); + var hasInput = false; + if (blockIndex == 0) + { + MergeDefinitions(input, initialDefinitions, ref hasInput); + } + + foreach (var predecessor in predecessors[blockIndex]) + { + if (hasOutput[predecessor]) + { + MergeDefinitions( + input, + blockOutputs[predecessor], + ref hasInput); + } + } + + if (!hasInput) + { + continue; + } + + var output = (long[])input.Clone(); + ApplyScalarDefinitions(output, blocks[blockIndex], instructions); + if (!hasOutput[blockIndex] || + !blockInputs[blockIndex].AsSpan().SequenceEqual(input) || + !blockOutputs[blockIndex].AsSpan().SequenceEqual(output)) + { + blockInputs[blockIndex] = input; + blockOutputs[blockIndex] = output; + hasOutput[blockIndex] = true; + changed = true; + } + } + } + + _scalarDefinitionsBeforePc.Clear(); + for (var blockIndex = 0; blockIndex < blocks.Count; blockIndex++) + { + if (!hasOutput[blockIndex]) + { + continue; + } + + var definitions = (long[])blockInputs[blockIndex].Clone(); + var block = blocks[blockIndex]; + for (var instructionIndex = block.StartIndex; + instructionIndex < block.EndIndex; + instructionIndex++) + { + var instruction = instructions[instructionIndex]; + if (instruction.Control is Gen5ImageControl or + Gen5ScalarMemoryControl or + Gen5GlobalMemoryControl or + Gen5BufferMemoryControl) + { + _scalarDefinitionsBeforePc[instruction.Pc] = + (long[])definitions.Clone(); + } + foreach (var destination in instruction.Destinations) + { + if (destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value < ScalarRegisterCount) + { + definitions[destination.Value] = instruction.Pc + 1L; + } + } + } + } + } + private static int FindInstructionIndex( IReadOnlyList instructions, uint pc) diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs index 7626886..479a27e 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvFixedShaders.cs @@ -164,4 +164,100 @@ public static class SpirvFixedShaders module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); return module.Build(); } + + public static byte[] CreateSolidFragment(float red, float green, float blue, float alpha) + { + var module = new SpirvModuleBuilder(); + module.AddCapability(SpirvCapability.Shader); + + var voidType = module.TypeVoid(); + var floatType = module.TypeFloat(32); + var vec4Type = module.TypeVector(floatType, 4); + var outputVec4Pointer = module.TypePointer(SpirvStorageClass.Output, vec4Type); + var output = module.AddGlobalVariable(outputVec4Pointer, SpirvStorageClass.Output); + module.AddName(output, "outColor"); + module.AddDecoration(output, SpirvDecoration.Location, 0); + + var functionType = module.TypeFunction(voidType); + var main = module.BeginFunction(voidType, functionType); + module.AddName(main, "main"); + module.AddLabel(); + var color = module.ConstantComposite( + vec4Type, + module.ConstantFloat(floatType, red), + module.ConstantFloat(floatType, green), + module.ConstantFloat(floatType, blue), + module.ConstantFloat(floatType, alpha)); + module.AddStatement(SpirvOp.Store, output, color); + module.AddStatement(SpirvOp.Return); + module.EndFunction(); + + module.AddEntryPoint(SpirvExecutionModel.Fragment, main, "main", [output]); + module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); + return module.Build(); + } + + /// + /// Diagnostic fragment stage that exposes one interpolated vertex output + /// directly as color. This keeps the real guest vertex/index/depth path + /// intact while isolating fragment-shader translation from interface data. + /// + public static byte[] CreateAttributeFragment(uint location) + { + var module = new SpirvModuleBuilder(); + module.AddCapability(SpirvCapability.Shader); + + var voidType = module.TypeVoid(); + var floatType = module.TypeFloat(32); + var vec4Type = module.TypeVector(floatType, 4); + var inputPointer = module.TypePointer(SpirvStorageClass.Input, vec4Type); + var outputPointer = module.TypePointer(SpirvStorageClass.Output, vec4Type); + var input = module.AddGlobalVariable(inputPointer, SpirvStorageClass.Input); + module.AddName(input, $"attr{location}"); + module.AddDecoration(input, SpirvDecoration.Location, location); + var output = module.AddGlobalVariable(outputPointer, SpirvStorageClass.Output); + module.AddName(output, "outColor"); + module.AddDecoration(output, SpirvDecoration.Location, 0); + + var functionType = module.TypeFunction(voidType); + var main = module.BeginFunction(voidType, functionType); + module.AddName(main, "main"); + module.AddLabel(); + var value = module.AddInstruction(SpirvOp.Load, vec4Type, input); + module.AddStatement(SpirvOp.Store, output, value); + module.AddStatement(SpirvOp.Return); + module.EndFunction(); + + module.AddEntryPoint( + SpirvExecutionModel.Fragment, + main, + "main", + [input, output]); + module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); + return module.Build(); + } + + /// + /// Minimal fragment stage for fixed-function depth-only passes. The + /// guest has no pixel shader and therefore cannot export colour; keeping + /// this stage output-free preserves that contract while allowing Vulkan + /// to run early/late depth tests for the translated vertex shader. + /// + public static byte[] CreateDepthOnlyFragment() + { + var module = new SpirvModuleBuilder(); + module.AddCapability(SpirvCapability.Shader); + + var voidType = module.TypeVoid(); + var functionType = module.TypeFunction(voidType); + var main = module.BeginFunction(voidType, functionType); + module.AddName(main, "main"); + module.AddLabel(); + module.AddStatement(SpirvOp.Return); + module.EndFunction(); + + module.AddEntryPoint(SpirvExecutionModel.Fragment, main, "main", []); + module.AddExecutionMode(main, SpirvExecutionMode.OriginUpperLeft); + return module.Build(); + } } diff --git a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs index bbb7b0b..6295010 100644 --- a/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs +++ b/src/SharpEmu.ShaderCompiler.Vulkan/SpirvModuleBuilder.cs @@ -143,6 +143,7 @@ public enum SpirvOp : ushort ControlBarrier = 224, MemoryBarrier = 225, AtomicIAdd = 234, + AtomicUMax = 239, Phi = 245, LoopMerge = 246, SelectionMerge = 247, diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs index ba0db21..01734ed 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderIr.cs @@ -175,7 +175,9 @@ public sealed record Gen5ImageControl( uint Dimension, bool IsArray, bool Glc, - bool Slc) : Gen5InstructionControl + bool Slc, + bool A16, + bool D16) : Gen5InstructionControl { public uint GetAddressRegister(int component) => component < AddressRegisters.Count @@ -219,16 +221,21 @@ public sealed record Gen5Vop3Control( uint NegateMask, uint OutputModifier, bool Clamp, + uint OperandSelect, uint? ScalarDestination) : Gen5InstructionControl; public sealed record Gen5SdwaControl( uint DestinationSelect, + uint DestinationUnused, uint Source0Select, uint Source1Select, + bool Source0SignExtend, + bool Source1SignExtend, uint AbsoluteMask, uint NegateMask, uint OutputModifier, - bool Clamp) : Gen5InstructionControl; + bool Clamp, + uint? ScalarDestination) : Gen5InstructionControl; public sealed record Gen5DppControl( uint Control, @@ -239,6 +246,10 @@ public sealed record Gen5DppControl( uint BankMask, uint RowMask) : Gen5InstructionControl; +public sealed record Gen5Dpp8Control( + uint LaneSelectors, + bool FetchInactive) : Gen5InstructionControl; + public sealed record Gen5ScalarMemoryControl( uint DestinationCount, int ImmediateOffsetBytes, @@ -257,11 +268,27 @@ public sealed record Gen5ImageBinding( IReadOnlyList SamplerDescriptor, uint? MipLevel); +// Data arrays may be rented from ArrayPool (oversized): always slice with +// DataLength, never Data.Length. Ownership transfers to the presenter, which +// returns pooled arrays after uploading them into host-visible buffers. public sealed record Gen5GlobalMemoryBinding( uint ScalarAddress, ulong BaseAddress, IReadOnlyList InstructionPcs, - byte[] Data); + byte[] Data, + int DataLength, + bool DataPooled) +{ + public bool Writable { get; set; } + + // Writable describes shader access and is also used to decide whether a + // compute dispatch has observable work. A statically reachable resource + // can nevertheless be unbound for the current scalar path; the evaluator + // supplies zero-filled storage for Vulkan in that case. Such synthetic + // storage must remain shader-writable, but must never be copied to the + // descriptor's unmapped guest address. + public bool WriteBackToGuest { get; set; } = true; +} public sealed record Gen5VertexInputBinding( uint Pc, @@ -272,12 +299,13 @@ public sealed record Gen5VertexInputBinding( ulong BaseAddress, uint Stride, uint OffsetBytes, - byte[] Data); + byte[] Data, + int DataLength, + bool DataPooled); public sealed record Gen5ShaderEvaluation( IReadOnlyList InitialScalarRegisters, IReadOnlyList ScalarRegisters, - IReadOnlyDictionary> ScalarRegistersByPc, IReadOnlyList ImageBindings, IReadOnlyList GlobalMemoryBindings, Gen5ComputeSystemRegisters? ComputeSystemRegisters = null, @@ -297,8 +325,58 @@ public sealed record Gen5ShaderProgram( ulong Address, IReadOnlyList Instructions) { + private const int ScalarRegisterCount = 256; + private IReadOnlySet? _runtimeScalarRegisters; + public IEnumerable ImageResources => Instructions .Select(instruction => instruction.Control) .OfType(); + + /// + /// The set of scalar registers the program reads or writes as runtime + /// values. It depends only on the (cached) decoded program, so it is + /// computed once and shared read-only across every draw that uses this + /// shader — the evaluator previously rebuilt this HashSet by scanning + /// every instruction on every draw, one of the largest per-draw + /// allocation and CPU sources. + /// + public IReadOnlySet RuntimeScalarRegisters => + _runtimeScalarRegisters ??= ComputeRuntimeScalarRegisters(); + + private IReadOnlySet ComputeRuntimeScalarRegisters() + { + var registers = new HashSet(); + foreach (var instruction in Instructions) + { + foreach (var operand in instruction.Sources) + { + if (operand.Kind == Gen5OperandKind.ScalarRegister && + operand.Value < ScalarRegisterCount) + { + registers.Add(operand.Value); + } + } + + foreach (var operand in instruction.Destinations) + { + if (operand.Kind == Gen5OperandKind.ScalarRegister && + operand.Value < ScalarRegisterCount) + { + registers.Add(operand.Value); + } + } + + if (instruction.Control is Gen5ScalarMemoryControl + { + DynamicOffsetRegister: { } offsetRegister, + } && + offsetRegister < ScalarRegisterCount) + { + registers.Add(offsetRegister); + } + } + + return registers; + } } diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs index 9880f66..0cdbacc 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderMetadataReader.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using System.Buffers.Binary; namespace SharpEmu.ShaderCompiler; @@ -17,9 +18,9 @@ public static class Gen5ShaderMetadataReader out Gen5ShaderMetadata metadata) { metadata = default!; - if (!ctx.TryReadUInt64(shaderHeaderAddress + ShaderUserDataOffset, out var userDataAddress) || + if (!TryReadUInt64(ctx, shaderHeaderAddress + ShaderUserDataOffset, out var userDataAddress) || userDataAddress == 0 || - !ctx.TryReadUInt64(userDataAddress, out var directResourceOffsetsAddress)) + !TryReadUInt64(ctx, userDataAddress, out var directResourceOffsetsAddress)) { return false; } @@ -27,7 +28,8 @@ public static class Gen5ShaderMetadataReader var resourceOffsets = new ulong[ResourceClassCount]; for (var resourceClass = 0; resourceClass < ResourceClassCount; resourceClass++) { - if (!ctx.TryReadUInt64( + if (!TryReadUInt64( + ctx, userDataAddress + 0x08 + (ulong)(resourceClass * sizeof(ulong)), out resourceOffsets[resourceClass])) { @@ -35,9 +37,9 @@ public static class Gen5ShaderMetadataReader } } - if (!ctx.TryReadUInt16(userDataAddress + 0x28, out var extendedUserDataSize) || - !ctx.TryReadUInt16(userDataAddress + 0x2A, out var shaderResourceTableSize) || - !ctx.TryReadUInt16(userDataAddress + 0x2C, out var directResourceCount) || + if (!TryReadUInt16(ctx, userDataAddress + 0x28, out var extendedUserDataSize) || + !TryReadUInt16(ctx, userDataAddress + 0x2A, out var shaderResourceTableSize) || + !TryReadUInt16(ctx, userDataAddress + 0x2C, out var directResourceCount) || directResourceCount > MaxMetadataEntries) { return false; @@ -46,7 +48,8 @@ public static class Gen5ShaderMetadataReader var resourceCounts = new ushort[ResourceClassCount]; for (var resourceClass = 0; resourceClass < ResourceClassCount; resourceClass++) { - if (!ctx.TryReadUInt16( + if (!TryReadUInt16( + ctx, userDataAddress + 0x2E + (ulong)(resourceClass * sizeof(ushort)), out resourceCounts[resourceClass]) || resourceCounts[resourceClass] > MaxMetadataEntries) @@ -65,7 +68,7 @@ public static class Gen5ShaderMetadataReader for (uint type = 0; type < directResourceCount; type++) { - if (!ctx.TryReadUInt16(directResourceOffsetsAddress + type * sizeof(ushort), out var offset)) + if (!TryReadUInt16(ctx, directResourceOffsetsAddress + type * sizeof(ushort), out var offset)) { return false; } @@ -93,7 +96,8 @@ public static class Gen5ShaderMetadataReader for (uint slot = 0; slot < count; slot++) { - if (!ctx.TryReadUInt16( + if (!TryReadUInt16( + ctx, resourceOffsets[resourceClass] + slot * sizeof(ushort), out var sharp)) { @@ -121,4 +125,30 @@ public static class Gen5ShaderMetadataReader resources); return true; } + + private static bool TryReadUInt16(CpuContext ctx, ulong address, out ushort value) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt16LittleEndian(bytes); + return true; + } + + private static bool TryReadUInt64(CpuContext ctx, ulong address, out ulong value) + { + Span bytes = stackalloc byte[sizeof(ulong)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt64LittleEndian(bytes); + return true; + } } diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs index 520fa3a..0dc2d25 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderScalarEvaluator.cs @@ -2,12 +2,52 @@ // SPDX-License-Identifier: GPL-2.0-or-later using SharpEmu.HLE; +using System.Buffers; +using System.Buffers.Binary; +using System.Diagnostics; using System.Numerics; namespace SharpEmu.ShaderCompiler; public static class Gen5ShaderScalarEvaluator { + // When a scalar POINTER load can't be resolved statically (its descriptor + // register read back garbage — e.g. 0 or 0xFFFFFFFF, a per-draw descriptor + // setup race), abort-and-drop-the-draw loses the whole pass. Demon's Souls' + // deferred-lighting / composite pixel shaders hit this intermittently, so + // the passes that would produce the composite's feeder targets get dropped + // and the frame stays black. Degrading instead (feed 0, keep translating, + // like the buffer-load path already does) lets the pass render with the + // unresolved resource missing rather than not at all. STRICT reverts. + private static readonly bool _strictScalarLoad = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_STRICT_SCALAR_LOAD"), + "1", + StringComparison.Ordinal); + + // A stale buffer descriptor should not discard an otherwise valid shader + // pass. Treat it as an all-zero buffer by default; callers that need + // strict diagnostics can restore the old failure behaviour explicitly. + private static readonly bool _strictBufferLoad = + string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_STRICT_BUFFER_LOAD"), + "1", + StringComparison.Ordinal); + private static readonly object _scalarFallbackTraceGate = new(); + private static readonly HashSet<(ulong Shader, uint Pc)> _tracedScalarFallbacks = []; + + // Uniform forward branches select material/resource bodies that remain + // statically present in the translated shader. Discover the skipped body's + // descriptors by default; SHARPEMU_CFG_RESOURCE_DISCOVERY=0 is a diagnostic + // opt-out. Conditional branches are deliberately not forked because their + // fall-through is already scanned and forking vector-mask conditions grows + // exponentially without adding descriptor coverage. + private static readonly bool _cfgResourceDiscovery = + !string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_CFG_RESOURCE_DISCOVERY"), + "0", + StringComparison.Ordinal); + /// /// Optional fallback for global-memory reads that ctx.Memory cannot satisfy (the /// emulator installs the HLE-tracked libc heap reader here at module load). Kept as @@ -17,17 +57,16 @@ public static class Gen5ShaderScalarEvaluator public delegate bool Gen5FallbackMemoryReader(ulong baseAddress, Span destination); + /// + /// Pool used for large draw-time guest-memory snapshots. The HLE host installs + /// its bounded transfer pool; standalone compiler tools use the shared pool. + /// + public static ArrayPool GlobalMemoryPool { get; set; } = ArrayPool.Shared; + private const int ScalarRegisterCount = 256; private const int ImageDescriptorDwords = 8; private const int SamplerDescriptorDwords = 4; private const int MaxGlobalMemoryBindingBytes = 16 * 1024 * 1024; - private static readonly int DefaultGlobalMemoryBindingBytes = - int.TryParse( - Environment.GetEnvironmentVariable("SHARPEMU_GLOBAL_BINDING_BYTES"), - out var configured) && configured >= sizeof(uint) - ? Math.Min(configured, MaxGlobalMemoryBindingBytes) - : 1 * 1024 * 1024; - public static long GlobalMemoryReadCount; public static long GlobalMemoryReadBytes; public static long GlobalMemoryReadCacheHits; @@ -35,12 +74,13 @@ public static class Gen5ShaderScalarEvaluator public static long GlobalMemoryReadLibcBytes; public static long GlobalMemoryReadReuses; - private const long CrossFrameReadCacheMaxBytes = 1024L * 1024 * 1024; - private static readonly object _crossFrameReadGate = new(); - private static readonly Dictionary<(ulong BaseAddress, int SizeBytes), byte[]> _crossFrameReadCache = new(); - private static long _crossFrameReadCacheBytes; private const ulong RdnaWaveMask = 0xFFFF_FFFFUL; + static Gen5ShaderScalarEvaluator() + { + RunScalarLoadSelfChecks(); + } + private readonly record struct BufferDescriptor( ulong BaseAddress, uint Stride, @@ -49,6 +89,31 @@ public static class Gen5ShaderScalarEvaluator uint NumberFormat, uint DataFormat); + private readonly record struct ScalarPathState( + uint StartPc, + uint[] ScalarRegisters, + ulong ExecMask, + bool ScalarConditionCode, + bool Supplemental); + + private readonly record struct ScalarPathKey(uint StartPc, ulong StateHash); + + private static ulong ComputeScalarStateHash( + IReadOnlyList registers, + ulong execMask, + bool scalarConditionCode) + { + const ulong prime = 1099511628211UL; + var hash = 14695981039346656037UL; + foreach (var value in registers) + { + hash = (hash ^ value) * prime; + } + + hash = (hash ^ execMask) * prime; + return (hash ^ (scalarConditionCode ? 1UL : 0UL)) * prime; + } + public static bool TryResolveImageBindings( CpuContext ctx, Gen5ShaderState state, @@ -71,7 +136,7 @@ public static class Gen5ShaderScalarEvaluator out Gen5ShaderEvaluation evaluation, out string error, bool resolveVertexInputs = false, - uint? vertexRecordLimit = null) + uint? requiredVertexRecordCount = null) { evaluation = default!; error = string.Empty; @@ -99,40 +164,114 @@ public static class Gen5ShaderScalarEvaluator var globalMemoryBindings = new List(); var globalMemoryByAddress = new Dictionary<(uint ScalarAddress, ulong BaseAddress), Gen5GlobalMemoryBinding>(); var vertexInputBindings = new List(); - var runtimeScalarRegisters = CollectRuntimeScalarRegisters(state.Program); - var scalarRegisterSnapshots = new Dictionary>(); - var scalarConditionCode = false; - uint? skipUntilPc = null; + // Shared, cached, read-only: computed once per decoded program. The + // set already includes every instruction's destination registers, so + // the per-load additions the loop used to make are redundant. + var runtimeScalarRegisters = state.Program.RuntimeScalarRegisters; + var resolvedImageByPc = new Dictionary(); + var finalScalarRegisters = (uint[])scalarRegisters.Clone(); + var pendingPaths = new Stack(); + var visitedPaths = new HashSet(); - foreach (var instruction in state.Program.Instructions) + void QueuePath( + uint pc, + uint[] registers, + ulong pathExecMask, + bool pathScc, + bool supplemental) { - if (skipUntilPc.HasValue) + var key = new ScalarPathKey( + pc, + ComputeScalarStateHash(registers, pathExecMask, pathScc)); + if (visitedPaths.Add(key)) { - if (instruction.Pc < skipUntilPc.Value) + pendingPaths.Push(new ScalarPathState( + pc, + registers, + pathExecMask, + pathScc, + supplemental)); + } + } + + if (state.Program.Instructions.Count != 0) + { + QueuePath( + state.Program.Instructions[0].Pc, + (uint[])scalarRegisters.Clone(), + execMask, + pathScc: false, + supplemental: false); + } + + while (pendingPaths.Count != 0) + { + var path = pendingPaths.Pop(); + scalarRegisters = path.ScalarRegisters; + execMask = path.ExecMask; + var scalarConditionCode = path.ScalarConditionCode; + uint? skipUntilPc = path.StartPc; + + foreach (var instruction in state.Program.Instructions) + { + if (skipUntilPc.HasValue) { - continue; + if (instruction.Pc < skipUntilPc.Value) + { + continue; + } + + skipUntilPc = null; } - skipUntilPc = null; - } + if (instruction.Opcode == "SEndpgm") + { + break; + } - scalarRegisterSnapshots[instruction.Pc] = (uint[])scalarRegisters.Clone(); + if (instruction.Opcode == "SBranch" && + TryGetSoppBranchTargetPc(instruction, out var targetPc)) + { + if (targetPc > instruction.Pc) + { + // The regular scalar evaluation follows the uniform + // branch. Evaluate its skipped fall-through region once + // as supplemental resource discovery: large shaders use + // forward S_BRANCH to select one material/resource body, + // and SPIR-V still needs descriptors for every body that + // remains in the statically translated CFG. Do not fork + // SC_BRANCH targets here; their fall-through regions are + // already visited linearly and forking every vector-mask + // condition causes exponential state growth. + if (_cfgResourceDiscovery) + { + var fallthroughPc = instruction.Pc + + (uint)(instruction.Words.Count * sizeof(uint)); + QueuePath( + fallthroughPc, + (uint[])scalarRegisters.Clone(), + execMask, + scalarConditionCode, + supplemental: true); + } - if (instruction.Opcode == "SEndpgm") - { - break; - } + skipUntilPc = targetPc; + continue; + } - if (instruction.Opcode == "SBranch" && - TryGetSoppBranchTargetPc(instruction, out var targetPc) && - targetPc > instruction.Pc) - { - skipUntilPc = targetPc; - continue; - } + // One linear pass over a supplemental body is sufficient + // to discover its static memory/image instructions. Do not + // iterate backward branches: loop-varying descriptors need + // runtime descriptor indexing and cannot be represented by + // cloning scalar states for an unbounded number of trips. + if (path.Supplemental) + { + break; + } + } - if (instruction.Encoding == Gen5ShaderEncoding.Sopc) - { + if (instruction.Encoding == Gen5ShaderEncoding.Sopc) + { if (!TryExecuteScalarCompare( instruction, scalarRegisters, @@ -145,7 +284,7 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Encoding == Gen5ShaderEncoding.Sopk && + if (instruction.Encoding == Gen5ShaderEncoding.Sopk && instruction.Opcode.StartsWith("SCmpk", StringComparison.Ordinal)) { if (!TryExecuteScalarCompareK( @@ -160,7 +299,7 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Encoding is + if (instruction.Encoding is Gen5ShaderEncoding.Sop1 or Gen5ShaderEncoding.Sop2 or Gen5ShaderEncoding.Sopk) @@ -184,25 +323,29 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Control is Gen5ScalarMemoryControl scalarMemory) - { - foreach (var destination in instruction.Destinations) + if (instruction.Control is Gen5ScalarMemoryControl scalarMemory) { - if (destination.Kind == Gen5OperandKind.ScalarRegister && destination.Value < ScalarRegisterCount) - { - runtimeScalarRegisters.Add(destination.Value); - } - } - - if (!TryExecuteScalarLoad(ctx, state, instruction, scalarMemory, scalarRegisters, globalMemoryBindings, globalMemoryByAddress, runtimeScalarRegisters, out error)) + // Destinations are already in the cached runtimeScalarRegisters + // set (it scans every instruction's destinations), so no + // per-load mutation is needed here. + var recordBinding = + !path.Supplemental || + !HasGlobalMemoryBindingForPc(globalMemoryBindings, instruction.Pc); + if (!TryExecuteScalarLoad(ctx, state, instruction, scalarMemory, scalarRegisters, globalMemoryBindings, globalMemoryByAddress, runtimeScalarRegisters, recordBinding, out error)) { return false; } continue; } - if (instruction.Control is Gen5GlobalMemoryControl globalMemory) - { + if (instruction.Control is Gen5GlobalMemoryControl globalMemory) + { + if (path.Supplemental && + HasGlobalMemoryBindingForPc(globalMemoryBindings, instruction.Pc)) + { + continue; + } + if (globalMemory.ScalarAddress >= ScalarRegisterCount - 1) { error = @@ -221,17 +364,24 @@ public static class Gen5ShaderScalarEvaluator } var key = (globalMemory.ScalarAddress, baseAddress); + var writable = instruction.Opcode.StartsWith( + "GlobalStore", + StringComparison.Ordinal) || + instruction.Opcode.StartsWith( + "GlobalAtomic", + StringComparison.Ordinal); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) { instructionPcs.Add(instruction.Pc); } + existingBinding.Writable |= writable; } else { - byte[] data; - if (!TryReadGlobalMemory(ctx, baseAddress, out data)) + if (!TryReadGlobalMemory(ctx, baseAddress, out var data, out var dataLength)) { error = $"global-memory-read-failed pc=0x{instruction.Pc:X} " + @@ -243,7 +393,10 @@ public static class Gen5ShaderScalarEvaluator globalMemory.ScalarAddress, baseAddress, new List { instruction.Pc }, - data); + data, + dataLength, + DataPooled: true); + binding.Writable = writable; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } @@ -251,8 +404,15 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Control is Gen5BufferMemoryControl bufferMemory) - { + if (instruction.Control is Gen5BufferMemoryControl bufferMemory) + { + if (path.Supplemental && + HasGlobalMemoryBindingForPc(globalMemoryBindings, instruction.Pc)) + { + continue; + } + + var writable = IsBufferMemoryWrite(instruction.Opcode); if (bufferMemory.ScalarResource >= ScalarRegisterCount - 3) { error = @@ -275,52 +435,89 @@ public static class Gen5ShaderScalarEvaluator if (bufferDescriptor.BaseAddress == 0) { - error = $"buffer-address-null pc=0x{instruction.Pc:X}"; - return false; + // A descriptor in a sibling block can be null for this + // invocation even though the GPU branch never executes the + // memory instruction. Vulkan still requires a descriptor + // for the statically present block, so bind a bounded zero + // buffer to this exact PC. It is never reused for another + // resource register or instruction. + var nullKey = (bufferMemory.ScalarResource, 0UL); + if (globalMemoryByAddress.TryGetValue(nullKey, out var nullBinding)) + { + nullBinding.Writable |= writable; + if (nullBinding.InstructionPcs is List nullInstructionPcs && + !nullInstructionPcs.Contains(instruction.Pc)) + { + nullInstructionPcs.Add(instruction.Pc); + } + } + else + { + var binding = new Gen5GlobalMemoryBinding( + bufferMemory.ScalarResource, + 0, + new List { instruction.Pc }, + new byte[sizeof(uint)], + sizeof(uint), + DataPooled: false) + { + Writable = writable, + }; + globalMemoryByAddress.Add(nullKey, binding); + globalMemoryBindings.Add(binding); + } + + continue; } if (resolveVertexInputs && IsVertexFetchCandidate(instruction, bufferMemory, bufferDescriptor)) { - var vertexReadSize = bufferDescriptor.SizeBytes; - if (vertexRecordLimit is { } recordLimit && - instruction.Sources.Count > 2 && - TryEvaluateScalarOperand( + if (instruction.Sources.Count <= 2 || + !TryEvaluateScalarOperand( instruction.Sources[2], scalarRegisters, out var scalarOffset)) { - var bindingOffset = unchecked((uint)bufferMemory.OffsetBytes + scalarOffset); - var requiredBytes = - (ulong)bindingOffset + - (ulong)(Math.Max(recordLimit, 1u) - 1u) * bufferDescriptor.Stride + - (ulong)bufferMemory.DwordCount * sizeof(uint); - vertexReadSize = Math.Min( - bufferDescriptor.SizeBytes, - Math.Max(requiredBytes, sizeof(uint))); + error = + $"vertex-input-offset-unresolved pc=0x{instruction.Pc:X} " + + $"s{bufferMemory.ScalarResource}"; + return false; } - if (!TryReadGlobalMemory( - ctx, - bufferDescriptor.BaseAddress, - vertexReadSize, - out var vertexData)) + var bindingOffset = unchecked( + (uint)bufferMemory.OffsetBytes + scalarOffset); + var vertexReadBytes = bufferDescriptor.SizeBytes; + if (requiredVertexRecordCount is > 0) { - error = - $"vertex-buffer-read-failed pc=0x{instruction.Pc:X} " + - $"address=0x{bufferDescriptor.BaseAddress:X16} " + - $"bytes={bufferDescriptor.SizeBytes} " + - $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords}"; - return false; + // Resource descriptors commonly span an entire UE vertex + // arena (several MiB), while one draw references only a + // few hundred records. Preserve the indexed draw's exact + // reachable byte range instead of snapshotting the full + // arena for every attribute of every draw. + var lastRecordOffset = + (ulong)(requiredVertexRecordCount.Value - 1) * + bufferDescriptor.Stride; + var elementBytes = + (ulong)Math.Max(bufferMemory.DwordCount, 1u) * sizeof(uint); + var recordSpan = Math.Max( + (ulong)bufferDescriptor.Stride, + SaturatingAdd(bindingOffset, elementBytes)); + var requiredBytes = SaturatingAdd( + lastRecordOffset, + recordSpan); + vertexReadBytes = Math.Min(vertexReadBytes, requiredBytes); } if (!TryCreateVertexInputBinding( instruction, bufferMemory, bufferDescriptor, - vertexData, + checked((int)Math.Min( + vertexReadBytes, + (ulong)MaxGlobalMemoryBindingBytes)), (uint)vertexInputBindings.Count, - scalarRegisters, + scalarOffset, out var vertexInputBinding)) { error = @@ -336,37 +533,62 @@ public static class Gen5ShaderScalarEvaluator var key = (bufferMemory.ScalarResource, bufferDescriptor.BaseAddress); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) + existingBinding.Writable |= writable; + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) { instructionPcs.Add(instruction.Pc); } } else { + var dataPooled = true; if (!TryReadGlobalMemory( ctx, bufferDescriptor.BaseAddress, bufferDescriptor.SizeBytes, - out var data)) + out var data, + out var dataLength)) { var descriptorWords = string.Join( ':', Enumerable.Range(0, 4).Select(index => $"{scalarRegisters[bufferMemory.ScalarResource + (uint)index]:X8}")); - error = - $"buffer-memory-read-failed pc=0x{instruction.Pc:X} " + - $"address=0x{bufferDescriptor.BaseAddress:X16} " + - $"bytes={bufferDescriptor.SizeBytes} " + - $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords} " + - $"s{bufferMemory.ScalarResource}=[{descriptorWords}]"; - return false; + if (_strictBufferLoad) + { + error = + $"buffer-memory-read-failed pc=0x{instruction.Pc:X} " + + $"address=0x{bufferDescriptor.BaseAddress:X16} " + + $"bytes={bufferDescriptor.SizeBytes} " + + $"stride={bufferDescriptor.Stride} records={bufferDescriptor.NumRecords} " + + $"s{bufferMemory.ScalarResource}=[{descriptorWords}]"; + return false; + } + + dataLength = checked((int)Math.Min( + bufferDescriptor.SizeBytes, + (ulong)MaxGlobalMemoryBindingBytes)); + data = new byte[Math.Max(dataLength, sizeof(uint))]; + dataLength = data.Length; + dataPooled = false; + Console.Error.WriteLine( + $"[LOADER][WARN] AGC buffer read unavailable; using zero buffer " + + $"pc=0x{instruction.Pc:X} address=0x{bufferDescriptor.BaseAddress:X16} " + + $"bytes={bufferDescriptor.SizeBytes} guest_writeback=disabled " + + $"s{bufferMemory.ScalarResource}=[{descriptorWords}]"); } var binding = new Gen5GlobalMemoryBinding( bufferMemory.ScalarResource, bufferDescriptor.BaseAddress, new List { instruction.Pc }, - data); + data, + dataLength, + DataPooled: dataPooled) + { + Writable = writable, + WriteBackToGuest = dataPooled, + }; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } @@ -374,7 +596,12 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (instruction.Control is not Gen5ImageControl image) + if (instruction.Control is not Gen5ImageControl image) + { + continue; + } + + if (path.Supplemental && resolvedImageByPc.ContainsKey(instruction.Pc)) { continue; } @@ -401,26 +628,69 @@ public static class Gen5ShaderScalarEvaluator return false; } - resolved.Add(new Gen5ImageBinding( - instruction.Pc, - instruction.Opcode, - image, - resourceDescriptor, - samplerDescriptor, - instruction.Opcode is "ImageLoadMip" or "ImageStoreMip" && - TryResolveVectorConstantBefore( - state.Program, + var imageBinding = new Gen5ImageBinding( instruction.Pc, - image.GetAddressRegister(2), - out var mipLevel) - ? mipLevel - : null)); + instruction.Opcode, + image, + resourceDescriptor, + samplerDescriptor, + instruction.Opcode is "ImageLoadMip" or "ImageStoreMip" && + TryResolveImageMipLevel( + state.Program, + instruction.Pc, + image, + out var mipLevel) + ? mipLevel + : null); + if (resolvedImageByPc.TryGetValue(instruction.Pc, out var existingIndex)) + { + var existing = resolved[existingIndex]; + var existingNull = existing.ResourceDescriptor.All(static word => word == 0); + var candidateNull = resourceDescriptor.All(static word => word == 0); + if (existingNull && !candidateNull) + { + resolved[existingIndex] = imageBinding; + } + else if (!candidateNull && + (!existing.ResourceDescriptor.SequenceEqual(resourceDescriptor) || + !existing.SamplerDescriptor.SequenceEqual(samplerDescriptor))) + { + error = + $"dynamic-image-descriptor pc=0x{instruction.Pc:X} " + + $"s{image.ScalarResource}/s{image.ScalarSampler}"; + return false; + } + } + else + { + resolvedImageByPc.Add(instruction.Pc, resolved.Count); + resolved.Add(imageBinding); + } + } + + if (!path.Supplemental) + { + finalScalarRegisters = (uint[])scalarRegisters.Clone(); + } + } + + if (vertexInputBindings.Count != 0) + { + if (!TryCaptureVertexInputData( + ctx, + vertexInputBindings, + out var capturedVertexInputs, + out error)) + { + return false; + } + + vertexInputBindings = capturedVertexInputs; } evaluation = new Gen5ShaderEvaluation( initialScalarRegisters, - scalarRegisters, - scalarRegisterSnapshots, + finalScalarRegisters, resolved, globalMemoryBindings, state.ComputeSystemRegisters, @@ -429,24 +699,32 @@ public static class Gen5ShaderScalarEvaluator return true; } + private static bool IsBufferMemoryWrite(string opcode) => + opcode.StartsWith("BufferStore", StringComparison.Ordinal) || + opcode.StartsWith("TBufferStore", StringComparison.Ordinal) || + opcode.StartsWith("BufferAtomic", StringComparison.Ordinal) || + opcode.StartsWith("TBufferAtomic", StringComparison.Ordinal); + + private static bool HasGlobalMemoryBindingForPc( + IReadOnlyList bindings, + uint pc) => + bindings.Any(binding => binding.InstructionPcs.Contains(pc)); + private static bool TryCreateVertexInputBinding( Gen5ShaderInstruction instruction, Gen5BufferMemoryControl control, BufferDescriptor descriptor, - byte[] data, + int desiredDataLength, uint location, - uint[] scalarRegisters, + uint scalarOffset, out Gen5VertexInputBinding binding) { binding = default!; - if (!IsVertexFetchCandidate(instruction, control, descriptor) || - instruction.Sources.Count <= 2 || - !TryEvaluateScalarOperand(instruction.Sources[2], scalarRegisters, out var scalarOffset)) + if (!IsVertexFetchCandidate(instruction, control, descriptor)) { return false; } - var bindingData = data; var bindingStride = descriptor.Stride; var bindingOffset = unchecked((uint)control.OffsetBytes + scalarOffset); var bindingDataFormat = descriptor.DataFormat; @@ -460,10 +738,138 @@ public static class Gen5ShaderScalarEvaluator descriptor.BaseAddress, bindingStride, bindingOffset, - bindingData); + Data: [], + DataLength: desiredDataLength, + DataPooled: false); return true; } + private static bool TryCaptureVertexInputData( + CpuContext ctx, + IReadOnlyList pending, + out List captured, + out string error) + { + captured = new List(pending.Count); + error = string.Empty; + var ordered = pending + .OrderBy(static binding => binding.Stride) + .ThenBy(static binding => binding.BaseAddress) + .ToArray(); + + for (var first = 0; first < ordered.Length;) + { + var stride = ordered[first].Stride; + var start = ordered[first].BaseAddress; + var end = SaturatingAdd(start, (ulong)ordered[first].DataLength); + var last = first + 1; + while (last < ordered.Length && ordered[last].Stride == stride) + { + var candidate = ordered[last]; + // Attribute descriptors for an interleaved stream commonly + // point a few bytes into the same record. Merge overlapping + // spans (and one-record adjacency) so all attributes share one + // captured array and one host vertex buffer. + if (candidate.BaseAddress > SaturatingAdd(end, stride)) + { + break; + } + + end = Math.Max( + end, + SaturatingAdd(candidate.BaseAddress, (ulong)candidate.DataLength)); + last++; + } + + var byteCount = end > start + ? Math.Min(end - start, (ulong)MaxGlobalMemoryBindingBytes) + : 0; + if (byteCount == 0 || + !TryReadGlobalMemory(ctx, start, byteCount, out var data, out var dataLength)) + { + foreach (var binding in captured) + { + if (binding.DataPooled) + { + GlobalMemoryPool.Return(binding.Data); + } + } + + error = + $"vertex-buffer-read-failed address=0x{start:X16} " + + $"bytes={byteCount} stride={stride}"; + captured.Clear(); + return false; + } + + for (var index = first; index < last; index++) + { + var binding = ordered[index]; + var delta = binding.BaseAddress - start; + captured.Add(binding with + { + BaseAddress = start, + OffsetBytes = checked((uint)(delta + binding.OffsetBytes)), + Data = data, + DataLength = dataLength, + DataPooled = index == first, + }); + } + + first = last; + } + + captured.Sort(static (left, right) => left.Location.CompareTo(right.Location)); + TraceTitleVertexInputs(captured); + return true; + } + + private static void TraceTitleVertexInputs(IReadOnlyList bindings) + { + if (!string.Equals( + Environment.GetEnvironmentVariable("SHARPEMU_TRACE_VERTEX_RAW"), + "1", + StringComparison.Ordinal) || + bindings.Count != 3 || + !bindings.Any(static binding => + binding.Pc == 0xF8 && binding.Stride == 16 && + binding.OffsetBytes == 12 && binding.DataFormat == 10) || + !bindings.Any(static binding => + binding.Pc == 0x204 && binding.Stride == 16 && + binding.OffsetBytes == 0 && binding.DataFormat == 12) || + !bindings.Any(static binding => + binding.Pc == 0x280 && binding.Stride == 16 && + binding.OffsetBytes == 8 && binding.DataFormat == 5)) + { + return; + } + + var shared = bindings[0]; + var records = new List(); + foreach (var vertexIndex in new uint[] { 0, 1, 2, 3, 131, 4095 }) + { + var recordOffset = (ulong)vertexIndex * shared.Stride; + if (recordOffset + shared.Stride > (ulong)shared.DataLength) + { + continue; + } + + records.Add( + $"{vertexIndex}:" + + Convert.ToHexString( + shared.Data, + checked((int)recordOffset), + checked((int)shared.Stride))); + } + + Console.Error.WriteLine( + $"[VERTEX-RAW] title base=0x{shared.BaseAddress:X16} " + + $"length={shared.DataLength} records={string.Join(',', records)}"); + } + + private static ulong SaturatingAdd(ulong left, ulong right) => + ulong.MaxValue - left < right ? ulong.MaxValue : left + right; + private static bool IsVertexFetchCandidate( Gen5ShaderInstruction instruction, Gen5BufferMemoryControl control, @@ -476,33 +882,6 @@ public static class Gen5ShaderScalarEvaluator (instruction.Opcode.StartsWith("BufferLoadFormat", StringComparison.Ordinal) || instruction.Opcode.StartsWith("TBufferLoadFormat", StringComparison.Ordinal)); - private static HashSet CollectRuntimeScalarRegisters(Gen5ShaderProgram program) - { - var registers = new HashSet(); - foreach (var instruction in program.Instructions) - { - foreach (var operand in instruction.Sources.Concat(instruction.Destinations)) - { - if (operand.Kind == Gen5OperandKind.ScalarRegister && - operand.Value < ScalarRegisterCount) - { - registers.Add(operand.Value); - } - } - - if (instruction.Control is Gen5ScalarMemoryControl - { - DynamicOffsetRegister: { } offsetRegister, - } && - offsetRegister < ScalarRegisterCount) - { - registers.Add(offsetRegister); - } - } - - return registers; - } - private static bool TryGetSoppBranchTargetPc( Gen5ShaderInstruction instruction, out uint targetPc) @@ -561,6 +940,26 @@ public static class Gen5ShaderScalarEvaluator return false; } + private static bool TryResolveImageMipLevel( + Gen5ShaderProgram program, + uint pc, + Gen5ImageControl image, + out uint mipLevel) + { + // For the currently translated 2D image path, mip follows x/y. A16 + // packs x/y into the first VGPR and places mip in the low half of the + // next; ordinary addresses use the third VGPR. + var register = image.GetAddressRegister(image.A16 ? 1 : 2); + if (!TryResolveVectorConstantBefore(program, pc, register, out var raw)) + { + mipLevel = 0; + return false; + } + + mipLevel = image.A16 ? raw & 0xFFFF : raw; + return true; + } + private static bool TryResolveConstantOperand(Gen5Operand operand, out uint value) { if (operand.Kind == Gen5OperandKind.LiteralConstant) @@ -578,118 +977,110 @@ public static class Gen5ShaderScalarEvaluator return false; } - [ThreadStatic] - private static Dictionary<(ulong BaseAddress, int SizeBytes), byte[]>? _globalMemoryReadCache; - + // Both readers rent from ArrayPool: these run per bound buffer per draw, + // and fresh multi-megabyte allocations here kept the background GC busy + // full-time. The rented array (possibly oversized) is handed to the + // presenter, which returns it to the pool after the host-buffer upload. public static void BeginGlobalMemoryReadScope() { - _globalMemoryReadCache = new Dictionary<(ulong, int), byte[]>(); } public static void EndGlobalMemoryReadScope() { - _globalMemoryReadCache = null; } - private static bool TryReadGlobalMemory( CpuContext ctx, ulong baseAddress, - out byte[] data) + out byte[] data, + out int dataLength) { - return TryReadGlobalMemory( - ctx, - baseAddress, - (ulong)DefaultGlobalMemoryBindingBytes, - out data); + var rented = GlobalMemoryPool.Rent(MaxGlobalMemoryBindingBytes); + for (var size = MaxGlobalMemoryBindingBytes; size >= 4096; size >>= 1) + { + if (ctx.Memory.TryRead(baseAddress, rented.AsSpan(0, size))) + { + Interlocked.Increment(ref GlobalMemoryReadCount); + Interlocked.Add(ref GlobalMemoryReadBytes, size); + Interlocked.Add(ref GlobalMemoryReadPvmBytes, size); + data = rented; + dataLength = size; + return true; + } + } + + GlobalMemoryPool.Return(rented); + data = []; + dataLength = 0; + return false; } private static bool TryReadGlobalMemory( CpuContext ctx, ulong baseAddress, ulong sizeBytes, - out byte[] data) + out byte[] data, + out int dataLength) { + data = []; + dataLength = 0; if (sizeBytes == 0) { - data = []; return false; } var cappedSize = Math.Min(sizeBytes, MaxGlobalMemoryBindingBytes); if (cappedSize > int.MaxValue) { - data = []; return false; } - var cache = _globalMemoryReadCache; - var cacheKey = (baseAddress, (int)cappedSize); - if (cache is not null && cache.TryGetValue(cacheKey, out var cached)) + var rented = GlobalMemoryPool.Rent( + Math.Max((int)cappedSize, sizeof(uint))); + if (cappedSize < sizeof(uint)) { - Interlocked.Increment(ref GlobalMemoryReadCacheHits); - data = cached; - return true; - } - - byte[]? previous; - lock (_crossFrameReadGate) - { - _crossFrameReadCache.TryGetValue(cacheKey, out previous); - } - - if (previous is not null && ctx.Memory.TryCompare(baseAddress, previous)) - { - Interlocked.Increment(ref GlobalMemoryReadReuses); - if (cache is not null) + rented.AsSpan(0, sizeof(uint)).Clear(); + var exact = rented.AsSpan(0, (int)cappedSize); + var readFromPvm = ctx.Memory.TryRead(baseAddress, exact); + if (readFromPvm || FallbackMemoryReader?.Invoke(baseAddress, exact) == true) { - cache[cacheKey] = previous; + Interlocked.Increment(ref GlobalMemoryReadCount); + Interlocked.Add(ref GlobalMemoryReadBytes, sizeof(uint)); + if (readFromPvm) + { + Interlocked.Add(ref GlobalMemoryReadPvmBytes, sizeof(uint)); + } + else + { + Interlocked.Add(ref GlobalMemoryReadLibcBytes, sizeof(uint)); + } + data = rented; + dataLength = sizeof(uint); + return true; } - data = previous; - return true; + GlobalMemoryPool.Return(rented); + return false; } var candidateSize = (int)cappedSize; while (candidateSize >= sizeof(uint)) { - data = GC.AllocateUninitializedArray(candidateSize); - var readFromPvm = ctx.Memory.TryRead(baseAddress, data); - if (readFromPvm || - FallbackMemoryReader?.Invoke(baseAddress, data) == true) + var span = rented.AsSpan(0, candidateSize); + var readFromPvm = ctx.Memory.TryRead(baseAddress, span); + if (readFromPvm || FallbackMemoryReader?.Invoke(baseAddress, span) == true) { Interlocked.Increment(ref GlobalMemoryReadCount); - Interlocked.Add(ref GlobalMemoryReadBytes, data.Length); + Interlocked.Add(ref GlobalMemoryReadBytes, candidateSize); if (readFromPvm) { - Interlocked.Add(ref GlobalMemoryReadPvmBytes, data.Length); + Interlocked.Add(ref GlobalMemoryReadPvmBytes, candidateSize); } else { - Interlocked.Add(ref GlobalMemoryReadLibcBytes, data.Length); + Interlocked.Add(ref GlobalMemoryReadLibcBytes, candidateSize); } - - if (cache is not null) - { - cache[cacheKey] = data; - } - - lock (_crossFrameReadGate) - { - if (_crossFrameReadCache.TryGetValue(cacheKey, out var replaced)) - { - _crossFrameReadCacheBytes -= replaced.Length; - } - - if (_crossFrameReadCacheBytes + data.Length > CrossFrameReadCacheMaxBytes) - { - _crossFrameReadCache.Clear(); - _crossFrameReadCacheBytes = 0; - } - - _crossFrameReadCache[cacheKey] = data; - _crossFrameReadCacheBytes += data.Length; - } - + data = rented; + dataLength = candidateSize; return true; } @@ -701,7 +1092,7 @@ public static class Gen5ShaderScalarEvaluator candidateSize = Math.Max(candidateSize / 2, sizeof(uint)); } - data = []; + GlobalMemoryPool.Return(rented); return false; } @@ -775,6 +1166,13 @@ public static class Gen5ShaderScalarEvaluator value = ~value; scalarConditionCode = value != 0; } + else if (instruction.Opcode == "SWqmB64") + { + var quadAny = (value | (value >> 1) | (value >> 2) | (value >> 3)) & + 0x1111_1111_1111_1111UL; + value = quadAny * 0xFUL; + scalarConditionCode = value != 0; + } WriteScalarPair(registers, destination.Value, value, ref execMask); return true; @@ -849,6 +1247,27 @@ public static class Gen5ShaderScalarEvaluator return true; } + if (instruction.Opcode == "SBfmB64") + { + if (instruction.Sources.Count < 2 || + destination.Value >= ScalarRegisterCount - 1 || + !TryEvaluateScalarOperand(instruction.Sources[0], registers, out var widthSource) || + !TryEvaluateScalarOperand(instruction.Sources[1], registers, out var offsetSource)) + { + error = $"scalar-source64 pc=0x{instruction.Pc:X} op={instruction.Opcode}"; + return false; + } + + var width = (int)widthSource & 63; + var offset = (int)offsetSource & 63; + var value = width == 0 + ? 0UL + : (ulong.MaxValue >> (64 - width)) << offset; + WriteScalarPair(registers, destination.Value, value, ref execMask); + scalarConditionCode = value != 0; + return true; + } + if (instruction.Opcode is "SCselectB64" or "SAndB64" or @@ -894,7 +1313,12 @@ public static class Gen5ShaderScalarEvaluator } if (instruction.Sources.Count == 0 || - !TryEvaluateScalarOperand(instruction.Sources[0], registers, out var left)) + !TryEvaluateScalarOperand( + instruction.Sources[0], + registers, + execMask, + scalarConditionCode, + out var left)) { var source = instruction.Sources.Count == 0 ? "" @@ -933,7 +1357,12 @@ public static class Gen5ShaderScalarEvaluator } if (instruction.Sources.Count < 2 || - !TryEvaluateScalarOperand(instruction.Sources[1], registers, out var right)) + !TryEvaluateScalarOperand( + instruction.Sources[1], + registers, + execMask, + scalarConditionCode, + out var right)) { var source = instruction.Sources.Count < 2 ? "" @@ -1134,6 +1563,44 @@ public static class Gen5ShaderScalarEvaluator out string error) { error = string.Empty; + if (instruction.Opcode.EndsWith("SaveexecB32", StringComparison.Ordinal)) + { + if (instruction.Destinations.Count != 1 || + instruction.Destinations[0] is not + { + Kind: Gen5OperandKind.ScalarRegister, + Value: < ScalarRegisterCount, + } destination32 || + instruction.Sources.Count == 0 || + !TryEvaluateScalarOperand(instruction.Sources[0], registers, out var source32)) + { + error = $"scalar-source32 pc=0x{instruction.Pc:X} op={instruction.Opcode}"; + return false; + } + + var oldExec32 = (uint)execMask; + var newExec32 = instruction.Opcode switch + { + "SAndSaveexecB32" => oldExec32 & source32, + "SOrSaveexecB32" => oldExec32 | source32, + "SXorSaveexecB32" => oldExec32 ^ source32, + "SAndn1SaveexecB32" => ~source32 & oldExec32, + "SAndn2SaveexecB32" => source32 & ~oldExec32, + "SOrn1SaveexecB32" => ~source32 | oldExec32, + "SOrn2SaveexecB32" => source32 | ~oldExec32, + "SNandSaveexecB32" => ~(source32 & oldExec32), + "SNorSaveexecB32" => ~(source32 | oldExec32), + "SXnorSaveexecB32" => ~(oldExec32 ^ source32), + _ => 0u, + }; + registers[destination32.Value] = oldExec32; + execMask = newExec32; + registers[126] = newExec32; + registers[127] = 0; + scalarConditionCode = newExec32 != 0; + return true; + } + if (instruction.Opcode is not ( "SAndSaveexecB64" or "SOrSaveexecB64" or @@ -1370,7 +1837,8 @@ public static class Gen5ShaderScalarEvaluator uint[] scalarRegisters, List globalMemoryBindings, Dictionary<(uint ScalarAddress, ulong BaseAddress), Gen5GlobalMemoryBinding> globalMemoryByAddress, - HashSet runtimeScalarRegisters, + IReadOnlySet runtimeScalarRegisters, + bool recordBinding, out string error) { error = string.Empty; @@ -1417,30 +1885,64 @@ public static class Gen5ShaderScalarEvaluator scalarBase.Value + 3 < ScalarRegisterCount && scalarRegisters[scalarBase.Value + 2] == 0 && scalarRegisters[scalarBase.Value + 3] == 0)); + var scalarPointerUnbound = ShouldTreatScalarPointerAsUnbound( + isBufferLoad, + address, + _strictScalarLoad); + if (scalarPointerUnbound) + { + TraceScalarPointerFallback( + state, + instruction, + scalarBase.Value, + scalarRegisters, + control, + baseAddress, + dynamicOffset); + } var bufferSize = ulong.MaxValue; - if (isBufferLoad) + if (recordBinding && isBufferLoad) { bufferSize = hasBufferDescriptor ? bufferDescriptor.SizeBytes : ulong.MaxValue; var key = (scalarBase.Value, bufferDescriptor.BaseAddress); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) instructionPcs.Add(instruction.Pc); + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) + { + instructionPcs.Add(instruction.Pc); + } } else { - TryReadGlobalMemory(ctx, bufferDescriptor.BaseAddress, bufferDescriptor.SizeBytes, out var data); - var binding = new Gen5GlobalMemoryBinding(scalarBase.Value, bufferDescriptor.BaseAddress, new List { instruction.Pc }, data); + var pooled = TryReadGlobalMemory( + ctx, + bufferDescriptor.BaseAddress, + bufferDescriptor.SizeBytes, + out var data, + out var dataLength); + var binding = new Gen5GlobalMemoryBinding( + scalarBase.Value, + bufferDescriptor.BaseAddress, + new List { instruction.Pc }, + data, + dataLength, + DataPooled: pooled) + { + WriteBackToGuest = pooled, + }; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } } - else if (baseAddress != 0) + else if (recordBinding && baseAddress != 0) { var key = (scalarBase.Value, baseAddress); if (globalMemoryByAddress.TryGetValue(key, out var existingBinding)) { - if (existingBinding.InstructionPcs is List instructionPcs) + if (existingBinding.InstructionPcs is List instructionPcs && + !instructionPcs.Contains(instruction.Pc)) { instructionPcs.Add(instruction.Pc); } @@ -1456,26 +1958,28 @@ public static class Gen5ShaderScalarEvaluator requiredBytes = Math.Min( (requiredBytes + 4095UL) & ~4095UL, MaxGlobalMemoryBindingBytes); - if (!TryReadGlobalMemory( - ctx, - baseAddress, - requiredBytes, - out var data)) - { - data = []; - } - + var pooled = TryReadGlobalMemory( + ctx, + baseAddress, + requiredBytes, + out var data, + out var dataLength); var binding = new Gen5GlobalMemoryBinding( scalarBase.Value, baseAddress, new List { instruction.Pc }, - data); + data, + dataLength, + DataPooled: pooled) + { + WriteBackToGuest = pooled, + }; globalMemoryByAddress.Add(key, binding); globalMemoryBindings.Add(binding); } } - if (!bufferUnbound && address == 0) + if (!bufferUnbound && !scalarPointerUnbound && address == 0) { error = FormatScalarLoadError( "invalid-load-address", @@ -1509,6 +2013,7 @@ public static class Gen5ShaderScalarEvaluator var componentOffset = unchecked(byteOffset + (ulong)(index * sizeof(uint))); if (bufferUnbound || + scalarPointerUnbound || isBufferLoad && (componentOffset >= bufferSize || bufferSize - componentOffset < sizeof(uint))) @@ -1517,18 +2022,12 @@ public static class Gen5ShaderScalarEvaluator continue; } - if (!ctx.TryReadUInt32( + if (!TryReadUInt32( + ctx, address + (ulong)(index * sizeof(uint)), - out var value) && - !TryReadUserDataScalarLoad( - state, - instruction, - control, - byteOffset, - index, - out value)) + out var value)) { - if (isBufferLoad) + if (isBufferLoad || !_strictScalarLoad) { scalarRegisters[destination.Value] = 0; continue; @@ -1552,6 +2051,87 @@ public static class Gen5ShaderScalarEvaluator return true; } + private static bool ShouldTreatScalarPointerAsUnbound( + bool isBufferLoad, + ulong address, + bool strictScalarLoad) => + !isBufferLoad && address == 0 && !strictScalarLoad; + + private static void TraceScalarPointerFallback( + Gen5ShaderState state, + Gen5ShaderInstruction instruction, + uint scalarBase, + IReadOnlyList scalarRegisters, + Gen5ScalarMemoryControl control, + ulong baseAddress, + ulong dynamicOffset) + { + lock (_scalarFallbackTraceGate) + { + if (!_tracedScalarFallbacks.Add((state.Program.Address, instruction.Pc))) + { + return; + } + } + + var definitions = state.Program.Instructions + .Where(candidate => + candidate.Pc < instruction.Pc && + candidate.Destinations.Any(destination => + destination.Kind == Gen5OperandKind.ScalarRegister && + destination.Value is var register && + register >= scalarBase && register <= scalarBase + 1)) + .TakeLast(8) + .Select(candidate => + $"0x{candidate.Pc:X}:{candidate.Opcode}[" + + string.Join(',', candidate.Words.Select(word => $"{word:X8}")) + "]") + .ToArray(); + var userData = string.Join( + ',', + state.UserData.Take(32).Select((value, index) => $"s{index}=0x{value:X8}")); + var high = scalarBase + 1 < scalarRegisters.Count + ? scalarRegisters[(int)scalarBase + 1] + : 0; + Console.Error.WriteLine( + $"[LOADER][WARN] agc.scalar_pointer_fallback " + + $"shader=0x{state.Program.Address:X16} pc=0x{instruction.Pc:X} " + + $"op={instruction.Opcode} base=s{scalarBase}" + + $"[0x{scalarRegisters[(int)scalarBase]:X8}:0x{high:X8}] " + + $"base_addr=0x{baseAddress:X16} imm={control.ImmediateOffsetBytes} " + + $"dynamic={dynamicOffset} definitions=[{string.Join(';', definitions)}] " + + $"user_data=[{userData}] metadata=" + + $"{(state.Metadata is null ? "missing" : $"srt={state.Metadata.ShaderResourceTableSizeDwords},eud={state.Metadata.ExtendedUserDataSizeDwords}")}"); + } + + [Conditional("DEBUG")] + private static void RunScalarLoadSelfChecks() + { + Debug.Assert( + ShouldTreatScalarPointerAsUnbound( + isBufferLoad: false, + address: 0, + strictScalarLoad: false), + "A null non-strict scalar pointer must read as zero instead of dropping the shader pass."); + Debug.Assert( + !ShouldTreatScalarPointerAsUnbound( + isBufferLoad: false, + address: 0, + strictScalarLoad: true), + "Strict scalar-load diagnostics must continue rejecting null pointers."); + Debug.Assert( + !ShouldTreatScalarPointerAsUnbound( + isBufferLoad: true, + address: 0, + strictScalarLoad: false), + "Buffer descriptor null handling must remain on the buffer-unbound path."); + Debug.Assert( + !ShouldTreatScalarPointerAsUnbound( + isBufferLoad: false, + address: 0x1000, + strictScalarLoad: false), + "A valid scalar pointer must not be treated as an unbound resource."); + } + private static bool TryDecodeBufferDescriptor( IReadOnlyList scalarRegisters, uint scalarBase, @@ -1592,8 +2172,14 @@ public static class Gen5ShaderScalarEvaluator var baseAddress = word0 | ((ulong)(word1 & 0xFFFFu) << 32); var stride = (word1 >> 16) & 0x3FFFu; var unifiedFormat = (word3 >> 12) & 0x7Fu; - var (dataFormat, numberFormat) = - DecodeGfx10BufferFormat(unifiedFormat); + if (!Gfx10UnifiedFormat.TryDecode( + unifiedFormat, + out var dataFormat, + out var numberFormat)) + { + return false; + } + var sizeBytes = stride == 0 ? word2 : (ulong)stride * word2; @@ -1601,111 +2187,6 @@ public static class Gen5ShaderScalarEvaluator return true; } - private static (uint DataFormat, uint NumberFormat) - DecodeGfx10BufferFormat(uint format) => - format switch - { - 0 => (0, 0), - >= 1 and <= 6 => (1, format - 1), - >= 7 and <= 13 => (2, DecodeUnifiedNumber(format - 7, 7)), - >= 14 and <= 19 => (3, format - 14), - >= 20 and <= 22 => (4, DecodeIntegerOrFloatNumber(format - 20)), - >= 23 and <= 29 => (5, DecodeUnifiedNumber(format - 23, 7)), - >= 30 and <= 36 => (6, DecodeUnifiedNumber(format - 30, 7)), - >= 37 and <= 43 => (7, DecodeUnifiedNumber(format - 37, 7)), - >= 44 and <= 49 => (8, format - 44), - >= 50 and <= 55 => (9, format - 50), - >= 56 and <= 61 => (10, format - 56), - >= 62 and <= 64 => (11, DecodeIntegerOrFloatNumber(format - 62)), - >= 65 and <= 71 => (12, DecodeUnifiedNumber(format - 65, 7)), - >= 72 and <= 74 => (13, DecodeIntegerOrFloatNumber(format - 72)), - >= 75 and <= 77 => (14, DecodeIntegerOrFloatNumber(format - 75)), - 128 => (1, 9), - 129 => (3, 9), - 130 => (10, 9), - 132 => (34, 7), - 133 => (16, 0), - 134 => (17, 0), - 135 => (18, 0), - 136 => (19, 0), - 140 => (4, 7), - _ => (0, 0), - }; - - private static uint DecodeUnifiedNumber(uint offset, uint formatCount) => - offset == formatCount - 1 ? 7u : offset; - - private static uint DecodeIntegerOrFloatNumber(uint offset) => - offset switch - { - 0 => 4, - 1 => 5, - _ => 7, - }; - - private static bool TryReadUserDataScalarLoad( - Gen5ShaderState state, - Gen5ShaderInstruction instruction, - Gen5ScalarMemoryControl control, - ulong byteOffset, - int componentIndex, - out uint value) - { - value = 0; - if (!instruction.Opcode.StartsWith("SLoadDword", StringComparison.Ordinal) || - state.Metadata is not { } metadata || - (byteOffset & 3) != 0) - { - return false; - } - - var baseDwordOffset = byteOffset >> 2; - if (baseDwordOffset > int.MaxValue) - { - return false; - } - - var dwordOffset = (long)baseDwordOffset + componentIndex; - if (dwordOffset < 0 || - dwordOffset >= state.UserData.Count || - !IsShaderUserDataResourceOffset(metadata, (uint)dwordOffset)) - { - return false; - } - - value = state.UserData[(int)dwordOffset]; - return true; - } - - private static bool IsShaderUserDataResourceOffset( - Gen5ShaderMetadata metadata, - uint dwordOffset) - { - if (dwordOffset < metadata.ShaderResourceTableSizeDwords) - { - return true; - } - - foreach (var resource in metadata.Resources) - { - var dwordCount = resource.Kind switch - { - Gen5ShaderResourceKind.ReadOnlyTexture or - Gen5ShaderResourceKind.ReadWriteTexture => 8u, - Gen5ShaderResourceKind.Sampler or - Gen5ShaderResourceKind.ConstantBuffer => 4u, - _ => 1u, - }; - if (dwordOffset >= resource.OffsetDwords && - dwordOffset < resource.OffsetDwords + dwordCount) - { - return true; - } - } - - return metadata.DirectResources.Values.Any(offset => offset == dwordOffset); - } - private static string FormatScalarLoadError( string reason, Gen5ShaderInstruction instruction, @@ -1764,6 +2245,34 @@ public static class Gen5ShaderScalarEvaluator return false; } + private static bool TryEvaluateScalarOperand( + Gen5Operand operand, + uint[] scalarRegisters, + ulong execMask, + bool scalarConditionCode, + out uint value) + { + if (operand.Kind == Gen5OperandKind.EncodedConstant) + { + switch (operand.Value) + { + // RDNA scalar-source special registers. These encodings share + // the inline-constant field but are evaluated from wave state. + case 251: // VCCZ + value = (scalarRegisters[106] | scalarRegisters[107]) == 0 ? 1u : 0u; + return true; + case 252: // EXECZ + value = execMask == 0 ? 1u : 0u; + return true; + case 253: // SCC + value = scalarConditionCode ? 1u : 0u; + return true; + } + } + + return TryEvaluateScalarOperand(operand, scalarRegisters, out value); + } + private static bool TryDecodeInlineConstant(uint encoded, out uint value) { if (encoded == 125) @@ -1838,4 +2347,16 @@ public static class Gen5ShaderScalarEvaluator opcode.StartsWith("ImageSample", StringComparison.Ordinal) || opcode.StartsWith("ImageGather", StringComparison.Ordinal); + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } } diff --git a/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs b/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs index 3df3d91..688bc84 100644 --- a/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs +++ b/src/SharpEmu.ShaderCompiler/Gen5ShaderTranslator.cs @@ -3,6 +3,7 @@ using SharpEmu.HLE; using System.Buffers.Binary; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Text; @@ -10,9 +11,83 @@ namespace SharpEmu.ShaderCompiler; public static class Gen5ShaderTranslator { + private static int _dppVectorsValidated; + /// + /// Bitmask (256 bits) of scalar registers whose values the program can + /// observe: scalar source operands (widened for 64-bit pairs), the + /// descriptor/sampler/address ranges named by instruction controls, and + /// the implicit state registers. Deterministic per instruction stream, so + /// the SPIR-V that loads exactly these registers from the per-draw + /// initial-state buffer is byte-stable across draws. + /// + public static ulong[] ComputeConsumedScalarMask(Gen5ShaderProgram program) + { + var mask = new ulong[4]; + AddConsumedScalar(mask, 106, 2); + AddConsumedScalar(mask, 124, 2); + AddConsumedScalar(mask, 126, 2); + foreach (var instruction in program.Instructions) + { + foreach (var source in instruction.Sources) + { + if (source.Kind == Gen5OperandKind.ScalarRegister) + { + AddConsumedScalar(mask, source.Value, 2); + } + } + + // Scalar memory bases can be a 4-dword buffer descriptor. + if (instruction.Encoding is Gen5ShaderEncoding.Smem or Gen5ShaderEncoding.Smrd && + instruction.Sources.Count > 0 && + instruction.Sources[0].Kind == Gen5OperandKind.ScalarRegister) + { + AddConsumedScalar(mask, instruction.Sources[0].Value, 4); + } + + switch (instruction.Control) + { + case Gen5ImageControl image: + AddConsumedScalar(mask, image.ScalarResource, 8); + AddConsumedScalar(mask, image.ScalarSampler, 4); + break; + case Gen5ScalarMemoryControl { DynamicOffsetRegister: { } offsetRegister }: + AddConsumedScalar(mask, offsetRegister, 2); + break; + case Gen5GlobalMemoryControl global: + AddConsumedScalar(mask, global.ScalarAddress, 2); + break; + case Gen5BufferMemoryControl buffer: + AddConsumedScalar(mask, buffer.ScalarResource, 4); + break; + } + } + + return mask; + } + + private static void AddConsumedScalar(ulong[] mask, uint register, uint count) + { + for (uint index = 0; index < count; index++) + { + var target = register + index; + if (target < 256) + { + mask[target >> 6] |= 1UL << (int)(target & 63); + } + } + } + + public static bool IsScalarConsumed(ulong[] mask, uint register) => + register < 256 && (mask[register >> 6] & (1UL << (int)(register & 63))) != 0; + private const int MaxInstructions = 4096; - private const int MinimumUserDataDwords = 16; - private const int MaximumUserDataDwords = 256; + private const uint PsUserDataRegister = 0x0C; + private const uint VsUserDataRegister = 0x4C; + private const uint GsUserDataRegister = 0x8C; + private const uint EsUserDataRegister = 0xCC; + private const uint ComputeUserDataRegister = 0x240; + private const uint ComputePgmRsrc2Register = 0x213; + private const int MaximumHardwareUserSgprs = 64; private static readonly ConditionalWeakTable _decodeCaches = new(); private sealed class ShaderDecodeCache @@ -124,6 +199,7 @@ public static class Gen5ShaderTranslator Gen5ComputeSystemRegisters? computeSystemRegisters = null, uint userDataScalarRegisterBase = 0) { + ValidateUserSgprCountDecoding(); state = default!; error = string.Empty; var cache = _decodeCaches.GetValue(ctx.Memory, static _ => new ShaderDecodeCache()); @@ -172,7 +248,20 @@ public static class Gen5ShaderTranslator } } - var userData = new uint[GetUserDataDwordCount(metadata)]; + if (!TryGetUserSgprCount( + shaderRegisters, + userDataBaseRegister, + out var userSgprCount, + out var rsrc2Register, + out _)) + { + error = + $"missing-user-sgpr-count ud_reg=0x{userDataBaseRegister:X} " + + $"rsrc2_reg=0x{rsrc2Register:X}"; + return false; + } + + var userData = new uint[userSgprCount]; for (uint index = 0; index < userData.Length; index++) { shaderRegisters.TryGetValue(userDataBaseRegister + index, out userData[index]); @@ -187,35 +276,72 @@ public static class Gen5ShaderTranslator return true; } - private static int GetUserDataDwordCount(Gen5ShaderMetadata? metadata) + private static bool TryGetUserSgprCount( + IReadOnlyDictionary shaderRegisters, + uint userDataBaseRegister, + out int count, + out uint rsrc2Register, + out uint rsrc2) { - var count = MinimumUserDataDwords; - if (metadata is null) + rsrc2Register = userDataBaseRegister == ComputeUserDataRegister + ? ComputePgmRsrc2Register + : userDataBaseRegister - 1; + if (!shaderRegisters.TryGetValue(rsrc2Register, out rsrc2)) { + count = 0; + return false; + } + + count = checked((int)((rsrc2 >> 1) & 0x1Fu)); + // GFX10 PS/VS/GS expose a sixth USER_SGPR bit. ES and compute do not. + // AGC's logical user-data layout (including its SRT pointer and back + // user data) describes memory reached through these SGPRs; it does not + // increase the hardware register window. + var hasUserSgprMsb = userDataBaseRegister is + PsUserDataRegister or VsUserDataRegister or GsUserDataRegister; + if (hasUserSgprMsb && + (rsrc2 & (1u << 27)) != 0) + { + count |= 0x20; + } + + if (userDataBaseRegister is not (PsUserDataRegister or + VsUserDataRegister or + GsUserDataRegister or + EsUserDataRegister or + ComputeUserDataRegister) || + count > MaximumHardwareUserSgprs) + { + count = 0; + return false; + } + + return true; + } + + [Conditional("DEBUG")] + private static void ValidateUserSgprCountDecoding() + { + static int Decode(uint baseRegister, uint rsrc2) + { + var registers = new Dictionary + { + [baseRegister == ComputeUserDataRegister + ? ComputePgmRsrc2Register + : baseRegister - 1] = rsrc2, + }; + var decoded = + TryGetUserSgprCount(registers, baseRegister, out var count, out _, out _); + Debug.Assert(decoded); return count; } - count = Math.Max(count, checked((int)Math.Min(metadata.ExtendedUserDataSizeDwords, MaximumUserDataDwords))); - count = Math.Max(count, checked((int)Math.Min(metadata.ShaderResourceTableSizeDwords, MaximumUserDataDwords))); - - foreach (var offset in metadata.DirectResources.Values) - { - count = Math.Max(count, checked((int)Math.Min(offset + 1u, MaximumUserDataDwords))); - } - - foreach (var resource in metadata.Resources) - { - var size = resource.Kind switch - { - Gen5ShaderResourceKind.ReadOnlyTexture or Gen5ShaderResourceKind.ReadWriteTexture => 8u, - Gen5ShaderResourceKind.Sampler => 4u, - Gen5ShaderResourceKind.ConstantBuffer => 4u, - _ => 1u, - }; - count = Math.Max(count, checked((int)Math.Min(resource.OffsetDwords + size, MaximumUserDataDwords))); - } - - return count; + Debug.Assert(Decode(PsUserDataRegister, 2u << 1) == 2); + Debug.Assert(Decode(PsUserDataRegister, (3u << 1) | (1u << 27)) == 35); + Debug.Assert(Decode(VsUserDataRegister, (1u << 1) | (1u << 27)) == 33); + Debug.Assert(Decode(GsUserDataRegister, (4u << 1) | (1u << 27)) == 36); + Debug.Assert(Decode(EsUserDataRegister, (7u << 1) | (1u << 27)) == 7); + Debug.Assert(Decode(ComputeUserDataRegister, (11u << 1) | (1u << 27)) == 11); } public static string DescribeState(Gen5ShaderState state) @@ -229,7 +355,8 @@ public static class Gen5ShaderTranslator if (state.Metadata is not { } metadata) { return - $"ud_base=s{state.UserDataScalarRegisterBase} ud[{userData}]" + + $"ud_base=s{state.UserDataScalarRegisterBase} hw_ud={state.UserData.Count} " + + $"ud[{userData}]" + $"{systemRegisters} metadata=missing"; } @@ -242,7 +369,8 @@ public static class Gen5ShaderTranslator $"{resource.Kind}[{resource.Slot}]@{resource.OffsetDwords}" + (resource.SizeFlag ? "+" : string.Empty))); return - $"ud_base=s{state.UserDataScalarRegisterBase} ud[{userData}]" + + $"ud_base=s{state.UserDataScalarRegisterBase} hw_ud={state.UserData.Count} " + + $"ud[{userData}]" + $"{systemRegisters} metadata[eud={metadata.ExtendedUserDataSizeDwords}," + $"srt={metadata.ShaderResourceTableSizeDwords},direct={direct},resources={resources}]"; } @@ -283,6 +411,7 @@ public static class Gen5ShaderTranslator out Gen5ShaderProgram program, out string error) { + ValidateDppControlVectors(); program = new Gen5ShaderProgram(address, []); error = string.Empty; if (address == 0) @@ -295,7 +424,7 @@ public static class Gen5ShaderTranslator var instructionCount = 0; for (uint pc = 0; instructionCount < MaxInstructions;) { - if (!ctx.TryReadUInt32(address + pc, out var word)) + if (!TryReadUInt32(ctx, address + pc, out var word)) { error = $"read-failed pc=0x{pc:X}"; return false; @@ -318,7 +447,7 @@ public static class Gen5ShaderTranslator words[0] = word; for (uint wordIndex = 1; wordIndex < sizeDwords; wordIndex++) { - if (!ctx.TryReadUInt32(address + pc + wordIndex * sizeof(uint), out words[wordIndex])) + if (!TryReadUInt32(ctx, address + pc + wordIndex * sizeof(uint), out words[wordIndex])) { error = $"read-failed pc=0x{pc + wordIndex * sizeof(uint):X}"; return false; @@ -340,6 +469,52 @@ public static class Gen5ShaderTranslator return false; } + [Conditional("DEBUG")] + private static void ValidateDppControlVectors() + { + if (System.Threading.Interlocked.Exchange(ref _dppVectorsValidated, 1) != 0) + { + return; + } + + static (uint Lane, bool InRange) Resolve(uint control, uint lane) + { + var rowBase = lane & ~15u; + var rowLane = lane & 15u; + return control switch + { + >= 0x101 and <= 0x10F => ( + rowBase + ((rowLane + (control & 15)) & 15), + rowLane + (control & 15) < 16), + >= 0x111 and <= 0x11F => ( + rowBase + ((rowLane - (control & 15)) & 15), + rowLane >= (control & 15)), + >= 0x121 and <= 0x12F => ( + rowBase + ((rowLane - (control & 15)) & 15), + true), + 0x140 => (rowBase + 15 - rowLane, true), + 0x141 => ((lane & ~7u) + 7 - (lane & 7), true), + >= 0x150 and <= 0x15F => (rowBase + (control & 15), true), + >= 0x160 and <= 0x16F => (rowBase + (rowLane ^ (control & 15)), true), + _ => (lane, false), + }; + } + + Debug.Assert(Resolve(0x101, 0) == (1u, true)); + Debug.Assert(Resolve(0x101, 15) == (0u, false)); + Debug.Assert(Resolve(0x112, 1) == (15u, false)); + Debug.Assert(Resolve(0x123, 0) == (13u, true)); + Debug.Assert(Resolve(0x140, 18) == (29u, true)); + Debug.Assert(Resolve(0x141, 9) == (14u, true)); + Debug.Assert(Resolve(0x153, 20) == (19u, true)); + Debug.Assert(Resolve(0x163, 22) == (21u, true)); + const uint dpp8 = + (7u << 0) | (6u << 3) | (5u << 6) | (4u << 9) | + (3u << 12) | (2u << 15) | (1u << 18) | (0u << 21); + Debug.Assert(((dpp8 >> (0 * 3)) & 7) == 7); + Debug.Assert(((dpp8 >> (7 * 3)) & 7) == 0); + } + private static bool TryDecodeInstruction( CpuContext ctx, ulong baseAddress, @@ -398,7 +573,7 @@ public static class Gen5ShaderTranslator case 0x34: case 0x35: encoding = Gen5ShaderEncoding.Vop3; - if (!ctx.TryReadUInt32(baseAddress + pc + sizeof(uint), out var vop3Extra)) + if (!TryReadUInt32(ctx, baseAddress + pc + sizeof(uint), out var vop3Extra)) { error = $"vop3-extra-read-failed pc=0x{pc:X}"; return false; @@ -419,7 +594,7 @@ public static class Gen5ShaderTranslator return DecodeFlat(word, out name, out sizeDwords, out error); case 0x38: encoding = Gen5ShaderEncoding.Mubuf; - if (!ctx.TryReadUInt32(baseAddress + pc + sizeof(uint), out var mubufExtra)) + if (!TryReadUInt32(ctx, baseAddress + pc + sizeof(uint), out var mubufExtra)) { error = $"mubuf-extra-read-failed pc=0x{pc:X}"; return false; @@ -428,7 +603,7 @@ public static class Gen5ShaderTranslator return DecodeMubuf(word, mubufExtra, out name, out sizeDwords, out error); case 0x3A: encoding = Gen5ShaderEncoding.Mtbuf; - if (!ctx.TryReadUInt32(baseAddress + pc + sizeof(uint), out var mtbufExtra)) + if (!TryReadUInt32(ctx, baseAddress + pc + sizeof(uint), out var mtbufExtra)) { error = $"mtbuf-extra-read-failed pc=0x{pc:X}"; return false; @@ -455,6 +630,26 @@ public static class Gen5ShaderTranslator } } + // Kept beside the production decoder so offline compatibility tools use + // precisely the same opcode tables and instruction-width rules as runtime + // shader translation. + internal static bool TryDecodeInstructionForPreflight( + CpuContext ctx, + uint pc, + uint word, + out string name, + out uint sizeDwords, + out string error) => + TryDecodeInstruction( + ctx, + 0, + pc, + word, + out _, + out name, + out sizeDwords, + out error); + private static bool DecodeSop(uint word, out string name, out uint sizeDwords, out string error) { var opcode = (word >> 23) & 0x7F; @@ -498,6 +693,16 @@ public static class Gen5ShaderTranslator 0x2B => "SXnorSaveexecB64", 0x37 => "SAndn1SaveexecB64", 0x38 => "SOrn1SaveexecB64", + 0x3C => "SAndSaveexecB32", + 0x3D => "SOrSaveexecB32", + 0x3E => "SXorSaveexecB32", + 0x3F => "SAndn2SaveexecB32", + 0x40 => "SOrn2SaveexecB32", + 0x41 => "SNandSaveexecB32", + 0x42 => "SNorSaveexecB32", + 0x43 => "SXnorSaveexecB32", + 0x44 => "SAndn1SaveexecB32", + 0x45 => "SOrn1SaveexecB32", _ => string.Empty, }; @@ -662,7 +867,7 @@ public static class Gen5ShaderTranslator { var opcode = (word >> 9) & 0xFF; var src0 = word & 0x1FF; - sizeDwords = src0 is 0xF9 or 0xFA or 0xFF ? 2u : 1u; + sizeDwords = src0 is 0xE9 or 0xEA or 0xF9 or 0xFA or 0xFF ? 2u : 1u; error = string.Empty; name = opcode switch { @@ -721,7 +926,8 @@ public static class Gen5ShaderTranslator } var src0 = word & 0x1FF; - sizeDwords = opcode is 0x20 or 0x21 or 0x2C or 0x2D || src0 is 0xF9 or 0xFA or 0xFF ? 2u : 1u; + sizeDwords = opcode is 0x20 or 0x21 or 0x2C or 0x2D || + src0 is 0xE9 or 0xEA or 0xF9 or 0xFA or 0xFF ? 2u : 1u; error = string.Empty; name = opcode switch { @@ -748,6 +954,7 @@ public static class Gen5ShaderTranslator 0x1B => "VAndB32", 0x1C => "VOrB32", 0x1D => "VXorB32", + 0x1E => "VXnorB32", 0x1F => "VMacF32", 0x20 => "VMadMkF32", 0x21 => "VMadAkF32", @@ -761,8 +968,8 @@ public static class Gen5ShaderTranslator 0x29 => "VSubbU32", 0x2A => "VSubbrevU32", 0x2B => "VFmacF32", - 0x2C => "VFmamkF32", - 0x2D => "VFmaakF32", + 0x2C => "VFmaMkF32", + 0x2D => "VFmaAkF32", 0x2F => "VCvtPkrtzF16F32", 0x30 => "VCvtPkU16U32", 0x31 => "VCvtPkI16I32", @@ -776,7 +983,7 @@ public static class Gen5ShaderTranslator { var opcode = (word >> 17) & 0xFF; var src0 = word & 0x1FF; - sizeDwords = src0 is 0xF9 or 0xFA or 0xFF ? 2u : 1u; + sizeDwords = src0 is 0xE9 or 0xEA or 0xF9 or 0xFA or 0xFF ? 2u : 1u; error = string.Empty; name = opcode switch { @@ -868,9 +1075,11 @@ public static class Gen5ShaderTranslator name = isVop3B ? opcode switch { + 0x128 => "VAddCoCiU32", 0x30F => "VAddCoU32", 0x310 => "VSubCoU32", 0x319 => "VSubrevCoU32", + 0x176 => "VMadU64U32", _ => $"Vop3bRaw{opcode:X3}", } : opcode switch @@ -914,6 +1123,12 @@ public static class Gen5ShaderTranslator 0x360 => "VReadlaneB32", 0x361 => "VWritelaneB32", 0x362 => "VLdexpF32", + 0x363 => "VBfmB32", + 0x364 => "VBcntU32B32", + 0x365 => "VMbcntLoU32B32", + 0x366 => "VMbcntHiU32B32", + 0x368 => "VCvtPknormI16F32", + 0x369 => "VCvtPknormU16F32", 0x373 => "VMadU32U16", 0x346 => "VLshlAddU32", 0x347 => "VAddLshlU32", @@ -930,7 +1145,7 @@ public static class Gen5ShaderTranslator } private static bool IsVop3BOpcode(uint opcode) => - opcode is 0x16D or 0x16E or 0x176 or 0x177 or 0x30F or 0x310 or 0x319; + opcode is 0x128 or 0x16D or 0x16E or 0x176 or 0x177 or 0x30F or 0x310 or 0x319; private static bool DecodeRaw2( uint word, @@ -956,6 +1171,7 @@ public static class Gen5ShaderTranslator error = string.Empty; name = opcode switch { + 0x00 => "DsAddU32", 0x0D => "DsWriteB32", 0x0E => "DsWrite2B32", 0x0F => "DsWrite2St64B32", @@ -964,6 +1180,10 @@ public static class Gen5ShaderTranslator 0x37 => "DsRead2B32", 0x38 => "DsRead2St64B32", 0x4D => "DsWriteB64", + 0xDE => "DsWriteB96", + 0xDF => "DsWriteB128", + 0xFE => "DsReadB96", + 0xFF => "DsReadB128", _ => string.Empty, }; @@ -1026,15 +1246,34 @@ public static class Gen5ShaderTranslator 0x01 => "BufferLoadFormatXy", 0x02 => "BufferLoadFormatXyz", 0x03 => "BufferLoadFormatXyzw", + 0x04 => "BufferStoreFormatX", + 0x05 => "BufferStoreFormatXy", + 0x06 => "BufferStoreFormatXyz", + 0x07 => "BufferStoreFormatXyzw", + 0x08 => "BufferLoadUbyte", + 0x09 => "BufferLoadSbyte", + 0x0A => "BufferLoadUshort", + 0x0B => "BufferLoadSshort", 0x0C => "BufferLoadDword", 0x0D => "BufferLoadDwordx2", 0x0E => "BufferLoadDwordx4", 0x0F => "BufferLoadDwordx3", + 0x18 => "BufferStoreByte", + 0x19 => "BufferStoreByteD16Hi", + 0x1A => "BufferStoreShort", + 0x1B => "BufferStoreShortD16Hi", 0x1C => "BufferStoreDword", 0x1D => "BufferStoreDwordx2", 0x1E => "BufferStoreDwordx4", 0x1F => "BufferStoreDwordx3", + 0x20 => "BufferLoadUbyteD16", + 0x21 => "BufferLoadUbyteD16Hi", + 0x22 => "BufferLoadSbyteD16", + 0x23 => "BufferLoadSbyteD16Hi", + 0x24 => "BufferLoadShortD16", + 0x25 => "BufferLoadShortD16Hi", 0x32 => "BufferAtomicAdd", + 0x38 => "BufferAtomicUMax", _ => $"MubufRaw{opcode:X2}", }; sizeDwords = (extra >> 24) == 0xFF ? 3u : 2u; @@ -1055,10 +1294,30 @@ public static class Gen5ShaderTranslator name = segment == 0x2 ? opcode switch { + 0x08 => "GlobalLoadUbyte", + 0x09 => "GlobalLoadSbyte", + 0x0A => "GlobalLoadUshort", + 0x0B => "GlobalLoadSshort", 0x0C => "GlobalLoadDword", 0x0D => "GlobalLoadDwordx2", 0x0E => "GlobalLoadDwordx4", 0x0F => "GlobalLoadDwordx3", + 0x18 => "GlobalStoreByte", + 0x19 => "GlobalStoreByteD16Hi", + 0x1A => "GlobalStoreShort", + 0x1B => "GlobalStoreShortD16Hi", + 0x1C => "GlobalStoreDword", + 0x1D => "GlobalStoreDwordx2", + 0x1E => "GlobalStoreDwordx4", + 0x1F => "GlobalStoreDwordx3", + 0x20 => "GlobalLoadUbyteD16", + 0x21 => "GlobalLoadUbyteD16Hi", + 0x22 => "GlobalLoadSbyteD16", + 0x23 => "GlobalLoadSbyteD16Hi", + 0x24 => "GlobalLoadShortD16", + 0x25 => "GlobalLoadShortD16Hi", + 0x32 => "GlobalAtomicAdd", + 0x38 => "GlobalAtomicUMax", _ => string.Empty, } : string.Empty; @@ -1132,6 +1391,7 @@ public static class Gen5ShaderTranslator 0x10 => "ImageAtomicCmpswap", 0x1C => "ImageAtomicDec", 0x20 => "ImageSample", + 0x22 => "ImageSampleD", 0x24 => "ImageSampleL", 0x25 => "ImageSampleB", 0x27 => "ImageSampleLz", @@ -1143,6 +1403,7 @@ public static class Gen5ShaderTranslator 0x47 => "ImageGather4Lz", 0x48 => "ImageGather4C", 0x4E => "ImageGather4CBCl", + 0x57 => "ImageGather4LzO", 0x5F => "ImageGather4CLzO", _ => string.Empty, }; @@ -1223,6 +1484,7 @@ public static class Gen5ShaderTranslator name.StartsWith("Image", StringComparison.Ordinal); public static bool IsStorageImageOperation(string name) => + name.StartsWith("ImageLoad", StringComparison.Ordinal) || name.StartsWith("ImageStore", StringComparison.Ordinal) || name.StartsWith("ImageAtomic", StringComparison.Ordinal); @@ -1239,7 +1501,11 @@ public static class Gen5ShaderTranslator var isDpp = encoding is Gen5ShaderEncoding.Vop1 or Gen5ShaderEncoding.Vop2 or Gen5ShaderEncoding.Vopc && (word & 0x1FF) == 0xFA; - var literal = !isSdwa && !isDpp && words.Length > MinimumEncodingDwords(encoding) + var isDpp8 = + encoding is Gen5ShaderEncoding.Vop1 or Gen5ShaderEncoding.Vop2 or Gen5ShaderEncoding.Vopc && + (word & 0x1FF) is 0xE9 or 0xEA; + var literal = !isSdwa && !isDpp && !isDpp8 && + words.Length > MinimumEncodingDwords(encoding) ? words[^1] : (uint?)null; IReadOnlyList sources = []; @@ -1314,20 +1580,36 @@ public static class Gen5ShaderTranslator var scalarOffset = (extra >> 25) & 0x7F; var offset = SignExtend(extra & 0x1FFFFF, 21); var count = ScalarLoadDwordCount(opcode); + var scalarOffsetOperand = Gen5Operand.Source(scalarOffset); + var dynamicOffsetRegister = scalarOffsetOperand.Kind == + Gen5OperandKind.ScalarRegister + ? scalarOffsetOperand.Value + : (uint?)null; sources = [ Gen5Operand.Scalar(scalarBase), - Gen5Operand.Scalar(scalarOffset), + scalarOffsetOperand, ]; destinations = Enumerable .Range((int)scalarDestination, checked((int)count)) .Select(index => Gen5Operand.Scalar((uint)index)) .ToArray(); - control = new Gen5ScalarMemoryControl(count, offset, scalarOffset); + control = new Gen5ScalarMemoryControl( + count, + offset, + dynamicOffsetRegister); break; } case Gen5ShaderEncoding.Vop1: - if (isDpp) + if (isDpp8) + { + var extra = words[1]; + sources = [Gen5Operand.Vector(extra & 0xFF)]; + control = new Gen5Dpp8Control( + extra >> 8, + (word & 0x1FF) == 0xEA); + } + else if (isDpp) { var extra = words[1]; sources = [Gen5Operand.Vector(extra & 0xFF)]; @@ -1339,24 +1621,36 @@ public static class Gen5ShaderTranslator var source0 = (extra & 0xFF) + ((((extra >> 23) & 1) == 0) ? 256u : 0u); sources = [Gen5Operand.Source(source0)]; - control = new Gen5SdwaControl( - (extra >> 8) & 0x7, - (extra >> 16) & 0x7, - 6, - (extra >> 21) & 1, - (extra >> 20) & 1, - (extra >> 14) & 0x3, - ((extra >> 13) & 1) != 0); + control = CreateSdwaControl(extra, isCompare: false, hasSource1: false); } else { sources = [Gen5Operand.Source(word & 0x1FF, literal)]; } - destinations = [Gen5Operand.Vector((word >> 17) & 0xFF)]; + // V_READFIRSTLANE_B32 is encoded as VOP1, but its destination + // field names an SGPR rather than a VGPR. Treating it like an + // ordinary vector destination leaves every invocation with a + // different value and corrupts scalar addresses derived from + // lane data. + destinations = opcode == "VReadfirstlaneB32" + ? [Gen5Operand.Scalar((word >> 17) & 0x7F)] + : [Gen5Operand.Vector((word >> 17) & 0xFF)]; break; case Gen5ShaderEncoding.Vop2: - if (isDpp) + if (isDpp8) + { + var extra = words[1]; + sources = + [ + Gen5Operand.Vector(extra & 0xFF), + Gen5Operand.Vector((word >> 9) & 0xFF), + ]; + control = new Gen5Dpp8Control( + extra >> 8, + (word & 0x1FF) == 0xEA); + } + else if (isDpp) { var extra = words[1]; sources = @@ -1378,14 +1672,7 @@ public static class Gen5ShaderTranslator Gen5Operand.Source(source0), Gen5Operand.Source(source1), ]; - control = new Gen5SdwaControl( - (extra >> 8) & 0x7, - (extra >> 16) & 0x7, - (extra >> 24) & 0x7, - ((extra >> 21) & 1) | (((extra >> 29) & 1) << 1), - ((extra >> 20) & 1) | (((extra >> 28) & 1) << 1), - (extra >> 14) & 0x3, - ((extra >> 13) & 1) != 0); + control = CreateSdwaControl(extra, isCompare: false, hasSource1: true); } else { @@ -1394,7 +1681,7 @@ public static class Gen5ShaderTranslator Gen5Operand.Source(word & 0x1FF, literal), Gen5Operand.Vector((word >> 9) & 0xFF), ]; - if (opcode is "VMadMkF32" or "VFmamkF32" && literal.HasValue) + if ((opcode is "VMadMkF32" or "VFmaMkF32") && literal.HasValue) { sources = [ @@ -1403,7 +1690,7 @@ public static class Gen5ShaderTranslator sources[1], ]; } - else if (opcode is "VMadAkF32" or "VFmaakF32" && literal.HasValue) + else if ((opcode is "VMadAkF32" or "VFmaAkF32") && literal.HasValue) { sources = [ @@ -1416,7 +1703,19 @@ public static class Gen5ShaderTranslator destinations = [Gen5Operand.Vector((word >> 17) & 0xFF)]; break; case Gen5ShaderEncoding.Vopc: - if (isDpp) + if (isDpp8) + { + var extra = words[1]; + sources = + [ + Gen5Operand.Vector(extra & 0xFF), + Gen5Operand.Vector((word >> 9) & 0xFF), + ]; + control = new Gen5Dpp8Control( + extra >> 8, + (word & 0x1FF) == 0xEA); + } + else if (isDpp) { var extra = words[1]; sources = @@ -1439,14 +1738,13 @@ public static class Gen5ShaderTranslator Gen5Operand.Source(source0), Gen5Operand.Source(source1), ]; - control = new Gen5SdwaControl( - (extra >> 8) & 0x7, - (extra >> 16) & 0x7, - (extra >> 24) & 0x7, - ((extra >> 21) & 1) | (((extra >> 29) & 1) << 1), - ((extra >> 20) & 1) | (((extra >> 28) & 1) << 1), - (extra >> 14) & 0x3, - ((extra >> 13) & 1) != 0); + var sdwa = CreateSdwaControl(extra, isCompare: true, hasSource1: true); + control = sdwa; + if (sdwa.ScalarDestination is { } scalarDestination && + scalarDestination != 106) + { + destinations = [Gen5Operand.Scalar(scalarDestination)]; + } } else { @@ -1479,6 +1777,7 @@ public static class Gen5ShaderTranslator (extra >> 29) & 0x7, (extra >> 27) & 0x3, ((word >> 15) & 1) != 0, + isVop3B ? 0 : (word >> 11) & 0xF, isVop3B ? (word >> 8) & 0x7F : null); break; } @@ -1495,6 +1794,10 @@ public static class Gen5ShaderTranslator ((word >> 17) & 1) != 0); sources = opcode switch { + "DsAddU32" => [ + Gen5Operand.Vector(vectorAddress), + Gen5Operand.Vector(vectorData0), + ], "DsWriteB32" => [ Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorData0), @@ -1504,6 +1807,19 @@ public static class Gen5ShaderTranslator Gen5Operand.Vector(vectorData0), Gen5Operand.Vector(vectorData0 + 1), ], + "DsWriteB96" => [ + Gen5Operand.Vector(vectorAddress), + Gen5Operand.Vector(vectorData0), + Gen5Operand.Vector(vectorData0 + 1), + Gen5Operand.Vector(vectorData0 + 2), + ], + "DsWriteB128" => [ + Gen5Operand.Vector(vectorAddress), + Gen5Operand.Vector(vectorData0), + Gen5Operand.Vector(vectorData0 + 1), + Gen5Operand.Vector(vectorData0 + 2), + Gen5Operand.Vector(vectorData0 + 3), + ], "DsWrite2B32" or "DsWrite2St64B32" => [ Gen5Operand.Vector(vectorAddress), Gen5Operand.Vector(vectorData0), @@ -1521,6 +1837,17 @@ public static class Gen5ShaderTranslator Gen5Operand.Vector(vectorDestination), Gen5Operand.Vector(vectorDestination + 1), ], + "DsReadB96" => [ + Gen5Operand.Vector(vectorDestination), + Gen5Operand.Vector(vectorDestination + 1), + Gen5Operand.Vector(vectorDestination + 2), + ], + "DsReadB128" => [ + Gen5Operand.Vector(vectorDestination), + Gen5Operand.Vector(vectorDestination + 1), + Gen5Operand.Vector(vectorDestination + 2), + Gen5Operand.Vector(vectorDestination + 3), + ], _ => [], }; break; @@ -1540,10 +1867,30 @@ public static class Gen5ShaderTranslator var scalarAddress = (extra >> 16) & 0x7F; var dwordCount = opcode switch { + "GlobalLoadUbyte" or + "GlobalLoadSbyte" or + "GlobalLoadUshort" or + "GlobalLoadSshort" or + "GlobalLoadUbyteD16" or + "GlobalLoadUbyteD16Hi" or + "GlobalLoadSbyteD16" or + "GlobalLoadSbyteD16Hi" or + "GlobalLoadShortD16" or + "GlobalLoadShortD16Hi" or + "GlobalStoreByte" or + "GlobalStoreByteD16Hi" or + "GlobalStoreShort" or + "GlobalStoreShortD16Hi" or + "GlobalStoreDword" or + "GlobalAtomicAdd" or + "GlobalAtomicUMax" => 1u, "GlobalLoadDword" => 1u, "GlobalLoadDwordx2" => 2u, "GlobalLoadDwordx3" => 3u, "GlobalLoadDwordx4" => 4u, + "GlobalStoreDwordx2" => 2u, + "GlobalStoreDwordx3" => 3u, + "GlobalStoreDwordx4" => 4u, _ => 0u, }; sources = @@ -1551,10 +1898,12 @@ public static class Gen5ShaderTranslator Gen5Operand.Vector(vectorAddress), Gen5Operand.Scalar(scalarAddress), ]; - destinations = Enumerable - .Range((int)vectorData, checked((int)dwordCount)) - .Select(index => Gen5Operand.Vector((uint)index)) - .ToArray(); + destinations = opcode.StartsWith("GlobalLoad", StringComparison.Ordinal) + ? Enumerable + .Range((int)vectorData, checked((int)dwordCount)) + .Select(index => Gen5Operand.Vector((uint)index)) + .ToArray() + : []; control = new Gen5GlobalMemoryControl( dwordCount, vectorAddress, @@ -1578,6 +1927,24 @@ public static class Gen5ShaderTranslator "BufferLoadFormatXy" => 2u, "BufferLoadFormatXyz" => 3u, "BufferLoadFormatXyzw" => 4u, + "BufferStoreFormatX" => 1u, + "BufferStoreFormatXy" => 2u, + "BufferStoreFormatXyz" => 3u, + "BufferStoreFormatXyzw" => 4u, + "BufferLoadUbyte" or + "BufferLoadSbyte" or + "BufferLoadUshort" or + "BufferLoadSshort" or + "BufferStoreByte" or + "BufferStoreByteD16Hi" or + "BufferStoreShort" or + "BufferStoreShortD16Hi" or + "BufferLoadUbyteD16" or + "BufferLoadUbyteD16Hi" or + "BufferLoadSbyteD16" or + "BufferLoadSbyteD16Hi" or + "BufferLoadShortD16" or + "BufferLoadShortD16Hi" => 1u, "BufferLoadDword" => 1u, "BufferLoadDwordx2" => 2u, "BufferLoadDwordx3" => 3u, @@ -1587,6 +1954,7 @@ public static class Gen5ShaderTranslator "BufferStoreDwordx3" => 3u, "BufferStoreDwordx4" => 4u, "BufferAtomicAdd" => 1u, + "BufferAtomicUMax" => 1u, _ => 0u, }; sources = @@ -1690,7 +2058,9 @@ public static class Gen5ShaderTranslator dimension, dimension is 4 or 5 or 7, ((word >> 13) & 1) != 0, - ((word >> 25) & 1) != 0); + ((word >> 25) & 1) != 0, + ((extra >> 30) & 1) != 0, + ((extra >> 31) & 1) != 0); break; } case Gen5ShaderEncoding.Exp: @@ -1726,6 +2096,30 @@ public static class Gen5ShaderTranslator (word >> 24) & 0xF, (word >> 28) & 0xF); + private static Gen5SdwaControl CreateSdwaControl( + uint word, + bool isCompare, + bool hasSource1) + { + var scalarDestination = isCompare + ? ((word >> 15) & 1) != 0 + ? (word >> 8) & 0x7Fu + : 106u + : (uint?)null; + return new Gen5SdwaControl( + isCompare ? 6u : (word >> 8) & 0x7u, + isCompare ? 0u : (word >> 11) & 0x3u, + (word >> 16) & 0x7u, + hasSource1 ? (word >> 24) & 0x7u : 6u, + ((word >> 19) & 1) != 0, + hasSource1 && ((word >> 27) & 1) != 0, + ((word >> 21) & 1) | (hasSource1 ? ((word >> 29) & 1) << 1 : 0), + ((word >> 20) & 1) | (hasSource1 ? ((word >> 28) & 1) << 1 : 0), + isCompare ? 0u : (word >> 14) & 0x3u, + !isCompare && ((word >> 13) & 1) != 0, + scalarDestination); + } + private static int MinimumEncodingDwords(Gen5ShaderEncoding encoding) => encoding switch { Gen5ShaderEncoding.Vop3 or @@ -1762,6 +2156,19 @@ public static class Gen5ShaderTranslator counts[key] = count + 1; } + private static bool TryReadUInt32(CpuContext ctx, ulong address, out uint value) + { + Span bytes = stackalloc byte[sizeof(uint)]; + if (!ctx.Memory.TryRead(address, bytes)) + { + value = 0; + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(bytes); + return true; + } + private readonly record struct ShaderDecodeInfo( int InstructionCount, Dictionary Counts, @@ -1824,6 +2231,7 @@ public static class Gen5ShaderTranslator $"va={addressRegisters},vd=v{image.VectorData}," + $"sr=s{image.ScalarResource},ss=s{image.ScalarSampler}," + $"dim={image.Dimension},da={(image.IsArray ? 1 : 0)}," + + $"a16={(image.A16 ? 1 : 0)},d16={(image.D16 ? 1 : 0)}," + $"glc={(image.Glc ? 1 : 0)}," + $"slc={(image.Slc ? 1 : 0)}"; } diff --git a/src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs b/src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs new file mode 100644 index 0000000..146ff0d --- /dev/null +++ b/src/SharpEmu.ShaderCompiler/Gfx10UnifiedFormat.cs @@ -0,0 +1,138 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +namespace SharpEmu.ShaderCompiler; + +/// +/// Converts the RDNA2 unified FORMAT field used by GFX10 buffer and image +/// descriptors into the data-format/number-format pair used by the rest of +/// SharpEmu's AGC pipeline. +/// +public static class Gfx10UnifiedFormat +{ + public static bool TryDecode( + uint unifiedFormat, + out uint dataFormat, + out uint numberFormat) + { + // RDNA2 ISA table 47 is intentionally sparse. In particular, the + // integer/normalized spellings that some assemblers expose for + // 10_11_11 and 11_11_10 are reserved by the hardware data-format + // table; so are 10_10_10_2 USCALED/SSCALED. Keep this an exact table + // instead of deriving ranges that accidentally make reserved encodings + // look valid. + (dataFormat, numberFormat) = unifiedFormat switch + { + 0 => (0u, 0u), + 1 => (1u, 0u), + 2 => (1u, 1u), + 3 => (1u, 2u), + 4 => (1u, 3u), + 5 => (1u, 4u), + 6 => (1u, 5u), + 7 => (2u, 0u), + 8 => (2u, 1u), + 9 => (2u, 2u), + 10 => (2u, 3u), + 11 => (2u, 4u), + 12 => (2u, 5u), + 13 => (2u, 7u), + 14 => (3u, 0u), + 15 => (3u, 1u), + 16 => (3u, 2u), + 17 => (3u, 3u), + 18 => (3u, 4u), + 19 => (3u, 5u), + 20 => (4u, 4u), + 21 => (4u, 5u), + 22 => (4u, 7u), + 23 => (5u, 0u), + 24 => (5u, 1u), + 25 => (5u, 2u), + 26 => (5u, 3u), + 27 => (5u, 4u), + 28 => (5u, 5u), + 29 => (5u, 7u), + 36 => (6u, 7u), + 43 => (7u, 7u), + 44 => (8u, 0u), + 45 => (8u, 1u), + 48 => (8u, 4u), + 49 => (8u, 5u), + 50 => (9u, 0u), + 51 => (9u, 1u), + 52 => (9u, 2u), + 53 => (9u, 3u), + 54 => (9u, 4u), + 55 => (9u, 5u), + 56 => (10u, 0u), + 57 => (10u, 1u), + 58 => (10u, 2u), + 59 => (10u, 3u), + 60 => (10u, 4u), + 61 => (10u, 5u), + 62 => (11u, 4u), + 63 => (11u, 5u), + 64 => (11u, 7u), + 65 => (12u, 0u), + 66 => (12u, 1u), + 67 => (12u, 2u), + 68 => (12u, 3u), + 69 => (12u, 4u), + 70 => (12u, 5u), + 71 => (12u, 7u), + 72 => (13u, 4u), + 73 => (13u, 5u), + 74 => (13u, 7u), + 75 => (14u, 4u), + 76 => (14u, 5u), + 77 => (14u, 7u), + 128 => (1u, 9u), + 129 => (3u, 9u), + 130 => (10u, 9u), + // Image-only encodings without a legacy DATA_FORMAT equivalent + // retain the unified identifier for exact downstream dispatch. + 131 => (131u, 7u), + 132 => (34u, 7u), + 133 => (16u, 0u), + 134 => (17u, 0u), + 135 => (18u, 0u), + 136 => (19u, 0u), + 137 => (137u, 0u), + 138 => (138u, 0u), + 139 => (139u, 0u), + 140 => (4u, 7u), + 141 => (20u, 0u), + 142 => (20u, 4u), + 143 => (21u, 0u), + 144 => (21u, 4u), + 145 => (22u, 4u), + 146 => (22u, 7u), + 147 => (32u, 0u), + 148 => (32u, 1u), + 149 => (32u, 4u), + 150 => (32u, 9u), + 151 => (33u, 0u), + 152 => (33u, 1u), + 153 => (33u, 4u), + 154 => (33u, 9u), + 169 => (169u, 0u), + 170 => (170u, 9u), + 171 => (171u, 0u), + 172 => (172u, 9u), + 173 => (173u, 0u), + 174 => (174u, 9u), + 175 => (175u, 0u), + 176 => (176u, 1u), + 177 => (177u, 0u), + 178 => (178u, 1u), + 179 => (179u, 7u), + 180 => (180u, 7u), + 181 => (181u, 0u), + 182 => (182u, 9u), + _ => (0u, 0u), + }; + + return unifiedFormat == 0 || dataFormat != 0; + } +} diff --git a/tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs b/tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs new file mode 100644 index 0000000..db00bf1 --- /dev/null +++ b/tests/SharpEmu.Libs.Tests/Tls/GuestTlsTemplateTests.cs @@ -0,0 +1,32 @@ +// Copyright (C) 2026 SharpEmu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +using SharpEmu.HLE; +using Xunit; + +namespace SharpEmu.Libs.Tests.Tls; + +public sealed class GuestTlsTemplateTests +{ + [Fact] + public void StartupReservationAcceptsTlsSpansLargerThanOneHostPage() + { + try + { + GuestTlsTemplate.Reset(); + + var staticOffset = GuestTlsTemplate.RegisterModule( + moduleId: 1, + initImage: new byte[0x20], + memorySize: 0x1870, + alignment: 0x10); + + Assert.Equal(0x1870UL, staticOffset); + Assert.True(staticOffset <= GuestTlsTemplate.StartupStaticTlsReservation); + } + finally + { + GuestTlsTemplate.Reset(); + } + } +} diff --git a/tools/SharpEmu.Tools.ShaderDump/Program.cs b/tools/SharpEmu.Tools.ShaderDump/Program.cs index 5da8bcf..47612f1 100644 --- a/tools/SharpEmu.Tools.ShaderDump/Program.cs +++ b/tools/SharpEmu.Tools.ShaderDump/Program.cs @@ -192,14 +192,22 @@ foreach (var (name, expectTranslate, words) in testPrograms) // the hand-assembled buffer_store words, and the 64-byte backing store // must cover every hand-assembled store offset. var globalBindings = storePcs.Count > 0 - ? new[] { new Gen5GlobalMemoryBinding(8u, 0UL, storePcs, new byte[64]) } + ? new[] + { + new Gen5GlobalMemoryBinding( + 8u, + 0UL, + storePcs, + new byte[64], + 64, + false), + } : Array.Empty(); var state = new Gen5ShaderState(program, new uint[16], Metadata: null); var evaluation = new Gen5ShaderEvaluation( new uint[256], new uint[256], - new Dictionary>(), Array.Empty(), globalBindings);